Update README.md

changed links to the community help repo

.dockerignore

@@ -1 +0,0 @@
-.*

.github/ISSUE_TEMPLATE.md

@@ -1,16 +0,0 @@
-For bugs, please provide the following:
-
-### Expected behavior
-
-### Actual behavior
-
-### Steps to reproduce the behavior
-
-### Information about your environment (OS/architecture, CPU, RAM, cluster/solo, configuration, etc.)
-
-
-For feature requests, please provide the following:
-
-### Description
-
-### Success criteria (What criteria will consider this ticket closeable?)

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,24 +0,0 @@
-## Overview
-
-[Describe what this pull request addresses.]
-
-Fixes #
-
-## Pull request checklist
-
-- [ ] I have read the [contributing guide](https://github.com/pilosa/pilosa/blob/master/CONTRIBUTING.md).
-- [ ] I have agreed to the [Contributor License Agreement](https://cla-assistant.io/pilosa/pilosa).
-- [ ] I have updated the [documentation](https://github.com/pilosa/pilosa/tree/master/docs).
-- [ ] I have resolved any merge conflicts.
-- [ ] I have included tests that cover my changes.
-- [ ] All new and existing tests pass.
-
-## Code review checklist
-This is the checklist that the reviewer will follow while reviewing your pull request. You do not need to do anything with this checklist, but be aware of what the reviewer will be looking for.
-
-- [ ] Ensure that any changes to external docs have been included in this pull request.
-- [ ] If the changes require that minor/major versions need to be updated, tag the PR appropriately.
-- [ ] Ensure the new code is [properly commented](https://github.com/golang/go/wiki/CodeReviewComments#doc-comments) and follows [Idiomatic Go](https://dmitri.shuralyov.com/idiomatic-go).
-- [ ] Check that tests have been written and that they cover the new functionality.
-- [ ] Run tests and ensure they pass.
-- [ ] Build and run the code, performing any applicable integration testing.

.github/workflows/cd.yml

@@ -0,0 +1,77 @@
+name: CD
+
+on:
+  push:
+    branches: ["master"]
+
+  workflow_dispatch:
+
+jobs:
+  validate:
+    runs-on: ubuntu-latest
+
+    steps:
+      # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
+      - uses: actions/checkout@v3
+
+      - uses: actions/setup-go@v3
+        with:
+          go-version: "^1.19.1"
+
+      - run: go version
+
+      - name: golangci-lint
+        uses: golangci/golangci-lint-action@v3
+        with:
+          args: --timeout=5m
+
+      - name: go vet
+        run: go vet ./...
+
+      - name: test
+        run: go test ./...
+
+  release:
+    needs: validate
+    runs-on: ubuntu-latest
+    outputs:
+      version: ${{ steps.semrel.outputs.version }}
+
+    steps:
+      - name: go-semantic-release
+        id: semrel
+        uses: go-semantic-release/action@v1.17.0
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+
+  build:
+    needs: release
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        goos:
+          - "darwin"
+          - "linux"
+          - "windows"
+        goarch:
+          - "amd64"
+          - "arm64"
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - uses: actions/setup-go@v3
+        with:
+          go-version: "^1.19.1"
+
+      - name: build
+        run: GOOS=${{ matrix.goos }} GOARCH=${{ matrix.goarch }} go build -o ./build/${{ matrix.goos }}-${{ matrix.goarch }}
+
+      - name: Upload binaries to release
+        uses: svenstaro/upload-release-action@v2
+        with:
+          repo_token: ${{ secrets.GITHUB_TOKEN }}
+          file: ./build/${{ matrix.goos }}-${{ matrix.goarch }}
+          asset_name: ${{ matrix.goos }}-${{ matrix.goarch }}
+          tag: ${{ github.ref }}
+

.github/workflows/ci.yml

@@ -0,0 +1,57 @@
+# This is a basic workflow to help you get started with Actions
+
+name: CI
+
+# Controls when the workflow will run
+on:
+  pull_request:
+    branches: ["master"]
+
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+
+# A workflow run is made up of one or more jobs that can run sequentially or in parallel
+jobs:
+  validate-title:
+    name: Validate PR title
+    runs-on: ubuntu-latest
+    steps:
+      - uses: go-semantic-release/action@v1
+        id: semrel
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          dry: true
+
+  # golangci-lint must be run separately from "validate" as there are go mod issues if you run it after the vet
+  golangci:
+    name: lint
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/setup-go@v3
+        with:
+          go-version: 1.19
+      - uses: actions/checkout@v3
+      - name: golangci-lint
+        uses: golangci/golangci-lint-action@v3
+        with:
+          # Optional: version of golangci-lint to use in form of v1.2 or v1.2.3 or `latest` to use the latest version
+          # version: v1.29
+          args: --timeout=8m
+
+  validate:
+    name: Code Checks
+    runs-on: ubuntu-latest
+
+    # Steps represent a sequence of tasks that will be executed as part of the job
+    steps:
+      # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
+      - uses: actions/checkout@v3
+
+      - uses: actions/setup-go@v3
+        with:
+          go-version: "^1.19.1"
+
+      - run: go version
+
+      - name: go vet
+        run: go vet ./...

.github/workflows/release.yml

@@ -0,0 +1,25 @@
+name: Release
+
+on: workflow_dispatch
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        goos:
+          - "darwin"
+          - "linux"
+        goarch:
+          - "amd64"
+          - "arm64"
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - uses: actions/setup-go@v3
+        with:
+          go-version: "^1.19.1"
+
+      - name: build
+        run: GOOS=${{ matrix.goos }} GOARCH=${{ matrix.goarch }} go build -o ./build/${{ matrix.goos }}-${{ matrix.goarch }}

.gitignore

@@ -4,3 +4,84 @@ vendor
 .protoc-gen-gofast
 .DS_Store
 build
+*~
+release-pilosa-fsck.*.*.tar.gz
+/log.*
+/tourna.log.*
+pilosa
+/featurebase
+*.dot
+.idea/
+.*.swp
+.terraform/
+*.tfstate
+launch.json
+.terraform.lock.hcl
+__pycache__/
+report.xml
+outputs.json
+builds/
+*.tfstate.backup
+.vscode
+
+batch/testdata/batch*.out
+
+idk/testdata/idk*.out
+idk/testenv/certs/*
+
+# copy of .gitignore from archived idk repo
+# Compiled Object files, Static and Dynamic libs (Shared Objects)
+*.o
+*.a
+*.so
+
+# Folders
+_obj
+_test
+
+# Architecture specific extensions/prefixes
+*.[568vq]
+[568vq].out
+
+*.cgo1.go
+*.cgo2.c
+_cgo_defun.c
+_cgo_gotypes.go
+_cgo_export.*
+
+_testmain.go
+
+*.exe
+*.test
+*.prof
+
+vendor
+
+.terraform
+terraform.tfstate*
+
+bin
+build
+testenv
+.pulled
+
+pilosa-sec-data-idk
+
+.idea/
+tags.dot
+*.log
+*.swp
+*__debug_bin
+
+# SQL3
+/sql3/sql3.html
+
+staticcheck.conf
+
+
+.quick
+dax/dax-data
+
+coverage-from-docker
+*.client_id.txt
+

.gitlab/.gitlab-ci.yml

@@ -0,0 +1,747 @@
+# You will see a couple of instances of:
+# PKG_LIST=$(go list ./... | grep -Ev 'internal/clustertests|simulacraData|batch|idk|v3/dax/test/dax' | paste -s -d, -)
+# This gets us a package list, comma-separated, which excludes the batch
+# and IDK tests, and a couple of subdirs with specialized stuff. We can
+# then use this with -coverpkg, or we can use ${PKG_LIST//,/ } to get a
+# space-separated list for use with `go test` to run the tests for those
+# directories only.
+
+include:
+  - local: /.gitlab/batch-ci.yml
+  - template: Security/SAST.gitlab-ci.yml
+  - template: Security/License-Scanning.gitlab-ci.yml
+  - template: Security/Dependency-Scanning.gitlab-ci.yml
+
+default:
+  retry:
+    max: 2 # This is confusing but this means "3 runs at max".
+    when:
+      - unknown_failure
+      - api_failure
+      - runner_system_failure
+      - job_execution_timeout
+      - stuck_or_timeout_failure
+
+variables:
+  GOVERSION: "1.19.3"
+  GOFUTURE: "latest"
+  CI_IMAGE: "${CI_REGISTRY_IMAGE}/ci-builder:0.0.1"
+  CI_PRE_CLONE_SCRIPT: |
+    set -x
+
+stages:
+  - ci_image_build
+  - lint
+  - test
+  - build
+  - post build
+  - integration
+  - gauntlet
+  - performance
+  - nonblocking
+  - cleanup_build
+
+gosec-sast:
+  allow_failure: false
+  before_script:
+    - export GOPRIVATE=github.com/molecula/*
+    - apk add openssh-client
+    - eval $(ssh-agent -s)
+    - echo "$FB_SSH_PRIVATE_KEY" | tr -d '\r' | ssh-add -
+    - mkdir -p ~/.ssh
+    - chmod 700 ~/.ssh
+    - git config --global --add url."ssh://git@github.com/".insteadOf "https://github.com/"
+    - ssh-keygen -F github.com || echo "$SSH_KNOWN_HOSTS_HASHED" >> ~/.ssh/known_hosts
+    - chmod 644 ~/.ssh/known_hosts
+  script:
+    ## securego/gosec works for scanning, but not converting to the gitlab report format.
+    - go install github.com/securego/gosec/v2/cmd/gosec@v2.12.0
+    - gosec -fmt=json -out=gosec.json -tests ./... || true
+    ## gitlab's wrapper for gosec works for converting, but not for scanning.
+    - go install 'gitlab.com/gitlab-org/security-products/analyzers/gosec@v1.4.0'
+    - gosec convert gosec.json > gl-sast-report.json
+
+.go-cache:
+  variables:
+    GOPATH: $CI_PROJECT_DIR/.go
+  before_script:
+    - mkdir -p .go
+  cache:
+    # this caching strategy makes it so each branch uses the same cache
+    key: "$CI_COMMIT_REF_SLUG"
+    paths:
+      - .go/pkg/mod/
+
+smoke build:
+  image: golang:$GOVERSION
+  extends: .go-cache
+  stage: lint
+  allow_failure: false
+  rules:
+    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
+  script:
+    - echo "Let's just see if it compiles... (sometimes the linter gives unclear errors if it doesn't)"
+    - go build ./...
+
+golangci-lint:
+  image: golangci/golangci-lint:v1.46.2
+  extends: .go-cache
+  stage: lint
+  allow_failure: true
+  rules:
+    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
+  script:
+    - echo "Checking for issues in new code"
+    - golangci-lint run -v --timeout=8m
+
+go mod tidy:
+  stage: lint
+  image: golang:$GOVERSION
+  extends: .go-cache
+  rules:
+    - if: '$CI_COMMIT_TAG == null && ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web")'
+  script:
+    - go mod tidy
+    - git diff --exit-code -- go.mod go.sum
+
+build lattice:
+  stage: test
+  image: node:14
+  variables:
+    AWS_PROFILE: "service-fb-ci"
+    AWS_ACCESS_KEY_ID: $AWS_FBCI_ACCESS_KEY_ID
+    AWS_SECRET_ACCESS_KEY: $AWS_FBCI_SECRET_ACCESS_KEY
+    CI: "false"
+  rules:
+    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
+  before_script:
+    - curl -sS "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
+    - unzip -qq awscliv2.zip
+    - ./aws/install
+    - aws --version
+    - aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID
+    - aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY
+    - aws configure set region "us-east-2"
+    - aws configure set aws_profile $AWS_PROFILE
+    - aws sts get-caller-identity # ensure we have a valid AWS login
+  script:
+    - cd lattice
+    - cache=$(find . -type f -print0 | sort -z | xargs -0 sha1sum | sha1sum | cut -d ' ' -f 1)
+    - echo "'$cache'"
+    - echo "looking for s3://molecula-artifact-storage/lattice/$cache/build.tar.gz"
+    # if is for if we had a cache object in S3
+    # else is for if we didn't have a cache object (and have to build).
+    - |
+      if aws s3api head-object --bucket molecula-artifact-storage --key "lattice/$cache/build.tar.gz"; then
+        # download object, extract, name the folder `build`
+        aws s3 cp "s3://molecula-artifact-storage/lattice/$cache/build.tar.gz" build.tar.gz
+        tar -xf build.tar.gz
+      else # cache file not found
+        yarn install --frozen-lockfile # CI needs to enforce that the lockfile doesn't need to be updated
+        yarn build
+        tar -czvf "$cache.tar.gz" build/
+        aws s3 mv "$cache.tar.gz" "s3://molecula-artifact-storage/lattice/$cache/build.tar.gz"
+        touch "$CI_COMMIT_SHA"
+        aws s3 mv "$CI_COMMIT_SHA" "s3://molecula-artifact-storage/lattice/$cache/$CI_COMMIT_SHA"
+      fi
+    - | # Ensure that we have build directory after the caching step
+      if [ ! -d build ]; then
+        echo "no build directory, erroring out" || exit 1
+      fi
+    - mv build ../
+    - cd ../
+    - rm -r lattice
+    - mv build lattice
+    - tar -czvf lattice.tar.gz lattice
+  artifacts:
+    paths:
+      - lattice.tar.gz
+
+build featurebase:
+  stage: test
+  image: golang:$GOVERSION
+  extends: .go-cache
+  rules:
+    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
+  script:
+    - rm -r lattice
+    - tar -xvf lattice.tar.gz
+    - go install github.com/rakyll/statik@v0.1.7
+    - $GOPATH/bin/statik -src=lattice
+    - export SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct)
+    - |
+      for goos in "darwin" "linux"; do
+        for goarch in "amd64" "arm64"; do
+          GOOS="${goos}" GOARCH="${goarch}" make build FLAGS="-o featurebase_${goos}_${goarch}"
+        done
+      done
+  artifacts:
+    paths:
+      - featurebase_*
+  needs:
+    - job: build lattice
+
+build fbsql amd64:
+  stage: test
+  variables:
+    BUILD_NAME: build_${CI_COMMIT_SHA}_${CI_CONCURRENT_ID}
+  tags:
+    - shell
+  rules:
+    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
+  script:
+    - export SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct)
+    - date
+    - GOOS="linux" GOARCH="amd64" make docker-build-fbsql BUILD_CGO=1
+    - GOOS="darwin" GOARCH="amd64" make docker-build-fbsql
+  artifacts:
+    paths:
+      - ./build/fbsql_*
+
+build fbsql arm64:
+  stage: test
+  variables:
+    BUILD_NAME: build_${CI_COMMIT_SHA}_${CI_CONCURRENT_ID}
+  tags:
+    - shell-arm64
+  rules:
+    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
+  script:
+    - export SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct)
+    - date
+    - GOOS="linux" GOARCH="arm64" make docker-build-fbsql BUILD_CGO=1
+    - GOOS="darwin" GOARCH="arm64" make docker-build-fbsql
+  artifacts:
+    paths:
+      - ./build/fbsql_*
+
+build amd container fb:
+  stage: test
+  tags:
+    - shell
+  rules:
+    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
+  before_script:
+    - echo "${DOCKER_DEPLOY_TOKEN}" | docker login -u ${DOCKER_DEPLOY_USER} --password-stdin ${CI_REGISTRY}
+  script:
+    - tag=${CI_REGISTRY_IMAGE}/featurebase:linux-amd64-${CI_COMMIT_REF_SLUG}
+    - export SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct)
+    - docker build --build-arg GO_VERSION=$GOVERSION --build-arg ARCH=amd64 --build-arg SOURCE_DATE_EPOCH=$SOURCE_DATE_EPOCH -t $tag -f .gitlab/Dockerfile .
+    - docker push $tag
+    - echo Created docker featurebase image with tag "$tag"
+  needs:
+    - job: build featurebase
+
+run jest tests:
+  stage: test
+  image: node:14
+  variables:
+    CI: "true"
+  rules:
+    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
+  script:
+    - echo "Testing lattice..."
+    - cd lattice
+    - npm install --force
+    - npm test -- --coverage --testResultsProcessor=jest-sonar-reporter
+  artifacts:
+    paths:
+      - lattice/coverage/lcov.info
+
+# We run go test -race on all the standard packages, skipping the ones that have their
+# own separate tests. we spin this off as nonblocking because it used to take a really
+# long time and even now it's pretty slow.
+run go tests race:
+  stage: nonblocking # don't let this job block any other jobs because it takes much longer than the other tests.
+  image: golang:$GOVERSION
+  extends: .go-cache
+  variables:
+    FEATUREBASE_CONTROLLER_CONFIG_SQLDB_DATABASE: run_go_tests_race
+    POSTGRES_DB: run_go_tests_race
+    FEATUREBASE_CONTROLLER_CONFIG_SQLDB_USER: postgres
+    POSTGRES_USER: postgres
+    FEATUREBASE_CONTROLLER_CONFIG_SQLDB_PASSWORD: $POSTGRES_PASSWORD
+    POSTGRES_PASSWORD: $POSTGRES_PASSWORD
+    FEATUREBASE_CONTROLLER_CONFIG_SQLDB_HOST: postgres
+  services:
+    - postgres:14.7
+  rules:
+    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
+  needs: ["smoke build"] # we do block on smoke build though bc it's pretty dumb to test stuff if it doesn't build
+  script:
+    - echo "Running featurebase race tests..."
+    - PKG_LIST=$(go list ./... | grep -Ev 'internal/clustertests|simulacraData|batch|idk|v3/dax/test/dax' | paste -s -d, -)
+    - export TMPDIR=/mnt/ramdisk/test-$CI_JOB_ID
+    - mkdir -p $TMPDIR
+    - go test -race -v -timeout=10m ${PKG_LIST//,/ }
+  after_script:
+    - rm -rf /mnt/ramdisk/test-$CI_JOB_ID
+  tags:
+    - docker
+
+# We run our base tests against $GOVERSION (a reasonably current version that we trust)
+# and use shardwidth22 for them. This gives us a canary for things breaking for
+# unusual shard widths.
+run go tests:
+  stage: test
+  image: golang:$GOVERSION
+  extends: .go-cache
+  variables:
+    FEATUREBASE_CONTROLLER_CONFIG_SQLDB_DATABASE: run_go_tests
+    POSTGRES_DB: run_go_tests
+    FEATUREBASE_CONTROLLER_CONFIG_SQLDB_USER: postgres
+    POSTGRES_USER: postgres
+    FEATUREBASE_CONTROLLER_CONFIG_SQLDB_PASSWORD: $POSTGRES_PASSWORD
+    POSTGRES_PASSWORD: $POSTGRES_PASSWORD
+    FEATUREBASE_CONTROLLER_CONFIG_SQLDB_HOST: postgres
+  services:
+    - postgres:14.7
+  rules:
+    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
+  script:
+    - echo "Running featurebase unit tests..."
+    - PKG_LIST=$(go list ./... | grep -Ev 'internal/clustertests|simulacraData|batch|idk|v3/dax/test/dax' | paste -s -d, -)
+    - export TMPDIR=/mnt/ramdisk/test-$CI_JOB_ID
+    - mkdir -p $TMPDIR
+    - go test -tags=shardwidth22 -timeout=10m -coverprofile=coverage.out -covermode=atomic -coverpkg=${PKG_LIST} ${PKG_LIST//,/ }
+  after_script:
+    - rm -rf /mnt/ramdisk/test-$CI_JOB_ID
+  artifacts:
+    paths:
+      - coverage.out
+  tags:
+    - docker
+
+run go tests dax/test/dax:
+  stage: test
+  image: golang:$GOVERSION
+  extends: .go-cache
+  tags:
+    - docker
+  variables:
+    FEATUREBASE_CONTROLLER_CONFIG_SQLDB_DATABASE: run_go_tests_dax
+    POSTGRES_DB: run_go_tests_dax
+    FEATUREBASE_CONTROLLER_CONFIG_SQLDB_USER: postgres
+    POSTGRES_USER: postgres
+    FEATUREBASE_CONTROLLER_CONFIG_SQLDB_PASSWORD: $POSTGRES_PASSWORD
+    POSTGRES_PASSWORD: $POSTGRES_PASSWORD
+    FEATUREBASE_CONTROLLER_CONFIG_SQLDB_HOST: postgres
+  services:
+    - postgres:14.7
+  rules:
+    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
+  script:
+    - echo "Building FB and Datagen docker images for DAX tests"
+    - PKG_LIST=$(go list ./... | grep -Ev 'internal/clustertests|simulacraData' | paste -s -d, -)
+    - export TMPDIR=/mnt/ramdisk/test-$CI_JOB_ID
+    - mkdir -p $TMPDIR
+    - go test -coverprofile=coverage-dax-integration.out -covermode=atomic -coverpkg=${PKG_LIST} -timeout=20m ./dax/test/dax
+  after_script:
+    - rm -rf /mnt/ramdisk/test-$CI_JOB_ID
+  artifacts:
+    paths:
+      - coverage-dax-integration.out
+
+# idk tests
+run go tests idk race:
+  variables:
+    PROJECT: race_${CI_CONCURRENT_ID}
+  stage: nonblocking
+  retry: 1
+  script:
+    - echo "Running test-all-race"
+    - cd ./idk/
+    - echo $PROJECT
+    - echo $CI_COMMIT_REF_SLUG
+    - BRANCH_NAME=${CI_COMMIT_REF_SLUG} IDK_FEATUREBASE_TAG=${CI_COMMIT_TAG} IDK_FEATUREBASE_HASH=${CI_COMMIT_SHA} make test-all-race
+  after_script:
+    - cd ./idk/
+    - make save-pilosa-logs
+    - make shutdown
+  artifacts:
+    paths:
+      - ./idk/testdata/*_coverage.out
+  tags:
+    - shell
+    - aws
+  needs:
+    - job: build amd container fb
+
+run go tests idk shard transactional:
+  variables:
+    IDK_DEFAULT_SHARD_TRANSACTIONAL: 1
+    PROJECT: shardttrans_${CI_CONCURRENT_ID}
+  stage: nonblocking
+  retry: 1
+  script:
+    - echo "Running shard transactional tests"
+    - cd ./idk/
+    - echo $PROJECT
+    - echo $CI_COMMIT_REF_SLUG
+    - BRANCH_NAME=${CI_COMMIT_REF_SLUG} IDK_FEATUREBASE_TAG=${CI_COMMIT_TAG} IDK_FEATUREBASE_HASH=${CI_COMMIT_SHA} make test-all
+  after_script:
+    - cd ./idk/
+    - make save-pilosa-logs
+    - make shutdown
+  artifacts:
+    paths:
+      - ./idk/testdata/*_coverage.out
+      - ./idk/testdata/*_logs.txt
+  tags:
+    - shell
+    - aws
+  needs:
+    - job: build amd container fb
+
+run go tests idk 533:
+  variables:
+    USERNAME: fb-idk-access
+    PROJECT: test533_${CI_CONCURRENT_ID}
+  stage: nonblocking
+  retry: 1
+  script:
+    - echo "Running confluent 5.3.3 test-all"
+    - cd ./idk/
+    - echo $PROJECT
+    - echo $CI_COMMIT_REF_SLUG
+    - CONFLUENT_VERSION=5.3.3 BRANCH_NAME=${CI_COMMIT_REF_SLUG} IDK_FEATUREBASE_TAG=${CI_COMMIT_TAG} IDK_FEATUREBASE_HASH=${CI_COMMIT_SHA} make test-all
+  after_script:
+    - cd ./idk/
+    - make save-pilosa-logs
+    - make shutdown
+  tags:
+    - shell
+    - aws
+  artifacts:
+    paths:
+      - ./idk/testdata/*_coverage.out
+  needs:
+    - job: build amd container fb
+
+run go tests idk sasl:
+  variables:
+    PROJECT: sasl_${CI_CONCURRENT_ID}
+  stage: nonblocking
+  retry: 1
+  script:
+    - echo "Running test-all-kafka-sasl"
+    - cd ./idk/
+    - echo $PROJECT
+    - echo $CI_COMMIT_REF_SLUG
+    - BRANCH_NAME=${CI_COMMIT_REF_SLUG} IDK_FEATUREBASE_TAG=${CI_COMMIT_TAG} IDK_FEATUREBASE_HASH=${CI_COMMIT_SHA} make test-all-kafka-sasl
+  after_script:
+    - cd ./idk/
+    - make save-pilosa-logs
+    - make shutdown
+  tags:
+    - shell
+    - aws
+  artifacts:
+    paths:
+      - ./idk/testdata/*_coverage.out
+  needs:
+    - job: build amd container fb
+
+upload to sonarcloud:
+  stage: nonblocking
+  image: sonarsource/sonar-scanner-cli:4.7
+  variables:
+    SONAR_TOKEN: $SONAR_TOKEN
+  rules:
+    - if: '$CI_COMMIT_TAG == null && ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web")'
+  script:
+    - sonar-scanner -Dsonar.projectKey=molecula_featurebase -Dsonar.organization=molecula -Dsonar.sources=. -Dsonar.host.url=https://sonarcloud.io -Dsonar.go.coverage.reportPaths=coverage*.out,results/coverage*out,idk/testdata/*coverage.out,batch/testdata/*coverage.out,coverage-from-docker/*.out -Dsonar.javascript.lcov.reportPaths=lattice/coverage/lcov.info
+  needs:
+    - job: run go tests
+    - job: run jest tests
+    - job: external lookup tests
+    - job: run go tests idk race
+      optional: true
+    - job: run go tests idk shard transactional
+      optional: true
+    - job: run go tests idk sasl
+      optional: true
+    - job: run go tests idk 533
+      optional: true
+    - job: run go tests batch
+      optional: true
+    - job: run go tests dax/test/dax
+      optional: true
+
+package for linux amd64:
+  stage: build
+  image: golang:$GOVERSION
+  extends: .go-cache
+  rules:
+    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
+  variables:
+    GOOS: "linux"
+    GOARCH: "amd64"
+  script:
+    - echo 'deb [trusted=yes] https://repo.goreleaser.com/apt/ /' | tee /etc/apt/sources.list.d/goreleaser.list
+    - apt update && apt install nfpm=2.11.3
+    - make package
+  artifacts:
+    paths:
+      - "*.deb"
+      - "*.rpm"
+
+upload_artifacts_to_nexus:
+  stage: post build
+  image: golang:$GOVERSION
+  rules:
+    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
+  script:
+    - curl -v --user "$NEXUS_YUM_CREDS" --upload-file *.arm64.rpm https://nexus.molecula.com/repository/molecula-yum/release/
+    - curl -v --user "$NEXUS_YUM_CREDS" --upload-file *.amd64.rpm https://nexus.molecula.com/repository/molecula-yum/release/
+  dependencies:
+    - package for linux amd64
+    - package for linux arm64
+
+trigger_m-cloud-images:
+  stage: post build
+  rules:
+    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
+  needs:
+    - upload_artifacts_to_nexus
+  trigger: molecula/m-cloud-images
+
+package for linux arm64:
+  stage: build
+  image: golang:$GOVERSION
+  extends: .go-cache
+  rules:
+    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
+  variables:
+    GOOS: "linux"
+    GOARCH: "arm64"
+  script:
+    - echo 'deb [trusted=yes] https://repo.goreleaser.com/apt/ /' | tee /etc/apt/sources.list.d/goreleaser.list
+    - apt update && apt install nfpm=2.11.3
+    - make package
+  artifacts:
+    paths:
+      - "*.deb"
+      - "*.rpm"
+
+build arm container fb:
+  stage: build
+  needs:
+    - "build featurebase"
+  tags:
+    - shell
+  rules:
+    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
+  before_script:
+    - echo "${DOCKER_DEPLOY_TOKEN}" | docker login -u ${DOCKER_DEPLOY_USER} --password-stdin ${CI_REGISTRY}
+  script:
+    - echo $CI_COMMIT_REF_SLUG
+    - tag=${CI_REGISTRY_IMAGE}/featurebase:linux-arm64-${CI_COMMIT_REF_SLUG}
+    - export SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct)
+    - docker build --build-arg GO_VERSION=$GOVERSION --build-arg ARCH=arm64 --build-arg SOURCE_DATE_EPOCH=$SOURCE_DATE_EPOCH -t $tag -f .gitlab/Dockerfile .
+    - docker push $tag
+    - echo Created docker featurebase image with tag "$tag"
+
+### start idk builds ###
+# building them all serially because otherwise you get container name conflicts.
+idk build_amd64:
+  stage: build
+  variables:
+    BUILD_NAME: build_${CI_COMMIT_SHA}_${CI_CONCURRENT_ID}
+  tags:
+    - shell
+  rules:
+    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
+  script:
+    - export SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct)
+    - cd ./idk/
+    - date
+    - make docker-build GOOS="linux" GOARCH="amd64" BUILD_CGO=1
+    - date
+    - make docker-build GOOS="darwin" GOARCH="amd64"
+    - date
+  artifacts:
+    paths:
+      - ./idk/build/*
+  needs:
+    # doesn't actually need this... just want it to start executing before *all* the tests finish
+    - job: run go tests
+
+idk build_arm64:
+  stage: build
+  tags:
+    - shell-arm64
+  rules:
+    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
+  script:
+    - export SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct)
+    - cd ./idk/
+    - date
+    - make docker-build GOOS="linux" GOARCH="arm64" BUILD_CGO=1 BUILD_NAME="linux-arm64"
+    - date
+    - make docker-build GOOS="darwin" GOARCH="arm64"
+    - date
+  artifacts:
+    paths:
+      - ./idk/build/*
+  needs:
+    # doesn't actually need this... just want it to start executing before *all* the tests finish
+    - job: run go tests
+
+# building them all serially because otherwise you get container name conflicts.
+# only do containers on default branch
+idk package_docker_all:
+  stage: build
+  tags:
+    - shell
+  rules:
+    - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web")'
+  script:
+    - export SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct)
+    - make docker-idk GOOS="linux" GOARCH="amd64"
+    - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
+    - make docker-idk-tag-push GOOS="linux" GOARCH="amd64"
+  needs:
+    - job: idk build_amd64
+    - job: idk build_arm64
+
+idk s3 dump:
+  stage: post build
+  allow_failure: false
+  variables:
+    PROFILE: "service-fb-ci"
+    AWS_SSH_PRIVATE_KEY: $AWS_FBCI_SSH_KEY
+    AWS_ACCESS_KEY_ID: $AWS_FBCI_ACCESS_KEY_ID
+    AWS_SECRET_ACCESS_KEY: $AWS_FBCI_SECRET_ACCESS_KEY
+  tags:
+    - shell
+  rules:
+    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
+  script:
+    - aws configure set aws_access_key_id $AWS_FBCI_ACCESS_KEY_ID
+    - aws configure set aws_secret_access_key $AWS_FBCI_SECRET_ACCESS_KEY
+    - aws configure set region "us-east-2"
+    - aws configure set aws_profile $PROFILE
+    - aws s3 cp ./idk/build/ s3://molecula-artifact-storage/idk/${CI_COMMIT_BRANCH}/${CI_COMMIT_SHORT_SHA}/ --recursive
+    - aws s3 cp ./idk/build/ s3://molecula-artifact-storage/idk/${CI_COMMIT_BRANCH}/_latest/ --recursive
+  needs:
+    - job: idk build_amd64
+    - job: idk build_arm64
+
+idk s3 dump tag:
+  stage: post build
+  variables:
+    PROFILE: "service-fb-ci"
+    AWS_SSH_PRIVATE_KEY: $AWS_FBCI_SSH_KEY
+    AWS_ACCESS_KEY_ID: $AWS_FBCI_ACCESS_KEY_ID
+    AWS_SECRET_ACCESS_KEY: $AWS_FBCI_SECRET_ACCESS_KEY
+    LOCATION: molecula-artifact-storage/idk/_tags
+  tags:
+    - shell
+  rules:
+    - if: '$CI_COMMIT_TAG != null && ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web")'
+  script:
+    - aws configure set aws_access_key_id $AWS_FBCI_ACCESS_KEY_ID
+    - aws configure set aws_secret_access_key $AWS_FBCI_SECRET_ACCESS_KEY
+    - aws configure set region "us-east-2"
+    - aws configure set aws_profile $PROFILE
+    - |
+      for goos in "darwin" "linux"; do
+        for goarch in "amd64" "arm64"; do
+          dir=idk-${CI_COMMIT_TAG}-${goos}-${goarch}
+          echo "Directory ${dir}"
+          mkdir ${dir}
+          mv ./idk/build/idk-${goos}-${goarch}/molecula-consumer-* ${dir}/
+          tar cvzf ${dir}.tar.gz ${dir}
+          aws s3 cp ${dir} s3://${LOCATION}/${CI_COMMIT_TAG}/${dir}/ --recursive
+          aws s3 cp ${dir}.tar.gz s3://${LOCATION}/${CI_COMMIT_TAG}/
+        done
+      done
+  needs:
+    - job: idk build_amd64
+    - job: idk build_arm64
+### end idk builds ###
+
+external lookup tests:
+  stage: integration
+  image: golang:$GOVERSION
+  extends: .go-cache
+  # TODO: no rules here, do we need to add the rules line?
+  variables:
+    POSTGRES_DB: $POSTGRES_DB
+    POSTGRES_USER: $POSTGRES_USER
+    POSTGRES_PASSWORD: $POSTGRES_PASSWORD
+    POSTGRES_HOST_AUTH_METHOD: trust
+  services:
+    - postgres:13.5
+  script:
+    - apt-get update --allow-releaseinfo-change -y
+    - apt-get install -y postgresql-client
+    - go test . -run "^TestExternalLookup" -externalLookupDSN postgresql://$POSTGRES_USER:$POSTGRES_PASSWORD@postgres/$POSTGRES_DB?sslmode=disable
+
+s3 dump:
+  stage: post build
+  variables:
+    PROFILE: "service-fb-ci"
+    AWS_SSH_PRIVATE_KEY: $AWS_FBCI_SSH_KEY
+    AWS_ACCESS_KEY_ID: $AWS_FBCI_ACCESS_KEY_ID
+    AWS_SECRET_ACCESS_KEY: $AWS_FBCI_SECRET_ACCESS_KEY
+  tags:
+    - shell
+  rules:
+    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "web"'
+  script:
+    - aws configure set aws_access_key_id $AWS_FBCI_ACCESS_KEY_ID
+    - aws configure set aws_secret_access_key $AWS_FBCI_SECRET_ACCESS_KEY
+    - aws configure set region "us-east-2"
+    - aws configure set aws_profile $PROFILE
+    - |
+      for goos in "darwin" "linux"; do
+        for goarch in "amd64" "arm64"; do
+          aws s3 cp featurebase_${goos}_${goarch} s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/${CI_COMMIT_SHORT_SHA}/featurebase_${goos}_${goarch}
+          aws s3 cp featurebase_${goos}_${goarch} s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/_latest/featurebase_${goos}_${goarch}
+          aws s3 cp ./build/fbsql_${goos}_${goarch} s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/${CI_COMMIT_SHORT_SHA}/fbsql_${goos}_${goarch}
+          aws s3 cp ./build/fbsql_${goos}_${goarch} s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/_latest/fbsql_${goos}_${goarch}
+        done
+      done
+  needs:
+    - job: build featurebase
+    - job: build fbsql amd64
+    - job: build fbsql arm64
+
+s3 dump tag:
+  stage: post build
+  variables:
+    PROFILE: "service-fb-ci"
+    AWS_SSH_PRIVATE_KEY: $AWS_FBCI_SSH_KEY
+    AWS_ACCESS_KEY_ID: $AWS_FBCI_ACCESS_KEY_ID
+    AWS_SECRET_ACCESS_KEY: $AWS_FBCI_SECRET_ACCESS_KEY
+    LOCATION: molecula-artifact-storage/featurebase/_tags
+  tags:
+    - shell
+  rules:
+    - if: '$CI_COMMIT_TAG != null && ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "web")'
+  script:
+    - aws configure set aws_access_key_id $AWS_FBCI_ACCESS_KEY_ID
+    - aws configure set aws_secret_access_key $AWS_FBCI_SECRET_ACCESS_KEY
+    - aws configure set region "us-east-2"
+    - aws configure set aws_profile $PROFILE
+    - |
+      for goos in "darwin" "linux"; do
+        for goarch in "amd64" "arm64"; do
+          dir=featurebase-${CI_COMMIT_TAG}-${goos}-${goarch}
+          echo "Directory ${dir}"
+          mkdir $dir
+          mv featurebase_${goos}_${goarch} ${dir}/featurebase
+          mv ./build/fbsql_${goos}_${goarch} ${dir}/fbsql
+          cp NOTICE install/featurebase.conf install/featurebase.*.service ${dir}/
+          tar cvzf ${dir}.tar.gz ${dir}
+          aws s3 cp ${dir} s3://${LOCATION}/${CI_COMMIT_TAG}/${dir}/ --recursive
+          aws s3 cp ${dir}.tar.gz s3://${LOCATION}/${CI_COMMIT_TAG}/
+        done
+      done
+
+  needs:
+    - job: build featurebase
+    - job: build fbsql amd64
+    - job: build fbsql arm64

.gitlab/Dockerfile

@@ -0,0 +1,24 @@
+FROM alpine:3.14.2
+
+LABEL maintainer "dev@molecula.com"
+LABEL org.opencontainers.image.authors="dev@molecula.com"
+
+ARG ARCH
+
+WORKDIR /
+
+RUN apk add --no-cache curl jq
+
+COPY NOTICE .
+COPY featurebase_linux_$ARCH featurebase
+
+RUN chmod ugo+x .
+EXPOSE 10101
+VOLUME /data
+
+ENV PILOSA_DATA_DIR /data
+ENV PILOSA_BIND 0.0.0.0:10101
+ENV PILOSA_BIND_GRPC 0.0.0.0:20101
+
+ENTRYPOINT ["/featurebase"]
+CMD ["server"]

.gitlab/batch-ci.yml

@@ -0,0 +1,28 @@
+run go tests batch:
+  variables:
+    PROJECT: batch_${CI_CONCURRENT_ID}
+  # this test relies on stuff that happens after build-lattice, which
+  # makes it pause the entire CI run waiting for this. we accept the
+  # small risk of wasting a build against the near certainty of spending
+  # five minutes running only one job.
+  stage: nonblocking
+  retry: 1
+  script:
+    - echo "Running test-all"
+    - cd ./batch/
+    - echo $PROJECT
+    - make build-featurebase
+    - make test-all
+  after_script:
+    - cd ./batch/
+    - make save-featurebase-logs
+    - make shutdown
+  artifacts:
+    paths:
+      - ./batch/testdata/*_coverage.out
+      - ./batch/testdata/*_logs.txt
+  tags:
+    - shell
+    - aws
+  needs:
+    - job: build amd container fb

.golangci.yml

@@ -0,0 +1,83 @@
+run:
+  deadline: 5m
+  timeout: 5m
+  skip-dirs-use-default: true
+  #skip the protobuf generated files
+  skip-dirs:
+    - pb
+    - proto
+  skip-files:
+    - pql/pql.peg.go
+linters:
+  enable:
+    # Recommended to be enabled by default (https://golangci-lint.run).
+    # - errcheck (lots to fix)
+    - gosimple
+    - govet
+    - ineffassign
+    - staticcheck
+    - typecheck
+    # - unused (about 20 to fix)
+
+    # Additional linters we choose to enable.
+    # - bodyclose (lots to fix, but we should)
+    - errchkjson
+    - errname
+    - gofmt
+    # - misspell (lots to fix, but we should)
+    - prealloc
+    # - predeclared (20 to fix)
+    # - stylecheck (quite a lot to fix, but we should definitely work on this)
+    - stylecheck
+    # - unconvert (not at all critical, but makes for cleaner code)
+  enable-all: false
+  disable-all: true
+
+output:
+  # colored-line-number|line-number|json|tab|checkstyle|code-climate, default is "colored-line-number"
+  format: tab
+  # print lines of code with issue, default is true
+  print-issued-lines: true
+  # print linter name in the end of issue text, default is true
+  print-linter-name: true
+
+linters-settings:
+  gofmt:
+    simplify: true
+  govet:
+    # report about shadowed variables
+    check-shadowing: true
+
+
+    # settings per analyzer
+    settings:
+      printf: # analyzer name, run `go tool vet help` to see all analyzers
+        funcs: # run `go tool vet help printf` to see available settings for `printf` analyzer
+          - (github.com/golangci/golangci-lint/pkg/logutils.Log).Infof
+          - (github.com/golangci/golangci-lint/pkg/logutils.Log).Warnf
+          - (github.com/golangci/golangci-lint/pkg/logutils.Log).Errorf
+          - (github.com/golangci/golangci-lint/pkg/logutils.Log).Fatalf
+
+    # enable or disable analyzers by name
+    # run `go tool vet help` to see all analyzers
+    enable:
+      - atomicalign
+    enable-all: false
+    disable:
+      - shadow
+    disable-all: false
+
+  stylecheck:
+    # ST1000: at least one file in a package should have a package comment
+    # ST1003: golang naming standards
+    # ST1016: methods on the same type should have the same receiver name
+    # ST1020: comment on exported function
+    checks: ["all", "-ST1000", "-ST1003", "-ST1016", "-ST1020"]
+
+issues:
+  exclude-use-default: false
+  max-issues-per-linter: 0
+  max-same-issues: 0
+  exclude:
+    - 'declaration of "(err|ctx)" shadows declaration at'
+    - 'Error return value of .(.*\.Help|.*\.MarkFlagRequired|(os\.)?std(out|err)\..*|.*Close|.*Flush|os\.Remove(All)?|.*printf?|os\.(Un)?Setenv). is not checked'

.travis.yml

@@ -1,28 +0,0 @@
-language: go
-go:
-  - 1.8
-  - 1.9
-  - master
-env:
-  global: # AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY
-    - secure: "VnBFmFfBOrrf7ONLN9WpAFCcV8SEt5G5VPnnHv97TP7PlJG8LWR6k6O+vRJOvf8V4vDMfKCTDonwWLgbssVf3yygo3C8ZoftY2phehEkWGffCgsd9ML/YBNbGq4LYLSE5HKvBqrZjQaOrVby71BAsP8W7RhC6hqzFQ00M/z8dZVfwaQQFwew2eEcSxLEaaDFS8Wgc3/UuwxDRPBq6u3cCN5RxfB+q70HvGVq4TT+0dqS4eCvz688+Z0GIGYx9olNjh0F2Kc8R2Po0lnUNa0GiHrZ21zeQ1DxIK04QABrWWmjL4h+bx3VHNKPFR4GYSKDf+pj1kfaqbfrAg6rMAJdGejgoS+QyjhgCoN4d3qRp8s+1nrxtp0TvezEdjwyxt4quGHbP5TxWUszssbGhWqf4mx6OeJ8MmdTaJjfu0f3NWJXMycqT6J73WKORk4rHeIqF9CIdxdmcpkwYj8rk0TEMTPTsd7WA8w2HIDsCz/jQnRmEgLUiNnTAofYc/uUi/Wg/T2hllkp+oBDTzxk9NTelkqx8TJ0bDmYYL9JWUi1siFHTHiVYTJgyirSfGNpe61u8OLmT0Hak/D399IfL7qgFLlMXk8q92typfO2xEduq6G+8KygeqiOMSsOY+xcDvZf5xtcEihYd21vjtrxRSqFsup/o8DIxEurQnfXBx1B+WA="
-    - secure: "U4fpHWDVOG4viqZsiVgUDW7OW1JW60uPOZy0q9pfbs86iHvmZq0PaScsZ+YdlYaN2GETVr7endDf6DCcZs1PWfg0F6VQfkOXcShX8HVS9O58lUZA5tyvbDVql9DQs4PbnkZo+ktz+Z0YaXqq2RdtMDOUz4bgZwspLPMA14if+N6w0tqCFpB7bEtpptTGsdbIQPG1n07yvSeNmK4mvrEEs77tWmhulN5iilpOqhpIvD39bJvtCYVALuJpzLd/OjLTPV9l/fl+hJkMXSj+X5ilO1DHINAcCM648iEX2phXAIWmi0O0Rbg2cI4kV9T5ysOIw8ux+YCm9bZDGTCt+VGBW5Fg+Z5iaXXexyKYCGiHleOJ7kCj9kXxh2u8NiYVNgb19dGJV5/HgQ6pcGWjeVEqr8yY1546zMjpTX+SYGQF+XZe+uggEjeAsk53ueXa0pyZTrlrqSvR7BBtWPx47s/dTg2L19FQYv3XpGMxEXLw92RplExQKi1h7QgihRxFpjGgURHhrt7d9eiNiNqBt3ZsHjmh2AkXZHnaDjlgSnFFWaMqP3UtDBWIuO+2BMbZUJVfP+gpQGBZ4gtpUSmV2JDCHgZgX5OAnLD4usxh+ATQ4rvUXF/tf8nMqEKHlGKd8hxpYSyMX21BoqfSfY4/IA0ejVE9BITqlrvqewqkP1yxe7o="
-addons:
-before_install:
-  - go get github.com/mattn/goveralls
-script:
-  - make vendor && $HOME/gopath/bin/goveralls -service=travis-ci -ignore "internal/internal.go,internal/public.pb.go,internal/private.pb.go"
-before_deploy:
-  - pip install awscli --user `whoami`
-deploy:
-  - provider: script
-    script: make prerelease-upload
-    skip_cleanup: true
-    on:
-      branch: master
-matrix:
-  allow_failures:
-    - go: master
-notifications:
-  slack:
-    secure: "SceWannxoGzeSu9PlEhl6icQFGuTmwax870k20nB2ZGYLjo77UEcwYoFwWvFsdYPa/HCo3JorMTYvMJ15VDJcnKEfzDr+kyXbHWBzUumclIOU/Im3ArEN6waQgyGbbWUQhvJjy4ATaxiOlmCyDV+KhKC9P3+WB33/OQtM3ngjAdTXYHAkfEcpeoOP75um+KsQgbi+hlnqfZdgDa6yIkFjaS3KZEJW1vmcOYYzNsXOA1Ip8j1NY6AjjWZlQorZJ/SYFqdhIv8ST3+a6cQk12u3t6TwZdcr3wmm1qmiW/SaK7UesWlT/YfElIuK8BBq9w1oZHxNKoAmLWTOe7MMisdItmtwgA14eMGl1rvNFlVf9sjsxs4AAzFvSZBZdDfx9XeLCBU5I2WUc/PKUgNQBPMVChxA7gEhtZLndsDdye7LsZASD2yYqjlVlgoZpzRexee/cJgCqUcNKDBHF39ZJYxV4KtZ0prjcSnVmLvuapplzTV4LZ+LyFapCyhiuM/oMJvxgmd7jTtFb5e5EkaHBPN1XwQWZw87yCjKsunTlTe1f1a5qoH/xvJHNpqE/jxOHU3DTLDgTxhb+FwC1Qj9a8bp+UYLw5F4P46ZnHlBGc2O74klv17EqvUMn3JhzASUtyxLGOgJulJ+o83rxJvhSiWt3GQIfkExVPzmz11641ElJI="

CHANGELOG.md

@@ -1,213 +0,0 @@
-# Changelog
-
-All notable changes to this project will be documented in this file.
-
-The format is based on [Keep a Changelog](http://keepachangelog.com/)
-and this project adheres to [Semantic Versioning](http://semver.org/).
-
-## [0.8.0] - 2017-11-15
-
-This version contains 31 contributions from 8 contributors. There are 84 files changed, 3,732 insertions, and 1,428 deletions.
-
-### Added
-
-- Diagnostics ([#895](https://github.com/pilosa/pilosa/pull/895))
-- Add docker-build make target for repeatable Docker-based builds ([#933](https://github.com/pilosa/pilosa/pull/933))
-- Add documentation on importing field values; fixes #924 ([#938](https://github.com/pilosa/pilosa/pull/938))
-- Add flag documentation and tests, remove "plugins.path" ([#942](https://github.com/pilosa/pilosa/pull/942))
-- Add TLS support ([#867](https://github.com/pilosa/pilosa/pull/867))
-- Add TLS cluster how to ([#898](https://github.com/pilosa/pilosa/pull/898))
-- Add support for gossip encryption ([#889](https://github.com/pilosa/pilosa/pull/889))
-- Add Recalculate Caches endpoint ([#881](https://github.com/pilosa/pilosa/pull/881))
-- Add search-friendly documentation for BSI range query syntax ([#955](https://github.com/pilosa/pilosa/pull/955))
-
-### Changed
-
-- Remove unneeded Gopkg.toml constraints and update all dependencies ([#943](https://github.com/pilosa/pilosa/pull/943))
-- Remove row and column labels in webUI ([#884](https://github.com/pilosa/pilosa/pull/884))
-- Internal Client refactoring ([#892](https://github.com/pilosa/pilosa/pull/892))
-- Remove column/row labels for input definition ([#945](https://github.com/pilosa/pilosa/pull/945))
-- Update dependencies and Go version ([#878](https://github.com/pilosa/pilosa/pull/878))
-
-### Fixed
-
-- Skip permissions test when run as root. Fixes #940 ([#941](https://github.com/pilosa/pilosa/pull/941))
-- Address "connection reset" issues in client ([#934](https://github.com/pilosa/pilosa/pull/934))
-- Fix field value import: Use signed int and respect field minimum ([#919](https://github.com/pilosa/pilosa/pull/919))
-- Constrain BoltDB to version rather than specific revision ([#887](https://github.com/pilosa/pilosa/pull/887))
-- Fix bug in environment variable format ([#882](https://github.com/pilosa/pilosa/pull/882))
-- Fix overflow in differenceRunBitmap ([#949](https://github.com/pilosa/pilosa/pull/949))
-
-### Performance
-
-- Use FieldNotNull to improve efficiency of BETWEEN queries ([#874](https://github.com/pilosa/pilosa/pull/874))
-
-## [0.7.2] - 2017-11-15
-
-This version contains 1 contribution from 1 contributor. There is 1 file changed, 16 insertions, and 1 deletion.
-
-### Changed
-
-- Bump HTTP client's MaxIdleConns and MaxIdleConnsPerHost ([#920](https://github.com/pilosa/pilosa/pull/920))
-
-## [0.7.1] - 2017-10-09
-
-This version contains 3 contributions from 3 contributors. There are 14 files changed, 221 insertions, and 52 deletions.
-
-### Changed
-
-- Update dependencies and Go version ([#878](https://github.com/pilosa/pilosa/pull/878))
-
-### Performance
-
-- Leverage not-null field to make BETWEEN queries more efficient ([#874](https://github.com/pilosa/pilosa/pull/874))
-
-## [0.7.0] - 2017-10-03
-
-This version contains 59 contributions from 9 contributors. There are 61 files changed, 5207 insertions, and 1054 deletions.
-
-### Added
-
-- Add HTTP API for fields ([#811](https://github.com/pilosa/pilosa/pull/811), [#856](https://github.com/pilosa/pilosa/pull/856))
-- Add HTTP API for delete views ([#785](https://github.com/pilosa/pilosa/pull/785))
-- Modify import endpoint to handle BSI field values ([#840](https://github.com/pilosa/pilosa/pull/840))
-- Add field Range() support to Executor ([#791](https://github.com/pilosa/pilosa/pull/791))
-- Support PQL Range() queries for fields ([#755](https://github.com/pilosa/pilosa/pull/755))
-- Add Sum() field query ([#778](https://github.com/pilosa/pilosa/pull/778))
-- Add documentation for BSI ([#861](https://github.com/pilosa/pilosa/pull/861))
-- Add BETWEEN for Range queries ([#847](https://github.com/pilosa/pilosa/pull/847))
-- Add Xor support for PQL ([#789](https://github.com/pilosa/pilosa/pull/789))
-- Enable auto-creating the schema on imports ([#837](https://github.com/pilosa/pilosa/pull/837))
-- Update client library docs ([#831](https://github.com/pilosa/pilosa/pull/831))
-- Handle SIGTERM signal ([#830](https://github.com/pilosa/pilosa/pull/830))
-- Add cluster config example to docs ([#806](https://github.com/pilosa/pilosa/pull/806))
-- Add ability to exclude attributes and bits in Bitmap queries ([#783](https://github.com/pilosa/pilosa/pull/783))
-
-### Fixed
-
-- Fix panic when iterating over an empty run container ([#860](https://github.com/pilosa/pilosa/pull/860))
-- Fix row id zero bug ([#814](https://github.com/pilosa/pilosa/pull/814))
-- Fix cache invalidation bug ([#795](https://github.com/pilosa/pilosa/pull/795))
-- Set container.n in differenceRunRun ([#794](https://github.com/pilosa/pilosa/pull/794))
-- Fix infinite loop in bitmap-to-array conversion ([#779](https://github.com/pilosa/pilosa/pull/779))
-- Fix CountRange bug ([#773](https://github.com/pilosa/pilosa/pull/773))
-
-### Deprecated
-
-- Remove support for row/column labels ([#839](https://github.com/pilosa/pilosa/pull/839))
-
-### Performance
-
-- Refactor differenceRunArray ([#859](https://github.com/pilosa/pilosa/pull/859))
-- Update fragment.FieldSum to use roaring IntersectionCount() ([#841](https://github.com/pilosa/pilosa/pull/841))
-- Add roaring optimizations ([#842](https://github.com/pilosa/pilosa/pull/842))
-- Convert lock to read lock ([#848](https://github.com/pilosa/pilosa/pull/848))
-- Reduce Lock calls in executor ([#846](https://github.com/pilosa/pilosa/pull/846))
-- Implement container.flipBitmap() to improve differenceRunBitmap() ([#849](https://github.com/pilosa/pilosa/pull/849))
-- Reuse container storage on UnmarshalBinary to improve memory utilization ([#820](https://github.com/pilosa/pilosa/pull/820))
-- Improve WriteTo performance ([#812](https://github.com/pilosa/pilosa/pull/812))
-
-## [0.6.0] - 2017-08-11
-
-This version contains 14 contributions from 5 contributors. There are 28 files changed, 4,936 insertions, and 692 deletions.
-
-### Added
-
-- Add Run-length Encoding ([#758](https://github.com/pilosa/pilosa/pull/758))
-
-### Changed
-
-- Make gossip the default broadcast type ([#750](https://github.com/pilosa/pilosa/pull/750))
-
-### Fixed
-
-- Fix CountRange ([#759](https://github.com/pilosa/pilosa/pull/759))
-- Fix `differenceArrayRun` logic ([#674](https://github.com/pilosa/pilosa/pull/674))
-
-## [0.5.0] - 2017-08-02
-
-This version contains 65 contributions from 8 contributors (including 1 volunteer contributor). There are 79 files changed, 7,972 insertions, and 2,800 deletions.
-
-### Added
-
-- Set open file limit during Pilosa startup ([#748](https://github.com/pilosa/pilosa/pull/748))
-- Add Input Definition ([#646](https://github.com/pilosa/pilosa/pull/646))
-- Add cache type: None ([#745](https://github.com/pilosa/pilosa/pull/745))
-- Add panic recovery in top level HTTP handler ([#741](https://github.com/pilosa/pilosa/pull/741))
-- Count open file handles as a StatsD metric ([#636](https://github.com/pilosa/pilosa/pull/636))
-- Add coverage tools to Makefile ([#635](https://github.com/pilosa/pilosa/pull/635))
-- Add Holder test coverage ([#629](https://github.com/pilosa/pilosa/pull/629))
-- Add runtime memory metrics ([#600](https://github.com/pilosa/pilosa/pull/600))
-- Add sorting flag to import command ([#606](https://github.com/pilosa/pilosa/pull/606))
-- Add PQL support for field values (WIP) ([#721](https://github.com/pilosa/pilosa/pull/721))
-- Set and retrieve field values (WIP) ([#702](https://github.com/pilosa/pilosa/pull/702))
-- Add BSI range-encoding schema support (WIP) ([#670](https://github.com/pilosa/pilosa/pull/670))
-
-### Changed
-
-- Move InternalPort config option to top-level ([#747](https://github.com/pilosa/pilosa/pull/747))
-- Switch from glide to dep for dependency management ([#744](https://github.com/pilosa/pilosa/pull/744))
-- Remove QueryRequest.Quantum since it is no longer used ([#699](https://github.com/pilosa/pilosa/pull/699))
-- Refactor test utilities into importable package ([#675](https://github.com/pilosa/pilosa/pull/675))
-
-### Fixed
-
-- Add mutex for attribute cache ([#729](https://github.com/pilosa/pilosa/pull/729))
-- Use log-path flag to specify log file ([#678](https://github.com/pilosa/pilosa/pull/678))
-
-## [0.4.0] - 2017-06-08
-
-This version contains 53 contributions from 13 contributors (including 4 volunteer contributors). There are 96 files changed, 6373 insertions, and 770 deletions.
-
-*Note that data files created in Pilosa < 0.4.0 are not compatible with Pilosa 0.4.0 as a result of [#520](https://github.com/pilosa/pilosa/pull/520).*
-
-### Added
-- Support metric reporting through StatsD protocol ([#468](https://github.com/pilosa/pilosa/pull/468), [#568](https://github.com/pilosa/pilosa/pull/568), [#580](https://github.com/pilosa/pilosa/pull/580))
-- Improve test coverage for ctl package ([#586](https://github.com/pilosa/pilosa/pull/586))
-- Add support for bit flip (negate) in roaring ([#592](https://github.com/pilosa/pilosa/pull/592))
-- Add xor support to roaring ([#571](https://github.com/pilosa/pilosa/pull/571))
-- Improve WebUI autocomplete ([#560](https://github.com/pilosa/pilosa/pull/560))
-- Add syntax hints tooltip to WebUI ([#537](https://github.com/pilosa/pilosa/pull/537))
-- Implement 'config' CLI command ([#541](https://github.com/pilosa/pilosa/pull/541))
-- Move docs into repo ([#563](https://github.com/pilosa/pilosa/pull/563))
-- Add inverse TopN() support ([#551](https://github.com/pilosa/pilosa/pull/551))
-- Add various Makefile updates ([#540](https://github.com/pilosa/pilosa/pull/540))
-- Provide details on Glide checksum mismatch ([#546](https://github.com/pilosa/pilosa/pull/546))
-- Add Docker multi-stage build ([#535](https://github.com/pilosa/pilosa/pull/535))
-- Support inverse Range() queries ([#533](https://github.com/pilosa/pilosa/pull/533))
-- Support colon commands in WebUI ([#529](https://github.com/pilosa/pilosa/pull/529), [#510](https://github.com/pilosa/pilosa/pull/510))
-
-### Changed
-- Increase default partition count from 16 to 256 (BREAKING CHANGE) ([#520](https://github.com/pilosa/pilosa/pull/520))
-- Validate unknown query params ([#578](https://github.com/pilosa/pilosa/pull/578))
-- Validate configuration file ([#573](https://github.com/pilosa/pilosa/pull/573))
-- Change default cache type to ranked ([#524](https://github.com/pilosa/pilosa/pull/524))
-- Add max-writes-per-requests limit ([#525](https://github.com/pilosa/pilosa/pull/525))
-
-### Fixed
-- Add "make test" to PHONY section of Makefile ([#605](https://github.com/pilosa/pilosa/pull/605))
-- Fix failing tests when IPv6 is disabled ([#594](https://github.com/pilosa/pilosa/pull/594))
-- Add minor docs fix, indent in JSON ([#599](https://github.com/pilosa/pilosa/pull/599))
-- Fix BroadcastHandler handle missing index error ([#597](https://github.com/pilosa/pilosa/pull/597))
-- Add WebUI fixes ([#589](https://github.com/pilosa/pilosa/pull/589))
-- Fix support for 32-bit Linux ([#549](https://github.com/pilosa/pilosa/pull/549), [#565](https://github.com/pilosa/pilosa/pull/565))
-- Fix 3 separate bugs in bitmapCountRange ([#559](https://github.com/pilosa/pilosa/pull/559))
-- Add client support for MaxInverseSliceByIndex ([#555](https://github.com/pilosa/pilosa/pull/555))
-- Fix bug in `handleGetSliceMax` ([#554](https://github.com/pilosa/pilosa/pull/554))
-- Default to `standard` view in export command ([#548](https://github.com/pilosa/pilosa/pull/548))
-- Fix vet issues with the assembly code in Roaring ([#528](https://github.com/pilosa/pilosa/pull/528))
-- Prevent row labels that match the column label ([#503](https://github.com/pilosa/pilosa/pull/503))
-- Fix roaring test: TestBitmap_Quick_Array1 ([#507](https://github.com/pilosa/pilosa/pull/507))
-- Don't try to create inverse views on Import() when inverseEnabled is false ([#462](https://github.com/pilosa/pilosa/pull/462))
-
-### Performance
-- Set n based on array length instead of incrementing repeatedly ([#590](https://github.com/pilosa/pilosa/pull/590))
-- Rewrite intersectCountArrayBitmap for perf test ([#577](https://github.com/pilosa/pilosa/pull/577))
-- Check for duplicate attributes under read lock on insert ([#562](https://github.com/pilosa/pilosa/pull/562))
-
-[Unreleased]: https://github.com/pilosa/pilosa/compare/v0.5...HEAD
-[0.4.0]: https://github.com/pilosa/pilosa/compare/v0.3...v0.4
-[0.5.0]: https://github.com/pilosa/pilosa/compare/v0.4...v0.5
-[0.6.0]: https://github.com/pilosa/pilosa/compare/v0.5...v0.6
-[0.7.0]: https://github.com/pilosa/pilosa/compare/v0.6...v0.7
-[0.8.0]: https://github.com/pilosa/pilosa/compare/v0.7...v0.8

CODE_OF_CONDUCT.md

@@ -0,0 +1,133 @@
+
+# Contributor Covenant Code of Conduct
+
+## Our Pledge
+
+We as members, contributors, and leaders pledge to make participation in our
+community a harassment-free experience for everyone, regardless of age, body
+size, visible or invisible disability, ethnicity, sex characteristics, gender
+identity and expression, level of experience, education, socio-economic status,
+nationality, personal appearance, race, caste, color, religion, or sexual
+identity and orientation.
+
+We pledge to act and interact in ways that contribute to an open, welcoming,
+diverse, inclusive, and healthy community.
+
+## Our Standards
+
+Examples of behavior that contributes to a positive environment for our
+community include:
+
+* Demonstrating empathy and kindness toward other people
+* Being respectful of differing opinions, viewpoints, and experiences
+* Giving and gracefully accepting constructive feedback
+* Accepting responsibility and apologizing to those affected by our mistakes,
+  and learning from the experience
+* Focusing on what is best not just for us as individuals, but for the overall
+  community
+
+Examples of unacceptable behavior include:
+
+* The use of sexualized language or imagery, and sexual attention or advances of
+  any kind
+* Trolling, insulting or derogatory comments, and personal or political attacks
+* Public or private harassment
+* Publishing others' private information, such as a physical or email address,
+  without their explicit permission
+* Other conduct which could reasonably be considered inappropriate in a
+  professional setting
+
+## Enforcement Responsibilities
+
+Community leaders are responsible for clarifying and enforcing our standards of
+acceptable behavior and will take appropriate and fair corrective action in
+response to any behavior that they deem inappropriate, threatening, offensive,
+or harmful.
+
+Community leaders have the right and responsibility to remove, edit, or reject
+comments, commits, code, wiki edits, issues, and other contributions that are
+not aligned to this Code of Conduct, and will communicate reasons for moderation
+decisions when appropriate.
+
+## Scope
+
+This Code of Conduct applies within all community spaces, and also applies when
+an individual is officially representing the community in public spaces.
+Examples of representing our community include using an official e-mail address,
+posting via an official social media account, or acting as an appointed
+representative at an online or offline event.
+
+## Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be
+reported to the community leaders responsible for enforcement at
+community@featurebase.com.
+All complaints will be reviewed and investigated promptly and fairly.
+
+All community leaders are obligated to respect the privacy and security of the
+reporter of any incident.
+
+## Enforcement Guidelines
+
+Community leaders will follow these Community Impact Guidelines in determining
+the consequences for any action they deem in violation of this Code of Conduct:
+
+### 1. Correction
+
+**Community Impact**: Use of inappropriate language or other behavior deemed
+unprofessional or unwelcome in the community.
+
+**Consequence**: A private, written warning from community leaders, providing
+clarity around the nature of the violation and an explanation of why the
+behavior was inappropriate. A public apology may be requested.
+
+### 2. Warning
+
+**Community Impact**: A violation through a single incident or series of
+actions.
+
+**Consequence**: A warning with consequences for continued behavior. No
+interaction with the people involved, including unsolicited interaction with
+those enforcing the Code of Conduct, for a specified period of time. This
+includes avoiding interactions in community spaces as well as external channels
+like social media. Violating these terms may lead to a temporary or permanent
+ban.
+
+### 3. Temporary Ban
+
+**Community Impact**: A serious violation of community standards, including
+sustained inappropriate behavior.
+
+**Consequence**: A temporary ban from any sort of interaction or public
+communication with the community for a specified period of time. No public or
+private interaction with the people involved, including unsolicited interaction
+with those enforcing the Code of Conduct, is allowed during this period.
+Violating these terms may lead to a permanent ban.
+
+### 4. Permanent Ban
+
+**Community Impact**: Demonstrating a pattern of violation of community
+standards, including sustained inappropriate behavior, harassment of an
+individual, or aggression toward or disparagement of classes of individuals.
+
+**Consequence**: A permanent ban from any sort of public interaction within the
+community.
+
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage],
+version 2.1, available at
+[https://www.contributor-covenant.org/version/2/1/code_of_conduct.html][v2.1].
+
+Community Impact Guidelines were inspired by
+[Mozilla's code of conduct enforcement ladder][Mozilla CoC].
+
+For answers to common questions about this code of conduct, see the FAQ at
+[https://www.contributor-covenant.org/faq][FAQ]. Translations are available at
+[https://www.contributor-covenant.org/translations][translations].
+
+[homepage]: https://www.contributor-covenant.org
+[v2.1]: https://www.contributor-covenant.org/version/2/1/code_of_conduct.html
+[Mozilla CoC]: https://github.com/mozilla/diversity
+[FAQ]: https://www.contributor-covenant.org/faq
+[translations]: https://www.contributor-covenant.org/translations

CONTRIBUTING.md

@@ -1,62 +0,0 @@
-# Contributing to Pilosa
-
-## Reporting a bug
-
-If you have discovered a bug and don't see it in the [github issue tracker][5], [open a new issue][1]
-
-## Submitting a feature request
-
-Feature requests are managed in Github issues. New features typically go through a [Proposal Process][4]
-which starts by [opening a new issue][1] that describes the new feature proposal.
-
-## Submitting code changes
-
-Before you start working on new features, you should [open a new issue][1] to let others know what
-you're doing before you start working, otherwise you run the risk of duplicating effort. This also
-gives others an opportunity to provide input for your feature.
-
-If you want to help but you aren't sure where to start, check out our [github label for low-effort issues][6].
-
-- Fork the [Pilosa repository][2] and then clone your fork:
-
-    ```shell
-    git clone git@github.com:<your-name>/pilosa.git
-    ```
-
-- Create a local feature branch:
-
-    ```shell
-    git checkout -b something-amazing
-    ```
-
-- Commit your changes locally using `git add` and `git commit`.
-
-- Make sure that you've written tests for your new feature, and then run the tests:
-
-    ```shell
-    make test
-    ```
-
-- Verify that your pull request is applied to the latest version of code on github:
-
-    ```shell
-    git remote add upstream git@github.com:pilosa/pilosa.git
-    git fetch upstream
-    git rebase -i upstream/master
-    ```
-
-- Push to your fork:
-
-    ```shell
-    git push -u <yourfork> something-amazing
-    ```
-
-- Submit a [pull request][3]
-
-
-[1]: https://github.com/pilosa/pilosa/issues/new
-[2]: https://github.com/pilosa/pilosa
-[3]: https://github.com/pilosa/pilosa/compare/
-[4]: https://github.com/pilosa/general/blob/master/proposal.md
-[5]: https://github.com/pilosa/pilosa/issues
-[6]: https://github.com/pilosa/pilosa/issues?q=is%3Aopen+is%3Aissue+label%3Anewcomer

Dockerfile

@@ -1,21 +1,58 @@
-FROM golang:1.9.2 as builder
+ARG GO_VERSION=latest
 
-ARG ldflags=''
+#######################
+### Lattice builder ###
+#######################
 
-COPY . /go/src/github.com/pilosa/pilosa
+FROM ghcr.io/featurebasedb/nodejs:0.0.1 as lattice-builder
+WORKDIR /lattice
 
-RUN cd /go/src/github.com/pilosa/pilosa \
-    && make vendor \
-    && CGO_ENABLED=0 go install -a -ldflags "$ldflags" github.com/pilosa/pilosa/cmd/pilosa
+COPY lattice/package.json ./
+COPY lattice/yarn.lock ./
+RUN yarn install
 
-FROM scratch
+COPY lattice ./
+RUN yarn build
 
-LABEL maintainer "dev@pilosa.com"
+######################
+### Pilosa builder ###
+######################
 
-COPY --from=builder /go/bin/pilosa /pilosa
+FROM golang:${GO_VERSION} as pilosa-builder
+ARG MAKE_FLAGS
+ARG SOURCE_DATE_EPOCH
+
+WORKDIR /pilosa
+
+RUN go install github.com/rakyll/statik@v0.1.7
+
+COPY . ./
+COPY --from=lattice-builder /lattice/build /lattice
+RUN /go/bin/statik -src=/lattice -dest=/pilosa
+
+ENV SOURCE_DATE_EPOCH=${SOURCE_DATE_EPOCH}
+RUN make build FLAGS="-o build/featurebase" ${MAKE_FLAGS}
+
+#####################
+### Pilosa runner ###
+#####################
+
+FROM alpine:3.13.2 as runner
+
+LABEL maintainer "dev@molecula.com"
+
+RUN apk add --no-cache curl jq tree
+
+COPY --from=pilosa-builder /pilosa/build/featurebase /
+
+COPY NOTICE /NOTICE
 
 EXPOSE 10101
 VOLUME /data
 
-ENTRYPOINT ["/pilosa"]
-CMD ["server", "--data-dir", "/data", "--bind", "http://0.0.0.0:10101"]
+ENV PILOSA_DATA_DIR /data
+ENV PILOSA_BIND 0.0.0.0:10101
+ENV PILOSA_BIND_GRPC 0.0.0.0:20101
+
+ENTRYPOINT ["/featurebase"]
+CMD ["server"]

Dockerfile-clustertests

@@ -0,0 +1,38 @@
+# This Dockerfile is used for cluster testing - it produces a much larger image
+# and includes all of Go as well as some utilities.
+
+FROM golang:1.19
+
+LABEL maintainer "dev@pilosa.com"
+
+COPY . /go/src/github.com/featurebasedb/featurebase/
+
+
+# download pumba for fault injection
+ADD https://github.com/alexei-led/pumba/releases/download/0.6.0/pumba_linux_amd64 /pumba
+RUN chmod +x /pumba
+
+# add docker client to pause/unpause nodes
+RUN apt update
+RUN apt install -y docker.io
+
+# add docker-compose so tests can use it for stuff
+ADD https://github.com/docker/compose/releases/latest/download/docker-compose-Linux-x86_64 /usr/local/bin/docker-compose
+RUN chmod +x /usr/local/bin/docker-compose
+
+WORKDIR /go/src/github.com/featurebasedb/featurebase/cmd/featurebase
+
+# generate an instrumented binary to allow for calculating code coverage for clustertests
+# the entrypoint for the binary is TestRunMain, which is wrapper for main
+RUN go test -covermode=atomic -coverpkg=../../... -c -tags testrunmain -o featurebase
+RUN cp /go/src/github.com/featurebasedb/featurebase/cmd/featurebase/featurebase /featurebase
+
+COPY NOTICE /NOTICE
+
+EXPOSE 10101
+VOLUME /data
+
+
+# use e.g. "-test.coverprofile=/results/coverage.out"
+CMD ["/featurebase", "-test.run=TestRunMain", "server"]
+

Dockerfile-clustertests-client

@@ -0,0 +1,37 @@
+# This Dockerfile is used for cluster testing - it produces a much larger image
+# and includes all of Go as well as some utilities.
+
+FROM golang:1.19
+
+LABEL maintainer "dev@pilosa.com"
+
+COPY . /go/src/github.com/featurebasedb/featurebase/
+
+# download pumba for fault injection
+ADD https://github.com/alexei-led/pumba/releases/download/0.6.0/pumba_linux_amd64 /pumba
+RUN chmod +x /pumba
+
+# add docker client to pause/unpause nodes
+RUN apt update
+RUN apt install -y docker.io
+
+# add docker-compose so tests can use it for stuff
+ADD https://github.com/docker/compose/releases/latest/download/docker-compose-Linux-x86_64 /usr/local/bin/docker-compose
+RUN chmod +x /usr/local/bin/docker-compose
+
+WORKDIR /go/src/github.com/featurebasedb/featurebase/cmd/featurebase
+
+RUN go test -covermode=atomic -coverpkg=../../... -c -tags testrunmain -o featurebase
+RUN cp /go/src/github.com/featurebasedb/featurebase/cmd/featurebase/featurebase /featurebase
+
+
+COPY NOTICE /NOTICE
+
+COPY ./internal/clustertests /go/src/github.com/featurebasedb/featurebase/internal/clustertests
+
+EXPOSE 10101
+VOLUME /data
+
+WORKDIR /go/src/github.com/featurebasedb/featurebase
+
+CMD ["/featurebase", "-test.run=TestRunMain", "-test.coverprofile=/results/coverage.out", "server", "--data-dir", "/data", "--bind", "http://0.0.0.0:10101"]

Dockerfile-datagen

@@ -0,0 +1,47 @@
+# syntax=docker/dockerfile:1
+
+##########################
+### datagen builder    ###
+##########################
+
+FROM golang:alpine as builder
+
+WORKDIR /featurebase
+
+COPY . ./
+
+RUN apk add --no-cache build-base bash git make librdkafka pkgconfig
+
+# install librdkafka
+RUN git clone https://github.com/edenhill/librdkafka.git
+RUN cd librdkafka && ./configure --prefix /usr && make && make install
+
+ENV PKG_CONFIG_PATH=/usr/lib/pkgconfig/
+
+RUN cd idk && make build-datagen
+
+# ENTRYPOINT ["tail", "-f", "/dev/null"]
+
+#########################
+### datagen runner    ###
+#########################
+
+FROM alpine:3.15.3 as runner
+
+WORKDIR /
+
+LABEL maintainer "dev@molecula.com"
+
+RUN apk add --no-cache curl jq
+
+COPY --from=builder /featurebase/idk/build/datagen /bin/
+COPY --from=builder /usr/lib/librdkafka* /usr/lib/
+COPY idk/datagen/testdata/* /testdata/
+
+EXPOSE 8080
+
+# VOLUME /data
+# ENV ADDR 0.0.0.0:8080
+
+#ENTRYPOINT ["sleep", "infinity"]
+ENTRYPOINT ["datagen"]

Dockerfile-dax

@@ -0,0 +1,32 @@
+ARG GO_VERSION=latest
+
+
+###########################
+### FeatureBase Builder ###
+###########################
+
+FROM golang:${GO_VERSION} as featurebase-builder
+ARG MAKE_FLAGS
+WORKDIR /fb
+
+COPY . ./
+RUN make build FLAGS="-o build/featurebase" ${MAKE_FLAGS}
+
+##########################
+### FeatureBase runner ###
+##########################
+
+FROM golang:alpine as runner
+
+LABEL maintainer "dev@featurebase.com"
+
+RUN apk add --no-cache curl jq tree
+
+COPY --from=featurebase-builder /fb/build/featurebase /
+
+COPY NOTICE /NOTICE
+
+EXPOSE 8080
+
+ENTRYPOINT ["/featurebase"]
+CMD ["dax"]

Dockerfile-dax-quick

@@ -0,0 +1,18 @@
+ARG GO_VERSION=latest
+
+##########################
+### FeatureBase runner ###
+##########################
+
+FROM alpine:3.13.2 as runner
+
+LABEL maintainer "dev@featurebase.com"
+
+RUN apk add --no-cache curl jq tree
+
+COPY ./fb_linux /featurebase
+
+EXPOSE 8080
+
+ENTRYPOINT ["/featurebase"]
+CMD ["dax"]

Dockerfile-fbsql

@@ -0,0 +1,48 @@
+ARG GO_VERSION=1.19
+
+FROM golang:1.19-buster as builder
+
+WORKDIR /
+RUN apt-get update -y -qq && apt-get install -y -qq \
+  build-essential \
+  git \
+  musl-tools \
+  netcat \
+  unixodbc \
+  unixodbc-dev \
+  && rm -rf /var/lib/apt/lists/*
+RUN ["git", "clone", "https://github.com/edenhill/librdkafka.git"]
+WORKDIR /librdkafka
+RUN ./configure --prefix /usr && \
+  make && \
+  make install
+
+WORKDIR /featurebase
+
+COPY . .
+
+ARG MAKE_FLAGS
+ARG GO_BUILD_FLAGS
+ARG SOURCE_DATE_EPOCH
+
+WORKDIR /featurebase/
+
+ENV SOURCE_DATE_EPOCH=${SOURCE_DATE_EPOCH}
+RUN make build-fbsql GO_BUILD_FLAGS="-mod=vendor ${GO_BUILD_FLAGS}" ${MAKE_FLAGS}
+
+FROM ubuntu:20.04 as runner
+
+RUN apt-get update -y -qq && apt-get install -y -qq \
+  ca-certificates \
+  musl-tools \
+  netcat \
+  unixodbc-dev \
+  && rm -rf /var/lib/apt/lists/*
+
+COPY --from=builder /featurebase/fbsql /usr/local/bin/
+
+# Verify that the linker can find everything.
+FROM runner as linkcheck
+RUN if [ -e /usr/local/bin/fbsql ] ; then ldd /usr/local/bin/fbsql; fi
+
+FROM runner

Gopkg.lock

@@ -1,243 +0,0 @@
-# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'.
-
-
-[[projects]]
-  name = "github.com/BurntSushi/toml"
-  packages = ["."]
-  revision = "b26d9c308763d68093482582cea63d69be07a0f0"
-  version = "v0.3.0"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/CAFxX/gcnotifier"
-  packages = ["."]
-  revision = "39b0596a2da3c92787b3319c6b5425a474b4e0da"
-
-[[projects]]
-  name = "github.com/DataDog/datadog-go"
-  packages = ["statsd"]
-  revision = "0ddda6bee21174ef6c4873647cb0d6ec9cba996f"
-  version = "1.1.0"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/armon/go-metrics"
-  packages = ["."]
-  revision = "9a4b6e10bed6220a1665955aa2b75afc91eb10b3"
-
-[[projects]]
-  name = "github.com/boltdb/bolt"
-  packages = ["."]
-  revision = "2f1ce7a837dcb8da3ec595b1dac9d0632f0f99e8"
-  version = "v1.3.1"
-
-[[projects]]
-  name = "github.com/davecgh/go-spew"
-  packages = ["spew"]
-  revision = "346938d642f2ec3594ed81d874461961cd0faa76"
-  version = "v1.1.0"
-
-[[projects]]
-  name = "github.com/fsnotify/fsnotify"
-  packages = ["."]
-  revision = "629574ca2a5df945712d3079857300b5e4da0236"
-  version = "v1.4.2"
-
-[[projects]]
-  name = "github.com/gogo/protobuf"
-  packages = ["proto"]
-  revision = "342cbe0a04158f6dcb03ca0079991a51a4248c02"
-  version = "v0.5"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/golang/groupcache"
-  packages = ["lru"]
-  revision = "84a468cf14b4376def5d68c722b139b881c450a4"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/golang/protobuf"
-  packages = ["proto"]
-  revision = "1643683e1b54a9e88ad26d98f81400c8c9d9f4f9"
-
-[[projects]]
-  name = "github.com/gorilla/context"
-  packages = ["."]
-  revision = "1ea25387ff6f684839d82767c1733ff4d4d15d0a"
-  version = "v1.1"
-
-[[projects]]
-  name = "github.com/gorilla/mux"
-  packages = ["."]
-  revision = "7f08801859139f86dfafd1c296e2cba9a80d292e"
-  version = "v1.6.0"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/hashicorp/errwrap"
-  packages = ["."]
-  revision = "7554cd9344cec97297fa6649b055a8c98c2a1e55"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/hashicorp/go-immutable-radix"
-  packages = ["."]
-  revision = "8aac2701530899b64bdea735a1de8da899815220"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/hashicorp/go-msgpack"
-  packages = ["codec"]
-  revision = "fa3f63826f7c23912c15263591e65d54d080b458"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/hashicorp/go-multierror"
-  packages = ["."]
-  revision = "83588e72410abfbe4df460eeb6f30841ae47d4c4"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/hashicorp/go-sockaddr"
-  packages = ["."]
-  revision = "9b4c5fa5b10a683339a270d664474b9f4aee62fc"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/hashicorp/golang-lru"
-  packages = ["simplelru"]
-  revision = "0a025b7e63adc15a622f29b0b2c4c3848243bbf6"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/hashicorp/hcl"
-  packages = [".","hcl/ast","hcl/parser","hcl/scanner","hcl/strconv","hcl/token","json/parser","json/scanner","json/token"]
-  revision = "23c074d0eceb2b8a5bfdbb271ab780cde70f05a8"
-
-[[projects]]
-  name = "github.com/hashicorp/memberlist"
-  packages = ["."]
-  revision = "ce8abaa0c60c2d6bee7219f5ddf500e0a1457b28"
-  version = "v0.1.0"
-
-[[projects]]
-  name = "github.com/inconshreveable/mousetrap"
-  packages = ["."]
-  revision = "76626ae9c91c4f2a10f34cad8ce83ea42c93bb75"
-  version = "v1.0"
-
-[[projects]]
-  name = "github.com/magiconair/properties"
-  packages = ["."]
-  revision = "be5ece7dd465ab0765a9682137865547526d1dfb"
-  version = "v1.7.3"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/miekg/dns"
-  packages = [".","internal/socket"]
-  revision = "9fc4eb252eedf0ef8adc05169ce35da5e31beaba"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/mitchellh/mapstructure"
-  packages = ["."]
-  revision = "06020f85339e21b2478f756a78e295255ffa4d6a"
-
-[[projects]]
-  name = "github.com/pelletier/go-toml"
-  packages = ["."]
-  revision = "16398bac157da96aa88f98a2df640c7f32af1da2"
-  version = "v1.0.1"
-
-[[projects]]
-  name = "github.com/rakyll/statik"
-  packages = ["fs"]
-  revision = "fd36b3595eb2ec8da4b8153b107f7ea08504899d"
-  version = "v0.1.1"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/sean-/seed"
-  packages = ["."]
-  revision = "e2103e2c35297fb7e17febb81e49b312087a2372"
-
-[[projects]]
-  name = "github.com/sony/gobreaker"
-  packages = ["."]
-  revision = "e9556a45379ef1da12e54847edb2fb3d7d566f36"
-  version = "0.3.0"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/spf13/afero"
-  packages = [".","mem"]
-  revision = "5660eeed305fe5f69c8fc6cf899132a459a97064"
-
-[[projects]]
-  name = "github.com/spf13/cast"
-  packages = ["."]
-  revision = "acbeb36b902d72a7a4c18e8f3241075e7ab763e4"
-  version = "v1.1.0"
-
-[[projects]]
-  name = "github.com/spf13/cobra"
-  packages = ["."]
-  revision = "7b2c5ac9fc04fc5efafb60700713d4fa609b777b"
-  version = "v0.0.1"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/spf13/jwalterweatherman"
-  packages = ["."]
-  revision = "12bd96e66386c1960ab0f74ced1362f66f552f7b"
-
-[[projects]]
-  name = "github.com/spf13/pflag"
-  packages = ["."]
-  revision = "e57e3eeb33f795204c1ca35f56c44f83227c6e66"
-  version = "v1.0.0"
-
-[[projects]]
-  name = "github.com/spf13/viper"
-  packages = ["."]
-  revision = "25b30aa063fc18e48662b86996252eabdcf2f0c7"
-  version = "v1.0.0"
-
-[[projects]]
-  branch = "master"
-  name = "golang.org/x/net"
-  packages = ["context"]
-  revision = "a337091b0525af65de94df2eb7e98bd9962dcbe2"
-
-[[projects]]
-  branch = "master"
-  name = "golang.org/x/sync"
-  packages = ["errgroup"]
-  revision = "fd80eb99c8f653c847d294a001bdf2a3a6f768f5"
-
-[[projects]]
-  branch = "master"
-  name = "golang.org/x/sys"
-  packages = ["unix"]
-  revision = "1e2299c37cc91a509f1b12369872d27be0ce98a6"
-
-[[projects]]
-  branch = "master"
-  name = "golang.org/x/text"
-  packages = ["internal/gen","internal/triegen","internal/ucd","transform","unicode/cldr","unicode/norm"]
-  revision = "88f656faf3f37f690df1a32515b479415e1a6769"
-
-[[projects]]
-  branch = "v2"
-  name = "gopkg.in/yaml.v2"
-  packages = ["."]
-  revision = "eb3733d160e74a9c7e442f435eb3bea458e1d19f"
-
-[solve-meta]
-  analyzer-name = "dep"
-  analyzer-version = 1
-  inputs-digest = "75badb0bcc3bb356b04af17979e0af61b4b66c5e0a483f09e39cf1f9b5e5de2c"
-  solver-name = "gps-cdcl"
-  solver-version = 1

Gopkg.toml

@@ -1,3 +0,0 @@
-# This file intentionally left blank as all needed dependencies are imported by
-# the project and thus tracked by `dep`.
-# See https://github.com/golang/dep/blob/master/docs/Gopkg.toml.md for details.

LICENSE

@@ -1,4 +1,3 @@
-
                                  Apache License
                            Version 2.0, January 2004
                         http://www.apache.org/licenses/
@@ -187,7 +186,7 @@
       same "printed page" as the copyright notice for easier
       identification within third-party archives.
 
-   Copyright [yyyy] [name of copyright owner]
+   Copyright 2023 Molecula Corp. All rights reserved.
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.

Makefile

@@ -1,115 +1,405 @@
-.PHONY: dep docker pilosa release-build prerelease-build release prerelease prerelease-upload install generate statik test cover cover-pkg cover-viz clean docker-build docker-test
+.PHONY: build clean build-lattice cover cover-viz default docker docker-build docker-build-fbsql docker-tag-push generate generate-protoc generate-pql generate-statik generate-stringer install install-protoc-gen-gofast install-protoc install-statik install-peg test docker-login
 
-DEP := $(shell command -v dep 2>/dev/null)
-STATIK := $(shell command -v statik 2>/dev/null)
-PROTOC := $(shell command -v protoc 2>/dev/null)
+SHELL := /bin/bash
 VERSION := $(shell git describe --tags 2> /dev/null || echo unknown)
-IDENTIFIER := $(VERSION)-$(GOOS)-$(GOARCH)
-CLONE_URL=github.com/pilosa/pilosa
-PKGS := $(shell cd $(GOPATH)/src/$(CLONE_URL); go list ./... | grep -v vendor)
-BUILD_TIME=`date -u +%FT%T%z`
-LDFLAGS="-X github.com/pilosa/pilosa.Version=$(VERSION) -X github.com/pilosa/pilosa.BuildTime=$(BUILD_TIME)"
-DOCKER_GOLANG_IMAGE=golang:latest
+VARIANT = Molecula
+GO=go
+GOOS=$(shell $(GO) env GOOS)
+GOARCH=$(shell $(GO) env GOARCH)
+VERSION_ID=$(if $(TRIAL_DEADLINE),trial-$(TRIAL_DEADLINE)-,)$(VERSION)-$(GOOS)-$(GOARCH)
+DATE_FMT="+%FT%T%z"
+# set SOURCE_DATE_EPOCH like this to use the last git commit timestamp
+# export SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct) instead of the current time from running `date`
+ifdef SOURCE_DATE_EPOCH
+	BUILD_TIME ?= $(shell date -u -d "@$(SOURCE_DATE_EPOCH)" "$(DATE_FMT)" 2>/dev/null || date -u -r "$(SOURCE_DATE_EPOCH)" "$(DATE_FMT)" 2>/dev/null || date -u "$(DATE_FMT)")
+else
+	BUILD_TIME ?= $(shell date -u "$(DATE_FMT)")
+endif
+SHARD_WIDTH = 20
+COMMIT := $(shell git describe --exact-match >/dev/null 2>&1 || git rev-parse --short HEAD)
+LDFLAGS="-X github.com/featurebasedb/featurebase/v3.Version=$(VERSION) -X github.com/featurebasedb/featurebase/v3.BuildTime=$(BUILD_TIME) -X github.com/featurebasedb/featurebase/v3.Variant=$(VARIANT) -X github.com/featurebasedb/featurebase/v3.Commit=$(COMMIT) -X github.com/featurebasedb/featurebase/v3.TrialDeadline=$(TRIAL_DEADLINE)"
+GO_VERSION=1.19
+GO_BUILD_FLAGS=
+DOCKER_BUILD= # set to 1 to use `docker-build` instead of `build` when creating a release
+BUILD_TAGS += 
+TEST_TAGS = roaringparanoia
+TEST_TIMEOUT=10m
+RACE_TEST_TIMEOUT=10m
+# size in GB to use for ramdisk, ?= so you can override it with env
+# 4GB is not enough for `make test`, 8GB usually is.
+RAMDISK_SIZE ?= 8
 
-default: test pilosa
+export GO111MODULE=on
+export GOPRIVATE=github.com/molecula
+export CGO_ENABLED=0
+AWS_ACCOUNTID ?= undefined
 
+# Run tests and compile Pilosa
+default: test build
+
+# Remove build directories
 clean:
 	rm -rf vendor build
+	rm -f *.rpm *.deb
 
-$(GOPATH)/bin:
-	mkdir $(GOPATH)/bin
+# Set up vendor directory using `go mod vendor`
+vendor: go.mod
+	$(GO) mod vendor
 
-dep: $(GOPATH)/bin
-	go get -u github.com/golang/dep/cmd/dep
+version:
+	@echo $(VERSION)
 
-vendor: Gopkg.toml
-ifndef DEP
-	make dep
-endif
-	dep ensure
-	touch vendor
+# We build a list of packages that omits the IDK and batch packages because
+# those packages require fancy environment setup.
+GOPACKAGES := $(shell $(GO) list ./... | grep -v "/v3/idk" | grep -v "/v3/batch")
 
-Gopkg.lock: dep Gopkg.toml
-	dep ensure
+# Run test suite
+test:
+	$(GO) test $(GOPACKAGES) -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) -v -timeout $(TEST_TIMEOUT) -count=1
 
-test: vendor
-	go test $(PKGS) $(TESTFLAGS)
+# Run test suite with race flag
+test-race:
+	CGO_ENABLED=1 $(GO) test $(GOPACKAGES) -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) -race -timeout $(RACE_TEST_TIMEOUT) -v
 
-cover: vendor
-	mkdir -p build/coverage
-	echo "mode: set" > build/coverage/all.out
-	for pkg in $(PKGS) ; do \
-		make cover-pkg PKG=$$pkg ; \
-	done
+testv: testvsub
 
-cover-pkg:
-	mkdir -p build/coverage
-	touch build/coverage/$(subst /,-,$(PKG)).out
-	go test -coverprofile=build/coverage/$(subst /,-,$(PKG)).out $(PKG)
-	tail -n +2 build/coverage/$(subst /,-,$(PKG)).out >> build/coverage/all.out
+testv-race: testvsub-race
 
+# testvsub: run go test -v in sub-directories in "local mode" with incremental output,
+#            avoiding go -test ./... "package list mode" which doesn't give output
+#            until the test run finishes. Package list mode makes it hard to
+#            find which test is hung/deadlocked.
+#
+testvsub:
+	@set -e; for pkg in $(GOPACKAGES); do \
+			if [ $${pkg:0:38} == "github.com/featurebasedb/featurebase/v3/idk" ]; then \
+				echo; echo "___ skipping subpkg $$pkg"; \
+				continue; \
+			fi; \
+			echo; echo "___ testing subpkg $$pkg"; \
+			$(GO) test -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) -v -timeout $(RACE_TEST_TIMEOUT) $$pkg || break; \
+			echo; echo "999 done testing subpkg $$pkg"; \
+		done
+
+# make a $(RAMDISK_SIZE)GB RAMDisk. Speed up tests by running
+# them with TMPDIR=/mnt/ramdisk.
+ramdisk-linux:
+	mount -o size=$(RAMDISK__SIZE)G -t tmpfs none /mnt/ramdisk
+
+# make a $(RAMDISK_SIZE)GB RAMDisk. Speed up tests by running
+# them with TMPDIR=/Volumes/RAMDisk. This is more important on
+# OS X than it is on Linux, because there's performance issues
+# with fsync on OS X that can make the SSD slow down to moving-platters
+# drive speeds. Oops.
+ramdisk-osx:
+	diskutil erasevolume HFS+ 'RAMDisk' $$(hdiutil attach -nobrowse -nomount ram://$$(expr 2097152 \* $(RAMDISK_SIZE)))
+
+detach-ramdisk-osx:
+	hdiutil detach /Volumes/RAMDisk
+
+testvsub-race:
+	@set -e; for pkg in $(GOPACKAGES); do \
+           echo; echo "___ testing subpkg $$pkg"; \
+           CGO_ENABLED=1 $(GO) test -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) -v -race -timeout $(RACE_TEST_TIMEOUT) $$pkg || break; \
+           echo; echo "999 done testing subpkg $$pkg"; \
+        done
+
+
+bench:
+	$(GO) test $(GOPACKAGES) -bench=. -run=NoneZ -timeout=127m $(TESTFLAGS)
+
+# Run test suite with coverage enabled
+cover:
+	mkdir -p build
+	$(MAKE) test TESTFLAGS="-coverprofile=build/coverage.out"
+
+# Run test suite with coverage enabled and view coverage results in browser
 cover-viz: cover
-	go tool cover -html=build/coverage/all.out
+	$(GO) tool cover -html=build/coverage.out
 
-pilosa: vendor
-	go build -ldflags $(LDFLAGS) $(FLAGS) $(CLONE_URL)/cmd/pilosa
+# Build featurebase
+build:
+	$(GO) build -tags='$(BUILD_TAGS)' -ldflags $(LDFLAGS) $(FLAGS) ./cmd/featurebase
 
-release-build: vendor
-ifdef DOCKER_BUILD
-	make docker-build FLAGS="-o build/pilosa-$(IDENTIFIER)/pilosa"
-else
-	make pilosa FLAGS="-o build/pilosa-$(IDENTIFIER)/pilosa"
+package:
+	GOOS=$(GOOS) GOARCH=$(GOARCH) $(MAKE) build
+	GOOS=$(GOOS) GOARCH=$(GOARCH) $(MAKE) build-fbsql
+	GOARCH=$(GOARCH) VERSION=$(VERSION) nfpm package --packager deb --target featurebase.$(VERSION).$(GOARCH).deb
+	GOARCH=$(GOARCH) VERSION=$(VERSION) nfpm package --packager rpm --target featurebase.$(VERSION).$(GOARCH).rpm
+
+# We allow setting a custom docker-compose "project". Multiple of the
+# same docker-compose environment can exist simultaneously as long as
+# they use different projects (the project name is prepended to
+# container names and such). This is useful in a CI environment where
+# we might be running multiple instances of the tests concurrently.
+PROJECT ?= clustertests
+DOCKER_COMPOSE = docker-compose -p $(PROJECT)
+
+# Run cluster integration tests using docker. Requires docker daemon to be
+# running and docker-compose to be installed.
+clustertests: vendor
+	$(DOCKER_COMPOSE) -f internal/clustertests/docker-compose.yml down
+	$(DOCKER_COMPOSE) -f internal/clustertests/docker-compose.yml build
+	$(DOCKER_COMPOSE) -f internal/clustertests/docker-compose.yml up -d pilosa1 pilosa2 pilosa3
+	PROJECT=$(PROJECT) $(DOCKER_COMPOSE) -f internal/clustertests/docker-compose.yml run client1
+	$(DOCKER_COMPOSE) -f internal/clustertests/docker-compose.yml down
+
+# Run the cluster tests with authentication enabled
+AUTH_ARGS="-c /go/src/github.com/featurebasedb/featurebase/internal/clustertests/testdata/featurebase.conf"
+authclustertests: vendor
+	CLUSTERTESTS_FB_ARGS=$(AUTH_ARGS) $(DOCKER_COMPOSE) -f internal/clustertests/docker-compose.yml down
+	CLUSTERTESTS_FB_ARGS=$(AUTH_ARGS) $(DOCKER_COMPOSE) -f internal/clustertests/docker-compose.yml build
+	CLUSTERTESTS_FB_ARGS=$(AUTH_ARGS) $(DOCKER_COMPOSE) -f internal/clustertests/docker-compose.yml up -d pilosa1 pilosa2 pilosa3
+	PROJECT=$(PROJECT) ENABLE_AUTH=1 $(DOCKER_COMPOSE) -f internal/clustertests/docker-compose.yml run client1
+	CLUSTERTESTS_FB_ARGS=$(AUTH_ARGS) $(DOCKER_COMPOSE) -f internal/clustertests/docker-compose.yml down
+
+# Install FeatureBase and IDK
+install: install-featurebase install-idk install-fbsql
+
+install-featurebase:
+	$(GO) install -tags='$(BUILD_TAGS)' -ldflags $(LDFLAGS) $(FLAGS) ./cmd/featurebase
+
+install-idk:
+	$(MAKE) -C ./idk install
+
+install-fbsql:
+	CGO_ENABLED=1 $(GO) install ./cmd/fbsql
+
+# Build the lattice assets
+build-lattice:
+	docker build -t lattice:build ./lattice
+	export LATTICE=`docker create lattice:build`; docker cp $$LATTICE:/lattice/. ./lattice/build && docker rm $$LATTICE
+
+# `go generate` protocol buffers
+generate-protoc: require-protoc require-protoc-gen-gofast
+	$(GO) generate github.com/featurebasedb/featurebase/v3/pb
+
+# `go generate` statik assets (lattice UI)
+generate-statik: build-lattice require-statik
+	$(GO) generate github.com/featurebasedb/featurebase/v3/statik
+
+# `go generate` statik assets (lattice UI) in Docker
+generate-statik-docker: build-lattice
+	docker run --rm -t -v $(PWD):/pilosa golang:1.15.8 sh -c "go get github.com/rakyll/statik && /go/bin/statik -src=/pilosa/lattice/build -dest=/pilosa -f"
+
+# `go generate` stringers
+generate-stringer:
+	$(GO) generate github.com/featurebasedb/featurebase/v3
+
+generate-pql: require-peg
+	cd pql && peg -inline pql.peg && cd ..
+
+generate-proto-grpc: require-protoc require-protoc-gen-go
+	protoc -I proto proto/pilosa.proto --go_out=plugins=grpc:proto
+#	address re-generation here only if we need to	
+#	protoc -I proto proto/vdsm.proto --go_out=plugins=grpc:proto
+
+# `go generate` all needed packages
+generate: generate-protoc generate-statik generate-stringer generate-pql
+
+# Create release using Docker
+docker-release:
+	$(MAKE) docker-build GOOS=linux GOARCH=amd64
+	$(MAKE) docker-build GOOS=linux GOARCH=arm64
+	$(MAKE) docker-build GOOS=darwin GOARCH=amd64
+	$(MAKE) docker-build GOOS=darwin GOARCH=arm64
+
+# Build a release in Docker
+docker-build: vendor
+	docker build \
+	    --build-arg GO_VERSION=$(GO_VERSION) \
+	    --build-arg MAKE_FLAGS="TRIAL_DEADLINE=$(TRIAL_DEADLINE) GOOS=$(GOOS) GOARCH=$(GOARCH)" \
+	    --build-arg SOURCE_DATE_EPOCH=$(SOURCE_DATE_EPOCH) \
+	    --target pilosa-builder \
+	    --tag featurebase:build .
+	docker create --name featurebase-build featurebase:build
+	mkdir -p build/featurebase-$(VERSION_ID)
+	docker cp featurebase-build:/pilosa/build/. ./build/featurebase-$(VERSION_ID)
+	cp NOTICE install/featurebase.conf install/featurebase*.service ./build/featurebase-$(VERSION_ID)
+	docker rm featurebase-build
+	tar -cvz -C build -f build/featurebase-$(VERSION_ID).tar.gz featurebase-$(VERSION_ID)/
+
+# Create Docker image from Dockerfile
+docker-image: vendor
+	docker build \
+	    --build-arg GO_VERSION=$(GO_VERSION) \
+	    --build-arg MAKE_FLAGS="TRIAL_DEADLINE=$(TRIAL_DEADLINE)" \
+	    --tag featurebase:$(VERSION) .
+	@echo Created docker image: featurebase:$(VERSION)
+
+docker-image-featurebase: vendor
+	docker build \
+	    --build-arg GO_VERSION=$(GO_VERSION) \
+		--file Dockerfile-dax \
+		--tag dax/featurebase .
+
+docker-image-featurebase-linux-amd64: vendor
+	docker build \
+	    --build-arg GO_VERSION=$(GO_VERSION) \
+	    --platform linux/amd64 \
+	    --file Dockerfile-dax \
+	    --tag dax/featurebase .
+
+docker-image-featurebase-test: vendor
+	docker build \
+	    --build-arg GO_VERSION=$(GO_VERSION) \
+		--file Dockerfile-clustertests \
+		--tag dax/featurebase-test .
+
+
+# build-for-quick builds a linux featurebase binary outside of docker
+# (which is much faster for some reason), and places it in the .quick
+# subdirectory.
+build-for-quick:
+	GOOS=linux $(MAKE) build FLAGS="-o .quick/fb_linux"
+
+# docker-image-featurebase-quick uses a pre-built featurebase binary
+# to quickly create a fresh docker image without needing to send the
+# context of the featurebase top level directory.
+docker-image-featurebase-quick: build-for-quick
+	docker build \
+	    --build-arg GO_VERSION=$(GO_VERSION) \
+	    --file Dockerfile-dax-quick \
+	    --tag dax/featurebase ./.quick/
+
+
+docker-image-datagen: vendor
+	docker build --tag dax/datagen --file Dockerfile-datagen .
+
+get-account-id:
+	$(eval AWS_ACCOUNTID := $(shell aws sts get-caller-identity --output=json | jq -r .Account))
+
+
+ecr-push-featurebase: docker-login
+	echo "Pushing to account $(AWS_ACCOUNTID), profile $(AWS_PROFILE)"
+	docker tag dax/featurebase:latest $(AWS_ACCOUNTID).dkr.ecr.us-east-2.amazonaws.com/dax/featurebase:latest
+	docker push $(AWS_ACCOUNTID).dkr.ecr.us-east-2.amazonaws.com/dax/featurebase:latest
+
+ecr-push-datagen: docker-login
+	docker tag dax/datagen:latest $(AWS_ACCOUNTID).dkr.ecr.us-east-2.amazonaws.com/dax/datagen:latest
+	docker push $(AWS_ACCOUNTID).dkr.ecr.us-east-2.amazonaws.com/dax/datagen:latest
+
+docker-login: get-account-id
+	aws ecr get-login-password --region us-east-2 | docker login --username AWS --password-stdin $(AWS_ACCOUNTID).dkr.ecr.us-east-2.amazonaws.com
+
+# Create docker image (alias)
+docker: docker-image # alias
+
+# Tag and push a Docker image
+docker-tag-push: vendor
+	docker tag "featurebase:$(VERSION)" $(DOCKER_TARGET)
+	docker push $(DOCKER_TARGET)
+	@echo Pushed docker image: $(DOCKER_TARGET)
+
+# These commands (docker-idk and docker-idk-tag-push)
+# are designed to be used in CI.
+# docker-idk builds idk docker images and tags them - intended for use in CI.
+docker-idk: vendor
+	docker build \
+		-f idk/Dockerfile \
+		--build-arg GO_VERSION=$(GO_VERSION) \
+		--build-arg MAKE_FLAGS="GOOS=$(GOOS) GOARCH=$(GOARCH) BUILD_CGO=$(BUILD_CGO)" \
+		--tag registry.gitlab.com/molecula/featurebase/idk:$(VERSION_ID) .
+	@echo Created docker image: registry.gitlab.com/molecula/featurebase/idk:$(VERSION_ID)
+# docker-idk-tag-push pushes tagged docker images to the GitLab container
+# registry - intended for use in CI.
+docker-idk-tag-push:
+	docker push registry.gitlab.com/molecula/featurebase/idk:$(VERSION_ID)
+	@echo Pushed docker image: registry.gitlab.com/molecula/featurebase/idk:$(VERSION_ID)
+
+# Run golangci-lint
+golangci-lint: require-golangci-lint
+	golangci-lint run --timeout 3m --skip-files '.*\.peg\.go'
+
+# Alias
+linter: golangci-lint
+
+# Better alias
+ocd: golangci-lint
+
+######################
+# Build dependencies #
+######################
+
+# Verifies that needed build dependency is installed. Errors out if not installed.
+require-%:
+	$(if $(shell command -v $* 2>/dev/null),\
+		$(info Verified build dependency "$*" is installed.),\
+		$(error Build dependency "$*" not installed. To install, try `make install-$*`))
+
+install-build-deps: install-protoc-gen-gofast install-protoc install-statik install-peg
+
+install-statik:
+	go install github.com/rakyll/statik@latest
+
+install-protoc-gen-gofast:
+	GO111MODULE=off $(GO) get -u github.com/gogo/protobuf/protoc-gen-gofast
+
+install-protoc:
+	@echo This tool cannot automatically install protoc. Please download and install protoc from https://google.github.io/proto-lens/installing-protoc.html
+	@echo On mac, brew install protobuf seems to work.
+	@echo As of the commit that added this line, protoc-gen-gofast was at 226206f39bd7, and the protoc version in use was:
+	@echo $$ protoc --version
+	@echo libprotoc 3.19.4
+
+install-peg:
+	GO111MODULE=off $(GO) get github.com/pointlander/peg
+
+install-golangci-lint:
+	GO111MODULE=off $(GO) get github.com/golangci/golangci-lint/cmd/golangci-lint
+
+test-external-lookup:
+	$(GO) test . -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) -run ^TestExternalLookup$$ -externalLookupDSN $(EXTERNAL_LOOKUP_DSN)
+
+bnf:
+	ebnf2railroad --no-overview-diagram  --no-optimizations ./sql3/sql3.ebnf
+
+#################################
+# fbsql builds in docker
+#################################
+
+# This allows multiple concurrent builds to happen in CI without
+# creating container name conflicts and such. (different BUILD_NAMEs
+# are passed in from gitlab-ci.yml)
+BUILD_NAME ?= fbsql-build
+
+LDFLAGS_STATIC="-linkmode external -extldflags \"-static\" -X 'github.com/featurebasedb/featurebase/v3/fbsql.Version=$(VERSION)' -X 'github.com/featurebasedb/featurebase/v3/fbsql.BuildTime=$(BUILD_TIME)' "
+
+UNAME_P := $(shell uname -p)
+BUILD_CGO ?= 0
+
+# Build fbsql
+build-fbsql:
+	@echo GOOS=$(GOOS) GOARCH=$(GOARCH) uname -p=$(UNAME_P) build_cgo=$(BUILD_CGO)
+ifeq ($(BUILD_CGO), 0)
+	make build-fbsql-non-cgo
 endif
-	cp LICENSE README.md build/pilosa-$(IDENTIFIER)
-	tar -cvz -C build -f build/pilosa-$(IDENTIFIER).tar.gz pilosa-$(IDENTIFIER)/
-	@echo "Created release build: build/pilosa-$(IDENTIFIER).tar.gz"
-
-release:
-	make release-build GOOS=darwin GOARCH=amd64
-	make release-build GOOS=linux GOARCH=amd64 DOCKER_BUILD=1
-	make release-build GOOS=linux GOARCH=386 DOCKER_BUILD=1
-
-prerelease-build: vendor
-	make pilosa FLAGS="-o build/pilosa-master-$(GOOS)-$(GOARCH)/pilosa"
-	cp LICENSE README.md build/pilosa-master-$(GOOS)-$(GOARCH)
-	tar -cvz -C build -f build/pilosa-master-$(GOOS)-$(GOARCH).tar.gz pilosa-master-$(GOOS)-$(GOARCH)/
-	@echo "Created pre-release build: build/pilosa-master-$(GOOS)-$(GOARCH).tar.gz"
-
-prerelease:
-	make prerelease-build GOOS=linux GOARCH=amd64
-
-prerelease-upload: prerelease
-	aws s3 cp build/pilosa-master-linux-amd64.tar.gz s3://build.pilosa.com/pilosa-master-linux-amd64.tar.gz --acl public-read
-
-install: vendor
-	go install -ldflags $(LDFLAGS) $(FLAGS) $(CLONE_URL)/cmd/pilosa
-
-.protoc-gen-gofast: vendor
-ifndef PROTOC
-	$(error "protoc is not available. please install protoc from https://github.com/google/protobuf/releases")
-endif
-	go build -o .protoc-gen-gofast ./vendor/github.com/gogo/protobuf/protoc-gen-gofast
-	cp ./.protoc-gen-gofast $(GOPATH)/bin/protoc-gen-gofast
-
-generate-protoc: .protoc-gen-gofast
-	go generate github.com/pilosa/pilosa/internal
-
-generate-statik: statik
-	go generate github.com/pilosa/pilosa
-
-generate: generate-protoc generate-statik
-
-statik:
-ifndef STATIK
-	go get github.com/rakyll/statik
+ifeq ($(BUILD_CGO), 1)
+	make build-fbsql-cgo
 endif
 
-docker:
-	docker build -t "pilosa:$(VERSION)" --build-arg ldflags=$(LDFLAGS) .
-	@echo "Created image: pilosa:$(VERSION)"
+build-fbsql-non-cgo:
+	CGO_ENABLED=0 $(GO) build -ldflags $(LDFLAGS) $(GO_BUILD_FLAGS) -o fbsql ./cmd/fbsql
 
-docker-build:
-	docker run --rm -v $(PWD):/go/src/$(CLONE_URL) -w /go/src/$(CLONE_URL) -e GOOS=$(GOOS) -e GOARCH=$(GOARCH) $(DOCKER_GOLANG_IMAGE) go build -ldflags $(LDFLAGS) $(FLAGS) $(CLONE_URL)/cmd/pilosa
+build-fbsql-cgo:
+ifeq ($(GOARCH), arm64)
+	CGO_ENABLED=1 $(GO) build -tags dynamic $(GO_BUILD_FLAGS) -o fbsql ./cmd/fbsql
+endif
+ifeq ($(GOARCH), amd64)
+	CC=/usr/bin/musl-gcc CGO_ENABLED=1 $(GO) build -tags "musl static" -ldflags $(LDFLAGS_STATIC) $(GO_BUILD_FLAGS) -o fbsql ./cmd/fbsql
+endif
+
+docker-build-fbsql: vendor
+	DOCKER_BUILDKIT=0 docker build \
+		--file Dockerfile-fbsql \
+	    --build-arg GO_VERSION=$(GO_VERSION) \
+	    --build-arg MAKE_FLAGS="GOOS=$(GOOS) GOARCH=$(GOARCH) BUILD_CGO=$(BUILD_CGO)" \
+	    --build-arg GO_BUILD_FLAGS=$(GO_BUILD_FLAGS) \
+		--build-arg SOURCE_DATE_EPOCH=$(SOURCE_DATE_EPOCH) \
+	    --target builder \
+	    --tag fbsql:$(BUILD_NAME) .
+	mkdir -p build
+	docker create --name $(BUILD_NAME) fbsql:$(BUILD_NAME)
+	docker cp $(BUILD_NAME):/featurebase/fbsql ./build/fbsql_$(GOOS)_$(GOARCH)
+	docker rm $(BUILD_NAME)
 
-docker-test:
-	docker run --rm -v $(PWD):/go/src/$(CLONE_URL) -w /go/src/$(CLONE_URL) $(DOCKER_GOLANG_IMAGE) go test $(TESTFLAGS) $(PKGS)

NOTES

@@ -1,26 +0,0 @@
-                                                  
-     Index        Column                          
-        ┌───────────▼────────────────────────────┐
-        │0000000000000000000000000000000000000000│
-        │0000000000000000000000000000000000000000│
-        │0000000000000000000000000000000000000000│
-   Row──▶0000000000000000000000000000000000000000│
-        │0000000000000000000000000000000000000000│
-        │────────────────────────────────────────┤
-        │0000000000000000000000000000000000000000│
-        │0000000000000000000000000000000000000000│
-        │0000000000000000000000000000000000000000│
-        │0000000000000000000000000000000000000000│
-        │0000000000000000000000000000000000000000│
-        │────────────────────────────────────────┤
-    F  ▶│0000000000000000000000000000000000000000│
-    r  ││0000000000000000000000000000000000000000│
-    a  ││0000000000000000000000000000000000000000│
-    m  ││0000000000000000000000000000000000000000│
-    e  ▶│0000000000000000000000000000000000000000│
-        └────────────────────────────────────────┘
-                                     ▲───────────▲
-                                         Slice    
-
-
-Fragment=intersection of frame & slice

NOTICE

@@ -0,0 +1,109 @@
+Software license
+================
+
+Copyright (C) 2017-2021 Molecula Corp. All rights reserved.
+
+Third-party software licenses
+=============================
+
+The file /lru/lru.go contains a redistribution of lru
+(github.com/golang/groupcache/lru); the license follows:
+
+    Copyright 2013 Google Inc.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+         http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+
+The file /roaring/btree.go contains a modified redistribution of b
+(https://github.com/cznic/b); the license follows:
+
+    Copyright (c) 2014 The b Authors. All rights reserved.
+
+    Redistribution and use in source and binary forms, with or without
+    modification, are permitted provided that the following conditions are
+    met:
+
+       * Redistributions of source code must retain the above copyright
+    notice, this list of conditions and the following disclaimer.
+       * Redistributions in binary form must reproduce the above
+    copyright notice, this list of conditions and the following disclaimer
+    in the documentation and/or other materials provided with the
+    distribution.
+       * Neither the names of the authors nor the names of the
+    contributors may be used to endorse or promote products derived from
+    this software without specific prior written permission.
+
+    THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+    "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+    LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+    A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+    OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+    SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+    LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+    DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+    THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+    (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+    OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+The file /server/tlsconfig.go contains a modified redistribution of bridge 
+(https://github.com/robustirc/bridge); the license follows:
+  
+    Copyright © 2014-2015 The RobustIRC Authors. All rights reserved.
+
+    Redistribution and use in source and binary forms, with or without
+    modification, are permitted provided that the following conditions are met:
+
+        * Redistributions of source code must retain the above copyright
+          notice, this list of conditions and the following disclaimer.
+
+        * Redistributions in binary form must reproduce the above copyright
+          notice, this list of conditions and the following disclaimer in the
+          documentation and/or other materials provided with the distribution.
+
+        * Neither the name of RobustIRC nor the names of contributors may be used
+          to endorse or promote products derived from this software without
+          specific prior written permission.
+
+    THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+    ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+    WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+    DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE
+    FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+    DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+    SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+    CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+    OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+    OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+The files /logger/filewriter.go and /logger/filewriter_test.go contain a modified redistribution of reopen (github.com/client9/reopen); the license follows:
+
+    The MIT License (MIT)
+
+    Copyright (c) 2015 Nick Galbreath
+
+    Permission is hereby granted, free of charge, to any person obtaining a copy
+    of this software and associated documentation files (the "Software"), to deal
+    in the Software without restriction, including without limitation the rights
+    to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+    copies of the Software, and to permit persons to whom the Software is
+    furnished to do so, subject to the following conditions:
+
+    The above copyright notice and this permission notice shall be included in all
+    copies or substantial portions of the Software.
+
+    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+    OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+    SOFTWARE.

OPENSOURCE.md

@@ -0,0 +1,45 @@
+## User Contribution Guidelines for FeatureBase
+
+Thank you for your interest in contributing to FeatureBase! We appreciate your support in making this open-source project even better. Here are some guidelines to help you get started with contributing to FeatureBase:
+
+1. Familiarize Yourself with the Project:
+   - Visit the FeatureBase website at www.featurebase.com to understand the project's goals, capabilities, and features.
+   - Read the documentation available on the website, including the installation guide, configuration options, and data modeling concepts.
+   - Explore the codebase by cloning the repository and reviewing the source code.
+
+2. Join the Community:
+   - Visit the FeatureBase community page at https://www.featurebase.com/community to learn more about the project's community and how to get involved.
+   - Join the Discord server at https://discord.gg/FBn2vEp7Na to chat with other contributors and users, ask questions, and share your ideas.
+
+3. Set Up Your Development Environment:
+   - Ensure you have Go installed on your machine. Make sure your shell's search path includes the go/bin directory.
+   - Clone the FeatureBase repository or download it as a zip file from the repository's page.
+   - Follow the "Build FeatureBase Server from source" instructions in the README file to compile the server binary and the ingester binaries.
+
+4. Choose a Contribution Area:
+   - Identify the area you'd like to contribute to, such as bug fixes, new features, performance improvements, documentation updates, or community support.
+   - Check the issue tracker on the repository or the FeatureBase community for open issues or feature requests that align with your interests and skills. Alternatively, propose your own idea by creating a new issue.
+
+5. Create a New Branch:
+   - Before making any changes, create a new branch in the repository's Git repository. This branch will contain your contributions.
+   - Give your branch a descriptive name that reflects the nature of your contribution.
+
+6. Make Your Changes:
+   - Follow the coding style and conventions used in the existing codebase.
+   - Write clear and concise commit messages for each logical change.
+   - If you're introducing new features or modifying existing behavior, make sure to update the documentation to reflect the changes.
+
+7. Test Your Changes:
+   - Run the existing test suite to ensure that your modifications do not introduce any regressions.
+   - If applicable, write additional tests to cover the changes you made.
+   - Document any new testing procedures required for your contribution.
+
+8. Submitting Your Contribution:
+   - Push your branch to the main repository or create a fork and submit a pull request to the main repository.
+   - Provide a detailed description of your changes, including the problem you solved and the approach you took.
+   - Be responsive to any feedback or suggestions provided by the project maintainers or other contributors.
+   - Once your contribution is approved, it will be reviewed and merged into the main codebase.
+
+Please note that by contributing to FeatureBase, you agree that your contributions will be licensed under the Apache 2.0 license, which governs the project.
+
+Thank you for considering contributing to FeatureBase! Your contributions are valuable and help improve the project for everyone.

README-dev.md

@@ -1,69 +0,0 @@
-
-Development Environment
-=======================
-
-Install Go versions 1.6.2+ or 1.7 for your platform.
-
-Fork `github.com/pilosa/pilosa` to your own account. The forked repo will be private.
-
-Make sure `$GOPATH` environment variable points to your Go working directory and `$PATH` incudes `$GOPATH/bin`.
-
-Create a directory (note that we use `github.com/pilosa`, NOT `github.com/USER`) and clone your own Pilosa repo:
-
-```sh
-mkdir -p ${GOPATH}/src/github.com/pilosa && cd $_
-git clone git@github.com:${USER}/pilosa.git
-```
-
-`cd` to your pilosa directory:
-
-```sh
-cd ${GOPATH}/src/github.com/pilosa/pilosa
-```
-
-Install `dep` to manage dependencies:
-
-```sh
-go get -u github.com/golang/dep/cmd/dep
-```
-
-Install Pilosa command line tools:
-
-```sh
-make install
-# or:
-# dep ensure && go install github.com/pilosa/pilosa/cmd/...
-```
-
-Running `pilosa` should now run a Pilosa instance.
-
-In order to sync your fork with upstream Pilosa repo, add an *upstream* to your repo:
-
-```sh
-cd ${GOPATH}/src/github.com/pilosa/pilosa
-git remote add upstream git@github.com:pilosa/pilosa.git
-```
-
-Before starting to work on a task, sync your branch with the upstream:
-
-```sh
-git fetch upstream
-git checkout master
-git merge upstream/master
-```
-
-Create a branch for the task:
-
-```sh
-git checkout -b a-branch-for-the-task
-```
-
-Update the code in the branch, and commit it.
-
-Push it to your own repo:
-
-```sh
-git push --set-upstream origin a-branch-for-the-task
-```
-
-All left to do is creating a pull request on github.com.

README.md

@@ -1,71 +1,72 @@
-<p>
-    <a href="https://www.pilosa.com">
-        <img src="https://www.pilosa.com/img/logo.svg" width="50%">
-    </a>
-</p>
+# FeatureBase Community
 
-[![Build Status](https://travis-ci.org/pilosa/pilosa.svg?branch=master)](https://travis-ci.org/pilosa/pilosa)
-[![GoDoc](https://godoc.org/github.com/pilosa/pilosa?status.svg)](https://godoc.org/github.com/pilosa/pilosa)
-[![Go Report Card](https://goreportcard.com/badge/github.com/pilosa/pilosa)](https://goreportcard.com/report/github.com/pilosa/pilosa)
-[![license](https://img.shields.io/github/license/pilosa/pilosa.svg)](https://github.com/pilosa/pilosa/blob/master/LICENSE)
-[![CLA Assistant](https://cla-assistant.io/readme/badge/pilosa/pilosa)](https://cla-assistant.io/pilosa/pilosa)
-[![GitHub release](https://img.shields.io/github/release/pilosa/pilosa.svg)](https://github.com/pilosa/pilosa/releases)
+FeatureBase Community is now archived and no longer maintained.
 
-## An open source, distributed bitmap index.
-- [Docs](#docs)
-- [Getting Started](#getting-started)
-- [Data Model](#data-model)
-- [Query Language](#query-language)
-- [Client Libraries](#client-libraries)
-- [Get Support](#get-support)
-- [Contributing](#contributing)
+* [FeatureBase Community Help](https://github.com/FeatureBaseDB/FB-community-help)
 
 
-## Docs
 
-See our [Documentation](https://www.pilosa.com/docs/) for information about installing and working with Pilosa.
+## Pilosa is now FeatureBase
 
+As of September 7, 2022, the Pilosa project is now FeatureBase. The core of the project remains the same: FeatureBase is the first real-time distributed database built entirely on bitmaps. (More information about updated capabilities and improvements below.)
+
+FeatureBase delivers low-latency query results, regardless of throughput or query volumes, on fresh data with extreme efficiency. It works because bitmaps are faster, simpler, and far more I/O efficient than traditional column-oriented data formats. With FeatureBase, you can ingest data from batch data sources (e.g. S3, CSV, Snowflake, BigQuery, etc.) and/or streaming data sources (e.g. Kafka/Confluent, Kinesis, Pulsar).
+
+For more information about FeatureBase, please visit [www.featurebase.com][HomePage].
 
 ## Getting Started
 
-1.  [Install Pilosa](https://www.pilosa.com/docs/installation/).
+* [Learn how to install FeatureBase Community](https://github.com/FeatureBaseDB/FB-community-help/blob/main/docs/community/com-getstart/com-getstart-home.md)
 
-2.  [Start Pilosa](https://www.pilosa.com/docs/getting-started/#starting-pilosa) with the default configuration:
+### Build FeatureBase Server from source
 
-    ```shell
-    pilosa server
-    ```
-    
-    and verify that it's running:
-    
-    ```shell
-    curl localhost:10101/nodes
-    ```
+0. Install go. Ensure that your shell's search path includes the go/bin directory.
+1. Clone the FeatureBase repository (or download as zip).
+2. In the featurebase directory, run `make install` to compile the FeatureBase server binary. By default, it will be installed in the go/bin directory.
+3. In the idk directory, run `make install` to compile the ingester binaries. By default, they will be installed in the go/bin directory.
+4. Run `featurebase server --handler.allowed-origins=http://localhost:3000` to run FeatureBase server with default settings (learn more about configuring FeatureBase at the link below). The `--handler.allowed-origins` parameter allows the standalone web UI to talk to the server; this can be omitted if the web UI is not needed.
+5. Run `curl localhost:10101/status` to verify the server is running and accessible.
 
-3.  Follow along with the [Sample Project](https://www.pilosa.com/docs/getting-started/#sample-project) to get a better understanding of Pilosa's capabilities.
+### Data Model
+
+Because FeatureBase is built on bitmaps, there is bit of a learning curve to grasp how your data is represented. 
+
+* [Learn about Data Modeling](https://github.com/FeatureBaseDB/FB-community-help/blob/main/docs/concepts/concepts-home.md)
 
 
-## Data Model
+### Ingest Data and Query
 
-Check out how the Pilosa [Data Model](https://www.pilosa.com/docs/data-model/) works.
+* [Learn how to ingest data from multiple data sources](https://github.com/FeatureBaseDB/FB-community-help/blob/main/docs/community/com-ingest/com-ingest-manage.md)
 
+## Community
 
-## Query Language
+You can email us at community@featurebase.com and [learn more about contributing](https://github.com/FeatureBaseDB/featurebase/blob/master/OPENSOURCE.md).
 
-You can interact with Pilosa directly in the console using the [Pilosa Query Language](https://www.pilosa.com/docs/query-language/) (PQL).
+Chat with us: [https://discord.gg/FBn2vEp7Na][Discord]
 
+## What's Changed Since the Pilosa Days? 
 
-## Client Libraries
+A lot has changed since the days of Pilosa. This list highlights some new capabilites included in FeatureBase. We have also made signficant improvements to the performance, scalability, and stability of the FeatureBase product. 
 
-There are supported libraries for the following languages:
-- [Go](https://www.pilosa.com/docs/client-libraries/#go)
-- [Java](https://www.pilosa.com/docs/client-libraries/#java)
-- [Python](https://www.pilosa.com/docs/client-libraries/#python)
+* Query Languages: FeatureBase supports Pilosa Query Language (PQL), as well as SQL
+* Stream and Batch Ingest: Combine real-time data streams with batch historical data and act on it within milliseconds.
+* Mutable: Perform inserts, updates, and deletes at scale, in real time and on-the-fly. This is key for meeting data compliance requirements, and for reflecting the constantly-changing nature of high-volume data.
+* Multi-Valued Set Fields: Store multiple comma-delimited values within a single field while *increasing* query performance of counts, TopKs, etc.
+* Time Quantums: Setting a time quantum on a field creates extra views which allow ranged Row queries down to the time interval specified. For example, if the time quantum is set to YMD, ranged Row queries down to the granularity of a day are supported.
+* RBF storage backend: this is a new compressed bitmap format which improves performance in a number of ways: ACID support on a per shard basis, prevents issues with the number of open files, reduces memory allocation and lock contention for reads, provides more consistent garbage collection, and allows backups to run concurrently with writes. However, because of this change, Pilosa backup files cannot be restored into FeatureBase.
 
-## Get Support
+## License
 
-There are [several channels](https://www.pilosa.com/community/#support) available for you to reach out to us for support.
+FeatureBase is licensed under the [Apache License, Version 2.0][License]
 
-## Contributing
-
-Pilosa is an open source project. Please see our [Contributing Guide](CONTRIBUTING.md) for information about how to get involved.
+[Community]: https://github.com/FeatureBaseDB/FB-community-help/tree/main
+[Install]:https://github.com/FeatureBaseDB/FB-community-help/blob/main/docs/community/com-getstart/com-getstart-home.md
+[Config]: https://github.com/FeatureBaseDB/FB-community-help/tree/main/docs/community/com-config
+[DataModel]: https://github.com/FeatureBaseDB/FB-community-help/blob/main/docs/concepts/concepts-home.md
+[Discord]: https://discord.gg/FBn2vEp7Na
+[HomePage]: http://featurebase.com?utm_campaign=Open%20Source&utm_source=GitHub
+[Ingest]: https://github.com/FeatureBaseDB/FB-community-help/blob/main/docs/community/com-ingest/com-ingest-manage.md
+ 
+[License]: http://www.apache.org/licenses/LICENSE-2.0
+[PQL]: https://docs.featurebase.com/docs/pql-guide/pql-home/?utm_campaign=Open%20Source&utm_source=GitHub
+[SQL]: https://docs.featurebase.com/docs/sql-guide/sql-guide-home/?utm_campaign=Open%20Source&utm_source=GitHub

api/client/grpc.go

@@ -0,0 +1,204 @@
+// Copyright 2022 Molecula Corp. (DBA FeatureBase).
+// SPDX-License-Identifier: Apache-2.0
+package client
+
+import (
+	"context"
+	"crypto/tls"
+	"sync"
+
+	"github.com/featurebasedb/featurebase/v3/logger"
+	pb "github.com/featurebasedb/featurebase/v3/proto"
+	"github.com/pkg/errors"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/connectivity"
+	"google.golang.org/grpc/credentials"
+	"google.golang.org/grpc/credentials/insecure"
+)
+
+const maxMsgSize = 1024 * 1024 * 100 // 100 megs ought to be enough for anybody!
+
+// GRPCClient is a client for working with the gRPC server.
+type GRPCClient struct {
+	dialTargets []string
+	tlsConfig   *tls.Config
+	logger      logger.Logger
+
+	mu          sync.RWMutex
+	conn        *grpc.ClientConn
+	targetIndex int
+}
+
+// NewGRPCClient returns a new instance of GRPCClient.
+func NewGRPCClient(dialTargets []string, tlsConfig *tls.Config, logger logger.Logger) (*GRPCClient, error) {
+	c := &GRPCClient{
+		dialTargets: dialTargets,
+		tlsConfig:   tlsConfig,
+		logger:      logger,
+	}
+	// resetConn sets GRPCClient.conn when it doesn't
+	// exist yet.
+	if err := c.resetConn(); err != nil {
+		return nil, errors.Wrap(err, "setting connection")
+	}
+
+	return c, nil
+}
+
+// resetConn resets the gRPC client connection. This method
+// can also be used to initially set the client connection
+// because it only tries to first close the connection if
+// the connection already exists.
+func (c *GRPCClient) resetConn() error {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	// If an existing connection exists, close it first.
+	if c.conn != nil {
+		if err := c.conn.Close(); err != nil {
+			return errors.Wrap(err, "closing existing connection")
+		}
+	}
+
+	var opts []grpc.DialOption
+	if c.tlsConfig != nil {
+		creds := credentials.NewTLS(c.tlsConfig)
+		opts = append(opts, grpc.WithTransportCredentials(creds))
+	} else {
+		opts = append(opts, grpc.WithTransportCredentials(insecure.NewCredentials()))
+	}
+
+	opts = append(opts, grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(maxMsgSize)))
+
+	var err error
+	if c.conn, err = grpc.Dial(c.dialTargets[c.getTargetIndex()], opts...); err != nil {
+		return errors.Wrap(err, "creating new grpc client")
+	}
+
+	return nil
+}
+
+// getTargetIndex gets the current target index, then increments it for
+// next time. Unprotected.
+func (c *GRPCClient) getTargetIndex() int {
+	if len(c.dialTargets) == 0 {
+		return 0
+	}
+	ret := c.targetIndex
+	c.targetIndex = (c.targetIndex + 1) % len(c.dialTargets) // cycle through dialTargets
+	return ret
+}
+
+// Close closes any connections the client has opened.
+func (c *GRPCClient) Close() error {
+	c.mu.RLock()
+	defer c.mu.RUnlock()
+
+	if c.conn != nil {
+		return c.conn.Close()
+	}
+	return nil
+}
+
+// Conn returns the gRPC client connection. If the connection
+// has gone into state `TransientFailure`, this method tries
+// to reset the connection and return that new connection.
+func (c *GRPCClient) Conn() *grpc.ClientConn {
+	c.mu.RLock()
+	if c.conn == nil {
+		c.mu.RUnlock()
+		return nil
+	} else if c.conn.GetState() != connectivity.TransientFailure {
+		defer c.mu.RUnlock()
+		return c.conn
+	}
+	c.mu.RUnlock()
+
+	if err := c.resetConn(); err != nil {
+		c.logger.Errorf("error resetting connection: %s", err)
+	}
+
+	c.mu.RLock()
+	defer c.mu.RUnlock()
+	return c.conn
+}
+
+// Query returns a stream of RowResponse for the given index and PQL string.
+func (c *GRPCClient) Query(ctx context.Context, index string, pql string) (pb.StreamClient, error) {
+	conn := c.Conn()
+
+	if conn == nil {
+		return nil, errors.New("client has not established a grpc connection")
+	}
+
+	grpcClient := pb.NewPilosaClient(conn)
+
+	stream, err := grpcClient.QueryPQL(ctx, &pb.QueryPQLRequest{
+		Index: index,
+		Pql:   pql,
+	})
+	if err != nil {
+		return nil, errors.Wrap(err, "getting stream")
+	} else if stream == nil {
+		return nil, errors.New("could not create stream")
+	}
+
+	return stream, err
+}
+
+// QueryUnary returns a TableResponse for the given index and PQL string.
+func (c *GRPCClient) QueryUnary(ctx context.Context, index string, pql string) (*pb.TableResponse, error) {
+	conn := c.Conn()
+
+	if conn == nil {
+		return nil, errors.New("client has not established a grpc connection")
+	}
+
+	grpcClient := pb.NewPilosaClient(conn)
+
+	return grpcClient.QueryPQLUnary(ctx, &pb.QueryPQLRequest{
+		Index: index,
+		Pql:   pql,
+	})
+}
+
+// Inspect returns a stream of RowResponse for the given index, columns, and filters.
+// It is intended to mimic something like "select [fields] from table where recordID IN (...)".
+func (c *GRPCClient) Inspect(ctx context.Context, index string, columnIDs []uint64, columnKeys []string, query string, fieldFilters []string, limit, offset uint64) (pb.StreamClient, error) {
+	conn := c.Conn()
+
+	if conn == nil {
+		return nil, errors.New("client has not established a grpc connection")
+	}
+
+	if len(columnIDs) > 0 && len(columnKeys) > 0 {
+		return nil, errors.New("only provide column ids or keys, not both")
+	}
+
+	// Convert columns to proto type IdsOrKeys.
+	idsOrKeys := &pb.IdsOrKeys{}
+	if len(columnKeys) > 0 {
+		idsOrKeys.Type = &pb.IdsOrKeys_Keys{Keys: &pb.StringArray{Vals: columnKeys}}
+	} else {
+		idsOrKeys.Type = &pb.IdsOrKeys_Ids{Ids: &pb.Uint64Array{Vals: columnIDs}}
+	}
+
+	grpcClient := pb.NewPilosaClient(conn)
+
+	stream, err := grpcClient.Inspect(ctx, &pb.InspectRequest{
+		Index:        index,
+		Columns:      idsOrKeys,
+		FilterFields: fieldFilters,
+		Limit:        limit,
+		Offset:       offset,
+		Query:        query,
+	})
+
+	if err != nil {
+		return nil, errors.Wrap(err, "getting stream")
+	} else if stream == nil {
+		return nil, errors.New("could not create stream")
+	}
+
+	return stream, err
+}

api_directive.go

@@ -0,0 +1,987 @@
+// Copyright 2021 Molecula Corp. All rights reserved.
+package pilosa
+
+import (
+	"context"
+	"io"
+	"log"
+	"sync"
+
+	"github.com/featurebasedb/featurebase/v3/dax"
+	"github.com/featurebasedb/featurebase/v3/dax/computer"
+	"github.com/featurebasedb/featurebase/v3/dax/storage"
+	"github.com/featurebasedb/featurebase/v3/disco"
+	"github.com/pkg/errors"
+)
+
+// ApplyDirective applies a Directive received, from the Controller, at the
+// /directive endpoint.
+func (api *API) ApplyDirective(ctx context.Context, d *dax.Directive) error {
+	// Get the current directive for comparison.
+	previousDirective := api.holder.Directive()
+
+	// Check that incoming version is newer.
+	// Note: 0 is an invalid Directive version. This decision was made because
+	// previousDirective is not a pointer to a directive, but a concrete
+	// Directive. Which means we can't check for nil, and by default it has a
+	// version of 0. So in order to ensure the version has increased, we need to
+	// require that incoming directive versions are greater than 0.
+	if d.Version == 0 {
+		return errors.Errorf("directive version cannot be 0")
+	} else if previousDirective.Version >= d.Version {
+		return errors.Errorf("directive version mismatch, got %d, but already have %d", d.Version, previousDirective.Version)
+	}
+
+	// Handle the operations based on the directive method.
+	switch d.Method {
+	case dax.DirectiveMethodDiff:
+		// In order to prevent adding too much code specific to handling a diff
+		// directive (e.g. adding something like an `enactDirectiveDiff()`
+		// method), we are instead going to build a full Directive based on the
+		// diff, and then proceed normally as if we had received a full
+		// Directive. We do that by copying the previous Directive and then
+		// applying the diffs to the copy.
+		newD := previousDirective.Copy()
+
+		// Apply the diffs from the incoming Directive to the new, copied
+		// Directive.
+		newD.ApplyDiff(d)
+
+		// Now proceed with the new diff as if we had received it as a full diff.
+		d = newD
+
+	case dax.DirectiveMethodFull:
+		// pass: normal operation
+
+	case dax.DirectiveMethodReset:
+		// Delete all tables.
+		if err := api.deleteAllIndexes(ctx); err != nil {
+			return errors.Wrap(err, "deleting all indexes")
+		}
+		// Set previousDirective to empty so the diff handles everything as new.
+		previousDirective = dax.Directive{}
+
+	case dax.DirectiveMethodSnapshot:
+		// TODO(tlt): this was the existing logic, but we should really diff the
+		// directive and ensure that overwriting the value in the cache doesn't
+		// have a negative effect.
+		api.holder.SetDirective(d)
+		return nil
+
+	default:
+		return errors.Errorf("invalid directive method: %s", d.Method)
+	}
+
+	// Cache this directive as the latest applied. There is functionality within
+	// the "enactDirective" stage of ApplyDirective which validates against this
+	// cached Directive, so it's important that it be set before calling
+	// enactDirective(). An example: when loading partition data from the
+	// Writelogger, there are validations to ensure that the partition being
+	// loaded is meant to be handled by this node; that validation is done
+	// against the cached Directive.
+	// TODO(tlt): despite what this comment says, this logic is not sound; we
+	// shouldn't be setting the directive until enactiveDirective() succeeds.
+	api.holder.SetDirective(d)
+	defer api.holder.SetDirectiveApplied(true)
+
+	return api.enactDirective(ctx, &previousDirective, d)
+}
+
+// deleteAllIndexes deletes all indexes handled by this node.
+func (api *API) deleteAllIndexes(ctx context.Context) error {
+	indexes, err := api.Schema(ctx, false)
+	if err != nil {
+		return errors.Wrap(err, "getting schema")
+	}
+
+	for i := range indexes {
+		if err := api.DeleteIndex(ctx, indexes[i].Name); err != nil {
+			return errors.Wrapf(err, "deleting index: %s", indexes[i].Name)
+		}
+	}
+
+	return nil
+}
+
+// directiveJobType allows us to switch on jobType in the directiveWorker in
+// order to use a single worker pool for all job types (as opposed to having a
+// separate worker pool for each job type).
+type directiveJobType interface {
+	// We have this method just to prevent *any* struct from implementing this
+	// interface automatically. But, interestingly enough, we don't actually
+	// have to have this method on the implementation because we embed the
+	// interface.
+	isJobType() bool
+}
+
+type directiveJobTableKeys struct {
+	directiveJobType
+	idx       *Index
+	tkey      dax.TableKey
+	partition dax.PartitionNum
+}
+
+type directiveJobFieldKeys struct {
+	directiveJobType
+	tkey  dax.TableKey
+	field dax.FieldName
+}
+
+type directiveJobShards struct {
+	directiveJobType
+	tkey  dax.TableKey
+	shard dax.ShardNum
+}
+
+// directiveWorker is a worker in a worker pool which handles portions of a
+// directive. Multiple instances of directiveWorker run in goroutines in order
+// to load data from snapshotter and writelogger concurrently. Note: unlike the
+// api.ingestWorkerPool, of which one pool is always running, the
+// directiveWorker pool is only running during the life of the
+// api.ApplyDirective call. Technically, this means that multiple
+// directiveWorker pools could be active at the same time, but we should never
+// be running more than once instance of ApplyDirective concurrently.
+func (api *API) directiveWorker(ctx context.Context, jobs <-chan directiveJobType, errs chan<- error) {
+	for j := range jobs {
+		switch job := j.(type) {
+		case directiveJobTableKeys:
+			if err := api.loadTableKeys(ctx, job.idx, job.tkey, job.partition); err != nil {
+				errs <- errors.Wrapf(err, "loading table keys: %s, %s", job.tkey, job.partition)
+			}
+		case directiveJobFieldKeys:
+			if err := api.loadFieldKeys(ctx, job.tkey, job.field); err != nil {
+				errs <- errors.Wrapf(err, "loading field keys: %s, %s", job.tkey, job.field)
+			}
+		case directiveJobShards:
+			if err := api.loadShard(ctx, job.tkey, job.shard); err != nil {
+				errs <- errors.Wrapf(err, "loading shard: %s, %s", job.tkey, job.shard)
+			}
+		default:
+			errs <- errors.Errorf("unsupported job type: %T %[1]v", job)
+		}
+
+		select {
+		case <-ctx.Done():
+			return
+		default:
+			// continue pulling jobs off the channel
+		}
+	}
+}
+
+func (api *API) enactDirective(ctx context.Context, fromD, toD *dax.Directive) error {
+	// enactTables is called before the jobs that run in the worker pool because
+	// it probably makes sense to apply the schema before trying to load data
+	// concurrently.
+	if err := api.enactTables(ctx, fromD, toD); err != nil {
+		return errors.Wrap(err, "enactTables")
+	}
+
+	// The following types use a shared pool of workers to run each
+	// directiveJobType.
+
+	var wg sync.WaitGroup
+
+	// open job channel
+	jobs := make(chan directiveJobType, api.directiveWorkerPoolSize)
+	errs := make(chan error)
+	done := make(chan struct{})
+
+	// Spin up n workers in goroutines that pull jobs from the jobs channel.
+	for i := 0; i < api.directiveWorkerPoolSize; i++ {
+		wg.Add(1)
+		go func() {
+			api.directiveWorker(ctx, jobs, errs)
+			defer wg.Done()
+		}()
+	}
+
+	// Wait for the WaitGroup counter to reach 0. When it has, indicate that
+	// we're done processing all jobs by closing the done channel.
+	go func() {
+		wg.Wait()
+		close(done)
+	}()
+
+	// Run through all the "enact" methods. These push jobs onto the jobs
+	// channel. Once all the jobs have been queued to the channel, we close the
+	// jobs channel. This allows the directiveWorkers to exit out of the
+	// function, which will then decrement the WaitGroup counter.
+	go func() {
+		api.pushJobsTableKeys(ctx, jobs, fromD, toD)
+		api.pushJobsFieldKeys(ctx, jobs, fromD, toD)
+		api.pushJobsShards(ctx, jobs, fromD, toD)
+		close(jobs)
+	}()
+
+	// Keep running until we get an error or until the done channel is closed.
+	// Note: the code is written such that only non-nil errors are pushed to the
+	// errs channel.
+	for {
+		select {
+		case err := <-errs:
+			return err
+		case <-done:
+			return nil
+		}
+	}
+}
+
+func (api *API) enactTables(ctx context.Context, fromD, toD *dax.Directive) error {
+	currentIndexes := api.holder.Indexes()
+
+	// Make a list of indexes that currently exist (from).
+	from := make(dax.TableKeys, 0, len(currentIndexes))
+	for _, idx := range currentIndexes {
+		qtid, err := dax.QualifiedTableIDFromKey(idx.Name())
+		if err != nil {
+			return errors.Wrap(err, "converting index name to qualified table id")
+		}
+		from = append(from, qtid.Key())
+	}
+
+	// TODO sanity check holder against fromD. We're getting existing
+	// indexes from holder, but in theory fromD should be
+	// identical. If we have an error in our directive-caching logic
+	// (it has happened before (just now, in fact!) and we'd be
+	// foolish to think it won't happen again), or we have schema
+	// mutations that are not going through the directive path, we
+	// could potentially catch them here.
+
+	// Make a list of tables that are in the directive (to) along with a map of
+	// tableKey to table (m).
+	m := make(map[dax.TableKey]*dax.QualifiedTable, len(toD.Tables))
+	to := make(dax.TableKeys, 0, len(toD.Tables))
+	for _, t := range toD.Tables {
+		m[t.Key()] = t
+		to = append(to, t.Key())
+	}
+
+	sc := newSliceComparer(from, to)
+
+	// Remove all indexes that are no longer part of the directive.
+	for _, tkey := range sc.removed() {
+		idx := string(tkey)
+		if err := api.DeleteIndex(ctx, idx); err != nil {
+			return errors.Wrapf(err, "deleting index: %s", tkey)
+		}
+	}
+
+	// Put partitions into a map by table.
+	partitionMap := toD.TranslatePartitionsMap()
+
+	// Add all indexes that weren't previously (but now are) a part of the
+	// directive.
+	for _, tkey := range sc.added() {
+		if qtbl, found := m[tkey]; !found {
+			return errors.Errorf("table '%s' was not in map", tkey)
+		} else if err := api.createTableAndFields(qtbl, partitionMap[tkey]); err != nil {
+			return err
+		}
+	}
+
+	// Check fields on all indexes present in both from and to.
+	for _, tkey := range sc.same() {
+		if err := api.enactFieldsForTable(ctx, tkey, fromD, toD); err != nil {
+			return errors.Wrapf(err, "enacting fields for table: '%s'", tkey)
+		}
+	}
+
+	return nil
+}
+
+func (api *API) enactFieldsForTable(ctx context.Context, tkey dax.TableKey, fromD, toD *dax.Directive) error {
+	qtid := tkey.QualifiedTableID()
+
+	fromT, err := fromD.Table(qtid)
+	if err != nil {
+		return errors.Wrap(err, "getting from table")
+	}
+	toT, err := toD.Table(qtid)
+	if err != nil {
+		return errors.Wrap(err, "getting to table")
+	}
+
+	// Get the index for tkey.
+	idx := api.holder.Index(string(tkey))
+	if idx == nil {
+		return errors.Errorf("index not found: %s", tkey)
+	}
+
+	sc := newSliceComparer(fromT.FieldNames(), toT.FieldNames())
+
+	// Add fields new to toT.
+	for _, fldName := range sc.added() {
+		if field, found := toT.Field(fldName); !found {
+			return dax.NewErrFieldDoesNotExist(fldName)
+		} else if err := createField(idx, field); err != nil {
+			return errors.Wrapf(err, "creating field: %s/%s", tkey, fldName)
+		}
+	}
+
+	// Remove fields which don't exist in toT.
+	for _, fldName := range sc.removed() {
+		if err := api.DeleteField(ctx, string(tkey), string(fldName)); err != nil {
+			return errors.Wrapf(err, "deleting field: %s/%s", tkey, fldName)
+		}
+	}
+
+	// // Update any field options which have changed for existing fields.
+	// for _, fldName := range sc.same() {
+	// 	// handle changed field options??
+	// }
+
+	return nil
+}
+
+func (api *API) pushJobsTableKeys(ctx context.Context, jobs chan<- directiveJobType, fromD, toD *dax.Directive) {
+	toPartitionsMap := toD.TranslatePartitionsMap()
+
+	// Get the diff between from/to directive.partitions.
+	partComp := newPartitionsComparer(fromD.TranslatePartitionsMap(), toPartitionsMap)
+
+	// Remove any partitions which are no longer assigned to this worker.
+	// TODO(tlt): currently, this is just removing the file lock on the
+	// resource; it's not actually removing the resource from the local
+	// computer. We should do that.
+	for tkey, partitions := range partComp.removed() {
+		qtid := tkey.QualifiedTableID()
+		for _, partition := range partitions {
+			api.serverlessStorage.RemoveTableKeyResource(qtid, partition)
+		}
+	}
+
+	// Loop over the partition map and load from Writelogger.
+	for tkey, partitions := range partComp.added() {
+		// Get index in order to find the translate stores (by partition) for
+		// the table.
+		idx := api.holder.Index(string(tkey))
+		if idx == nil {
+			log.Printf("index not found in holder: %s", tkey)
+			continue
+		}
+
+		// Update the cached version of translate partitions that we keep on the
+		// Index.
+		idx.SetTranslatePartitions(toPartitionsMap[tkey])
+
+		for _, partition := range partitions {
+			jobs <- directiveJobTableKeys{
+				idx:       idx,
+				tkey:      tkey,
+				partition: partition,
+			}
+		}
+	}
+}
+
+func (api *API) loadTableKeys(ctx context.Context, idx *Index, tkey dax.TableKey, partition dax.PartitionNum) error {
+	qtid := tkey.QualifiedTableID()
+
+	resource := api.serverlessStorage.GetTableKeyResource(qtid, partition)
+	if resource.IsLocked() {
+		api.logger().Warnf("skipping loadTableKeys (already held) %s %d", tkey, partition)
+		return nil
+	}
+
+	// load latest snapshot
+	if rc, err := resource.LoadLatestSnapshot(); err != nil {
+		return errors.Wrap(err, "loading table key snapshot")
+	} else if rc != nil {
+		defer rc.Close()
+		if err := api.TranslateIndexDB(ctx, string(tkey), int(partition), rc); err != nil {
+			return errors.Wrap(err, "restoring table keys")
+		}
+	}
+
+	// define write log loading in a function since we have to do it
+	// before and after locking
+	loadWriteLog := func() error {
+		writelog, err := resource.LoadWriteLog()
+		if err != nil {
+			return errors.Wrap(err, "getting write log reader for table keys")
+		}
+		if writelog == nil {
+			return nil
+		}
+		reader := storage.NewTableKeyReader(qtid, partition, writelog)
+		defer reader.Close()
+		store := idx.TranslateStore(int(partition))
+		for msg, err := reader.Read(); err != io.EOF; msg, err = reader.Read() {
+			if err != nil {
+				return errors.Wrap(err, "reading from log reader")
+			}
+			for key, id := range msg.StringToID {
+				if err := store.ForceSet(id, key); err != nil {
+					return errors.Wrapf(err, "forcing set id, key: %d, %s", id, key)
+				}
+			}
+		}
+		return nil
+	}
+	// 1st write log load
+	if err := loadWriteLog(); err != nil {
+		return err
+	}
+
+	// acquire lock on this partition's keys
+	if err := resource.Lock(); err != nil {
+		return errors.Wrap(err, "locking table key partition")
+	}
+
+	// reload writelog in case of changes between last load and
+	// lock. The resource object takes care of only loading new data.
+	return loadWriteLog()
+}
+
+func (api *API) pushJobsFieldKeys(ctx context.Context, jobs chan<- directiveJobType, fromD, toD *dax.Directive) {
+	// Get the diff between from/to directive.fields.
+	fieldComp := newFieldsComparer(fromD.TranslateFieldsMap(), toD.TranslateFieldsMap())
+
+	// Remove any field keys which are no longer assigned to this worker.
+	// TODO(tlt): currently, this is just removing the file lock on the
+	// resource; it's not actually removing the resource from the local
+	// computer. We should do that.
+	for tkey, fields := range fieldComp.removed() {
+		qtid := tkey.QualifiedTableID()
+		for _, field := range fields {
+			api.serverlessStorage.RemoveFieldKeyResource(qtid, field)
+		}
+	}
+
+	// Loop over the field map and load from Writelogger.
+	for tkey, fields := range fieldComp.added() {
+		for _, field := range fields {
+			jobs <- directiveJobFieldKeys{
+				tkey:  tkey,
+				field: field,
+			}
+		}
+	}
+}
+
+func (api *API) loadFieldKeys(ctx context.Context, tkey dax.TableKey, field dax.FieldName) error {
+	qtid := tkey.QualifiedTableID()
+
+	resource := api.serverlessStorage.GetFieldKeyResource(qtid, field)
+	if resource.IsLocked() {
+		api.logger().Warnf("skipping loadFieldKeys (already held) %s %s", tkey, field)
+		return nil
+	}
+
+	// load latest snapshot
+	if rc, err := resource.LoadLatestSnapshot(); err != nil {
+		return errors.Wrap(err, "loading field key snapshot")
+	} else if rc != nil {
+		defer rc.Close()
+		if err := api.TranslateFieldDB(ctx, string(tkey), string(field), rc); err != nil {
+			return errors.Wrap(err, "restoring field keys")
+		}
+	}
+
+	// define write log loading in a function since we have to do it
+	// before and after locking
+	loadWriteLog := func() error {
+		writelog, err := resource.LoadWriteLog()
+		if err != nil {
+			return errors.Wrap(err, "getting write log reader for field keys")
+		}
+		if writelog == nil {
+			return nil
+		}
+		reader := storage.NewFieldKeyReader(qtid, field, writelog)
+		defer reader.Close()
+		// Get field in order to find the translate store.
+		fld := api.holder.Field(string(tkey), string(field))
+		if fld == nil {
+			log.Printf("field not found in holder: %s", field)
+			return nil
+		}
+		store := fld.TranslateStore()
+
+		for msg, err := reader.Read(); err != io.EOF; msg, err = reader.Read() {
+			if err != nil {
+				return errors.Wrap(err, "reading from log reader")
+			}
+			for key, id := range msg.StringToID {
+				if err := store.ForceSet(id, key); err != nil {
+					return errors.Wrapf(err, "forcing set id, key: %d, %s", id, key)
+				}
+			}
+		}
+		return nil
+	}
+	// 1st write log load
+	if err := loadWriteLog(); err != nil {
+		return err
+	}
+
+	// acquire lock on this partition's keys
+	if err := resource.Lock(); err != nil {
+		return errors.Wrap(err, "locking field key partition")
+	}
+
+	// reload writelog in case of changes between last load and
+	// lock. The resource object takes care of only loading new data.
+	return loadWriteLog()
+}
+
+func (api *API) pushJobsShards(ctx context.Context, jobs chan<- directiveJobType, fromD, toD *dax.Directive) {
+	// Put shards into a map by table.
+	shardMap := toD.ComputeShardsMap()
+
+	// Get the diff between from/to directive shards.
+	shardComp := newShardsComparer(fromD.ComputeShardsMap(), shardMap)
+
+	// Remove any shards which are no longer assigned to this worker.
+	// TODO(tlt): currently, this is just removing the file lock on the
+	// resource; it's not actually removing the resource from the local
+	// computer. We should do that.
+	for tkey, shards := range shardComp.removed() {
+		qtid := tkey.QualifiedTableID()
+		for _, shard := range shards {
+			partition := dax.PartitionNum(disco.ShardToShardPartition(string(tkey), uint64(shard), disco.DefaultPartitionN))
+			api.serverlessStorage.RemoveShardResource(qtid, partition, shard)
+		}
+	}
+
+	// Loop over the shard map and load from Writelogger.
+	for tkey, shards := range shardComp.added() {
+		for _, shard := range shards {
+			jobs <- directiveJobShards{
+				tkey:  tkey,
+				shard: shard,
+			}
+		}
+	}
+}
+
+func (api *API) loadShard(ctx context.Context, tkey dax.TableKey, shard dax.ShardNum) error {
+	qtid := tkey.QualifiedTableID()
+
+	partition := dax.PartitionNum(disco.ShardToShardPartition(string(tkey), uint64(shard), disco.DefaultPartitionN))
+
+	resource := api.serverlessStorage.GetShardResource(qtid, partition, shard)
+	if resource.IsLocked() {
+		api.logger().Warnf("skipping loadShard (already held) %s %d", tkey, shard)
+		return nil
+	}
+
+	if rc, err := resource.LoadLatestSnapshot(); err != nil {
+		return errors.Wrap(err, "reading latest snapshot for shard")
+	} else if rc != nil {
+		defer rc.Close()
+		if err := api.RestoreShard(ctx, string(tkey), uint64(shard), rc); err != nil {
+			return errors.Wrap(err, "restoring shard data")
+		}
+	}
+
+	// define write log loading in a func because we do it twice.
+	loadWriteLog := func() error {
+		writelog, err := resource.LoadWriteLog()
+		if err != nil {
+			return errors.Wrap(err, "")
+		}
+		if writelog == nil {
+			return nil
+		}
+
+		reader := storage.NewShardReader(qtid, partition, shard, writelog)
+		defer reader.Close()
+		for logMsg, err := reader.Read(); err != io.EOF; logMsg, err = reader.Read() {
+			if err != nil {
+				return errors.Wrap(err, "reading from log reader")
+			}
+
+			switch msg := logMsg.(type) {
+			case *computer.ImportRoaringMessage:
+				req := &ImportRoaringRequest{
+					Clear:           msg.Clear,
+					Action:          msg.Action,
+					Block:           msg.Block,
+					Views:           msg.Views,
+					UpdateExistence: msg.UpdateExistence,
+					SuppressLog:     true,
+				}
+				if err := api.ImportRoaring(ctx, msg.Table, msg.Field, msg.Shard, true, req); err != nil {
+					return errors.Wrapf(err, "import roaring, table: %s, field: %s, shard: %d", msg.Table, msg.Field, msg.Shard)
+				}
+
+			case *computer.ImportMessage:
+				req := &ImportRequest{
+					Index:      msg.Table,
+					Field:      msg.Field,
+					Shard:      msg.Shard,
+					RowIDs:     msg.RowIDs,
+					ColumnIDs:  msg.ColumnIDs,
+					RowKeys:    msg.RowKeys,
+					ColumnKeys: msg.ColumnKeys,
+					Timestamps: msg.Timestamps,
+					Clear:      msg.Clear,
+				}
+
+				qcx := api.Txf().NewQcx()
+				defer qcx.Abort()
+
+				opts := []ImportOption{
+					OptImportOptionsClear(msg.Clear),
+					OptImportOptionsIgnoreKeyCheck(msg.IgnoreKeyCheck),
+					OptImportOptionsPresorted(msg.Presorted),
+					OptImportOptionsSuppressLog(true),
+				}
+				if err := api.Import(ctx, qcx, req, opts...); err != nil {
+					return errors.Wrapf(err, "import, table: %s, field: %s, shard: %d", msg.Table, msg.Field, msg.Shard)
+				}
+
+			case *computer.ImportValueMessage:
+				req := &ImportValueRequest{
+					Index:           msg.Table,
+					Field:           msg.Field,
+					Shard:           msg.Shard,
+					ColumnIDs:       msg.ColumnIDs,
+					ColumnKeys:      msg.ColumnKeys,
+					Values:          msg.Values,
+					FloatValues:     msg.FloatValues,
+					TimestampValues: msg.TimestampValues,
+					StringValues:    msg.StringValues,
+					Clear:           msg.Clear,
+				}
+
+				qcx := api.Txf().NewQcx()
+				defer qcx.Abort()
+
+				opts := []ImportOption{
+					OptImportOptionsClear(msg.Clear),
+					OptImportOptionsIgnoreKeyCheck(msg.IgnoreKeyCheck),
+					OptImportOptionsPresorted(msg.Presorted),
+					OptImportOptionsSuppressLog(true),
+				}
+				if err := api.ImportValue(ctx, qcx, req, opts...); err != nil {
+					return errors.Wrapf(err, "import value, table: %s, field: %s, shard: %d", msg.Table, msg.Field, msg.Shard)
+				}
+			case *computer.ImportRoaringShardMessage:
+				req := &ImportRoaringShardRequest{
+					Remote:      true,
+					Views:       make([]RoaringUpdate, len(msg.Views)),
+					SuppressLog: true,
+				}
+				for i, view := range msg.Views {
+					req.Views[i] = RoaringUpdate{
+						Field:        view.Field,
+						View:         view.View,
+						Clear:        view.Clear,
+						Set:          view.Set,
+						ClearRecords: view.ClearRecords,
+					}
+				}
+				if err := api.ImportRoaringShard(ctx, msg.Table, msg.Shard, req); err != nil {
+					return errors.Wrapf(err, "import roaring shard table: %s, shard: %d", msg.Table, msg.Shard)
+				}
+			}
+		}
+		return nil
+	}
+	// 1st write log load
+	if err := loadWriteLog(); err != nil {
+		return err
+	}
+
+	// acquire lock on this partition's keys
+	if err := resource.Lock(); err != nil {
+		return errors.Wrap(err, "locking field key partition")
+	}
+
+	// reload writelog in case of changes between last load and
+	// lock. The resource object takes care of only loading new data.
+	return loadWriteLog()
+}
+
+//////////////////////////////////////////////////////////////
+
+// sliceComparer is used to compare the differences between two slices of comparables.
+type sliceComparer[K comparable] struct {
+	from []K
+	to   []K
+}
+
+func newSliceComparer[K comparable](from []K, to []K) *sliceComparer[K] {
+	return &sliceComparer[K]{
+		from: from,
+		to:   to,
+	}
+}
+
+// added returns the items which are present in `to` but not in `from`.
+func (s *sliceComparer[K]) added() []K {
+	return thingsAdded(s.from, s.to)
+}
+
+// removed returns the items which are present in `from` but not in `to`.
+func (s *sliceComparer[K]) removed() []K {
+	return thingsAdded(s.to, s.from)
+}
+
+// same returns the items which are in both `to` and `from`.
+func (s *sliceComparer[K]) same() []K {
+	var same []K
+	for _, fromThing := range s.from {
+		for _, toThing := range s.to {
+			if fromThing == toThing {
+				same = append(same, fromThing)
+				break
+			}
+		}
+	}
+	return same
+}
+
+// thingsAdded returns the comparable things which are present in `to` but not
+// in `from`.
+func thingsAdded[K comparable](from []K, to []K) []K {
+	var added []K
+	for i := range to {
+		var found bool
+		for j := range from {
+			if from[j] == to[i] {
+				found = true
+				break
+			}
+		}
+		if !found {
+			added = append(added, to[i])
+		}
+	}
+	return added
+}
+
+// partitionsComparer is used to compare the differences between two maps of
+// table:[]partition.
+type partitionsComparer struct {
+	from map[dax.TableKey]dax.PartitionNums
+	to   map[dax.TableKey]dax.PartitionNums
+}
+
+func newPartitionsComparer(from map[dax.TableKey]dax.PartitionNums, to map[dax.TableKey]dax.PartitionNums) *partitionsComparer {
+	return &partitionsComparer{
+		from: from,
+		to:   to,
+	}
+}
+
+// added returns the partitions which are present in `to` but not in `from`. The
+// results remain in the format of a map of table:[]partition.
+func (p *partitionsComparer) added() map[dax.TableKey]dax.PartitionNums {
+	return partitionsAdded(p.from, p.to)
+}
+
+// removed returns the partitions which are present in `from` but not in `to`.
+// The results remain in the format of a map of table:[]partition.
+func (p *partitionsComparer) removed() map[dax.TableKey]dax.PartitionNums {
+	return partitionsAdded(p.to, p.from)
+}
+
+// partitionsAdded returns the partitions which are present in `to` but not in `from`.
+func partitionsAdded(from map[dax.TableKey]dax.PartitionNums, to map[dax.TableKey]dax.PartitionNums) map[dax.TableKey]dax.PartitionNums {
+	if from == nil {
+		return to
+	}
+
+	added := make(map[dax.TableKey]dax.PartitionNums)
+	for tt, tps := range to {
+		fps, found := from[tt]
+		if !found {
+			added[tt] = tps
+			continue
+		}
+
+		addedPartitions := dax.PartitionNums{}
+		for i := range tps {
+			var found bool
+			for j := range fps {
+				if fps[j] == tps[i] {
+					found = true
+					break
+				}
+			}
+			if !found {
+				addedPartitions = append(addedPartitions, tps[i])
+			}
+		}
+
+		if len(addedPartitions) > 0 {
+			added[tt] = addedPartitions
+		}
+	}
+	return added
+}
+
+// fieldsComparer is used to compare the differences between two maps of
+// table:[]fieldVersion.
+type fieldsComparer struct {
+	from map[dax.TableKey][]dax.FieldName
+	to   map[dax.TableKey][]dax.FieldName
+}
+
+func newFieldsComparer(from map[dax.TableKey][]dax.FieldName, to map[dax.TableKey][]dax.FieldName) *fieldsComparer {
+	return &fieldsComparer{
+		from: from,
+		to:   to,
+	}
+}
+
+// added returns the fields which are present in `to` but not in `from`. The
+// results remain in the format of a map of table:[]field.
+func (f *fieldsComparer) added() map[dax.TableKey][]dax.FieldName {
+	return fieldsAdded(f.from, f.to)
+}
+
+// removed returns the fields which are present in `from` but not in `to`.
+// The results remain in the format of a map of table:[]field.
+func (f *fieldsComparer) removed() map[dax.TableKey][]dax.FieldName {
+	return fieldsAdded(f.to, f.from)
+}
+
+// fieldsAdded returns the fields which are present in `to` but not in `from`.
+func fieldsAdded(from map[dax.TableKey][]dax.FieldName, to map[dax.TableKey][]dax.FieldName) map[dax.TableKey][]dax.FieldName {
+	if from == nil {
+		return to
+	}
+
+	added := make(map[dax.TableKey][]dax.FieldName)
+	for tt, tps := range to {
+		fps, found := from[tt]
+		if !found {
+			added[tt] = tps
+			continue
+		}
+
+		addedFieldVersions := []dax.FieldName{}
+		for i := range tps {
+			var found bool
+			for j := range fps {
+				if fps[j] == tps[i] {
+					found = true
+					break
+				}
+			}
+			if !found {
+				addedFieldVersions = append(addedFieldVersions, tps[i])
+			}
+		}
+
+		if len(addedFieldVersions) > 0 {
+			added[tt] = addedFieldVersions
+		}
+	}
+	return added
+}
+
+// shardsComparer is used to compare the differences between two maps of
+// table:[]shardV.
+type shardsComparer struct {
+	from map[dax.TableKey]dax.ShardNums
+	to   map[dax.TableKey]dax.ShardNums
+}
+
+func newShardsComparer(from map[dax.TableKey]dax.ShardNums, to map[dax.TableKey]dax.ShardNums) *shardsComparer {
+	return &shardsComparer{
+		from: from,
+		to:   to,
+	}
+}
+
+// added returns the shards which are present in `to` but not in `from`. The
+// results remain in the format of a map of table:[]shard.
+func (s *shardsComparer) added() map[dax.TableKey]dax.ShardNums {
+	return shardsAdded(s.from, s.to)
+}
+
+// removed returns the shards which are present in `from` but not in `to`. The
+// results remain in the format of a map of table:[]shard.
+func (s *shardsComparer) removed() map[dax.TableKey]dax.ShardNums {
+	return shardsAdded(s.to, s.from)
+}
+
+// shardsAdded returns the shards which are present in `to` but not in `from`.
+func shardsAdded(from map[dax.TableKey]dax.ShardNums, to map[dax.TableKey]dax.ShardNums) map[dax.TableKey]dax.ShardNums {
+	if from == nil {
+		return to
+	}
+
+	added := make(map[dax.TableKey]dax.ShardNums)
+	for tt, tss := range to {
+		fss, found := from[tt]
+		if !found {
+			added[tt] = tss
+			continue
+		}
+
+		addedShards := dax.ShardNums{}
+		for i := range tss {
+			var found bool
+			for j := range fss {
+				if fss[j] == tss[i] {
+					found = true
+					break
+				}
+			}
+			if !found {
+				addedShards = append(addedShards, tss[i])
+			}
+		}
+
+		if len(addedShards) > 0 {
+			added[tt] = addedShards
+		}
+	}
+	return added
+}
+
+// createTableAndFields creates the FeatureBase Tables and Fields provided in
+// the dax.Directive format.
+func (api *API) createTableAndFields(tbl *dax.QualifiedTable, partitions dax.PartitionNums) error {
+	cim := &CreateIndexMessage{
+		Index:     string(tbl.Key()),
+		CreatedAt: 0,
+		Meta: IndexOptions{
+			Keys:           tbl.StringKeys(),
+			TrackExistence: true,
+		},
+	}
+
+	// Create the index in etcd as the system of record.
+	if err := api.holder.persistIndex(context.Background(), cim); err != nil {
+		return errors.Wrap(err, "persisting index")
+	}
+
+	idx, err := api.holder.createIndexWithPartitions(cim, partitions)
+	if err != nil {
+		return errors.Wrapf(err, "adding index: %s", tbl.Name)
+	}
+
+	// Add the fields
+	for _, fld := range tbl.Fields {
+		if fld.IsPrimaryKey() {
+			continue
+		}
+		if err := createField(idx, fld); err != nil {
+			return errors.Wrapf(err, "creating field: %s", fld.Name)
+		}
+	}
+
+	return nil
+}
+
+// createField creates a FeatureBase Field in the provided FeatureBase Index
+// based on the provided field's type.
+func createField(idx *Index, fld *dax.Field) error {
+	opts, err := FieldOptionsFromField(fld)
+	if err != nil {
+		return errors.Wrapf(err, "creating field options from field: %s", fld.Name)
+	}
+
+	if _, err := idx.createNullableField(string(fld.Name), "", opts...); err != nil {
+		return errors.Wrapf(err, "creating field on index: %s", fld.Name)
+	}
+	return nil
+}

api_directive_internal_test.go

@@ -0,0 +1,25 @@
+package pilosa
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestThingsAddedGeneric(t *testing.T) {
+	from := []string{"a", "b", "c"}
+	to := []string{"b", "c", "d"}
+
+	added := thingsAdded(from, to)
+	assert.Equal(t, added, []string{"d"})
+}
+
+func TestSliceComparer(t *testing.T) {
+	from := []string{"a", "b", "c"}
+	to := []string{"b", "c", "d"}
+
+	sc := newSliceComparer(from, to)
+
+	added := sc.added()
+	assert.Equal(t, added, []string{"d"})
+}

api_directive_test.go

@@ -0,0 +1,97 @@
+package pilosa_test
+
+import (
+	"context"
+	"testing"
+
+	pilosa "github.com/featurebasedb/featurebase/v3"
+	"github.com/featurebasedb/featurebase/v3/dax"
+	daxtest "github.com/featurebasedb/featurebase/v3/dax/test"
+	"github.com/featurebasedb/featurebase/v3/test"
+	"github.com/stretchr/testify/assert"
+)
+
+// Ensure holder can handle an incoming directive.
+func TestAPI_Directive(t *testing.T) {
+
+	c := test.MustRunCluster(t, 1)
+	defer c.Close()
+
+	api := c.GetPrimary().API
+	ctx := context.Background()
+
+	qdbid := dax.NewQualifiedDatabaseID("acme", "db1")
+	tbl1 := daxtest.TestQualifiedTableWithID(t, qdbid, "1", "tbl1", 12, false)
+	tbl2 := daxtest.TestQualifiedTableWithID(t, qdbid, "2", "tbl2", 12, false)
+	tbl3 := daxtest.TestQualifiedTableWithID(t, qdbid, "3", "tbl3", 12, false)
+
+	t.Run("Schema", func(t *testing.T) {
+
+		// Empty directive (and empty holder).
+		{
+			d := &dax.Directive{
+				Method:  dax.DirectiveMethodFull,
+				Version: 1,
+			}
+			err := api.ApplyDirective(ctx, d)
+			assert.NoError(t, err)
+			assertTablesMatch(t, []string{}, api.Holder().Indexes())
+		}
+
+		// Add a new table.
+		{
+			d := &dax.Directive{
+				Method: dax.DirectiveMethodFull,
+				Tables: []*dax.QualifiedTable{
+					tbl1,
+				},
+				Version: 2,
+			}
+			err := api.ApplyDirective(ctx, d)
+			assert.NoError(t, err)
+			assertTablesMatch(t, []string{"tbl__acme__db1__1"}, api.Holder().Indexes())
+		}
+
+		// Add a new table, and keep the existing table.
+		{
+			d := &dax.Directive{
+				Method: dax.DirectiveMethodFull,
+				Tables: []*dax.QualifiedTable{
+					tbl1,
+					tbl2,
+				},
+				Version: 3,
+			}
+			err := api.ApplyDirective(ctx, d)
+			assert.NoError(t, err)
+			assertTablesMatch(t, []string{"tbl__acme__db1__1", "tbl__acme__db1__2"}, api.Holder().Indexes())
+		}
+
+		// Add a new table and remove one of the existing tables.
+		{
+			d := &dax.Directive{
+				Method: dax.DirectiveMethodFull,
+				Tables: []*dax.QualifiedTable{
+					tbl2,
+					tbl3,
+				},
+				Version: 4,
+			}
+			err := api.ApplyDirective(ctx, d)
+			assert.NoError(t, err)
+			assertTablesMatch(t, []string{"tbl__acme__db1__2", "tbl__acme__db1__3"}, api.Holder().Indexes())
+		}
+	})
+}
+
+// assertTablesMatch is a helper function which asserts that the list of index
+// names in `actual` match those provided in `expected`.
+func assertTablesMatch(t *testing.T, expected []string, actual []*pilosa.Index) {
+	t.Helper()
+
+	act := make([]string, len(actual))
+	for i := range actual {
+		act[i] = actual[i].Name()
+	}
+	assert.ElementsMatch(t, expected, act)
+}

apimethod_string.go

@@ -0,0 +1,57 @@
+// Code generated by "stringer -type=apiMethod"; DO NOT EDIT.
+
+package pilosa
+
+import "strconv"
+
+func _() {
+	// An "invalid array index" compiler error signifies that the constant values have changed.
+	// Re-run the stringer command to generate them again.
+	var x [1]struct{}
+	_ = x[apiClusterMessage-0]
+	_ = x[apiCreateField-1]
+	_ = x[apiCreateIndex-2]
+	_ = x[apiDeleteField-3]
+	_ = x[apiDeleteAvailableShard-4]
+	_ = x[apiDeleteIndex-5]
+	_ = x[apiDeleteView-6]
+	_ = x[apiExportCSV-7]
+	_ = x[apiFragmentBlockData-8]
+	_ = x[apiFragmentBlocks-9]
+	_ = x[apiFragmentData-10]
+	_ = x[apiTranslateData-11]
+	_ = x[apiFieldTranslateData-12]
+	_ = x[apiField-13]
+	_ = x[apiImport-14]
+	_ = x[apiImportValue-15]
+	_ = x[apiIndex-16]
+	_ = x[apiQuery-17]
+	_ = x[apiRecalculateCaches-18]
+	_ = x[apiSchema-19]
+	_ = x[apiShardNodes-20]
+	_ = x[apiState-21]
+	_ = x[apiViews-22]
+	_ = x[apiApplySchema-23]
+	_ = x[apiStartTransaction-24]
+	_ = x[apiFinishTransaction-25]
+	_ = x[apiTransactions-26]
+	_ = x[apiGetTransaction-27]
+	_ = x[apiActiveQueries-28]
+	_ = x[apiPastQueries-29]
+	_ = x[apiIDReserve-30]
+	_ = x[apiIDCommit-31]
+	_ = x[apiIDReset-32]
+	_ = x[apiPartitionNodes-33]
+	_ = x[apiMutexCheck-34]
+}
+
+const _apiMethod_name = "apiClusterMessageapiCreateFieldapiCreateIndexapiDeleteFieldapiDeleteAvailableShardapiDeleteIndexapiDeleteViewapiExportCSVapiFragmentBlockDataapiFragmentBlocksapiFragmentDataapiTranslateDataapiFieldTranslateDataapiFieldapiImportapiImportValueapiIndexapiQueryapiRecalculateCachesapiSchemaapiShardNodesapiStateapiViewsapiApplySchemaapiStartTransactionapiFinishTransactionapiTransactionsapiGetTransactionapiActiveQueriesapiPastQueriesapiIDReserveapiIDCommitapiIDResetapiPartitionNodesapiMutexCheck"
+
+var _apiMethod_index = [...]uint16{0, 17, 31, 45, 59, 82, 96, 109, 121, 141, 158, 173, 189, 210, 218, 227, 241, 249, 257, 277, 286, 299, 307, 315, 329, 348, 368, 383, 400, 416, 430, 442, 453, 463, 480, 493}
+
+func (i apiMethod) String() string {
+	if i < 0 || i >= apiMethod(len(_apiMethod_index)-1) {
+		return "apiMethod(" + strconv.FormatInt(int64(i), 10) + ")"
+	}
+	return _apiMethod_name[_apiMethod_index[i]:_apiMethod_index[i+1]]
+}

apply.go

@@ -0,0 +1,686 @@
+// Copyright 2021 Molecula Corp. All rights reserved.
+package pilosa
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+	"sync"
+
+	"github.com/apache/arrow/go/v10/arrow"
+	"github.com/apache/arrow/go/v10/arrow/array"
+	"github.com/apache/arrow/go/v10/arrow/memory"
+	"github.com/featurebasedb/featurebase/v3/pql"
+	"github.com/featurebasedb/featurebase/v3/tracing"
+	"github.com/featurebasedb/featurebase/v3/vprint"
+	"github.com/gomem/gomem/pkg/dataframe"
+	"github.com/pkg/errors"
+
+	ivy "robpike.io/ivy/arrow"
+	config "robpike.io/ivy/config"
+	"robpike.io/ivy/exec"
+	"robpike.io/ivy/parse"
+	"robpike.io/ivy/run"
+	"robpike.io/ivy/scan"
+	"robpike.io/ivy/value"
+)
+
+type (
+	ApplyResult *arrow.Column
+)
+
+func runIvyString(context value.Context, str string) (ok bool, err error) {
+	defer func() {
+		if r := recover(); r != nil {
+			err = r.(value.Error)
+		}
+	}()
+	scanner := scan.New(context, "<args>", strings.NewReader(str))
+	parser := parse.NewParser("<args>", scanner, context)
+	ok = run.Run(parser, context, false)
+	return
+}
+
+// Possibly combine all arrays together then apply some interesting
+// computation at the end?
+func IvyReduce(reduceCode string, opCode string, opt *ExecOptions) (func(ctx context.Context, prev, v interface{}) interface{}, func() (*dataframe.DataFrame, error)) {
+	var accumulator value.Value
+	mu := &sync.Mutex{}
+	concat := value.BinaryOps[opCode]
+	conf := getDefaultConfig()
+	ctxIvy := exec.NewContext(&conf)
+	// concat returned results at coordinating node.
+	reduceFn := func(ctx context.Context, prev, v interface{}) interface{} {
+		if v == nil {
+			return prev
+		}
+		if accumulator == nil {
+			switch val := v.(type) {
+			case *dataframe.DataFrame:
+				col := val.ColumnAt(0)
+				resolver := dataframe.NewChunkResolver(col)
+				accumulator = value.NewArrowVector(col, &conf, &resolver)
+			case value.Value:
+				accumulator = v.(value.Value)
+			default:
+				return errors.New(fmt.Sprintf("ivy reduction failed first unexpected type %T", v))
+			}
+			return nil
+		}
+		switch val := v.(type) {
+		case *dataframe.DataFrame:
+			col := val.ColumnAt(0)
+			resolver := dataframe.NewChunkResolver(col)
+			x := value.NewArrowVector(col, &conf, &resolver)
+			mu.Lock() // i'm being overyerly cautious..need to confirm this can be concurrent
+			accumulator = concat.EvalBinary(ctxIvy, accumulator, x)
+			mu.Unlock()
+		case value.Value:
+			mu.Lock()
+			accumulator = concat.EvalBinary(ctxIvy, accumulator, val)
+			mu.Unlock()
+		default:
+			return errors.New(fmt.Sprintf("ivy reduction failed unexpected type %T", v))
+		}
+
+		return nil
+	}
+	tablerFn := func() (*dataframe.DataFrame, error) {
+		pool := memory.NewGoAllocator() // TODO(twg) 2022/09/01 singledton?
+		if opt.Remote {
+			col := value.ToArrowColumn(accumulator, pool)
+			return dataframe.NewDataFrameFromColumns(pool, []arrow.Column{*col})
+		}
+		// only actually reduce on the initiating node i hate the network
+		// over head but oh well
+		ctxIvy.AssignGlobal("_", accumulator)
+		ok, err := runIvyString(ctxIvy, reduceCode)
+		if err != nil {
+			return nil, err
+		}
+		if ok {
+			v := ctxIvy.Global("_")
+			if v == nil {
+				return nil, errors.New("ivy reduction no result ")
+			}
+			col := value.ToArrowColumn(ctxIvy.Global("_"), pool)
+
+			return dataframe.NewDataFrameFromColumns(pool, []arrow.Column{*col})
+		}
+		return nil, errors.New("ivy reduction failed ")
+	}
+
+	return reduceFn, tablerFn
+}
+
+// executeApply executes a Apply() call.
+func (e *executor) executeApply(ctx context.Context, qcx *Qcx, index string, c *pql.Call, shards []uint64, opt *ExecOptions) (*dataframe.DataFrame, error) {
+	if !e.dataframeEnabled {
+		return nil, errors.New("Dataframe support not enabled")
+	}
+	span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeMax")
+	defer span.Finish()
+
+	if _, err := c.FirstStringArg("_ivy"); err != nil {
+		return nil, errors.Wrap(err, " no ivy program supplied")
+	}
+
+	if len(c.Children) > 1 {
+		return nil, errors.New("Apply() only accepts a single bitmap input filter")
+	}
+
+	// Execute calls in bulk on each remote node and merge.
+	mapFn := func(ctx context.Context, shard uint64, mopt *mapOptions) (_ interface{}, err error) {
+		return e.executeApplyShard(ctx, qcx, index, c, shard)
+	}
+	ivyReduce, ok, err := c.StringArg("_ivyReduce")
+	if err != nil {
+		return nil, err
+	}
+	reduceFn, tablerFn := IvyReduce("_", ",", opt)
+	if ok {
+		reduceFn, tablerFn = IvyReduce(ivyReduce, ",", opt)
+	}
+
+	_, err = e.mapReduce(ctx, index, shards, c, opt, mapFn, reduceFn)
+	if err != nil {
+		return nil, err
+	}
+	return tablerFn()
+}
+
+func getDefaultConfig() config.Config {
+	maxbits := uint(1e9)   // "maximum size of an integer, in bits; 0 means no limit")
+	maxdigits := uint(1e4) // "above this many `digits`, integers print as floating point; 0 disables")
+	maxstack := uint(100000)
+	origin := 1  // "set index origin to `n` (must be 0 or 1)")
+	prompt := "" // flag.String("prompt", "", "command `prompt`")
+	format := ""
+	//      debugFlag := "" // flag.String("debug", "", "comma-separated `names` of debug settings to enable")
+	conf := config.Config{}
+	conf.SetFormat(format)
+	conf.SetMaxBits(maxbits)
+	conf.SetMaxDigits(maxdigits)
+	conf.SetMaxStack(maxstack)
+	conf.SetOrigin(origin)
+	conf.SetPrompt(prompt)
+	conf.SetOutput(io.Discard)
+	conf.SetErrOutput(io.Discard)
+	conf.SetEmbedded(true) // needed to propagate panic
+	return conf
+}
+
+func filterDataframe(resolver dataframe.Resolver, pool memory.Allocator, filter []int64) (*dataframe.IndexResolver, error) {
+	if resolver.NumRows() == 0 {
+		return nil, errors.New("No data")
+	}
+	indexResolver := dataframe.NewIndexResolver(len(filter), uint32(ShardWidth-1))
+	for i, id := range filter {
+		if int(id) >= resolver.NumRows() {
+			continue
+		}
+		c, o := resolver.Resolve(int(id))
+		indexResolver.Set(i, c, o)
+
+	}
+	return indexResolver, nil
+}
+
+func (e *executor) executeApplyShard(ctx context.Context, qcx *Qcx, index string, c *pql.Call, shard uint64) (value.Value, error) {
+	span, _ := tracing.StartSpanFromContext(ctx, "Executor.executeApplyShard")
+	defer span.Finish()
+
+	ivyProgram, ok, err := c.StringArg("_ivy")
+	if err != nil || !ok {
+		return nil, errors.Wrap(err, "finding ivy program")
+	}
+	var filter *Row
+	if len(c.Children) == 1 {
+		row, err := e.executeBitmapCallShard(ctx, qcx, index, c.Children[0], shard)
+		if err != nil {
+			return nil, err
+		}
+		filter = row
+		if !filter.Any() {
+			// no need to actuall run the query for its not operating against any values
+			return value.NewVector([]value.Value{}), nil
+		}
+	}
+	//
+	pool := memory.NewGoAllocator() // TODO(twg) 2022/09/01 singledton?
+
+	ids := filter.ShardColumns() // needs to be shard columns
+	// Fetch index.
+	idx := e.Holder.Index(index)
+	if idx == nil {
+		return nil, newNotFoundError(ErrIndexNotFound, index)
+	}
+
+	fname := idx.GetDataFramePath(shard)
+
+	if !e.dataFrameExists(fname) {
+		return value.NewVector([]value.Value{}), nil
+	}
+
+	table, err := e.getDataTable(ctx, fname, pool)
+	if err != nil {
+		return nil, err
+	}
+	defer table.Release()
+	df, err := dataframe.NewDataFrameFromTable(pool, table)
+	if err != nil {
+		return nil, err
+	}
+	p := dataframe.NewChunkResolver(df.ColumnAt(0))
+	var resolver dataframe.Resolver
+	resolver = &p
+	if filter != nil {
+		if len(ids) == 0 {
+			return value.NewVector([]value.Value{}), nil
+		}
+		resolver, err = filterDataframe(resolver, pool, ids)
+		if err != nil {
+			return nil, err
+		}
+	}
+	conf := getDefaultConfig()
+	context, err := ivy.RunArrow(dataframe.NewTableFacade(df), ivyProgram, conf, resolver)
+	if err != nil {
+		return nil, fmt.Errorf("ivy map error: %w", err)
+	}
+	return context.Global("_"), nil
+}
+
+// ///////////////////////////////////////////////////////
+// all the ingest supporting functions
+// ///////////////////////////////////////////////////////
+
+func NewShardFile(ctx context.Context, name string, mem memory.Allocator, e *executor) (*ShardFile, error) {
+	if !e.dataFrameExists(name) {
+		return &ShardFile{dest: name, executor: e, strings: make(map[key][]string)}, nil
+	}
+	// else read in existing
+	table, err := e.getDataTable(ctx, name, mem)
+	if err != nil {
+		return nil, err
+	}
+	return &ShardFile{table: table, schema: table.Schema(), dest: name, executor: e, strings: make(map[key][]string)}, nil
+}
+
+type NameType struct {
+	Name     string
+	DataType arrow.DataType
+}
+type ChangesetRequest struct {
+	ShardIds     []int64 // only shardwidth bits to provide 0 indexing inside shard file
+	Columns      []interface{}
+	SimpleSchema []NameType
+}
+
+// TODO(twg) 2022/09/30 Needs a refactor
+func cast(v interface{}) arrow.DataType {
+	switch v.(type) {
+	case *arrow.Int64Type:
+		return arrow.PrimitiveTypes.Int64
+	case int64:
+		return arrow.PrimitiveTypes.Int64
+	case *arrow.Float64Type:
+		return arrow.PrimitiveTypes.Float64
+	case float64:
+		return arrow.PrimitiveTypes.Float64
+	case *arrow.StringType:
+		return arrow.BinaryTypes.String
+	default:
+		vprint.VV("%T .... %v", v, v)
+	}
+	return arrow.PrimitiveTypes.Int64
+}
+
+func (cr *ChangesetRequest) ArrowSchema() *arrow.Schema {
+	fields := make([]arrow.Field, len(cr.SimpleSchema))
+	for i := range cr.SimpleSchema {
+		fields[i] = arrow.Field{Name: cr.SimpleSchema[i].Name, Type: cast(cr.SimpleSchema[i].DataType)}
+	}
+	return arrow.NewSchema(fields, nil)
+}
+
+type key struct {
+	col   int
+	chunk int
+}
+
+type ShardFile struct {
+	table      arrow.Table
+	schema     *arrow.Schema
+	beforeRows int64
+	added      int64
+	columns    []interface{}
+	dest       string
+	executor   *executor
+	strings    map[key][]string
+}
+
+func compareSchema(s1, s2 *arrow.Schema) bool {
+	if s1 == nil || s2 == nil {
+		return false
+	}
+	if len(s1.Fields()) != len(s2.Fields()) {
+		return false
+	}
+	for i := 0; i < len(s1.Fields()); i++ {
+		f1 := s1.Field(i)
+		f2 := s2.Field(i)
+		if f1.Name != f2.Name {
+			return false
+		}
+		if f1.Type != f2.Type {
+			return false
+		}
+	}
+	return true
+}
+
+func (sf *ShardFile) EnsureSchema(cs *ChangesetRequest) error {
+	schema := cs.ArrowSchema()
+	if sf.schema == nil {
+		sf.schema = schema
+	} else {
+		if !compareSchema(sf.schema, schema) {
+			vprint.VV("incomeing schema", schema)
+			vprint.VV("existing schema", sf.schema)
+			return errors.New("dataframe schema's don't match")
+		}
+	}
+	sf.columns = make([]interface{}, len(sf.schema.Fields()))
+	return nil
+}
+
+func (sf *ShardFile) buildAppenders(maxid int64) {
+	if sf.table != nil {
+		sf.beforeRows = sf.table.NumRows()
+	}
+	if maxid < sf.beforeRows {
+		// no need to add new rows
+		return
+	}
+	newSize := maxid - sf.beforeRows + 1
+	for i := 0; i < len(sf.schema.Fields()); i++ {
+		switch sf.schema.Field(i).Type {
+		case arrow.PrimitiveTypes.Int64:
+			sf.columns[i] = make([]int64, newSize)
+		case arrow.PrimitiveTypes.Float64:
+			sf.columns[i] = make([]float64, newSize)
+		case arrow.BinaryTypes.String:
+			sf.columns[i] = make([]string, newSize)
+		}
+	}
+	sf.added = newSize
+}
+
+// the row offset must be reset to 0 for the slices being appended
+func (sf *ShardFile) SetIntValue(col int, row int64, val int64) {
+	v := sf.columns[col].([]int64)
+	v[row-sf.beforeRows] = val
+}
+
+func (sf *ShardFile) SetFloatValue(col int, row int64, val float64) {
+	v := sf.columns[col].([]float64)
+	v[row-sf.beforeRows] = val
+}
+
+func (sf *ShardFile) SetStringValue(col int, row int64, val string) {
+	v := sf.columns[col].([]string)
+	v[row-sf.beforeRows] = val
+}
+
+func (sf *ShardFile) Process(cs *ChangesetRequest) error {
+	err := sf.process(cs)
+	if err != nil {
+		return err
+	}
+	rtemp := sf.dest + ".temp"
+	err = sf.Save(rtemp)
+	if err != nil {
+		return err
+	}
+	return os.Rename(rtemp+sf.executor.TableExtension(), sf.dest+sf.executor.TableExtension())
+}
+
+func (sf *ShardFile) LoadBlobs() error {
+	for col := 0; col < len(sf.schema.Fields()); col++ {
+		column := sf.table.Column(col)
+		switch column.DataType() {
+		case arrow.BinaryTypes.String:
+			for i, chunk := range column.Data().Chunks() {
+				stringData := chunk.(*array.String)
+				k := key{col: col, chunk: i}
+				for j := 0; j < stringData.Len(); j++ {
+					v := stringData.Value(j)
+					sf.strings[k] = append(sf.strings[k], v)
+
+				}
+			}
+		}
+	}
+	return nil
+}
+
+func (sf *ShardFile) ReplaceString(col, chunk, l int, s string) {
+	sf.strings[key{col: col, chunk: chunk}][l] = s
+}
+
+func (sf *ShardFile) process(cs *ChangesetRequest) error {
+	offset := 0
+	if sf.table != nil {
+		// need to load blobs prior
+		sf.LoadBlobs()
+		column := sf.table.Column(0)
+		resolver := dataframe.NewChunkResolver(column)
+		for i, rowid := range cs.ShardIds {
+			offset = i
+			if rowid >= sf.table.NumRows() {
+				break
+			}
+			chunk, l := resolver.Resolve(int(rowid))
+			for col := 0; col < len(sf.schema.Fields()); col++ {
+				column := sf.table.Column(col)
+				switch column.DataType() {
+				case arrow.PrimitiveTypes.Int64:
+					v := column.Data().Chunk(chunk).(*array.Int64).Int64Values()
+					v[l] = cs.Columns[col].([]int64)[i]
+				case arrow.PrimitiveTypes.Float64:
+					v := column.Data().Chunk(chunk).(*array.Float64).Float64Values()
+					v[l] = cs.Columns[col].([]float64)[i]
+				case arrow.BinaryTypes.String:
+					// TODO(twg) 2023/01/09 How to update existing?
+					new := cs.Columns[col].([]string)[i]
+					sf.ReplaceString(col, chunk, l, new)
+				default:
+					panic(fmt.Sprintf("Unknown Type %v", column.DataType()))
+				}
+			}
+		}
+	}
+	max := cs.ShardIds[len(cs.ShardIds)-1]
+	sf.buildAppenders(max)
+	// need to check if only replace and no apend
+	if sf.added > 0 {
+		for i, rowid := range cs.ShardIds[offset:] {
+			i += offset
+
+			for col := 0; col < len(sf.schema.Fields()); col++ {
+				switch sf.schema.Field(col).Type {
+				case arrow.PrimitiveTypes.Int64:
+					sf.SetIntValue(col, rowid, cs.Columns[col].([]int64)[i])
+				case arrow.PrimitiveTypes.Float64:
+					sf.SetFloatValue(col, rowid, cs.Columns[col].([]float64)[i])
+				case arrow.BinaryTypes.String:
+					sf.SetStringValue(col, rowid, cs.Columns[col].([]string)[i])
+				default:
+					panic(fmt.Sprintf("2 Unknown Type %v", sf.schema.Field(col).Type))
+				}
+			}
+		}
+	}
+
+	return nil
+}
+
+type twoSlices struct {
+	id_slice    []int
+	lists_slice [][]string
+}
+
+type SortByOther twoSlices
+
+func (sbo SortByOther) Len() int {
+	return len(sbo.id_slice)
+}
+
+func (sbo SortByOther) Swap(i, j int) {
+	sbo.id_slice[i], sbo.id_slice[j] = sbo.id_slice[j], sbo.id_slice[i]
+	sbo.lists_slice[i], sbo.lists_slice[j] = sbo.lists_slice[j], sbo.lists_slice[i]
+}
+
+func (sbo SortByOther) Less(i, j int) bool {
+	return sbo.id_slice[i] < sbo.id_slice[j]
+}
+
+func (sf *ShardFile) buildFromStrings(idx int, mem memory.Allocator) []arrow.Array {
+	ids := make([]int, 0)
+	lists := make([][]string, 0)
+	for k, v := range sf.strings {
+		if k.col == idx { // ugh not ordered :(
+			ids = append(ids, k.chunk)
+			lists = append(lists, v)
+		}
+	}
+	// sort ids/lists
+	parts := twoSlices{id_slice: ids, lists_slice: lists}
+	sort.Sort(SortByOther(parts))
+
+	builder := array.NewStringBuilder(mem)
+	chunks := make([]arrow.Array, 0)
+	for _, v := range parts.lists_slice {
+		builder.AppendValues(v, nil)
+		newChunk := builder.NewArray()
+		chunks = append(chunks, newChunk)
+	}
+	return chunks
+}
+
+func (sf *ShardFile) Save(name string) error {
+	parts := make([]arrow.Array, 0)
+	mem := memory.NewGoAllocator()
+	for col := 0; col < len(sf.schema.Fields()); col++ {
+		chunks := make([]arrow.Array, 0)
+		if sf.table != nil {
+			// we append if there was existing file
+			column := sf.table.Column(col)
+			// if primitive type
+			switch column.DataType() {
+			case arrow.BinaryTypes.String:
+				chunks = sf.buildFromStrings(col, mem)
+			default:
+				chunks = append(chunks, column.Data().Chunks()...)
+			}
+			// else binary type
+		}
+		switch sf.schema.Field(col).Type {
+		case arrow.PrimitiveTypes.Int64:
+			// case *arrow.Int64Type:
+			if sf.added > 0 {
+				ibuild := array.NewInt64Builder(mem)
+				ibuild.AppendValues(sf.columns[col].([]int64), nil) // TODO(twg) 2022/09/28 need to handle null
+				newChunk := ibuild.NewArray()
+				chunks = append(chunks, newChunk)
+			}
+			record, err := array.Concatenate(chunks, mem)
+			if err != nil {
+				return err
+			}
+			parts = append(parts, record)
+		case arrow.PrimitiveTypes.Float64:
+			// case *arrow.Float64Type:
+			if sf.added > 0 {
+				fbuild := array.NewFloat64Builder(mem)
+				fbuild.AppendValues(sf.columns[col].([]float64), nil) // TODO(twg) 2022/09/28 need to handle null
+				newChunk := fbuild.NewArray()
+				chunks = append(chunks, newChunk)
+			}
+			record, err := array.Concatenate(chunks, mem)
+			if err != nil {
+				return err
+			}
+			parts = append(parts, record)
+		case arrow.BinaryTypes.String:
+			if sf.added > 0 {
+				fbuild := array.NewStringBuilder(mem)
+				fbuild.AppendValues(sf.columns[col].([]string), nil) // TODO(twg) 2022/09/28 need to handle null
+				newChunk := fbuild.NewArray()
+				chunks = append(chunks, newChunk)
+			}
+			record, err := array.Concatenate(chunks, mem)
+			if err != nil {
+				return err
+			}
+			parts = append(parts, record)
+		default:
+			vprint.VV("UNKNOWN %T", sf.schema.Field(col).Type)
+		}
+	}
+	rec := array.NewRecord(sf.schema, parts, sf.beforeRows+sf.added)
+	table := array.NewTableFromRecords(sf.schema, []arrow.Record{rec})
+
+	return sf.executor.SaveTable(name, table, mem)
+}
+
+// TODO(twg) 2022/10/03 Not a huge fan of the global variable will look at adding to executor structure
+// when dataframe is fully integrated
+var (
+	dataframeShardLocks map[uint64]*sync.Mutex
+	muWriteDataframe    sync.Mutex
+)
+
+func init() {
+	dataframeShardLocks = make(map[uint64]*sync.Mutex)
+}
+
+func getDataframeWritelock(shard uint64) *sync.Mutex {
+	muWriteDataframe.Lock()
+	defer muWriteDataframe.Unlock()
+	lock, ok := dataframeShardLocks[shard]
+	if ok {
+		return lock
+	}
+	newLock := sync.Mutex{}
+	dataframeShardLocks[shard] = &newLock
+	return &newLock
+}
+
+func (api *API) ApplyDataframeChangeset(ctx context.Context, index string, cs *ChangesetRequest, shard uint64) error {
+	// TODO(twg) 2022/09/29 need to validate api call
+	idx := api.Holder().Index(index)
+
+	// check if dataframe exists
+	fname := idx.GetDataFramePath(shard)
+
+	// only 1 shard writer allowed at at time so wait for it to be available
+	mu := getDataframeWritelock(shard)
+	mu.Lock()
+	defer mu.Unlock()
+	mem := memory.NewGoAllocator()
+	shardFile, err := NewShardFile(ctx, fname, mem, api.server.executor)
+	if err != nil {
+		return err
+	}
+
+	err = shardFile.EnsureSchema(cs)
+	if err != nil {
+		return err
+	}
+
+	return shardFile.Process(cs)
+}
+
+type column struct {
+	Name string
+	Type string
+}
+
+func (api *API) GetDataframeSchema(ctx context.Context, indexName string) (interface{}, error) {
+	idx, err := api.Index(ctx, indexName)
+	if err != nil {
+		return nil, err
+	}
+	base := idx.DataframesPath()
+	dir, _ := os.Open(base)
+	files, _ := dir.Readdir(0)
+	parts := make([]column, 0)
+	mem := memory.NewGoAllocator()
+	for i := range files {
+		file := files[i]
+		name := file.Name()
+		if api.server.executor.IsDataframeFile(name) {
+			// strip off the parquet extenison
+			name = strings.TrimSuffix(name, filepath.Ext(name))
+			// read the parquet file and extract the schema
+			fname := filepath.Join(base, name)
+			table, err := api.server.executor.getDataTable(ctx, fname, mem)
+			if err != nil {
+				return nil, err
+			}
+			for i := 0; i < int(table.NumCols()); i++ {
+				col := table.Column(i)
+				part := column{Name: col.Name(), Type: col.DataType().String()}
+				parts = append(parts, part)
+			}
+			break // only go on first file
+		}
+	}
+	return parts, nil
+}

arrow.go

@@ -0,0 +1,562 @@
+// Copyright 2021 Molecula Corp. All rights reserved.
+package pilosa
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"os"
+	"strings"
+	"sync"
+
+	"github.com/apache/arrow/go/v10/arrow"
+	"github.com/apache/arrow/go/v10/arrow/array"
+	"github.com/apache/arrow/go/v10/arrow/ipc"
+	"github.com/apache/arrow/go/v10/arrow/memory"
+	"github.com/apache/arrow/go/v10/parquet"
+	"github.com/apache/arrow/go/v10/parquet/file"
+	"github.com/apache/arrow/go/v10/parquet/pqarrow"
+	"github.com/featurebasedb/featurebase/v3/pql"
+	"github.com/featurebasedb/featurebase/v3/tracing"
+	"github.com/gomem/gomem/pkg/dataframe"
+	"github.com/pkg/errors"
+)
+
+/*
+The function Arrow provides filtered access to the raw values stored in the dataframe.
+If Arrow is just provided a bitmap filter, such as ConstRow or any Bitmap Operation,
+all the values associated with each column are returned.  This set can be limited with
+the addition of the header parameter
+Example:
+Arrow(ConstRow(columns=[2,4,6]),header=["fval"])
+*/
+
+// executeApply executes a Arrow() call.
+func (e *executor) executeArrow(ctx context.Context, qcx *Qcx, index string, c *pql.Call, shards []uint64, opt *ExecOptions) (arrow.Table, error) {
+	if !e.dataframeEnabled {
+		return nil, errors.New("Dataframe support not enabled")
+	}
+	span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeArrow")
+	defer span.Finish()
+	if len(c.Children) > 1 {
+		return nil, errors.New("Apply() only accepts a single bitmap input filter")
+	}
+	var columnFilter []string
+	if cols, ok := c.Args["header"].([]interface{}); ok {
+		columnFilter = make([]string, 0, len(cols))
+		for _, v := range cols {
+			columnFilter = append(columnFilter, v.(string))
+		}
+	}
+	mapcounter := 0
+	reducecounter := 0
+	pool := memory.NewGoAllocator() // TODO(twg) 2022/09/01 singledton?
+	// Execute calls in bulk on each remote node and merge.
+	mu := &sync.Mutex{}
+	mapFn := func(ctx context.Context, shard uint64, mopt *mapOptions) (_ interface{}, err error) {
+		mu.Lock()
+		mapcounter++
+		mu.Unlock()
+		return e.executeArrowShard(ctx, qcx, index, c, shard, pool, columnFilter)
+	}
+	tables := make([]*BasicTable, 0)
+
+	reduceFn := func(ctx context.Context, prev, v interface{}) interface{} {
+		mu.Lock()
+		reducecounter++
+		mu.Unlock()
+		if v == nil {
+			return prev
+		}
+		switch t := v.(type) {
+		case *BasicTable:
+
+			if t.resolver != nil {
+				mu.Lock()
+				tables = append(tables, t)
+				mu.Unlock()
+			}
+		case arrow.Table:
+			if t.NumRows() > 0 {
+				bt := BasicTableFromArrow(t, pool)
+				mu.Lock()
+				tables = append(tables, bt)
+				mu.Unlock()
+			}
+		}
+		return nil
+	}
+
+	_, err := e.mapReduce(ctx, index, shards, c, opt, mapFn, reduceFn)
+	if err != nil {
+		return nil, err
+	}
+	if len(tables) == 0 {
+		return &BasicTable{name: "empty"}, nil
+	}
+	tbl := Concat(tables[0].Schema(), tables, pool)
+	r := dataframe.NewChunkResolver(tbl.Column(0))
+	return &BasicTable{resolver: &r, table: tbl}, nil
+}
+
+type BasicTable struct {
+	resolver dataframe.Resolver
+	table    arrow.Table
+	filtered bool
+	name     string
+}
+
+func (st *BasicTable) Name() string {
+	return st.name
+}
+
+func (st *BasicTable) Schema() *arrow.Schema {
+	if st.table != nil {
+		return st.table.Schema()
+	}
+	return &arrow.Schema{}
+}
+
+func (st *BasicTable) IsFiltered() bool {
+	return st.filtered
+}
+
+func (st *BasicTable) NumRows() int64 {
+	if st.resolver == nil {
+		return 0
+	}
+	return int64(st.resolver.NumRows())
+}
+
+func (st *BasicTable) NumCols() int64 {
+	if st.table != nil {
+		return st.table.NumCols()
+	}
+	return 0
+}
+
+func (st *BasicTable) Column(i int) *arrow.Column {
+	if st.table != nil {
+		return st.table.Column(i)
+	}
+	return nil
+}
+
+func (st *BasicTable) Retain() {
+	if st.table != nil {
+		st.table.Retain()
+	}
+}
+
+func (st *BasicTable) Release() {
+	if st.table != nil {
+		st.table.Retain()
+	}
+}
+
+func (st *BasicTable) Get(column, row int) interface{} {
+	field := st.Schema().Field(column)
+	c, i := st.resolver.Resolve(row)
+	nullable := field.Nullable
+
+	chunk := st.Column(column).Data().Chunk(c)
+	// TODO(twg) 2023/01/26 potential NULL support?
+	if nullable && chunk.IsNull(i) {
+		return nil
+	}
+	switch field.Type.(type) {
+	case *arrow.BooleanType:
+		return chunk.(*array.Boolean).Value(i)
+	case *arrow.Int8Type:
+		v := chunk.(*array.Int8).Int8Values()
+		return int64(v[i])
+	case *arrow.Int16Type:
+		v := chunk.(*array.Int16).Int16Values()
+		return int64(v[i])
+	case *arrow.Int32Type:
+		v := chunk.(*array.Int32).Int32Values()
+		return int64(v[i])
+	case *arrow.Int64Type:
+		v := chunk.(*array.Int64).Int64Values()
+		return int64(v[i])
+	case *arrow.Uint8Type:
+		v := chunk.(*array.Uint8).Uint8Values()
+		return uint64(v[i])
+	case *arrow.Uint16Type:
+		v := chunk.(*array.Uint16).Uint16Values()
+		return uint64(v[i])
+	case *arrow.Uint32Type:
+		v := chunk.(*array.Uint32).Uint32Values()
+		return uint64(v[i])
+	case *arrow.Uint64Type:
+		v := chunk.(*array.Uint64).Uint64Values()
+		return v[i]
+	case *arrow.Float32Type:
+		v := chunk.(*array.Float32).Float32Values()
+		return float64(v[i])
+	case *arrow.Float64Type:
+		v := chunk.(*array.Float64).Float64Values()
+		return v[i]
+	case *arrow.StringType:
+		return chunk.(*array.String).Value(i)
+	}
+	return 0
+}
+
+func builderFrom(mem memory.Allocator, dt arrow.DataType, size int64) array.Builder {
+	var bldr array.Builder
+	switch dt := dt.(type) {
+	case *arrow.BooleanType:
+		bldr = array.NewBooleanBuilder(mem)
+	case *arrow.Int8Type:
+		bldr = array.NewInt8Builder(mem)
+	case *arrow.Int16Type:
+		bldr = array.NewInt16Builder(mem)
+	case *arrow.Int32Type:
+		bldr = array.NewInt32Builder(mem)
+	case *arrow.Int64Type:
+		bldr = array.NewInt64Builder(mem)
+	case *arrow.Uint8Type:
+		bldr = array.NewUint8Builder(mem)
+	case *arrow.Uint16Type:
+		bldr = array.NewUint16Builder(mem)
+	case *arrow.Uint32Type:
+		bldr = array.NewUint32Builder(mem)
+	case *arrow.Uint64Type:
+		bldr = array.NewUint64Builder(mem)
+	case *arrow.Float32Type:
+		bldr = array.NewFloat32Builder(mem)
+	case *arrow.Float64Type:
+		bldr = array.NewFloat64Builder(mem)
+	case *arrow.StringType:
+		bldr = array.NewStringBuilder(mem)
+	default:
+		panic(fmt.Errorf("builderFrom: invalid Arrow type %v", dt))
+	}
+	bldr.Reserve(int(size))
+	return bldr
+}
+
+func appendData(bldr array.Builder, v interface{}) {
+	switch bldr := bldr.(type) {
+	case *array.BooleanBuilder:
+		bldr.Append(v.(bool))
+	case *array.Int8Builder:
+		bldr.Append(v.(int8))
+	case *array.Int16Builder:
+		bldr.Append(v.(int16))
+	case *array.Int32Builder:
+		bldr.Append(v.(int32))
+	case *array.Int64Builder:
+		bldr.Append(v.(int64))
+	case *array.Uint8Builder:
+		bldr.Append(v.(uint8))
+	case *array.Uint16Builder:
+		bldr.Append(v.(uint16))
+	case *array.Uint32Builder:
+		bldr.Append(v.(uint32))
+	case *array.Uint64Builder:
+		bldr.Append(v.(uint64))
+	case *array.Float32Builder:
+		bldr.Append(v.(float32))
+	case *array.Float64Builder:
+		bldr.Append(v.(float64))
+	case *array.StringBuilder:
+		bldr.Append(v.(string))
+	default:
+		panic(fmt.Errorf("appendData: invalid Arrow builder type %T", bldr))
+	}
+}
+
+func Concat(schema *arrow.Schema, tables []*BasicTable, mem memory.Allocator) arrow.Table {
+	if len(tables) == 1 {
+		if !tables[0].IsFiltered() {
+			return tables[0]
+		}
+	}
+	cols := make([]arrow.Column, len(schema.Fields()))
+
+	defer func(cols []arrow.Column) {
+		for i := range cols {
+			cols[i].Release()
+		}
+	}(cols)
+	sz := 0
+	for i := range tables {
+		sz += int(tables[i].NumRows())
+	}
+	for i := range cols {
+		field := schema.Field(i)
+		arrs := make([]arrow.Array, 0)
+		builder := builderFrom(mem, field.Type, int64(sz))
+		for t := range tables {
+			table := tables[t]
+			if table.IsFiltered() {
+				for row := 0; row < int(table.NumRows()); row++ {
+					v := table.Get(i, row)
+					appendData(builder, v)
+				}
+				arrs = append(arrs, builder.NewArray())
+
+			} else {
+				parts := table.Column(i).Data()
+				arrs = append(arrs, parts.Chunks()...)
+			}
+		}
+		chunk := arrow.NewChunked(field.Type, arrs)
+		cols[i] = *arrow.NewColumn(field, chunk)
+		chunk.Release()
+	}
+	return array.NewTable(schema, cols, -1)
+}
+
+func (st *BasicTable) MarshalJSON() ([]byte, error) {
+	results := make(map[string]interface{})
+	n := 0
+	if st.table != nil {
+		n = int(st.table.NumCols())
+	}
+	for b := 0; b < n; b++ {
+		col := st.table.Column(b)
+		result := make([]interface{}, st.resolver.NumRows())
+		for n := st.resolver.NumRows() - 1; n >= 0; n-- {
+			v := st.Get(b, n)
+			result[n] = v
+
+		}
+		results[col.Name()] = result
+	}
+	return json.Marshal(results)
+}
+
+func BasicTableFromArrow(table arrow.Table, mem memory.Allocator) *BasicTable {
+	col := table.Column(0)
+	r := dataframe.NewChunkResolver(col)
+	return &BasicTable{resolver: &r, table: table}
+}
+
+func filterColumns(filters []string, table arrow.Table) arrow.Table {
+	filters = append(filters, "_ID")
+	schema := table.Schema()
+	// TODO(twg) 2022/11/09 add glob support
+	allFields := schema.Fields()
+	in := func(key string) bool {
+		for _, v := range filters {
+			if v == key {
+				return true
+			}
+		}
+		return false
+	}
+	cols := make([]arrow.Column, 0)
+	fields := make([]arrow.Field, 0)
+	for i := range allFields {
+		field := allFields[i]
+		if in(field.Name) {
+			cols = append(cols, *table.Column(i))
+			fields = append(fields, field)
+		}
+	}
+
+	filterdSchema := arrow.NewSchema(fields, nil) // TODO(twg) 2022/11/09 handle meta:w
+	return array.NewTable(filterdSchema, cols, table.NumRows())
+}
+
+func (e *executor) executeArrowShard(ctx context.Context, qcx *Qcx, index string, c *pql.Call, shard uint64, pool memory.Allocator, columnFilter []string) (*BasicTable, error) {
+	name := fmt.Sprintf("a. %v", shard)
+	span, _ := tracing.StartSpanFromContext(ctx, "Executor.executeArrowShard")
+	defer span.Finish()
+
+	var filter *Row
+	if len(c.Children) == 1 {
+		row, err := e.executeBitmapCallShard(ctx, qcx, index, c.Children[0], shard)
+		if err != nil {
+			return nil, err
+		}
+		filter = row
+		if !filter.Any() {
+			// no need to actuall run the query for its not operating against any values
+			return &BasicTable{name: name}, nil
+		}
+	}
+	//
+	ids := filter.ShardColumns() // needs to be shard columns
+	// Fetch index.
+	idx := e.Holder.Index(index)
+	if idx == nil {
+		return nil, newNotFoundError(ErrIndexNotFound, index)
+	}
+
+	fname := idx.GetDataFramePath(shard)
+
+	if !e.dataFrameExists(fname) {
+		return &BasicTable{name: name}, nil
+	}
+
+	table, err := e.getDataTable(ctx, fname, pool)
+	if err != nil {
+		return nil, errors.Wrap(err, "arrow readTableParquet")
+	}
+	defer table.Release()
+	if len(columnFilter) > 0 {
+		table = filterColumns(columnFilter, table)
+	}
+	df, err := dataframe.NewDataFrameFromTable(pool, table)
+	if err != nil {
+		return nil, errors.Wrap(err, "arrow NewDataFromTable")
+	}
+	p := dataframe.NewChunkResolver(df.ColumnAt(0))
+	var resolver dataframe.Resolver
+	resolver = &p
+	if filter != nil {
+		if len(ids) == 0 {
+			return &BasicTable{name: name}, nil
+		}
+		resolver, err = filterDataframe(resolver, pool, ids)
+		if err != nil {
+			return nil, errors.Wrap(err, "filtering dataframe")
+		}
+	}
+	table.Retain()
+	return &BasicTable{resolver: resolver, table: table, filtered: filter != nil, name: name}, nil
+}
+
+func (e *executor) dataFrameExists(fname string) bool {
+	if e.typeIsParquet() {
+		if _, err := os.Stat(fname + ".parquet"); os.IsNotExist(err) {
+			return false
+		}
+		return true
+	}
+	if _, err := os.Stat(fname + ".arrow"); os.IsNotExist(err) {
+		return false
+	}
+	return true
+}
+
+func (e *executor) getDataTable(ctx context.Context, fname string, mem memory.Allocator) (arrow.Table, error) {
+	if e.typeIsParquet() {
+		table, err := readTableParquetCtx(ctx, fname, mem)
+		return table, err
+	}
+	return readTableArrow(fname, mem)
+}
+
+func (e *executor) typeIsParquet() bool {
+	return e.datafameUseParquet
+}
+
+func (e *executor) IsDataframeFile(name string) bool {
+	if e.typeIsParquet() {
+		return strings.HasSuffix(name, ".parquet")
+	}
+	return strings.HasSuffix(name, ".arrow")
+}
+
+func (e *executor) SaveTable(name string, table arrow.Table, mem memory.Allocator) error {
+	if e.typeIsParquet() {
+		return writeTableParquet(table, name)
+	}
+	return writeTableArrow(table, name, mem)
+}
+
+func (e *executor) TableExtension() string {
+	if e.typeIsParquet() {
+		return ".parquet"
+	}
+	return ".arrow"
+}
+
+func readTableArrow(filename string, mem memory.Allocator) (arrow.Table, error) {
+	r, err := os.Open(filename + ".arrow")
+	if err != nil {
+		return nil, err
+	}
+	rr, err := ipc.NewFileReader(r, ipc.WithAllocator(mem))
+	if err != nil {
+		return nil, err
+	}
+	defer rr.Close()
+	records := make([]arrow.Record, rr.NumRecords())
+	i := 0
+	for {
+		rec, err := rr.Read()
+		if err == io.EOF {
+			break
+		} else if err != nil {
+			return nil, err
+		}
+		records[i] = rec
+		i++
+	}
+	records = records[:i]
+	table := array.NewTableFromRecords(rr.Schema(), records)
+	return table, nil
+}
+
+func readTableParquetCtx(ctx context.Context, filename string, mem memory.Allocator) (arrow.Table, error) {
+	r, err := os.Open(filename + ".parquet")
+	if err != nil {
+		return nil, err
+	}
+	defer r.Close()
+
+	pf, err := file.NewParquetReader(r)
+	if err != nil {
+		return nil, err
+	}
+
+	reader, err := pqarrow.NewFileReader(pf, pqarrow.ArrowReadProperties{}, mem)
+	if err != nil {
+		return nil, err
+	}
+	return reader.ReadTable(ctx)
+}
+
+func writeTableParquet(table arrow.Table, filename string) error {
+	f, err := os.Create(filename + ".parquet")
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+	props := parquet.NewWriterProperties(parquet.WithDictionaryDefault(false))
+	arrProps := pqarrow.DefaultWriterProps()
+	chunkSize := 10 * 1024 * 1024
+	err = pqarrow.WriteTable(table, f, int64(chunkSize), props, arrProps)
+	if err != nil {
+		return err
+	}
+	f.Sync()
+	return nil
+}
+
+func writeTableArrow(table arrow.Table, filename string, mem memory.Allocator) error {
+	f, err := os.Create(filename + ".arrow")
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+	writer, err := ipc.NewFileWriter(f, ipc.WithAllocator(mem), ipc.WithSchema(table.Schema()))
+	if err != nil {
+		panic(err)
+	}
+	chunkSize := int64(0)
+	tr := array.NewTableReader(table, chunkSize)
+	defer tr.Release()
+	n := 0
+	for tr.Next() {
+		arec := tr.Record()
+		err = writer.Write(arec)
+		if err != nil {
+			panic(err)
+		}
+		n++
+	}
+	err = writer.Close()
+	if err != nil {
+		panic(err)
+	}
+	f.Sync()
+	return nil
+}

arrow_test.go

@@ -0,0 +1,53 @@
+// Copyright 2021 Molecula Corp. All rights reserved.
+package pilosa
+
+import (
+	"context"
+	"encoding/hex"
+	"math/rand"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/apache/arrow/go/v10/arrow"
+	"github.com/apache/arrow/go/v10/arrow/array"
+	"github.com/apache/arrow/go/v10/arrow/memory"
+)
+
+func TempFileName(prefix string) string {
+	randBytes := make([]byte, 16)
+	rand.Read(randBytes)
+	return filepath.Join(os.TempDir(), prefix+hex.EncodeToString(randBytes))
+}
+
+func Test_TableParquet(t *testing.T) {
+	// create a arrow table
+	schema := arrow.NewSchema(
+		[]arrow.Field{
+			{Name: "num", Type: arrow.PrimitiveTypes.Float64},
+		},
+		nil, // no metadata
+	)
+	mem := memory.NewGoAllocator()
+	b := array.NewRecordBuilder(mem, schema)
+	defer b.Release()
+	b.Field(0).(*array.Float64Builder).AppendValues([]float64{1.0, 1.5, 2.0}, nil)
+	table := array.NewTableFromRecords(schema, []arrow.Record{b.NewRecord()})
+	defer table.Release()
+	fileName := TempFileName("pq-")
+	// save it as  a parquet file
+	err := writeTableParquet(table, fileName)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.Remove(fileName)
+
+	// read it back in and compare the result
+	got, err := readTableParquetCtx(context.Background(), fileName, mem)
+	if err != nil {
+		t.Fatalf("readTableParquetCtx() error = %v", err)
+	}
+	if got.NumCols() != table.NumCols() {
+		t.Errorf("got:%v expected:%v", got.NumCols(), table.NumCols())
+	}
+}

attr.go

@@ -1,554 +0,0 @@
-// Copyright 2017 Pilosa Corp.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package pilosa
-
-import (
-	"bytes"
-	"crypto/sha1"
-	"encoding/binary"
-	"fmt"
-	"sort"
-	"sync"
-	"time"
-
-	"github.com/boltdb/bolt"
-	"github.com/gogo/protobuf/proto"
-	"github.com/pilosa/pilosa/internal"
-)
-
-// AttrBlockSize is the size of attribute blocks for anti-entropy.
-const AttrBlockSize = 100
-
-// Attribute data type enum.
-const (
-	AttrTypeString = 1
-	AttrTypeInt    = 2
-	AttrTypeBool   = 3
-	AttrTypeFloat  = 4
-)
-
-// AttrCache represents a cache for attributes.
-type AttrCache struct {
-	mu    sync.RWMutex
-	attrs map[uint64]map[string]interface{}
-}
-
-// Get returns the cached attributes for a given id.
-func (c *AttrCache) Get(id uint64) map[string]interface{} {
-	c.mu.RLock()
-	defer c.mu.RUnlock()
-	attrs := c.attrs[id]
-	if attrs == nil {
-		return nil
-	}
-
-	// Make a copy for safety
-	ret := make(map[string]interface{})
-	for k, v := range attrs {
-		ret[k] = v
-	}
-	return ret
-}
-
-// Set updates the cached attributes for a given id.
-func (c *AttrCache) Set(id uint64, attrs map[string]interface{}) {
-	c.mu.Lock()
-	defer c.mu.Unlock()
-	c.attrs[id] = attrs
-}
-
-// AttrStore represents a storage layer for attributes.
-type AttrStore struct {
-	mu        sync.RWMutex
-	path      string
-	db        *bolt.DB
-	attrCache *AttrCache
-}
-
-// NewAttrCache returns a new instance of AttrCache.
-func NewAttrCache() *AttrCache {
-	return &AttrCache{
-		attrs: make(map[uint64]map[string]interface{}),
-	}
-}
-
-// NewAttrStore returns a new instance of AttrStore.
-func NewAttrStore(path string) *AttrStore {
-	return &AttrStore{
-		path:      path,
-		attrCache: NewAttrCache(),
-	}
-}
-
-// Path returns path to the store's data file.
-func (s *AttrStore) Path() string { return s.path }
-
-// Open opens and initializes the store.
-func (s *AttrStore) Open() error {
-	// Open storage.
-	db, err := bolt.Open(s.path, 0666, &bolt.Options{Timeout: 1 * time.Second})
-	if err != nil {
-		return err
-	}
-	s.db = db
-
-	// Initialize database.
-	if err := s.db.Update(func(tx *bolt.Tx) error {
-		if _, err := tx.CreateBucketIfNotExists([]byte("attrs")); err != nil {
-			return err
-		}
-		return nil
-	}); err != nil {
-		return err
-	}
-
-	return nil
-}
-
-// Close closes the store.
-func (s *AttrStore) Close() error {
-	if s.db != nil {
-		s.db.Close()
-	}
-	return nil
-}
-
-// Attrs returns a set of attributes by ID.
-func (s *AttrStore) Attrs(id uint64) (m map[string]interface{}, err error) {
-	s.mu.RLock()
-	defer s.mu.RUnlock()
-
-	// Check cache for map.
-	if m = s.attrCache.Get(id); m != nil {
-		return m, nil
-	}
-
-	// Find attributes from storage.
-	if err = s.db.View(func(tx *bolt.Tx) error {
-		m, err = txAttrs(tx, id)
-		if err != nil {
-			return err
-		}
-		return nil
-	}); err != nil {
-		return nil, err
-	}
-
-	// Add to cache.
-	s.attrCache.Set(id, m)
-
-	return
-}
-
-// SetAttrs sets attribute values for a given ID.
-func (s *AttrStore) SetAttrs(id uint64, m map[string]interface{}) error {
-	// Ignore empty maps.
-	if len(m) == 0 {
-		return nil
-	}
-
-	// Check if the attributes already exist under a read-only lock.
-	if attr, err := s.Attrs(id); err != nil {
-		return err
-	} else if attr != nil && mapContains(attr, m) {
-		return nil
-	}
-
-	// Obtain write lock.
-	s.mu.Lock()
-	defer s.mu.Unlock()
-
-	var attr map[string]interface{}
-	if err := s.db.Update(func(tx *bolt.Tx) error {
-		tmp, err := txUpdateAttrs(tx, id, m)
-		if err != nil {
-			return err
-		}
-		attr = tmp
-
-		return nil
-	}); err != nil {
-		return err
-	}
-
-	// Swap attributes map in cache.
-	s.attrCache.Set(id, attr)
-
-	return nil
-}
-
-// SetBulkAttrs sets attribute values for a set of ids.
-func (s *AttrStore) SetBulkAttrs(m map[uint64]map[string]interface{}) error {
-	s.mu.Lock()
-	defer s.mu.Unlock()
-
-	attrs := make(map[uint64]map[string]interface{})
-	if err := s.db.Update(func(tx *bolt.Tx) error {
-		// Collect and sort keys.
-		ids := make([]uint64, 0, len(m))
-		for id := range m {
-			ids = append(ids, id)
-		}
-		sort.Sort(uint64Slice(ids))
-
-		// Update attributes for each id.
-		for _, id := range ids {
-			attr, err := txUpdateAttrs(tx, id, m[id])
-			if err != nil {
-				return err
-			}
-			attrs[id] = attr
-		}
-
-		return nil
-	}); err != nil {
-		return err
-	}
-
-	// Swap attributes map in cache.
-	for id, attr := range attrs {
-		s.attrCache.Set(id, attr)
-	}
-
-	return nil
-}
-
-// Blocks returns a list of all blocks in the store.
-func (s *AttrStore) Blocks() ([]AttrBlock, error) {
-	tx, err := s.db.Begin(false)
-	if err != nil {
-		return nil, err
-	}
-	defer tx.Rollback()
-
-	// Wrap cursor to segment by block.
-	cur := newBlockCursor(tx.Bucket([]byte("attrs")).Cursor(), AttrBlockSize)
-
-	// Iterate over each block.
-	var blocks []AttrBlock
-	for cur.nextBlock() {
-		block := AttrBlock{ID: cur.blockID()}
-
-		// Compute checksum of every key/value in block.
-		h := sha1.New()
-		for k, v := cur.next(); k != nil; k, v = cur.next() {
-			h.Write(k)
-			h.Write(v)
-		}
-		block.Checksum = h.Sum(nil)
-
-		// Append block.
-		blocks = append(blocks, block)
-	}
-
-	return blocks, nil
-}
-
-// BlockData returns all data for a single block.
-func (s *AttrStore) BlockData(i uint64) (map[uint64]map[string]interface{}, error) {
-	m := make(map[uint64]map[string]interface{})
-
-	// Start read-only transaction.
-	tx, err := s.db.Begin(false)
-	if err != nil {
-		return nil, err
-	}
-	defer tx.Rollback()
-
-	// Move to the start of the block.
-	min := u64tob(uint64(i) * AttrBlockSize)
-	max := u64tob(uint64(i+1) * AttrBlockSize)
-	cur := tx.Bucket([]byte("attrs")).Cursor()
-	for k, v := cur.Seek(min); k != nil; k, v = cur.Next() {
-		// Exit if we're past the end of the block.
-		if bytes.Compare(k, max) != -1 {
-			break
-		}
-
-		// Decode attribute map and associate with id.
-		var pb internal.AttrMap
-		if err := proto.Unmarshal(v, &pb); err != nil {
-			return nil, err
-		}
-		m[btou64(k)] = decodeAttrs(pb.GetAttrs())
-	}
-
-	return m, nil
-}
-
-// txAttrs returns a map of attributes for an id.
-func txAttrs(tx *bolt.Tx, id uint64) (map[string]interface{}, error) {
-	v := tx.Bucket([]byte("attrs")).Get(u64tob(id))
-	if v == nil {
-		return emptyMap, nil
-	}
-
-	var pb internal.AttrMap
-	if err := proto.Unmarshal(v, &pb); err != nil {
-		return nil, err
-	}
-	return decodeAttrs(pb.GetAttrs()), nil
-}
-
-// txUpdateAttrs updates the attributes for an id.
-// Returns the new combined set of attributes for the id.
-func txUpdateAttrs(tx *bolt.Tx, id uint64, m map[string]interface{}) (map[string]interface{}, error) {
-	attr, err := txAttrs(tx, id)
-	if err != nil {
-		return nil, err
-	}
-
-	// Create a new map if it is empty so we don't update emptyMap.
-	if len(attr) == 0 {
-		attr = make(map[string]interface{}, len(m))
-	}
-
-	// Merge attributes with original values.
-	// Nil values should delete keys.
-	for k, v := range m {
-		if v == nil {
-			delete(attr, k)
-			continue
-		}
-
-		switch v := v.(type) {
-		case int:
-			attr[k] = int64(v)
-		case uint:
-			attr[k] = int64(v)
-		case uint64:
-			attr[k] = int64(v)
-		case string, int64, bool, float64:
-			attr[k] = v
-		default:
-			return nil, fmt.Errorf("invalid attr type: %T", v)
-		}
-	}
-
-	// Marshal and save new values.
-	buf, err := proto.Marshal(&internal.AttrMap{Attrs: encodeAttrs(attr)})
-	if err != nil {
-		return nil, err
-	}
-	if err := tx.Bucket([]byte("attrs")).Put(u64tob(id), buf); err != nil {
-		return nil, err
-	}
-	return attr, nil
-}
-
-func encodeAttrs(m map[string]interface{}) []*internal.Attr {
-	keys := make([]string, 0, len(m))
-	for k := range m {
-		keys = append(keys, k)
-	}
-	sort.Strings(keys)
-
-	a := make([]*internal.Attr, len(keys))
-	for i := range keys {
-		a[i] = encodeAttr(keys[i], m[keys[i]])
-	}
-	return a
-}
-
-func decodeAttrs(pb []*internal.Attr) map[string]interface{} {
-	m := make(map[string]interface{}, len(pb))
-	for i := range pb {
-		key, value := decodeAttr(pb[i])
-		m[key] = value
-	}
-	return m
-}
-
-// encodeAttr converts a key/value pair into an Attr internal representation.
-func encodeAttr(key string, value interface{}) *internal.Attr {
-	pb := &internal.Attr{Key: key}
-	switch value := value.(type) {
-	case string:
-		pb.Type = AttrTypeString
-		pb.StringValue = value
-	case float64:
-		pb.Type = AttrTypeFloat
-		pb.FloatValue = value
-	case uint64:
-		pb.Type = AttrTypeInt
-		pb.IntValue = int64(value)
-	case int64:
-		pb.Type = AttrTypeInt
-		pb.IntValue = value
-	case bool:
-		pb.Type = AttrTypeBool
-		pb.BoolValue = value
-	}
-	return pb
-}
-
-// decodeAttr converts from an Attr internal representation to a key/value pair.
-func decodeAttr(attr *internal.Attr) (key string, value interface{}) {
-	switch attr.Type {
-	case AttrTypeString:
-		return attr.Key, attr.StringValue
-	case AttrTypeInt:
-		return attr.Key, attr.IntValue
-	case AttrTypeBool:
-		return attr.Key, attr.BoolValue
-	case AttrTypeFloat:
-		return attr.Key, attr.FloatValue
-	default:
-		return attr.Key, nil
-	}
-}
-
-// cloneAttrs returns a shallow clone of m.
-func cloneAttrs(m map[string]interface{}) map[string]interface{} {
-	other := make(map[string]interface{}, len(m))
-	for k, v := range m {
-		other[k] = v
-	}
-	return other
-}
-
-// u64tob encodes v to big endian encoding.
-func u64tob(v uint64) []byte {
-	b := make([]byte, 8)
-	binary.BigEndian.PutUint64(b, v)
-	return b
-}
-
-// btou64 decodes b from big endian encoding.
-func btou64(b []byte) uint64 { return binary.BigEndian.Uint64(b) }
-
-// emptyMap is a reusable map that contains no keys.
-var emptyMap = make(map[string]interface{})
-
-// AttrBlock represents a checksummed block of the attribute store.
-type AttrBlock struct {
-	ID       uint64 `json:"id"`
-	Checksum []byte `json:"checksum"`
-}
-
-// AttrBlocks represents a list of blocks.
-type AttrBlocks []AttrBlock
-
-// Diff returns a list of block ids that are different or are new in other.
-// Block lists must be in sorted order.
-func (a AttrBlocks) Diff(other []AttrBlock) []uint64 {
-	var ids []uint64
-	for {
-		// Read next block from each list.
-		var blk0, blk1 *AttrBlock
-		if len(a) > 0 {
-			blk0 = &a[0]
-		}
-		if len(other) > 0 {
-			blk1 = &other[0]
-		}
-
-		// Exit if "a" contains no more blocks.
-		if blk0 == nil {
-			return ids
-		}
-
-		// Add block ID if it's different or if it's only in "a".
-		if blk1 == nil || blk0.ID < blk1.ID {
-			ids = append(ids, blk0.ID)
-			a = a[1:]
-		} else if blk1.ID < blk0.ID {
-			other = other[1:]
-		} else {
-			if !bytes.Equal(blk0.Checksum, blk1.Checksum) {
-				ids = append(ids, blk0.ID)
-			}
-			a, other = a[1:], other[1:]
-		}
-	}
-}
-
-// blockCursor represents a cursor for iterating over blocks of a bolt bucket.
-type blockCursor struct {
-	cur  *bolt.Cursor
-	base uint64
-	n    uint64
-
-	buf struct {
-		key    []byte
-		value  []byte
-		filled bool
-	}
-}
-
-// newBlockCursor returns a new block cursor that wraps cur using n sized blocks.
-func newBlockCursor(c *bolt.Cursor, n int) blockCursor {
-	cur := blockCursor{
-		cur: c,
-		n:   uint64(n),
-	}
-	cur.buf.key, cur.buf.value = c.First()
-	cur.buf.filled = true
-	return cur
-}
-
-// blockID returns the current block ID. Only valid after call to nextBlock().
-func (cur *blockCursor) blockID() uint64 { return cur.base }
-
-// nextBlock moves the cursor to the next block.
-// Returns true if another block exists, otherwise returns false.
-func (cur *blockCursor) nextBlock() bool {
-	if cur.buf.key == nil {
-		return false
-	}
-
-	cur.base = binary.BigEndian.Uint64(cur.buf.key) / cur.n
-	return true
-}
-
-// next returns the next key/value within the block.
-// Returns nils at the end of the block.
-func (cur *blockCursor) next() (key, value []byte) {
-	// Use buffered value, if set.
-	if cur.buf.filled {
-		key, value = cur.buf.key, cur.buf.value
-		cur.buf.filled = false
-		return key, value
-	}
-
-	// Read next key.
-	key, value = cur.cur.Next()
-
-	// Fill buffer for EOF.
-	if key == nil {
-		cur.buf.key, cur.buf.value, cur.buf.filled = key, value, false
-		return nil, nil
-	}
-
-	// Parse key and buffer if outside of block.
-	id := binary.BigEndian.Uint64(key)
-	if id/cur.n > cur.base {
-		cur.buf.key, cur.buf.value, cur.buf.filled = key, value, true
-		return nil, nil
-	}
-
-	return key, value
-}
-
-// mapContains returns true if all keys & values of subset are in m.
-func mapContains(m, subset map[string]interface{}) bool {
-	for k, v := range subset {
-		value, ok := m[k]
-		if !ok || value != v {
-			return false
-		}
-	}
-	return true
-}

attr_test.go

@@ -1,125 +0,0 @@
-// Copyright 2017 Pilosa Corp.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package pilosa_test
-
-import (
-	"reflect"
-	"testing"
-
-	"github.com/pilosa/pilosa/test"
-)
-
-// Ensure database can set and retrieve column attributes.
-func TestAttrStore_Attrs(t *testing.T) {
-	s := test.MustOpenAttrStore()
-	defer s.Close()
-
-	// Set attributes.
-	if err := s.SetAttrs(1, map[string]interface{}{"A": 100, "C": -27}); err != nil {
-		t.Fatal(err)
-	} else if err := s.SetAttrs(2, map[string]interface{}{"A": uint64(200)}); err != nil {
-		t.Fatal(err)
-	} else if err := s.SetAttrs(1, map[string]interface{}{"B": "VALUE"}); err != nil {
-		t.Fatal(err)
-	}
-
-	// Retrieve attributes for column #1.
-	if m, err := s.Attrs(1); err != nil {
-		t.Fatal(err)
-	} else if !reflect.DeepEqual(m, map[string]interface{}{"A": int64(100), "B": "VALUE", "C": int64(-27)}) {
-		t.Fatalf("unexpected attrs(1): %#v", m)
-	}
-
-	// Retrieve attributes for column #2.
-	if m, err := s.Attrs(2); err != nil {
-		t.Fatal(err)
-	} else if !reflect.DeepEqual(m, map[string]interface{}{"A": int64(200)}) {
-		t.Fatalf("unexpected attrs(2): %#v", m)
-	}
-}
-
-// Ensure database returns a non-nil empty map if unset.
-func TestAttrStore_Attrs_Empty(t *testing.T) {
-	s := test.MustOpenAttrStore()
-	defer s.Close()
-
-	if m, err := s.Attrs(100); err != nil {
-		t.Fatal(err)
-	} else if m == nil || len(m) > 0 {
-		t.Fatalf("unexpected attrs: %#v", m)
-	}
-}
-
-// Ensure database can unset attributes if explicitly set to nil.
-func TestAttrStore_Attrs_Unset(t *testing.T) {
-	s := test.MustOpenAttrStore()
-	defer s.Close()
-
-	// Set attributes.
-	if err := s.SetAttrs(1, map[string]interface{}{"A": "X", "B": "Y"}); err != nil {
-		t.Fatal(err)
-	} else if err := s.SetAttrs(1, map[string]interface{}{"B": nil}); err != nil {
-		t.Fatal(err)
-	}
-
-	// Verify attributes.
-	if m, err := s.Attrs(1); err != nil {
-		t.Fatal(err)
-	} else if !reflect.DeepEqual(m, map[string]interface{}{"A": "X"}) {
-		t.Fatalf("unexpected attrs: %#v", m)
-	}
-}
-
-// Ensure attribute block checksums can be returned.
-func TestAttrStore_Blocks(t *testing.T) {
-	s := test.MustOpenAttrStore()
-	defer s.Close()
-
-	// Set attributes.
-	if err := s.SetAttrs(1, map[string]interface{}{"A": uint64(100)}); err != nil {
-		t.Fatal(err)
-	} else if err := s.SetAttrs(2, map[string]interface{}{"A": uint64(200)}); err != nil {
-		t.Fatal(err)
-	} else if err := s.SetAttrs(100, map[string]interface{}{"B": "VALUE"}); err != nil {
-		t.Fatal(err)
-	} else if err := s.SetAttrs(350, map[string]interface{}{"C": "FOO"}); err != nil {
-		t.Fatal(err)
-	}
-
-	// Retrieve blocks.
-	blks0, err := s.Blocks()
-	if err != nil {
-		t.Fatal(err)
-	} else if len(blks0) != 3 || blks0[0].ID != 0 || blks0[1].ID != 1 || blks0[2].ID != 3 {
-		t.Fatalf("unexpected blocks: %#v", blks0)
-	}
-
-	// Change second block.
-	if err := s.SetAttrs(100, map[string]interface{}{"X": 12}); err != nil {
-		t.Fatal(err)
-	}
-
-	// Ensure second block changed.
-	blks1, err := s.Blocks()
-	if err != nil {
-		t.Fatal(err)
-	} else if !reflect.DeepEqual(blks0[0], blks1[0]) {
-		t.Fatalf("block 0 mismatch: %#v != %#v", blks0[0], blks1[0])
-	} else if reflect.DeepEqual(blks0[1], blks1[1]) {
-		t.Fatalf("block 1 match: %#v ", blks0[0])
-	} else if !reflect.DeepEqual(blks0[2], blks1[2]) {
-		t.Fatalf("block 2 mismatch: %#v != %#v", blks0[2], blks1[2])
-	}
-}

audit.go

@@ -0,0 +1,13 @@
+// Copyright 2022 Molecula Corp. (DBA FeatureBase).
+// SPDX-License-Identifier: Apache-2.0
+package pilosa
+
+import (
+	"github.com/featurebasedb/featurebase/v3/testhook"
+)
+
+var NewAuditor func() testhook.Auditor = NewNopAuditor
+
+func NewNopAuditor() testhook.Auditor {
+	return testhook.NewNopAuditor()
+}

audit_internal_test.go

@@ -0,0 +1,40 @@
+// Copyright 2022 Molecula Corp. (DBA FeatureBase).
+// SPDX-License-Identifier: Apache-2.0
+package pilosa
+
+import (
+	"fmt"
+	"reflect"
+
+	"github.com/featurebasedb/featurebase/v3/testhook"
+)
+
+// These audit hooks are desireable during testing, but not in
+// production.
+type auditorViewHooks struct{}
+type auditorFragmentHooks struct{}
+
+// static type checks
+var _ testhook.RegistryHookLive = &auditorViewHooks{}
+var _ testhook.RegistryHookLive = &auditorFragmentHooks{}
+
+func (*auditorViewHooks) Live(o interface{}, entry *testhook.RegistryEntry) error {
+	if entry != nil && entry.OpenCount != 0 {
+		return fmt.Errorf("view %s still open", o.(*view).name)
+	}
+	return nil
+}
+
+func (*auditorFragmentHooks) Live(o interface{}, entry *testhook.RegistryEntry) error {
+	if entry != nil && entry.OpenCount != 0 {
+		return fmt.Errorf("fragment %s still open", o.(*fragment).path())
+	}
+	return nil
+}
+
+func GetInternalTestHooks() testhook.RegistryHooks {
+	return map[reflect.Type]testhook.RegistryHook{
+		reflect.TypeOf((*view)(nil)):     &auditorViewHooks{},
+		reflect.TypeOf((*fragment)(nil)): &auditorFragmentHooks{},
+	}
+}

audit_test.go

@@ -0,0 +1,95 @@
+// Copyright 2022 Molecula Corp. (DBA FeatureBase).
+// SPDX-License-Identifier: Apache-2.0
+package pilosa_test
+
+import (
+	"fmt"
+	"os"
+	"reflect"
+
+	"github.com/featurebasedb/featurebase/v3"
+	"github.com/featurebasedb/featurebase/v3/testhook"
+)
+
+// AuditLeaksOn is a global switch to turn on resource
+// leak checking at the end of a test run.
+var AuditLeaksOn = true
+
+// for tests, we use a single shared auditor used by all of the holders.
+var globalTestAuditor = testhook.NewVerifyCloseAuditor(testHooks)
+
+// These audit hooks are desireable during testing, but not in
+// production.
+type auditorIndexHooks struct{}
+type auditorFieldHooks struct{}
+type auditorHolderHooks struct{}
+
+// static type checking
+var _ testhook.RegistryHookLive = &auditorIndexHooks{}
+var _ testhook.RegistryHookLive = &auditorFieldHooks{}
+var _ testhook.RegistryHookPostDestroy = &auditorHolderHooks{}
+var _ testhook.RegistryHookLive = &auditorHolderHooks{}
+
+var testHooks = map[reflect.Type]testhook.RegistryHook{
+	reflect.TypeOf((*pilosa.Index)(nil)):  &auditorIndexHooks{},
+	reflect.TypeOf((*pilosa.Field)(nil)):  &auditorFieldHooks{},
+	reflect.TypeOf((*pilosa.Holder)(nil)): &auditorHolderHooks{},
+}
+
+func init() {
+	if !AuditLeaksOn {
+		return
+	}
+	for k, v := range pilosa.GetInternalTestHooks() {
+		testHooks[k] = v
+	}
+	testhook.RegisterPreTestHook(func() error {
+		pilosa.NewAuditor = NewTestAuditor
+		return nil
+	})
+	testhook.RegisterPostTestHook(func() error {
+		err, errs := globalTestAuditor.FinalCheck()
+		if err != nil {
+			for i, e := range errs {
+				fmt.Fprintf(os.Stderr, "[%d]: %v\n", i, e)
+			}
+		}
+		return err
+	})
+}
+
+func NewTestAuditor() testhook.Auditor {
+	return globalTestAuditor
+}
+
+func (*auditorIndexHooks) Live(o interface{}, entry *testhook.RegistryEntry) error {
+	if entry != nil && entry.OpenCount != 0 {
+		return fmt.Errorf("index %s still open", o.(*pilosa.Index).Name())
+	}
+	return nil
+}
+
+func (*auditorFieldHooks) Live(o interface{}, entry *testhook.RegistryEntry) error {
+	if entry != nil && entry.OpenCount != 0 {
+		return fmt.Errorf("field %s still open", o.(*pilosa.Field).Name())
+	}
+	return nil
+}
+
+func (*auditorHolderHooks) WasDestroyed(o interface{}, kv testhook.KV, ent *testhook.RegistryEntry, err error) error {
+	path := o.(*pilosa.Holder).Path()
+	if path == "" {
+		fmt.Fprintf(os.Stderr, "OOPS: trying to destroy a holder with no path! created: %s\n",
+			ent.Stack)
+	} else {
+		os.RemoveAll(o.(*pilosa.Holder).Path())
+	}
+	return err
+}
+
+func (*auditorHolderHooks) Live(o interface{}, entry *testhook.RegistryEntry) error {
+	if entry != nil && entry.OpenCount != 0 {
+		return fmt.Errorf("holder %s still open", o.(*pilosa.Holder).Path())
+	}
+	return nil
+}

authn/authenticate.go

@@ -0,0 +1,445 @@
+// Copyright 2022 Molecula Corp. (DBA FeatureBase).
+// SPDX-License-Identifier: Apache-2.0
+
+// Package authn handles authentication
+package authn
+
+import (
+	"context"
+	"encoding/hex"
+	"encoding/json"
+	"fmt"
+	"net"
+	"net/http"
+	"net/url"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/golang-jwt/jwt"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/metadata"
+
+	"github.com/featurebasedb/featurebase/v3/logger"
+	"github.com/pkg/errors"
+	"golang.org/x/oauth2"
+)
+
+// AuthContextKey is a unique type to prevent collisions when using context.WithValue()
+type AuthContextKey string
+
+const (
+	// AccessCookieName is the name of the cookie that holds the access token.
+	AccessCookieName = "molecula-chip"
+
+	// RefreshCookieName is the name of the cookie that holds the refresh token.
+	RefreshCookieName = "refresh-molecula-chip"
+
+	// RefreshHeaderName is the name of the header that holds the refresh token.
+	RefreshHeaderName = "X-Molecula-Refresh-Token"
+
+	// ContextValueAccessToken is the key used to set AccessTokens in a ctx.
+	ContextValueAccessToken = AuthContextKey("Access")
+
+	// ContextValueRefreshToken is the key used to set RefreshTokens in a ctx.
+	ContextValueRefreshToken = AuthContextKey("Refresh")
+)
+
+// cachedGroups is used to hold groups and when they were last cached
+type cachedGroups struct {
+	cacheTime time.Time
+	groups    []Group
+}
+
+// UserInfo holds the information about the user from the token
+type UserInfo struct {
+	UserID       string    `json:"userid"`
+	UserName     string    `json:"username"`
+	Groups       []Group   `json:"groups"`
+	Expiry       time.Time `json:"expiry"`
+	Token        string    `json:"token"`
+	RefreshToken string    `json:"refreshtoken"`
+}
+
+// Group holds group information for an authenticated user
+type Group struct {
+	GroupID   string `json:"id"`
+	GroupName string `json:"displayName"`
+}
+
+// Groups holds a slice of Group for marshalling from JSON
+type Groups struct {
+	NextLink string  `json:"@odata.nextLink"`
+	Groups   []Group `json:"value"`
+}
+
+// Auth holds state, configuration, and utilities needed for authentication.
+type Auth struct {
+	logger            logger.Logger
+	accessCookieName  string
+	refreshCookieName string
+	secretKey         []byte
+	groupEndpoint     string
+	logoutEndpoint    string
+	fbURL             string // fbURL is the domain featurebase is hosted on, used for post logout redirection
+	oAuthConfig       *oauth2.Config
+	cacheTTL          time.Duration           // cacheTTL is used to determine if a cached item should be refreshed or not
+	groupsCache       map[string]cachedGroups // groupsCache is a map of accessToken -> group memberships
+	lastCacheClean    time.Time               // last cache clean is the time that the cache was last cleaned
+	allowedNetworks   []net.IPNet             // list of allowed networks for ingest
+}
+
+// NewAuth instantiates and returns a new Auth struct
+func NewAuth(logger logger.Logger, url string, scopes []string, authURL, tokenURL, groupEndpoint, logout, clientID, clientSecret, secretKey string, configuredIPs []string) (auth *Auth, err error) {
+	auth = &Auth{
+		logger:            logger,
+		accessCookieName:  AccessCookieName,
+		refreshCookieName: RefreshCookieName,
+		groupEndpoint:     groupEndpoint,
+		logoutEndpoint:    logout,
+		fbURL:             url,
+		oAuthConfig: &oauth2.Config{
+			RedirectURL:  fmt.Sprintf("%s/redirect", url),
+			ClientID:     clientID,
+			ClientSecret: clientSecret,
+			Scopes:       scopes,
+			Endpoint: oauth2.Endpoint{
+				AuthURL:  authURL,
+				TokenURL: tokenURL,
+			},
+		},
+		groupsCache:    map[string]cachedGroups{},
+		cacheTTL:       10 * time.Minute,
+		lastCacheClean: time.Now(),
+	}
+
+	if auth.secretKey, err = decodeHex(secretKey); err != nil {
+		return nil, errors.Wrap(err, "decoding secret key")
+	}
+
+	// convert IPs and add them to allowed networks
+	err = auth.convertIP(configuredIPs)
+	if err != nil {
+		return nil, err
+	}
+
+	return auth, nil
+}
+
+// CleanOAuthConfig returns a's oauthConfig without the client secret
+func (a Auth) CleanOAuthConfig() oauth2.Config {
+	b := *a.oAuthConfig
+	b.ClientSecret = ""
+	return b
+}
+
+// SecretKey is a convenient function to get the SecretKey from an Auth struct
+func (a Auth) SecretKey() []byte {
+	return a.secretKey
+}
+
+// refreshToken refreshes a given access/refresh token pair
+func (a *Auth) refreshToken(access, refresh string) (string, string, error) {
+	resp, err := http.PostForm(a.oAuthConfig.Endpoint.TokenURL,
+		url.Values{
+			"grant_type":    {"refresh_token"},
+			"refresh_token": {refresh},
+			"client_id":     {a.oAuthConfig.ClientID},
+			"client_secret": {a.oAuthConfig.ClientSecret},
+		},
+	)
+	if err != nil {
+		return "", "", errors.Wrap(err, "refreshing token")
+	}
+
+	if resp.StatusCode != http.StatusOK {
+		return "", "", fmt.Errorf("refreshing token: %s", resp.Status)
+	}
+
+	defer resp.Body.Close()
+
+	var t oauth2.Token
+	if err := json.NewDecoder(resp.Body).Decode(&t); err != nil {
+		return "", "", errors.Wrap(err, "decoding refreshed token")
+	}
+
+	// remove the old groups from the groups cache
+	delete(a.groupsCache, access)
+
+	return t.AccessToken, t.RefreshToken, nil
+}
+
+// Authenticate takes in a auth token `access` and returns UserInfo from that token
+// it is caller's responsibility to inform the user that the access token has been refreshed
+func (a *Auth) Authenticate(access, refresh string) (*UserInfo, error) {
+	// clean up the cache every 30 minutes or so
+	if time.Since(a.lastCacheClean) >= 30*time.Minute {
+		a.cleanCache()
+	}
+
+	if len(access) == 0 {
+		return nil, fmt.Errorf("auth token is empty")
+	}
+
+	// NOTE: we are using ParseUnverified here because the IDP validates the
+	// token's signature when we get the user's groups, we just need to make
+	// sure it's not expired and is well-formed
+	token, _, err := new(jwt.Parser).ParseUnverified(access, &jwt.MapClaims{})
+	// well-formed-ness check
+	if token == nil || token.Claims == nil || err != nil {
+		return nil, fmt.Errorf("parsing auth token: %v", err)
+	}
+
+	claims := *token.Claims.(*jwt.MapClaims)
+
+	// expiry check
+	if exp, ok := claims["exp"]; ok {
+		var expiry int64
+		switch v := exp.(type) {
+		case string:
+			expiry, err = strconv.ParseInt(v, 10, 64)
+			if err != nil {
+				return nil, fmt.Errorf("parsing exp string: %v", err)
+			}
+		case float64:
+			expiry = int64(v)
+		case int64:
+			expiry = v
+		}
+
+		if expiry < time.Now().UTC().Unix() {
+			access, refresh, err = a.refreshToken(access, refresh)
+			if err != nil {
+				return nil, fmt.Errorf("token is expired: %w", err)
+			}
+		}
+	}
+
+	userInfo := UserInfo{
+		Token:        access,
+		RefreshToken: refresh,
+		Groups:       []Group{},
+	}
+
+	if uid, ok := claims["oid"].(string); ok {
+		userInfo.UserID = uid
+	}
+	if name, ok := claims["name"].(string); ok {
+		userInfo.UserName = name
+	}
+
+	if userInfo.Groups, err = a.getGroups(access); err != nil {
+		return nil, errors.Wrap(err, "getting groups")
+	}
+
+	return &userInfo, nil
+}
+
+// cleanCache removes old items from our cache
+func (a *Auth) cleanCache() {
+	for access, tkn := range a.groupsCache {
+		// if it's been more than 24 hours since the groups were cached
+		if time.Since(tkn.cacheTime) >= 24*time.Hour {
+			// remove it from our cache
+			delete(a.groupsCache, access)
+		}
+	}
+	a.lastCacheClean = time.Now()
+}
+
+// Login redirects a user to login to their configured oAuth authorize endpoint
+func (a *Auth) Login(w http.ResponseWriter, r *http.Request) {
+	authURL := a.oAuthConfig.AuthCodeURL(a.oAuthConfig.Endpoint.AuthURL)
+	http.Redirect(w, r, authURL, http.StatusTemporaryRedirect)
+}
+
+// Logout clears out the user's cookie, removes the token from our cache, and
+// redirects user to IdP's logout endpoint
+func (a *Auth) Logout(w http.ResponseWriter, r *http.Request) {
+	// remove the access token from a.groupsCache
+	if access, err := r.Cookie(a.accessCookieName); err == nil {
+		delete(a.groupsCache, access.Value)
+	}
+	// clear cookie
+	http.SetCookie(w, &http.Cookie{
+		Name:     a.accessCookieName,
+		Value:    "",
+		Path:     "/",
+		Secure:   true,
+		HttpOnly: true,
+		SameSite: http.SameSiteStrictMode,
+		Expires:  time.Unix(0, 0),
+	})
+	http.SetCookie(w, &http.Cookie{
+		Name:     a.refreshCookieName,
+		Value:    "",
+		Path:     "/",
+		Secure:   true,
+		HttpOnly: true,
+		SameSite: http.SameSiteStrictMode,
+		Expires:  time.Unix(0, 0),
+	})
+
+	http.Redirect(w, r, fmt.Sprintf("%s?post_logout_redirect_uri=%s/", a.logoutEndpoint, a.fbURL), http.StatusTemporaryRedirect)
+}
+
+// Redirect handles the oAuth /redirect endpoint. It gets an access token and
+// returns it to the user in the form of a cookie
+func (a *Auth) Redirect(w http.ResponseWriter, r *http.Request) {
+	token, err := a.oAuthConfig.Exchange(r.Context(), r.FormValue("code"), oauth2.AccessTypeOffline)
+	if err != nil {
+		a.logger.Warnf("getting token from IdP: %+v", err)
+		http.Error(w, "Bad Request", http.StatusBadRequest)
+		return
+	}
+
+	a.SetCookie(w, token.AccessToken, token.RefreshToken, token.Expiry)
+	http.Redirect(w, r, "/", http.StatusTemporaryRedirect)
+}
+
+// getGroups gets the group membership for a given token from configured IdP
+func (a *Auth) getGroups(token string) ([]Group, error) {
+	var groups Groups
+
+	gc, ok := a.groupsCache[token]
+	if ok && (time.Since(gc.cacheTime) < a.cacheTTL) && len(gc.groups) > 0 {
+		return gc.groups, nil
+	}
+
+	nextLink := a.groupEndpoint
+	for nextLink != "" {
+		req, err := http.NewRequest("GET", nextLink, nil)
+		if err != nil {
+			return nil, errors.Wrap(err, "creating new request to group endpoint")
+		}
+
+		req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", token))
+		response, err := http.DefaultClient.Do(req)
+		if err != nil {
+			return nil, errors.Wrap(err, "getting group membership info")
+		}
+		if response.StatusCode != http.StatusOK {
+			return nil, fmt.Errorf("getting group membership info: %s", response.Status)
+		}
+
+		var g Groups
+		if err = json.NewDecoder(response.Body).Decode(&g); err != nil {
+			return groups.Groups, errors.Wrap(err, "failed unmarshalling group membership response")
+		}
+
+		response.Body.Close()
+		groups.Groups = append(groups.Groups, g.Groups...)
+		nextLink = g.NextLink
+	}
+
+	if len(groups.Groups) == 0 {
+		return nil, fmt.Errorf("no groups found")
+	}
+
+	a.groupsCache[token] = cachedGroups{
+		cacheTime: time.Now(),
+		groups:    groups.Groups,
+	}
+	return groups.Groups, nil
+}
+
+func (a *Auth) SetCookie(w http.ResponseWriter, access, refresh string, expiry time.Time) error {
+	http.SetCookie(w, &http.Cookie{
+		Name:     a.refreshCookieName,
+		Value:    refresh,
+		Path:     "/",
+		Secure:   true,
+		HttpOnly: true,
+		SameSite: http.SameSiteStrictMode,
+		Expires:  expiry,
+	})
+
+	http.SetCookie(w, &http.Cookie{
+		Name:     a.accessCookieName,
+		Value:    access,
+		Path:     "/",
+		Secure:   true,
+		HttpOnly: true,
+		SameSite: http.SameSiteStrictMode,
+		Expires:  expiry,
+	})
+	return nil
+}
+
+func (a *Auth) SetGRPCMetadata(ctx context.Context, md metadata.MD, access, refresh string) (context.Context, error) {
+	mCookies := map[string]string{}
+	if c, ok := md["cookie"]; ok {
+		for _, cookie := range c {
+			name, val := parseCookie(cookie)
+			mCookies[name] = val
+		}
+	}
+
+	mCookies[a.accessCookieName] = access
+	mCookies[a.refreshCookieName] = refresh
+
+	cookies := []string{}
+	for name, val := range mCookies {
+		cookies = append(cookies, name+"="+val)
+	}
+
+	md["cookie"] = cookies
+	return metadata.NewIncomingContext(ctx, md), grpc.SetHeader(ctx, md)
+}
+
+func decodeHex(hexstr string) ([]byte, error) {
+	data, err := hex.DecodeString(hexstr)
+	if err != nil {
+		return nil, errors.Wrap(err, "decoding hex string to byte slice")
+	}
+	if len(data) != 32 {
+		return nil, fmt.Errorf("invalid key length")
+	}
+	return data, nil
+}
+
+func (a *Auth) convertIP(configuredIPs []string) error {
+	sz := len(configuredIPs)
+	nets := make([]net.IPNet, sz)
+	for i, ip := range configuredIPs {
+		// skip empty strings
+		if ip == "" {
+			sz--
+			continue
+		}
+		// for IPs passed without a subnet, append /32 to only allow 1 IP
+		// this step is needed because ParseCIDR method assumes a CIDR address
+		if !strings.Contains(ip, "/") {
+			ip = ip + "/32"
+		}
+		_, subnet, err := net.ParseCIDR(ip)
+		if err != nil {
+			return errors.Wrapf(err, "parsing CIDR for %v", ip)
+		}
+		nets[i] = *subnet
+	}
+	a.allowedNetworks = nets[:sz]
+	return nil
+}
+
+// if IP is in allowed networks, then return true to grant admin permissions
+func (a *Auth) CheckAllowedNetworks(clientIP string) bool {
+	clientIP = strings.Split(clientIP, ":")[0]
+	convertedIP := net.ParseIP(clientIP)
+	for _, network := range a.allowedNetworks {
+		if network.Contains(convertedIP) {
+			return true
+		}
+	}
+	return false
+}
+
+func parseCookie(cookie string) (name, data string) {
+	vals := strings.Split(cookie, "=")
+	if len(vals) == 0 {
+		vals = []string{"", ""}
+	} else if len(vals) < 2 {
+		vals = append(vals, "")
+	}
+	return vals[0], vals[1]
+}

authn/authenticate_internal_test.go

@@ -0,0 +1,760 @@
+package authn
+
+import (
+	"bytes"
+	"context"
+	"encoding/hex"
+	"encoding/json"
+	"fmt"
+	"net"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"reflect"
+	"strconv"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/featurebasedb/featurebase/v3/logger"
+	"github.com/golang-jwt/jwt"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/metadata"
+)
+
+func NewTestAuth(t *testing.T) *Auth {
+	t.Helper()
+	var (
+		ClientID         = "e9088663-eb08-41d7-8f65-efb5f54bbb71"
+		ClientSecret     = "DEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEF"
+		AuthorizeURL     = "https://login.microsoftonline.com/4a137d66-d161-4ae4-b1e6-07e9920874b8/oauth2/v2.0/authorize"
+		TokenURL         = "https://login.microsoftonline.com/4a137d66-d161-4ae4-b1e6-07e9920874b8/oauth2/v2.0/token"
+		GroupEndpointURL = "https://graph.microsoft.com/v1.0/me/transitiveMemberOf/microsoft.graph.group?$count=true"
+		LogoutURL        = "https://login.microsoftonline.com/common/oauth2/v2.0/logout"
+		Scopes           = []string{"https://graph.microsoft.com/.default", "offline_access"}
+		Key              = "DEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEF"
+		configuredIPs    = []string{}
+	)
+
+	a, err := NewAuth(
+		logger.NewStandardLogger(os.Stdout),
+		"http://localhost:10101/",
+		Scopes,
+		AuthorizeURL,
+		TokenURL,
+		GroupEndpointURL,
+		LogoutURL,
+		ClientID,
+		ClientSecret,
+		Key,
+		configuredIPs,
+	)
+	if err != nil {
+		t.Fatalf("building auth object%s", err)
+	}
+	return a
+}
+
+func TestSetGRPCMetadata(t *testing.T) {
+	a := NewTestAuth(t)
+	for name, md := range map[string]metadata.MD{
+		"empty":     {},
+		"something": {"cookie": []string{a.accessCookieName + "=something"}},
+		"somethingElse": {"cookie": []string{
+			a.accessCookieName + "=something",
+			a.refreshCookieName + "=something",
+		}},
+		"otherCookies": {"cookie": []string{a.accessCookieName + "=something", "blah=blah"}},
+	} {
+		t.Run(name, func(t *testing.T) {
+			ogCookies := md["cookie"]
+			ctx := grpc.NewContextWithServerTransportStream(
+				metadata.NewIncomingContext(context.TODO(),
+					md,
+				),
+				NewServerTransportStream(),
+			)
+			md, ok := metadata.FromIncomingContext(ctx)
+			if !ok {
+				t.Fatalf("expected ok, got: %v", ok)
+			}
+			ctx, err := a.SetGRPCMetadata(ctx, md, "accesstoken!", "refreshtoken!")
+			if err != nil {
+				t.Fatalf("expected no errors, got: %v", err)
+			}
+			if err := grpc.SendHeader(ctx, md); err != nil {
+				t.Fatalf("expected no errors, got: %v", err)
+			}
+			md, ok = metadata.FromIncomingContext(ctx)
+			if !ok {
+				t.Fatalf("expected ok, got: %v", ok)
+			}
+			c, ok := md["cookie"]
+			if !ok {
+				t.Fatalf("expected ok, got: %v", ok)
+			}
+			var accessCookie, refreshCookie string
+			for _, cookie := range c {
+				if strings.HasPrefix(cookie, a.accessCookieName) {
+					accessCookie = cookie
+				} else if strings.HasPrefix(cookie, a.refreshCookieName) {
+					refreshCookie = cookie
+				}
+				if refreshCookie != "" && accessCookie != "" {
+					break
+				}
+			}
+
+			exp := a.accessCookieName + "=accesstoken!"
+			if accessCookie != exp {
+				t.Fatalf("expected '%v', got '%v'", exp, accessCookie)
+			}
+			exp = a.refreshCookieName + "=refreshtoken!"
+			if refreshCookie != exp {
+				t.Fatalf("expected '%v', got '%v'", exp, refreshCookie)
+			}
+
+			for _, cookie := range c {
+				if strings.HasPrefix(cookie, a.accessCookieName) || strings.HasPrefix(cookie, a.refreshCookieName) {
+					continue
+				}
+				found := false
+
+				for _, ogCookie := range ogCookies {
+					if cookie == ogCookie {
+						found = true
+						break
+					}
+				}
+				if !found {
+					t.Fatal("SetGRPCMetadata did not maintain the previous cookie list")
+				}
+			}
+		})
+	}
+}
+
+func TestAuth(t *testing.T) {
+	a := NewTestAuth(t)
+	t.Run("SetCookie", func(t *testing.T) {
+		w := httptest.NewRecorder()
+		err := a.SetCookie(w, "access", "refresh", time.Now().Add(time.Hour))
+		if err != nil {
+			t.Fatalf("expected no errors, got: %v", err)
+		}
+
+		if w.Result().Cookies()[0].Value == "" {
+			t.Errorf("expected something, got empty string")
+		}
+
+		if got, want := w.Result().Cookies()[0].Path, "/"; got != want {
+			t.Fatalf("path=%s, want %s", got, want)
+		}
+	})
+
+	t.Run("KeyLength", func(t *testing.T) {
+		_, err := NewAuth(
+			logger.NewStandardLogger(os.Stdout),
+			"http://localhost:10101/",
+			[]string{"https://graph.microsoft.com/.default", "offline_access"},
+			"https://login.microsoftonline.com/4a137d66-d161-4ae4-b1e6-07e9920874b8/oauth2/v2.0/authorize",
+			"https://login.microsoftonline.com/4a137d66-d161-4ae4-b1e6-07e9920874b8/oauth2/v2.0/token",
+			"https://graph.microsoft.com/v1.0/me/transitiveMemberOf/microsoft.graph.group?$count=true",
+			"https://login.microsoftonline.com/common/oauth2/v2.0/logout",
+			"e9088663-eb08-41d7-8f65-efb5f54bbb71",
+			"DEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEF",
+			"DEADBEEFD",
+			[]string{},
+		)
+		if err == nil || !strings.Contains(err.Error(), "decoding secret key") {
+			t.Fatalf("expected error decoding secret key got: %v", err)
+		}
+	})
+	t.Run("GetSecretKey", func(t *testing.T) {
+		want, _ := hex.DecodeString("DEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEF")
+		if got := a.SecretKey(); !bytes.Equal(got, want) {
+			t.Fatalf("expected %v, got %v", got, want)
+		}
+	})
+}
+
+func TestAuthenticate(t *testing.T) {
+	cases := []struct {
+		name         string
+		uid          string
+		uname        string
+		exp          int64
+		refresh      bool
+		refreshToken string
+		malformed    bool
+		empty        bool
+		groups       []Group
+		err          error
+	}{
+		{
+			name:  "GoodToken",
+			uid:   "42",
+			uname: "A. Token",
+			groups: []Group{
+				{
+					GroupID:   "ac97c9e2-346b-42a2-b6da-18bcb61a32fe",
+					GroupName: "adminGroup",
+				},
+			},
+		},
+		{
+			name:      "Malformed",
+			malformed: true,
+			err:       fmt.Errorf("parsing auth token: token contains an invalid number of segments"),
+		},
+		{
+			name:  "Empty",
+			empty: true,
+			err:   fmt.Errorf("auth token is empty"),
+		},
+		{
+			name:  "ExpiredTokenNoRefresh",
+			uid:   "42",
+			uname: "A. Token",
+			groups: []Group{
+				{
+					GroupID:   "ac97c9e2-346b-42a2-b6da-18bcb61a32fe",
+					GroupName: "adminGroup",
+				},
+			},
+			exp: -17764800,
+			err: fmt.Errorf("token is expired: refreshing token: 400 Bad Request"),
+		},
+		{
+			name:  "ExpiredTokenYesRefresh",
+			uid:   "42",
+			uname: "A. Token",
+			groups: []Group{
+				{
+					GroupID:   "ac97c9e2-346b-42a2-b6da-18bcb61a32fe",
+					GroupName: "adminGroup",
+				},
+			},
+			refresh:      true,
+			refreshToken: "refreshToken",
+			exp:          -17764800,
+		},
+		{
+			name:  "ExpiredTokenYesRefreshButError",
+			uid:   "42",
+			uname: "A. Token",
+			groups: []Group{
+				{
+					GroupID:   "ac97c9e2-346b-42a2-b6da-18bcb61a32fe",
+					GroupName: "adminGroup",
+				},
+			},
+			refresh:      true,
+			refreshToken: "blah!!",
+			exp:          -17764800,
+			err:          fmt.Errorf("token is expired: refreshing token: 403 Forbidden"),
+		},
+	}
+	for _, test := range cases {
+		t.Run(test.name, func(t *testing.T) {
+			// setup the test
+			a := NewTestAuth(t)
+			token := ""
+			var err error
+			if !test.malformed && !test.empty {
+				tkn := jwt.New(jwt.SigningMethodHS256)
+				claims := tkn.Claims.(jwt.MapClaims)
+				claims["oid"] = test.uid
+				claims["name"] = test.uname
+				if test.exp != 0 {
+					claims["exp"] = float64(test.exp)
+				}
+				token, err = tkn.SignedString(a.SecretKey())
+				if err != nil {
+					t.Fatalf("unexpected error when signing token %v", err)
+				}
+			} else if !test.empty {
+				token = "asdfasdfasdfasdF"
+			}
+			if len(test.groups) > 0 {
+				a.groupsCache[token] = cachedGroups{time.Now(), test.groups}
+			}
+			if test.refresh {
+				srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+					if err := r.ParseForm(); err != nil {
+						t.Fatalf("unexpected error: %v", err)
+					}
+					refresh := r.Form.Get("refresh_token")
+					if refresh != test.refreshToken {
+						t.Fatalf("refresh token not passed properly, expected %v, got %v", test.refreshToken, refresh)
+						return
+					}
+					if refresh != "refreshToken" {
+						http.Error(w, "bad token", http.StatusForbidden)
+					}
+
+					tkn := jwt.New(jwt.SigningMethodHS256)
+					claims := tkn.Claims.(jwt.MapClaims)
+					claims["oid"] = test.uid
+					claims["name"] = test.uname
+					expiry := float64(time.Now().Add(2 * time.Hour).Unix())
+					claims["exp"] = expiry
+					fresh, err := tkn.SignedString(a.SecretKey())
+					if err != nil {
+						t.Fatalf("unexpected error when signing token %v", err)
+					}
+
+					a.groupsCache[fresh] = cachedGroups{time.Now(), test.groups}
+					fmt.Fprintf(w, `{"access_token": "`+fresh+`",  "refresh_token": "blah",  "token_type": "bearer",  "expires": `+strconv.FormatFloat(expiry, 'f', 0, 64)+` }`)
+				}))
+				defer srv.Close()
+				a.oAuthConfig.Endpoint.TokenURL = srv.URL
+			}
+
+			// do the actual testing
+			uinfo, err := a.Authenticate(token, test.refreshToken)
+			// okay this part kind of sucks bc we need to check errors and i
+			// dont want to write a whole new test for things that should have
+			// errors just to avoid this mess. errors.Is doesn't work either
+			if (test.err == nil && err != nil) || (test.err != nil && err == nil) {
+				t.Fatalf("expected %v, but got %v", test.err, err)
+			} else if test.err != nil && err != nil {
+				if test.err.Error() != err.Error() {
+					t.Fatalf("expected %v, but got %v", test.err, err)
+				} else {
+					return
+				}
+			}
+
+			if !reflect.DeepEqual(uinfo.Groups, test.groups) {
+				t.Fatalf("expected %v, got %v", test.groups, uinfo.Groups)
+			}
+			if !reflect.DeepEqual(uinfo.UserID, test.uid) {
+				t.Fatalf("expected %v, got %v", test.uid, uinfo.UserID)
+			}
+			if !reflect.DeepEqual(uinfo.UserName, test.uname) {
+				t.Fatalf("expected %v, got %v", test.uname, uinfo.UserName)
+			}
+		})
+	}
+}
+
+func TestAuthenticate_CleanCache(t *testing.T) {
+	// this deserves its own test bc it has gross setup required
+	t.Run("should clean", func(t *testing.T) {
+		a := NewTestAuth(t)
+		now := time.Now()
+		a.groupsCache["oldy"] = cachedGroups{now.Add(-24 * time.Hour), []Group{}}
+		a.groupsCache["goldy"] = cachedGroups{now.Add(-4 * time.Hour), []Group{}}
+		a.lastCacheClean = now.Add(-45 * time.Minute)
+
+		_, _ = a.Authenticate("this doesn't matter", "this doesn't matter?")
+		if a.lastCacheClean.Sub(now) <= time.Nanosecond {
+			t.Fatalf("cache should have been cleaned")
+		}
+		if _, ok := a.groupsCache["oldy"]; ok {
+			t.Errorf("oldy should have been deleted")
+		}
+		if _, ok := a.groupsCache["goldy"]; !ok {
+			t.Errorf("goldy should not have been deleted")
+		}
+	})
+	t.Run("shouldn't clean", func(t *testing.T) {
+		a := NewTestAuth(t)
+		now := time.Now()
+		a.groupsCache["oldy"] = cachedGroups{now.Add(-24 * time.Hour), []Group{}}
+		a.groupsCache["goldy"] = cachedGroups{now.Add(-4 * time.Hour), []Group{}}
+		a.lastCacheClean = now
+
+		_, _ = a.Authenticate("this doesn't matter", "this doesn't matter?")
+		if a.lastCacheClean.Sub(now) >= time.Nanosecond {
+			t.Fatalf("cache should not have been cleaned")
+		}
+		if _, ok := a.groupsCache["oldy"]; !ok {
+			t.Errorf("oldy should not have been deleted")
+		}
+		if _, ok := a.groupsCache["goldy"]; !ok {
+			t.Errorf("goldy should not have been deleted")
+		}
+	})
+}
+
+func TestGetGroups(t *testing.T) {
+	a := NewTestAuth(t)
+	a.groupsCache = map[string]cachedGroups{
+		"the world is changed": {
+			cacheTime: time.Now(),
+			groups: []Group{
+				{
+					GroupID:   "a han noston ned wilith",
+					GroupName: "I smell it in the air",
+				},
+			},
+		},
+	}
+	srvNext := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		body, err := json.Marshal(
+			Groups{
+				Groups: []Group{
+					{
+						GroupID:   "han mathon ne chae",
+						GroupName: "I feel it in the earth",
+					},
+				},
+			},
+		)
+		if err != nil {
+			t.Fatalf("unexpected error marshalling groups response: %v", err)
+		}
+		fmt.Fprintf(w, "%s", body)
+	}))
+	defer srvNext.Close()
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		body, err := json.Marshal(
+			Groups{
+				NextLink: srvNext.URL,
+				Groups: []Group{
+					{
+						GroupID:   "han mathon ne nen",
+						GroupName: "i feel it in the water",
+					},
+				},
+			},
+		)
+		if err != nil {
+			t.Fatalf("unexpected error marshalling groups response: %v", err)
+		}
+		fmt.Fprintf(w, "%s", body)
+	}))
+	defer srv.Close()
+	a.groupEndpoint = srv.URL
+
+	for name, test := range map[string]struct {
+		token  string
+		groups []Group
+	}{
+		"InCache": {
+			token: "the world is changed",
+			groups: []Group{
+				{
+					GroupID:   "a han noston ned wilith",
+					GroupName: "I smell it in the air",
+				},
+			},
+		},
+		"NotInCache": {
+			token: "i smell it in the air",
+			groups: []Group{
+				{
+					GroupID:   "han mathon ne nen",
+					GroupName: "i feel it in the water",
+				},
+				{
+					GroupID:   "han mathon ne chae",
+					GroupName: "I feel it in the earth",
+				},
+			},
+		},
+	} {
+		t.Run(name, func(t *testing.T) {
+			if got, err := a.getGroups(test.token); err != nil || !reflect.DeepEqual(got, test.groups) {
+				t.Errorf("expected %v, nil, got %v, %v", test.groups, got, err)
+			}
+		})
+	}
+}
+
+func TestDecodeHex(t *testing.T) {
+	t.Run("cantDecode", func(t *testing.T) {
+		_, err := decodeHex("gggg")
+		if err == nil {
+			t.Fatalf("expected err cannot decode slice, got nil")
+		}
+	})
+	t.Run("tooSmall", func(t *testing.T) {
+		_, err := decodeHex("DEADBEEF")
+		if err == nil {
+			t.Fatalf("expected err wrong length, got nil")
+		}
+	})
+	t.Run("tooBig", func(t *testing.T) {
+		_, err := decodeHex("DEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEF")
+		if err == nil {
+			t.Fatalf("expected err wrong length, got nil")
+		}
+	})
+	t.Run("justRight", func(t *testing.T) {
+		_, err := decodeHex("DEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEF")
+		if err != nil {
+			t.Fatalf("expected nil, got %v", err)
+		}
+	})
+}
+
+func TestHandlers(t *testing.T) {
+	a := NewTestAuth(t)
+	t.Run("login", func(t *testing.T) {
+		req := httptest.NewRequest("GET", "/login", nil)
+		w := httptest.NewRecorder()
+		a.Login(w, req)
+		resp := w.Result()
+		if resp.StatusCode != http.StatusTemporaryRedirect {
+			t.Fatalf("expected redirect, got %v", resp.StatusCode)
+		}
+		redirect := a.oAuthConfig.AuthCodeURL(a.oAuthConfig.Endpoint.AuthURL)
+		if got, err := resp.Location(); err != nil || got.String() != redirect {
+			t.Fatalf("expected %v, got %v", redirect, got.Path)
+		}
+	})
+	t.Run("logout", func(t *testing.T) {
+		req := httptest.NewRequest("GET", "/logout", nil)
+		w := httptest.NewRecorder()
+		req.AddCookie(
+			&http.Cookie{
+				Name:     a.accessCookieName,
+				Value:    "test",
+				Path:     "/",
+				Secure:   true,
+				HttpOnly: true,
+				SameSite: http.SameSiteStrictMode,
+				Expires:  time.Unix(3000000, 0),
+			},
+		)
+
+		req.AddCookie(
+			&http.Cookie{
+				Name:     a.refreshCookieName,
+				Value:    "test",
+				Path:     "/",
+				Secure:   true,
+				HttpOnly: true,
+				SameSite: http.SameSiteStrictMode,
+				Expires:  time.Unix(3000000, 0),
+			},
+		)
+
+		a.groupsCache["test"] = cachedGroups{}
+		a.Logout(w, req)
+		resp := w.Result()
+		if resp.StatusCode != http.StatusTemporaryRedirect {
+			t.Fatalf("expected redirect, got %v", resp.StatusCode)
+		}
+		redirect := fmt.Sprintf("%s?post_logout_redirect_uri=%s/", a.logoutEndpoint, a.fbURL)
+		if got, err := resp.Location(); err != nil || got.String() != redirect {
+			t.Fatalf("expected %v, got %v", redirect, got.Path)
+		}
+		for _, c := range resp.Cookies() {
+			if c.Name == a.accessCookieName || c.Name == a.refreshCookieName {
+				if c.Value != "" {
+					t.Fatalf("cookie not set to empty value!")
+				}
+				want := time.Unix(0, 0).Unix()
+				got := c.Expires.Unix()
+				if want != got {
+					t.Fatalf("expected %v, got %v", want, got)
+				}
+			}
+		}
+		if _, ok := a.groupsCache["test"]; ok {
+			t.Fatalf("groups not deleted!")
+		}
+	})
+	t.Run("redirectGood", func(t *testing.T) {
+		req := httptest.NewRequest("GET", "/redirect", nil)
+		w := httptest.NewRecorder()
+		tkn := jwt.New(jwt.SigningMethodHS256)
+		claims := tkn.Claims.(jwt.MapClaims)
+		claims["oid"] = "user id"
+		claims["name"] = "user name"
+		expiresIn := 2 * time.Hour
+		exp := time.Now().Add(expiresIn)
+		expiry := float64(exp.Unix())
+		claims["exp"] = expiry
+		fresh, err := tkn.SignedString(a.SecretKey())
+		if err != nil {
+			t.Fatalf("unexpected error when signing token %v", err)
+		}
+
+		srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			body := `{"access_token": "` + fresh + `", "refresh_token": "blah", "expires_in": "` + strconv.Itoa(int(expiresIn.Seconds())) + `"}`
+			w.Header().Set("Content-Type", "application/json; charset=utf-8")
+			w.WriteHeader(http.StatusOK)
+			w.Write([]byte(body))
+		}))
+		a.oAuthConfig.Endpoint.TokenURL = srv.URL
+		a.Redirect(w, req)
+		resp := w.Result()
+		if resp.StatusCode != http.StatusTemporaryRedirect {
+			t.Fatalf("expected redirect, got %v", resp.StatusCode)
+		}
+		if got, err := resp.Location(); err != nil || got.String() != "/" {
+			t.Fatalf("expected %v, got %v", "/", got.Path)
+		}
+		cookies := resp.Cookies()
+		for _, c := range cookies {
+			if c.Name == a.accessCookieName && c.Value != fresh {
+				t.Fatalf("expected %v, got %v", exp, c.Value)
+			} else if c.Name == a.refreshCookieName && c.Value != "blah" {
+				t.Fatalf("expected %v, got %v", "blah", c.Value)
+			}
+		}
+	})
+
+	t.Run("redirectBad", func(t *testing.T) {
+		req := httptest.NewRequest("GET", "/redirect", nil)
+		w := httptest.NewRecorder()
+		srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			http.Error(w, "Server Error", http.StatusInternalServerError)
+		}))
+		a.oAuthConfig.Endpoint.TokenURL = srv.URL
+		a.Redirect(w, req)
+		resp := w.Result()
+		if resp.StatusCode != http.StatusBadRequest {
+			t.Fatalf("expected BadRequest, got %v", resp.StatusCode)
+		}
+	})
+}
+
+// This type is used for mocking ServerTransportStreams in tests
+type ServerTransportStream struct {
+	md     metadata.MD
+	method string
+}
+
+func NewServerTransportStream() *ServerTransportStream {
+	return &ServerTransportStream{
+		md:     metadata.MD{},
+		method: "test",
+	}
+}
+
+func (s *ServerTransportStream) Method() string {
+	return s.method
+}
+
+func (s *ServerTransportStream) SetHeader(md metadata.MD) error {
+	s.md = md
+	return nil
+}
+
+func (s *ServerTransportStream) SendHeader(md metadata.MD) error {
+	_ = md
+	return nil
+}
+
+func (s *ServerTransportStream) SetTrailer(md metadata.MD) error {
+	_ = md
+	return nil
+}
+
+func TestCleanOAuthConfig(t *testing.T) {
+	a := NewTestAuth(t)
+	res := a.CleanOAuthConfig()
+	assertEqual("", res.ClientSecret, t)
+	assertEqual(a.oAuthConfig.ClientID, res.ClientID, t)
+	assertEqual(a.oAuthConfig.RedirectURL, res.RedirectURL, t)
+	assertEqual(a.oAuthConfig.Scopes, res.Scopes, t)
+	assertEqual(a.oAuthConfig.Endpoint, res.Endpoint, t)
+}
+
+func assertEqual(exp, got interface{}, t *testing.T) {
+	if !reflect.DeepEqual(exp, got) {
+		t.Fatalf("expected %v, got %v", exp, got)
+	}
+}
+
+func TestCheckAllowedNetworks(t *testing.T) {
+	tests := []struct {
+		requestIP     string
+		configuredIPs []string
+		isAdmin       bool
+	}{
+		{
+			requestIP:     "10.0.0.1",
+			configuredIPs: []string{"10.0.0.1"},
+			isAdmin:       true,
+		},
+		{
+			requestIP:     "10.0.0.3",
+			configuredIPs: []string{"10.0.0.1", "10.0.0.2"},
+			isAdmin:       false,
+		},
+		{
+			requestIP:     "10.0.0.2",
+			configuredIPs: []string{"10.0.0.1/30"},
+			isAdmin:       true,
+		},
+		// it is possible for the client IP to have a port
+		{
+			requestIP:     "10.0.0.2:22",
+			configuredIPs: []string{"10.0.0.1/30"},
+			isAdmin:       true,
+		},
+		{
+			requestIP:     "10.1.0.3",
+			configuredIPs: []string{"10.0.0.1/32"},
+			isAdmin:       false,
+		},
+		{
+			requestIP:     "10.0.0.254",
+			configuredIPs: []string{"10.0.0.1/24"},
+			isAdmin:       true,
+		},
+	}
+
+	for i, test := range tests {
+		t.Run(fmt.Sprintf("network-%d", i), func(t *testing.T) {
+			a := NewTestAuth(t)
+			if err := a.convertIP(test.configuredIPs); err != nil {
+				t.Fatalf("failed to convert IPs from strings to net.IP: %v", err)
+			}
+			got := a.CheckAllowedNetworks(test.requestIP)
+			if got != test.isAdmin {
+				t.Fatalf("expected %v, got %v", test.isAdmin, got)
+			}
+		})
+	}
+}
+
+func TestConvertIP(t *testing.T) {
+	tests := []struct {
+		configuredIPs []string
+		convertedIPs  []net.IPNet
+	}{
+		{
+			configuredIPs: []string{"10.0.0.1"},
+			convertedIPs: []net.IPNet{
+				{IP: net.ParseIP("10.0.0.1"), Mask: net.CIDRMask(32, 32)},
+			},
+		},
+		{
+			configuredIPs: []string{"10.0.0.1/30"},
+			convertedIPs: []net.IPNet{
+				{IP: net.ParseIP("10.0.0.0"), Mask: net.CIDRMask(30, 32)},
+			},
+		},
+	}
+
+	for i, test := range tests {
+		t.Run(fmt.Sprintf("network-%d", i), func(t *testing.T) {
+			a := NewTestAuth(t)
+			if err := a.convertIP(test.configuredIPs); err != nil {
+				t.Fatalf("failed to convert IPs from strings to net.IP: %v", err)
+			}
+
+			if len(a.allowedNetworks) != len(test.convertedIPs) {
+				t.Fatalf("expected len of %v networks, got %v", len(test.convertedIPs), len(a.allowedNetworks))
+			}
+
+			for i := range a.allowedNetworks {
+				expected, got := test.convertedIPs[i], a.allowedNetworks[i]
+				if got.IP.String() != expected.IP.String() {
+					t.Fatalf("for IP, expected %v, got %v", expected.IP, got.IP)
+				}
+				if got.Mask.String() != expected.Mask.String() {
+					t.Fatalf("for mask, expected %v, got %v", expected.Mask.String(), got.Mask.String())
+				}
+			}
+		})
+	}
+}

authn/context.go

@@ -0,0 +1,54 @@
+// Copyright 2022 Molecula Corp (DBA FeatureBase). All rights reserved.
+package authn
+
+import "context"
+
+// Empty struct to avoid allocations
+type contextKeyAccessToken struct{}
+type contextKeyRefreshToken struct{}
+type contextKeyUserInfo struct{}
+type contextKeyIndexes struct{}
+
+// GetAccessToken gets the access token from a context.
+func GetAccessToken(ctx context.Context) (token string, ok bool) {
+	token, ok = ctx.Value(contextKeyAccessToken{}).(string)
+	return
+}
+
+// WithAccessToken makes a new Context with an access token.
+func WithAccessToken(ctx context.Context, token string) context.Context {
+	return context.WithValue(ctx, contextKeyAccessToken{}, token)
+}
+
+// GetRefreshToken gets the refresh token from a context.
+func GetRefreshToken(ctx context.Context) (token string, ok bool) {
+	token, ok = ctx.Value(contextKeyRefreshToken{}).(string)
+	return
+}
+
+// WithRefreshToken makes a new Context with a refresh token.
+func WithRefreshToken(ctx context.Context, token string) context.Context {
+	return context.WithValue(ctx, contextKeyRefreshToken{}, token)
+}
+
+// GetUserInfo gets the UserInfo from a context.
+func GetUserInfo(ctx context.Context) (userInfo *UserInfo, ok bool) {
+	userInfo, ok = ctx.Value(contextKeyUserInfo{}).(*UserInfo)
+	return
+}
+
+// WithUserInfo makes a new Context with UserInfo.
+func WithUserInfo(ctx context.Context, userInfo *UserInfo) context.Context {
+	return context.WithValue(ctx, contextKeyUserInfo{}, userInfo)
+}
+
+// GetIndexes get the indices from a context.
+func GetIndexes(ctx context.Context) (indexes []string, ok bool) {
+	indexes, ok = ctx.Value(contextKeyIndexes{}).([]string)
+	return
+}
+
+// WithIndexes makes a new Context with a []string containing the indicies.
+func WithIndexes(ctx context.Context, indexes []string) context.Context {
+	return context.WithValue(ctx, contextKeyUserInfo{}, indexes)
+}

authz/authorization.go

@@ -0,0 +1,130 @@
+// Copyright 2022 Molecula Corp. (DBA FeatureBase).
+// SPDX-License-Identifier: Apache-2.0
+
+package authz
+
+import (
+	"fmt"
+	"io"
+
+	"github.com/featurebasedb/featurebase/v3/authn"
+
+	"gopkg.in/yaml.v2"
+)
+
+type GroupPermissions struct {
+	Permissions map[string]map[string]Permission `yaml:"user-groups"`
+	Admin       string                           `yaml:"admin"`
+}
+
+type Permission string
+
+const (
+	None  Permission = ""
+	Read  Permission = "read"
+	Write Permission = "write"
+	Admin Permission = "admin"
+)
+
+// Satisfies returns whether `p` satisfies the permissions required by `b`
+func (p Permission) Satisfies(b Permission) bool {
+	switch p {
+	case "":
+		return b == ""
+	case "read":
+		return b == "" || b == "read"
+	case "write":
+		return b == "" || b == "read" || b == "write"
+	case "admin":
+		return b == "" || b == "read" || b == "write" || b == "admin"
+	}
+	return false
+}
+
+func (p *GroupPermissions) ReadPermissionsFile(permsFile io.Reader) (err error) {
+	permsData, err := io.ReadAll(permsFile)
+
+	if err != nil {
+		return fmt.Errorf("reading permissions failed with error: %s", err)
+	}
+
+	err = yaml.UnmarshalStrict(permsData, &p)
+	if err != nil {
+		return fmt.Errorf("unmarshalling permissions failed with error: %s", err)
+	}
+
+	return
+}
+
+func (p *GroupPermissions) GetPermissions(user *authn.UserInfo, index string) (permission Permission, errors error) {
+	groups := user.Groups
+	if admin := p.IsAdmin(groups); admin {
+		return Admin, nil
+	}
+
+	allPermissions := map[Permission]bool{
+		Write: false,
+		Read:  false,
+	}
+
+	if len(groups) == 0 {
+		return None, fmt.Errorf("user is not part of any groups in identity provider")
+	}
+
+	var groupsDenied []string
+	for _, group := range groups {
+		if _, ok := p.Permissions[group.GroupID]; ok {
+			if perm, ok := p.Permissions[group.GroupID][index]; ok {
+				allPermissions[perm] = true
+			} else {
+				return None, fmt.Errorf("user %s does not have permission to index %s", user.UserID, index)
+			}
+		} else {
+			groupsDenied = append(groupsDenied, group.GroupID)
+		}
+	}
+
+	if len(groupsDenied) == len(groups) {
+		return None, fmt.Errorf("group(s) %s does not have permission to FeatureBase", groupsDenied)
+	}
+
+	if allPermissions[Write] {
+		return Write, nil
+	} else if allPermissions[Read] {
+		return Read, nil
+	} else {
+		return None, fmt.Errorf("no permissions found")
+	}
+}
+
+func (p *GroupPermissions) IsAdmin(groups []authn.Group) bool {
+	for _, group := range groups {
+		if p.Admin == group.GroupID {
+			return true
+		}
+	}
+	return false
+}
+
+func (p *GroupPermissions) GetAuthorizedIndexList(groups []authn.Group, desiredPermission Permission) (indexList []string) {
+	// if user is admin, find all indexes in permissions file and return them
+	if p.IsAdmin(groups) {
+		for groupId := range p.Permissions {
+			for index := range p.Permissions[groupId] {
+				indexList = append(indexList, index)
+			}
+		}
+		return indexList
+	}
+
+	for _, group := range groups {
+		if _, ok := p.Permissions[group.GroupID]; ok {
+			for index, permission := range p.Permissions[group.GroupID] {
+				if permission.Satisfies(desiredPermission) {
+					indexList = append(indexList, index)
+				}
+			}
+		}
+	}
+	return indexList
+}

authz/authorization_test.go

@@ -0,0 +1,305 @@
+// Copyright 2022 Molecula Corp. (DBA FeatureBase).
+// SPDX-License-Identifier: Apache-2.0
+package authz_test
+
+import (
+	"fmt"
+	"reflect"
+	"sort"
+	"strings"
+	"testing"
+
+	"github.com/featurebasedb/featurebase/v3/authn"
+	"github.com/featurebasedb/featurebase/v3/authz"
+)
+
+func TestAuth_ReadPermissionsFile(t *testing.T) {
+
+	singleInput := `user-groups:
+  "dca35310-ecda-4f23-86cd-876aee55906b":
+    "test": "read"
+admin: "ac97c9e2-346b-42a2-b6da-18bcb61a32fe"`
+
+	multiInput := `user-groups:
+  "dca35310-ecda-4f23-86cd-876aee55906b":
+    "test": "read"
+    "test2": "write"
+  "dca35310-ecda-4f23-86cd-876aee559900":
+    "test": "write"
+admin: "ac97c9e2-346b-42a2-b6da-18bcb61a32fe"`
+
+	singlePermission := authz.GroupPermissions{
+		Permissions: map[string]map[string]authz.Permission{
+			"dca35310-ecda-4f23-86cd-876aee55906b": {"test": authz.Read},
+		},
+		Admin: "ac97c9e2-346b-42a2-b6da-18bcb61a32fe",
+	}
+
+	multiPermission := authz.GroupPermissions{
+		Permissions: map[string]map[string]authz.Permission{
+			"dca35310-ecda-4f23-86cd-876aee55906b": {"test": authz.Read, "test2": authz.Write},
+			"dca35310-ecda-4f23-86cd-876aee559900": {"test": authz.Write}},
+		Admin: "ac97c9e2-346b-42a2-b6da-18bcb61a32fe",
+	}
+
+	tests := []struct {
+		input  string
+		output authz.GroupPermissions
+	}{
+		{singleInput, singlePermission},
+		{multiInput, multiPermission},
+	}
+
+	for i, test := range tests {
+		t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
+			permFile := strings.NewReader(test.input)
+
+			var p authz.GroupPermissions
+			err := p.ReadPermissionsFile(permFile)
+			if err != nil {
+				t.Fatalf("readPermissionsFile error: %s", err)
+			}
+
+			if !reflect.DeepEqual(p, test.output) {
+				t.Fatalf("expected output %s, but got %s", test.output, p)
+			}
+		},
+		)
+	}
+}
+
+func TestAuth_GetPermissions(t *testing.T) {
+
+	// initializes different example of permissions file in yaml
+	permissions1 := `"user-groups":
+  "dca35310-ecda-4f23-86cd-876aee55906b":
+    "test": "read"
+admin: "ac97c9e2-346b-42a2-b6da-18bcb61a32fe"`
+
+	permissions2 := `"user-groups":
+  "dca35310-ecda-4f23-86cd-876aee559900":
+    "test": "write"
+admin: "ac97c9e2-346b-42a2-b6da-18bcb61a32fe"`
+
+	permissions3 := `"user-groups":
+  "dca35310-ecda-4f23-86cd-876aee55906b":
+    "test": "write"
+    "test2": "read"
+  "dca35310-ecda-4f23-86cd-876aee559900":
+    "test": "read"
+admin: "ac97c9e2-346b-42a2-b6da-18bcb61a32fe"`
+
+	permissions4 := `"user-groups":
+  "dca35310-ecda-4f23-86cd-876aee559900":
+    "test": ""
+admin: "ac97c9e2-346b-42a2-b6da-18bcb61a32fe"`
+
+	// initializes groups that are returned from identity provider
+	groupName := "name"
+	groupsList1 := []authn.Group{}
+	groupsList2 := []authn.Group{{
+		GroupID:   "fake-group",
+		GroupName: groupName}}
+	groupsList3 := []authn.Group{
+		{GroupID: "dca35310-ecda-4f23-86cd-876aee55906b", GroupName: groupName},
+		{GroupID: "dca35310-ecda-4f23-86cd-876aee559900", GroupName: groupName},
+	}
+	groupsList4 := []authn.Group{{GroupID: "ac97c9e2-346b-42a2-b6da-18bcb61a32fe", GroupName: groupName}}
+
+	tests := []struct {
+		yamlData   string
+		groups     []authn.Group
+		index      string
+		userAccess authz.Permission
+		err        string
+	}{
+		{
+			permissions1,
+			groupsList1,
+			"test",
+			authz.None,
+			"user is not part of any groups in identity provider",
+		},
+		{
+			permissions1,
+			groupsList3,
+			"test1",
+			authz.None,
+			"does not have permission to index",
+		},
+		{
+			permissions2,
+			groupsList2,
+			"test",
+			authz.None,
+			"does not have permission to FeatureBase",
+		},
+		{
+			permissions1,
+			groupsList3,
+			"test",
+			authz.Read,
+			"",
+		},
+		{
+			permissions2,
+			groupsList3,
+			"test",
+			authz.Write,
+			"",
+		},
+		{
+			permissions3,
+			groupsList4,
+			"test",
+			authz.Admin,
+			"",
+		},
+		{
+			permissions4,
+			groupsList3,
+			"test",
+			authz.None,
+			"no permissions found",
+		},
+	}
+
+	for i, test := range tests {
+		t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
+
+			permFile := strings.NewReader(test.yamlData)
+
+			var p authz.GroupPermissions
+			if err := p.ReadPermissionsFile(permFile); err != nil {
+				t.Errorf("Error: %s", err)
+			}
+
+			p1, err := p.GetPermissions(&authn.UserInfo{Groups: test.groups}, test.index)
+
+			if p1 != test.userAccess {
+				t.Errorf("expected permission to be %s, but got %s", test.userAccess, p1)
+			}
+
+			if err != nil {
+				if !strings.Contains(err.Error(), test.err) {
+					t.Errorf("expected error to contain %s, but got %s", test.err, err.Error())
+				}
+			}
+
+		})
+	}
+}
+
+func TestAuth_IsAdmin(t *testing.T) {
+
+	group1 := []authn.Group{
+		{GroupID: "ac97c9e2-346b-42a2-b6da-18bcb61a32fe", GroupName: "admin-group"},
+	}
+
+	group2 := []authn.Group{
+		{GroupID: "dca35310-ecda-4f23-86cd-876aee55906b", GroupName: "group-name"},
+	}
+
+	groupPermissions := authz.GroupPermissions{
+		Permissions: map[string]map[string]authz.Permission{
+			"dca35310-ecda-4f23-86cd-876aee55906b": {"test": authz.Write},
+		},
+		Admin: "ac97c9e2-346b-42a2-b6da-18bcb61a32fe",
+	}
+
+	tests := []struct {
+		groups           []authn.Group
+		groupPermissions authz.GroupPermissions
+		output           bool
+	}{
+		{
+			group1, groupPermissions, true,
+		},
+		{
+			group2, groupPermissions, false,
+		},
+	}
+
+	for i, test := range tests {
+		t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
+			p := test.groupPermissions
+			resp := p.IsAdmin(test.groups)
+			if resp != test.output {
+				t.Errorf("expected %t, but got %t", test.output, resp)
+			}
+		})
+	}
+}
+
+func TestAuth_GetAuthorizedIndexList(t *testing.T) {
+
+	group1 := []authn.Group{
+		{GroupID: "dca35310-ecda-4f23-86cd-876aee55906b", GroupName: "group-name"},
+	}
+
+	group2 := []authn.Group{
+		{GroupID: "ac97c9e2-346b-42a2-b6da-18bcb61a32fe", GroupName: "admin-group"},
+	}
+
+	group3 := []authn.Group{
+		{GroupID: "dca35310-ecda-4f23-86cd-876aee559900", GroupName: "group-name"},
+	}
+
+	p := authz.GroupPermissions{
+		Permissions: map[string]map[string]authz.Permission{
+			"dca35310-ecda-4f23-86cd-876aee55906b": {
+				"test1": authz.Read,
+				"test2": authz.Write,
+			},
+			"dca35310-ecda-4f23-86cd-876aee559900": {
+				"test3": authz.Read,
+			},
+		},
+		Admin: "ac97c9e2-346b-42a2-b6da-18bcb61a32fe",
+	}
+
+	tests := []struct {
+		groups     []authn.Group
+		permission authz.Permission
+		output     []string
+	}{
+		{
+			group1,
+			authz.Read,
+			[]string{"test1", "test2"},
+		},
+		{
+			group1,
+			authz.Write,
+			[]string{"test2"},
+		},
+		{
+			group3,
+			authz.Write,
+			nil,
+		},
+		{
+			group2,
+			authz.Read,
+			[]string{"test1", "test2", "test3"},
+		},
+		{
+			group2,
+			authz.Write,
+			[]string{"test1", "test2", "test3"},
+		},
+	}
+
+	for i, test := range tests {
+		t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
+
+			indexList := p.GetAuthorizedIndexList(test.groups, test.permission)
+			sort.Strings(indexList)
+
+			if !reflect.DeepEqual(indexList, test.output) {
+				t.Errorf("expected %s, but got %s", test.output, indexList)
+			}
+		})
+	}
+
+}

batch/Dockerfile-test

@@ -0,0 +1,11 @@
+ARG GO_VERSION=1.19
+
+FROM golang:${GO_VERSION}
+
+WORKDIR /go/src/github.com/featurebasedb/featurebase/
+
+COPY . .
+
+WORKDIR /go/src/github.com/featurebasedb/featurebase/batch/
+
+CMD ["go","test","-v","-mod=vendor","-tags=odbc,dynamic","./..."]

batch/Dockerfile-wait

@@ -0,0 +1,8 @@
+FROM ubuntu:18.04
+
+RUN ["apt-get", "update", "-y"]
+RUN ["apt-get", "install", "-y", "curl", "netcat"]
+
+ADD wait.sh /wait
+
+ENTRYPOINT ["/wait"]

batch/Makefile

@@ -0,0 +1,44 @@
+GO ?= go
+
+# We allow setting a custom docker-compose "project". Multiple of the
+# same docker-compose environment can exist simultaneously as long as
+# they use different projects (the project name is prepended to
+# container names and such). This is useful in a CI environment where
+# we might be running multiple instances of the tests concurrently.
+PROJECT ?= batch
+DOCKER_COMPOSE = docker-compose -p $(PROJECT)
+
+vendor: ../go.mod
+	$(GO) mod vendor
+
+build-%:
+	$(DOCKER_COMPOSE) build $*
+
+test-all:
+	$(MAKE) startup
+	$(MAKE) test-run
+	$(MAKE) shutdown
+
+start-all: build-wait
+	$(DOCKER_COMPOSE) up -d featurebase
+	$(DOCKER_COMPOSE) run -T wait featurebase curl --silent --fail http://featurebase:10101/status
+
+startup: start-all
+
+shutdown:
+	$(DOCKER_COMPOSE) down -v --remove-orphans
+
+save-%-logs:
+	$(DOCKER_COMPOSE) logs $* > ./testdata/$(PROJECT)_$*_logs.txt
+
+TCMD ?= ./...
+# do "make startup", then e.g. "make test-run-local TCMD='-run=MyFavTest ./kafka'"
+test-run-local: vendor
+	pwd
+	$(DOCKER_COMPOSE) build batch-test
+	$(DOCKER_COMPOSE) run -T batch-test go test -mod=vendor -tags=odbc,dynamic $(TCMD)
+
+TPKG ?= ../...
+test-run: vendor
+	$(DOCKER_COMPOSE) build batch-test
+	$(DOCKER_COMPOSE) run -T batch-test bash -c "set -o pipefail; go test -v -mod=vendor -tags=odbc,dynamic ./... -covermode=atomic -coverpkg=$(TPKG) -coverprofile=/testdata/$(PROJECT)_base_coverage.out"

batch/README.md

@@ -0,0 +1,46 @@
+# batch
+
+The `batch` package provides a standard tool set for batching records in a way
+that is most performant for ingesting those records into FeatureBase. The main
+implementation is `Batch` (which can be initated with the `NewBatch()`
+function). The `NewBatch()` function takes an `Importer` which contains all of
+ the methods required to interact with FeatureBase; these include methods for
+ doing string/id translation as well as for importing shards of data.
+
+ IDK uses the `batch` package internally. Another example where the `batch`
+ package is used in the `sql3` package. When an "INSERT INTO" statement is
+ executed, the SQL engine uses a `Batch` to do key translation and build import
+ batches prior to doing the final import.
+## Integration tests
+
+To run the tests, you will need to install the following dependencies:
+
+1. [Docker](https://docs.docker.com/install/)
+2. [Docker Compose](https://docs.docker.com/compose/install/)
+
+In addition to these dependancies, you will need to be added to the molecula [Gitlab](https://registry.gitlab.com/molecula) account.
+
+First start the test environment. This is a docker-compose environment that includes featurebase.
+
+    make startup
+
+To build and run the integration tests, run:
+
+    make test-run-local
+
+Then to shut down the test environment, run:
+
+    make shutdown
+
+The previous command is equivalent to running the following:
+
+    make startup
+    sleep 30 # wait for services to come up
+    make test-run
+    make shutdown
+
+To run an individual test, you can run the command directly using docker-compose. Note that you must run `docker-compose build batch-test` for docker to run the latest code. Modify the following as needed:
+
+    make startup
+    docker-compose build batch-test
+    docker-compose run batch-test /usr/local/go/bin/go test -count=1 -mod=vendor -run=TestCmdMainOne .

batch/batcher.go

@@ -0,0 +1,20 @@
+package batch
+
+import (
+	"time"
+
+	"github.com/featurebasedb/featurebase/v3/dax"
+)
+
+// Batcher is an interface implemented by anything which can allocate new
+// batches.
+type Batcher interface {
+	NewBatch(cfg Config, tbl *dax.Table, fields []*dax.Field) (RecordBatch, error)
+}
+
+// Config is the configuration options passed to NewBatch for any implementation
+// of the Batcher interface.
+type Config struct {
+	Size         int
+	MaxStaleness time.Duration
+}

batch/convert.go

@@ -0,0 +1,145 @@
+package batch
+
+import (
+	"time"
+
+	featurebase "github.com/featurebasedb/featurebase/v3"
+	"github.com/featurebasedb/featurebase/v3/errors"
+)
+
+var (
+	MinTimestampNano = time.Unix(-1<<32, 0).UTC()       // 1833-11-24T17:31:44Z
+	MaxTimestampNano = time.Unix(1<<32, 0).UTC()        // 2106-02-07T06:28:16Z
+	MinTimestamp     = time.Unix(-62135596799, 0).UTC() // 0001-01-01T00:00:01Z
+	MaxTimestamp     = time.Unix(253402300799, 0).UTC() // 9999-12-31T23:59:59Z
+
+	ErrTimestampOutOfRange = errors.New("", "value provided for timestamp field is out of range")
+)
+
+type TimeUnit string
+
+const (
+	TimeUnitSeconds      = TimeUnit(featurebase.TimeUnitSeconds)
+	TimeUnitMilliseconds = TimeUnit(featurebase.TimeUnitMilliseconds)
+	TimeUnitMicroseconds = TimeUnit(featurebase.TimeUnitMicroseconds)
+	TimeUnitUSeconds     = TimeUnit(featurebase.TimeUnitUSeconds)
+	TimeUnitNanoseconds  = TimeUnit(featurebase.TimeUnitNanoseconds)
+)
+
+// TimestampToInt64 converts the provided timestamp to an int64 as the number of
+// units past the epoch.
+func TimestampToInt64(unit TimeUnit, epoch time.Time, ts time.Time) (int64, error) {
+	var err error
+
+	unit, err = validateTimeUnit(unit)
+	if err != nil {
+		return 0, errors.Wrap(err, "validating time unit")
+	}
+
+	epoch, err = validateEpoch(epoch)
+	if err != nil {
+		return 0, errors.Wrap(err, "validating epoch")
+	}
+
+	// Check if the epoch alone is out-of-range. If so, ingest should halt,
+	// regardless of state of the timestamp out-of-range CLI option.
+	if err := validateTimestamp(unit, epoch); err != nil {
+		return 0, errors.Wrap(err, "validating epoch")
+	}
+
+	epochAsInt64 := timestampToInt(unit, epoch)
+
+	// Check if the timestamp is out-of-range.
+	if err := validateTimestamp(unit, ts); err != nil {
+		return 0, errors.Wrapf(ErrTimestampOutOfRange, "validating timestamp: %s", ts)
+	}
+
+	tsAsInt64 := timestampToInt(unit, ts)
+
+	return tsAsInt64 - epochAsInt64, nil
+}
+
+// validateTimeUnit checks if the time unit is supported. If the provided unit
+// is blank, validateTimeUnit returns the default TimeUnit.
+func validateTimeUnit(unit TimeUnit) (TimeUnit, error) {
+	switch unit {
+	case "":
+		return TimeUnitSeconds, nil
+
+	case TimeUnitSeconds,
+		TimeUnitMilliseconds,
+		TimeUnitMicroseconds,
+		TimeUnitUSeconds,
+		TimeUnitNanoseconds:
+		return unit, nil
+	}
+
+	return "", errors.Errorf("unsupported time unit: %s", unit)
+}
+
+// validateEpoch checks if the epoch is supported. If the provided epoch
+// is "zero", validateEpoch returns the default epoch value.
+func validateEpoch(epoch time.Time) (time.Time, error) {
+	if epoch.IsZero() {
+		return time.Unix(0, 0), nil
+	}
+	return epoch, nil
+}
+
+// validateTimestamp checks if the timestamp is within the range of what FB accepts.
+func validateTimestamp(unit TimeUnit, ts time.Time) error {
+	// Min and Max timestamps that Featurebase accepts
+	var minStamp, maxStamp time.Time
+	switch unit {
+	case TimeUnitNanoseconds:
+		minStamp = MinTimestampNano
+		maxStamp = MaxTimestampNano
+	default:
+		minStamp = MinTimestamp
+		maxStamp = MaxTimestamp
+	}
+
+	if ts.Before(minStamp) || ts.After(maxStamp) {
+		return errors.Errorf("timestamp value (%v) must be within min: %v and max: %v", ts, minStamp, maxStamp)
+	}
+	return nil
+}
+
+// timestampToInt takes a time unit and a time.Time and converts it to an
+// integer value.
+func timestampToInt(unit TimeUnit, ts time.Time) int64 {
+	switch unit {
+	case TimeUnitSeconds:
+		return ts.Unix()
+	case TimeUnitMilliseconds:
+		return ts.UnixMilli()
+	case TimeUnitMicroseconds, TimeUnitUSeconds:
+		return ts.UnixMicro()
+	case TimeUnitNanoseconds:
+		return ts.UnixNano()
+	}
+	return 0
+}
+
+// intToTimestamp takes a timeunit and an integer value and converts it to
+// time.Time.
+func intToTimestamp(unit TimeUnit, val int64) (time.Time, error) {
+	switch unit {
+	case TimeUnitSeconds:
+		return time.Unix(val, 0).UTC(), nil
+	case TimeUnitMilliseconds:
+		return time.UnixMilli(val).UTC(), nil
+	case TimeUnitMicroseconds, TimeUnitUSeconds:
+		return time.UnixMicro(val).UTC(), nil
+	case TimeUnitNanoseconds:
+		return time.Unix(0, val).UTC(), nil
+	default:
+		return time.Time{}, errors.Errorf("Unknown time unit: '%v'", unit)
+	}
+}
+
+// Int64ToTimestamp converts the provided int64 to a timestamp based on the time unit
+// and epoch.
+func Int64ToTimestamp(unit TimeUnit, epoch time.Time, val int64) (time.Time, error) {
+	return intToTimestamp(unit, timestampToInt(unit, epoch)+val)
+}

batch/docker-compose.yml

@@ -0,0 +1,29 @@
+version: '3'
+
+services:
+  featurebase:
+    build:
+      context: ../.
+      dockerfile: ./Dockerfile-clustertests
+    environment:
+      PILOSA_DATA_DIR: /data
+      PILOSA_BIND: 0.0.0.0:10101
+      PILOSA_BIND_GRPC: 0.0.0.0:20101
+      PILOSA_ADVERTISE: featurebase:10101
+    command: /featurebase -test.run=TestRunMain -test.coverprofile=/testdata/batch_coverage.out server
+    volumes:
+      - ./testdata:/testdata
+
+  batch-test:
+    build:
+      context: ../.
+      dockerfile: ./batch/Dockerfile-test
+    volumes:
+      - ./testdata:/testdata
+
+  wait:
+    depends_on:
+      - "featurebase"
+    build:
+      context: .
+      dockerfile: Dockerfile-wait

batch/egpool/egpool.go

@@ -0,0 +1,111 @@
+// Copyright 2022 Molecula Corp. (DBA FeatureBase).
+// SPDX-License-Identifier: Apache-2.0
+package egpool
+
+import (
+	"errors"
+	"fmt"
+	"sync"
+)
+
+type Group struct {
+	PoolSize int
+
+	jobs chan func() error
+
+	sema     chan struct{}
+	errMu    sync.Mutex
+	firstErr error
+	errs     []error
+}
+
+func (eg *Group) Go(f func() error) {
+	if eg.PoolSize <= 0 {
+		eg.PoolSize = 1
+	}
+
+	if eg.jobs == nil {
+		eg.jobs = make(chan func() error)
+		eg.sema = make(chan struct{}, eg.PoolSize)
+	}
+
+	// Start the job in an idle worker if possible.
+	select {
+	case eg.jobs <- f:
+		return
+	default:
+	}
+
+	// Start a new worker if necessary.
+	select {
+	case eg.jobs <- f:
+		// A worker finished its previous job and took this one over.
+		return
+	case eg.sema <- struct{}{}:
+		// Start a new worker.
+		go eg.processJobs()
+		eg.jobs <- f
+	}
+}
+
+func (eg *Group) err(err error) {
+	eg.errMu.Lock()
+	defer eg.errMu.Unlock()
+
+	if eg.firstErr == nil {
+		eg.firstErr = err
+	}
+	eg.errs = append(eg.errs, err)
+}
+
+type PanicError struct {
+	Value interface{}
+}
+
+func (p PanicError) Error() string {
+	return fmt.Sprintf("panic: %v", p.Value)
+}
+
+var ErrGoexit = errors.New("runtime.Goexit used in job function")
+
+func (eg *Group) processJobs() {
+	// Notify pool of shutdown.
+	defer func() { <-eg.sema }()
+
+	// Handle panic and Goexit.
+	var finished bool
+	defer func() {
+		if !finished {
+			if p := recover(); p != nil {
+				eg.err(PanicError{p})
+			} else {
+				eg.err(ErrGoexit)
+			}
+		}
+	}()
+
+	// Run jobs from queue.
+	for jobFn := range eg.jobs {
+		err := jobFn()
+		if err != nil {
+			eg.err(err)
+		}
+	}
+
+	finished = true
+}
+
+func (eg *Group) Wait() error {
+	if eg.jobs == nil {
+		return nil
+	}
+	close(eg.jobs)
+	for i := 0; i < eg.PoolSize; i++ {
+		eg.sema <- struct{}{}
+	}
+	return eg.firstErr
+}
+
+func (eg *Group) Errors() []error {
+	return eg.errs
+}

batch/egpool/egpool_test.go

@@ -0,0 +1,38 @@
+// Copyright 2022 Molecula Corp. (DBA FeatureBase).
+// SPDX-License-Identifier: Apache-2.0
+package egpool_test
+
+import (
+	"errors"
+	"testing"
+
+	"github.com/featurebasedb/featurebase/v3/batch/egpool"
+)
+
+func TestEGPool(t *testing.T) {
+	eg := egpool.Group{}
+
+	a := make([]int, 10)
+
+	for i := 0; i < 10; i++ {
+		i := i
+		eg.Go(func() error {
+			a[i] = i
+			if i == 7 {
+				return errors.New("blah")
+			}
+			return nil
+		})
+	}
+
+	err := eg.Wait()
+	if err == nil || err.Error() != "blah" {
+		t.Errorf("expected err blah, got: %v", err)
+	}
+
+	for i := 0; i < 10; i++ {
+		if a[i] != i {
+			t.Errorf("expected a[%d] to be %d, but is %d", i, i, a[i])
+		}
+	}
+}

batch/error.go

@@ -0,0 +1,8 @@
+package batch
+
+import "github.com/pkg/errors"
+
+// Predefined batch-related errors.
+var (
+	ErrPreconditionFailed = errors.New("Precondition failed")
+)

batch/metrics.go

@@ -0,0 +1,3 @@
+// Copyright 2022 Molecula Corp. (DBA FeatureBase).
+// SPDX-License-Identifier: Apache-2.0
+package batch

batch/testdata/README.md

@@ -0,0 +1,3 @@
+# testdata
+
+This directory is used in CI tests. I think.

batch/wait.sh

@@ -0,0 +1,26 @@
+#!/bin/sh
+
+name=$1
+shift
+
+_start_ts=$(date +%s)
+elapsed=0
+timeout=120
+while :
+do
+    $@ > /dev/null
+    _ret=$?
+    _end_ts=$(date +%s)
+    if [ $_ret -eq 0 ]; then
+        echo "$name is available after $((_end_ts - _start_ts)) seconds."
+        break
+    else
+        echo "Waiting for $name after $((_end_ts - _start_ts)) seconds."
+    fi
+    sleep 1s
+    elapsed=$((elapsed+1))
+    if [ $elapsed -ge $timeout ]; then
+	exit 110
+    fi
+done
+set -ex

bitmap.go

@@ -1,468 +0,0 @@
-// Copyright 2017 Pilosa Corp.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package pilosa
-
-// #cgo  CFLAGS:-mpopcnt
-
-import (
-	"encoding/json"
-	"sort"
-
-	"github.com/pilosa/pilosa/internal"
-	"github.com/pilosa/pilosa/roaring"
-)
-
-// Bitmap represents a set of bits.
-type Bitmap struct {
-	segments []BitmapSegment
-
-	// Attributes associated with the bitmap.
-	Attrs map[string]interface{}
-}
-
-// NewBitmap returns a new instance of Bitmap.
-func NewBitmap(bits ...uint64) *Bitmap {
-	bm := &Bitmap{}
-	for _, i := range bits {
-		bm.SetBit(i)
-	}
-	return bm
-}
-
-// Merge merges data from other into b.
-func (b *Bitmap) Merge(other *Bitmap) {
-	var segments []BitmapSegment
-
-	itr := newMergeSegmentIterator(b.segments, other.segments)
-	for s0, s1 := itr.next(); s0 != nil || s1 != nil; s0, s1 = itr.next() {
-		// Use the other bitmap's data if segment is missing.
-		if s0 == nil {
-			segments = append(segments, *s1)
-			continue
-		} else if s1 == nil {
-			segments = append(segments, *s0)
-			continue
-		}
-
-		// Otherwise merge.
-		s0.Merge(s1)
-		segments = append(segments, *s0)
-	}
-
-	b.segments = segments
-	b.InvalidateCount()
-}
-
-// IntersectionCount returns the number of intersections between b and other.
-func (b *Bitmap) IntersectionCount(other *Bitmap) uint64 {
-	var n uint64
-
-	itr := newMergeSegmentIterator(b.segments, other.segments)
-	for s0, s1 := itr.next(); s0 != nil || s1 != nil; s0, s1 = itr.next() {
-		// Ignore non-overlapping segments.
-		if s0 == nil || s1 == nil {
-			continue
-		}
-
-		n += s0.IntersectionCount(s1)
-	}
-	return n
-}
-
-// Intersect returns the itersection of b and other.
-func (b *Bitmap) Intersect(other *Bitmap) *Bitmap {
-	var segments []BitmapSegment
-
-	itr := newMergeSegmentIterator(b.segments, other.segments)
-	for s0, s1 := itr.next(); s0 != nil || s1 != nil; s0, s1 = itr.next() {
-		// Ignore non-overlapping segments.
-		if s0 == nil || s1 == nil {
-			continue
-		}
-		segments = append(segments, *s0.Intersect(s1))
-	}
-
-	return &Bitmap{segments: segments}
-}
-
-// Xor returns the xor of b and other.
-func (b *Bitmap) Xor(other *Bitmap) *Bitmap {
-	var segments []BitmapSegment
-
-	itr := newMergeSegmentIterator(b.segments, other.segments)
-	for s0, s1 := itr.next(); s0 != nil || s1 != nil; s0, s1 = itr.next() {
-		if s1 == nil {
-			segments = append(segments, *s0)
-			continue
-		} else if s0 == nil {
-			segments = append(segments, *s1)
-			continue
-		}
-
-		segments = append(segments, *s0.Xor(s1))
-	}
-
-	return &Bitmap{segments: segments}
-}
-
-// Union returns the bitwise union of b and other.
-func (b *Bitmap) Union(other *Bitmap) *Bitmap {
-	var segments []BitmapSegment
-	itr := newMergeSegmentIterator(b.segments, other.segments)
-	for s0, s1 := itr.next(); s0 != nil || s1 != nil; s0, s1 = itr.next() {
-		if s1 == nil {
-			segments = append(segments, *s0)
-			continue
-		} else if s0 == nil {
-			segments = append(segments, *s1)
-			continue
-		}
-		segments = append(segments, *s0.Union(s1))
-	}
-
-	return &Bitmap{segments: segments}
-}
-
-// Difference returns the diff of b and other.
-func (b *Bitmap) Difference(other *Bitmap) *Bitmap {
-	var segments []BitmapSegment
-
-	itr := newMergeSegmentIterator(b.segments, other.segments)
-	for s0, s1 := itr.next(); s0 != nil || s1 != nil; s0, s1 = itr.next() {
-		if s0 == nil {
-			continue
-		} else if s1 == nil {
-			segments = append(segments, *s0)
-			continue
-		}
-		segments = append(segments, *s0.Difference(s1))
-	}
-
-	return &Bitmap{segments: segments}
-}
-
-// SetBit sets the i-th bit of the bitmap.
-func (b *Bitmap) SetBit(i uint64) (changed bool) {
-	return b.createSegmentIfNotExists(i / SliceWidth).SetBit(i)
-}
-
-// ClearBit clears the i-th bit of the bitmap.
-func (b *Bitmap) ClearBit(i uint64) (changed bool) {
-	s := b.segment(i / SliceWidth)
-	if s == nil {
-		return false
-	}
-	return s.ClearBit(i)
-}
-
-// segment returns a segment for a given slice.
-// Returns nil if segment does not exist.
-func (b *Bitmap) segment(slice uint64) *BitmapSegment {
-	if i := sort.Search(len(b.segments), func(i int) bool {
-		return b.segments[i].slice >= slice
-	}); i < len(b.segments) && b.segments[i].slice == slice {
-		return &b.segments[i]
-	}
-	return nil
-}
-
-func (b *Bitmap) createSegmentIfNotExists(slice uint64) *BitmapSegment {
-	i := sort.Search(len(b.segments), func(i int) bool {
-		return b.segments[i].slice >= slice
-	})
-
-	// Return exact match.
-	if i < len(b.segments) && b.segments[i].slice == slice {
-		return &b.segments[i]
-	}
-
-	// Insert new segment.
-	b.segments = append(b.segments, BitmapSegment{})
-	if i < len(b.segments) {
-		copy(b.segments[i+1:], b.segments[i:])
-	}
-	b.segments[i] = BitmapSegment{
-		slice:    slice,
-		writable: true,
-	}
-
-	return &b.segments[i]
-}
-
-// InvalidateCount updates the cached count in the bitmap.
-func (b *Bitmap) InvalidateCount() {
-	for i := range b.segments {
-		b.segments[i].InvalidateCount()
-	}
-}
-
-// IncrementCount increments the bitmap cached counter, note this is an optimization that assumes that the caller is aware the size increased.
-func (b *Bitmap) IncrementCount(i uint64) {
-	seg := b.segment(i / SliceWidth)
-	if seg != nil {
-		seg.n++
-	}
-
-}
-
-// DecrementCount decrements the bitmap cached counter.
-func (b *Bitmap) DecrementCount(i uint64) {
-	seg := b.segment(i / SliceWidth)
-	if seg != nil {
-		if seg.n > 0 {
-			seg.n--
-		}
-	}
-}
-
-// Count returns the number of set bits in the bitmap.
-func (b *Bitmap) Count() uint64 {
-	var n uint64
-	for i := range b.segments {
-		n += b.segments[i].Count()
-	}
-	return n
-}
-
-// MarshalJSON returns a JSON-encoded byte slice of b.
-func (b *Bitmap) MarshalJSON() ([]byte, error) {
-	var o struct {
-		Attrs map[string]interface{} `json:"attrs"`
-		Bits  []uint64               `json:"bits"`
-	}
-	o.Bits = b.Bits()
-
-	o.Attrs = b.Attrs
-	if o.Attrs == nil {
-		o.Attrs = make(map[string]interface{})
-	}
-
-	return json.Marshal(&o)
-}
-
-// Bits returns the bits in b as a slice of ints.
-func (b *Bitmap) Bits() []uint64 {
-	a := make([]uint64, 0, b.Count())
-	for i := range b.segments {
-		a = append(a, b.segments[i].Bits()...)
-	}
-	return a
-}
-
-// encodeBitmap converts b into its internal representation.
-func encodeBitmap(b *Bitmap) *internal.Bitmap {
-	if b == nil {
-		return nil
-	}
-
-	return &internal.Bitmap{
-		Bits:  b.Bits(),
-		Attrs: encodeAttrs(b.Attrs),
-	}
-}
-
-// decodeBitmap converts b from its internal representation.
-func decodeBitmap(pb *internal.Bitmap) *Bitmap {
-	if pb == nil {
-		return nil
-	}
-
-	b := NewBitmap()
-	b.Attrs = decodeAttrs(pb.Attrs)
-	for _, v := range pb.Bits {
-		b.SetBit(v)
-	}
-	return b
-}
-
-// Union performs a union on a slice of bitmaps.
-func Union(bitmaps []*Bitmap) *Bitmap {
-	other := bitmaps[0]
-	for _, bm := range bitmaps[1:] {
-		other = other.Union(bm)
-	}
-	return other
-}
-
-// BitmapSegment holds a subset of a bitmap.
-// This could point to a mmapped roaring bitmap or an in-memory bitmap. The
-// width of the segment will always match the slice width.
-type BitmapSegment struct {
-	// Slice this segment belongs to
-	slice uint64
-
-	// Underlying raw bitmap implementation.
-	// This is an mmapped bitmap if writable is false. Otherwise
-	// it is a heap allocated bitmap which can be manipulated.
-	data     roaring.Bitmap
-	writable bool
-
-	// Bit count
-	n uint64
-}
-
-// Merge adds chunks from other to s.
-// Chunks in s are overwritten if they exist in other.
-func (s *BitmapSegment) Merge(other *BitmapSegment) {
-	s.ensureWritable()
-
-	itr := other.data.Iterator()
-	for v, eof := itr.Next(); !eof; v, eof = itr.Next() {
-		s.SetBit(v)
-	}
-}
-
-// IntersectionCount returns the number of intersections between s and other.
-func (s *BitmapSegment) IntersectionCount(other *BitmapSegment) uint64 {
-	return s.data.IntersectionCount(&other.data)
-}
-
-// Intersect returns the itersection of s and other.
-func (s *BitmapSegment) Intersect(other *BitmapSegment) *BitmapSegment {
-	data := s.data.Intersect(&other.data)
-
-	return &BitmapSegment{
-		data:  *data,
-		slice: s.slice,
-		n:     data.Count(),
-	}
-}
-
-// Union returns the bitwise union of s and other.
-func (s *BitmapSegment) Union(other *BitmapSegment) *BitmapSegment {
-	data := s.data.Union(&other.data)
-
-	return &BitmapSegment{
-		data:  *data,
-		slice: s.slice,
-		n:     data.Count(),
-	}
-}
-
-// Difference returns the diff of s and other.
-func (s *BitmapSegment) Difference(other *BitmapSegment) *BitmapSegment {
-	data := s.data.Difference(&other.data)
-
-	return &BitmapSegment{
-		data:  *data,
-		slice: s.slice,
-		n:     data.Count(),
-	}
-}
-
-// Xor returns the xor of s and other.
-func (s *BitmapSegment) Xor(other *BitmapSegment) *BitmapSegment {
-	data := s.data.Xor(&other.data)
-
-	return &BitmapSegment{
-		data:  *data,
-		slice: s.slice,
-		n:     data.Count(),
-	}
-}
-
-// SetBit sets the i-th bit of the bitmap.
-func (s *BitmapSegment) SetBit(i uint64) (changed bool) {
-	s.ensureWritable()
-	changed, _ = s.data.Add(i)
-	if changed {
-		s.n++
-	}
-	return changed
-}
-
-// ClearBit clears the i-th bit of the bitmap.
-func (s *BitmapSegment) ClearBit(i uint64) (changed bool) {
-	s.ensureWritable()
-
-	changed, _ = s.data.Remove(i)
-	if changed {
-		s.n--
-	}
-	return changed
-}
-
-// InvalidateCount updates the cached count in the bitmap.
-func (s *BitmapSegment) InvalidateCount() {
-	s.n = s.data.Count()
-}
-
-// Bits returns a list of all bits set in the segment.
-func (s *BitmapSegment) Bits() []uint64 {
-	a := make([]uint64, 0, s.Count())
-	itr := s.data.Iterator()
-	for v, eof := itr.Next(); !eof; v, eof = itr.Next() {
-		a = append(a, v)
-	}
-	return a
-}
-
-// Count returns the number of set bits in the bitmap.
-func (s *BitmapSegment) Count() uint64 { return s.n }
-
-// ensureWritable clones the segment if it is pointing to non-writable data.
-func (s *BitmapSegment) ensureWritable() {
-	if s.writable {
-		return
-	}
-
-	s.data = *s.data.Clone()
-	s.writable = true
-}
-
-// mergeSegmentIterator produces an iterator that loops through two sets of segments.
-type mergeSegmentIterator struct {
-	a0, a1 []BitmapSegment
-}
-
-// newMergeSegmentIterator returns a new instance of mergeSegmentIterator.
-func newMergeSegmentIterator(a0, a1 []BitmapSegment) mergeSegmentIterator {
-	return mergeSegmentIterator{a0: a0, a1: a1}
-}
-
-// next returns the next set of segments.
-func (itr *mergeSegmentIterator) next() (s0, s1 *BitmapSegment) {
-	// Find current segments.
-	if len(itr.a0) > 0 {
-		s0 = &itr.a0[0]
-	}
-	if len(itr.a1) > 0 {
-		s1 = &itr.a1[0]
-	}
-
-	// Return if either or both are nil.
-	if s0 == nil && s1 == nil {
-		return
-	} else if s0 == nil {
-		itr.a1 = itr.a1[1:]
-		return
-	} else if s1 == nil {
-		itr.a0 = itr.a0[1:]
-		return
-	}
-
-	// Otherwise determine which is first.
-	if s0.slice < s1.slice {
-		itr.a0 = itr.a0[1:]
-		return s0, nil
-	} else if s0.slice > s1.slice {
-		itr.a1 = itr.a1[1:]
-		return s1, nil
-	}
-
-	// Return both if slices are equal.
-	itr.a0, itr.a1 = itr.a0[1:], itr.a1[1:]
-	return s0, s1
-}

bitmap_test.go

@@ -1,92 +0,0 @@
-// Copyright 2017 Pilosa Corp.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package pilosa_test
-
-import (
-	"reflect"
-	"testing"
-
-	"github.com/pilosa/pilosa"
-)
-
-// Ensure a bitmap can be merged
-func TestBitmap_Merge(t *testing.T) {
-	bm1 := pilosa.NewBitmap(1, 2, 3, SliceWidth+1, 2*SliceWidth)
-	bm2 := pilosa.NewBitmap(3, 4, 5)
-	bm1.Merge(bm2)
-
-	if bm1.Count() != 7 {
-		t.Fatalf("Count after merge %d != 7\n", bm1.Count())
-	}
-
-}
-
-// Ensure a bitmap can Xor'ed
-func TestBitmap_Xor(t *testing.T) {
-	bm1 := pilosa.NewBitmap(0, 1, SliceWidth)
-	bm2 := pilosa.NewBitmap(0, 2*SliceWidth)
-	exp := []uint64{1, SliceWidth, 2 * SliceWidth}
-
-	res := bm1.Xor(bm2)
-	if res.Count() != 3 {
-		t.Fatalf("Test 1 Count after xor %d != 3\n", res.Count())
-	}
-
-	if !reflect.DeepEqual(res.Bits(), exp) {
-		t.Fatalf("Test 2 Results %v != expected %v\n", res.Bits(), exp)
-	}
-	res = bm2.Xor(bm1)
-	if res.Count() != 3 {
-		t.Fatalf("Test 3 Count after xor %d != 3\n", res.Count())
-	}
-	if !reflect.DeepEqual(res.Bits(), exp) {
-		t.Fatalf("Test 4 Results %v != expected %v\n", res.Bits(), exp)
-	}
-
-}
-func TestBitmap_Union_Segment(t *testing.T) {
-	bm1 := pilosa.NewBitmap(0, 1, SliceWidth)
-	bm2 := pilosa.NewBitmap(0, 2*SliceWidth)
-	exp := []uint64{0, 1, SliceWidth, 2 * SliceWidth}
-	res := bm1.Union(bm2)
-
-	if res.Count() != 4 {
-		t.Fatalf("Test 1 Count after Union %d != 5\n", res.Count())
-	}
-	if !reflect.DeepEqual(res.Bits(), exp) {
-		t.Fatalf("Test 2 Union Results %v != expected %v\n", res.Bits(), exp)
-	}
-	res = bm2.Union(bm1)
-	if res.Count() != 4 {
-		t.Fatalf("Test 3 Count after xor %d != 5\n", res.Count())
-	}
-	if !reflect.DeepEqual(res.Bits(), exp) {
-		t.Fatalf("Test 2 Union Results %v != expected %v\n", res.Bits(), exp)
-	}
-}
-
-func TestBitmap_Difference_Segment(t *testing.T) {
-	bm1 := pilosa.NewBitmap(0, 1, SliceWidth)
-	bm2 := pilosa.NewBitmap(0, 2*SliceWidth)
-	exp := []uint64{1, SliceWidth}
-	res := bm1.Difference(bm2)
-
-	if res.Count() != 2 {
-		t.Fatalf("Test 1 Count after Difference %d != 5\n", res.Count())
-	}
-	if !reflect.DeepEqual(res.Bits(), exp) {
-		t.Fatalf("Test 2 Difference Results %v != expected %v\n", res.Bits(), exp)
-	}
-}

broadcast.go

@@ -1,181 +1,165 @@
-// Copyright 2017 Pilosa Corp.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
+// Copyright 2022 Molecula Corp. (DBA FeatureBase).
+// SPDX-License-Identifier: Apache-2.0
 package pilosa
 
 import (
 	"fmt"
-	"reflect"
 
-	"github.com/gogo/protobuf/proto"
-	"github.com/pilosa/pilosa/internal"
+	"github.com/featurebasedb/featurebase/v3/disco"
+	"github.com/pkg/errors"
 )
 
-// NodeSet represents an interface for Node membership and inter-node communication.
-type NodeSet interface {
-	// Returns a list of all Nodes in the cluster
-	Nodes() []*Node
-
-	// Open starts any network activity implemented by the NodeSet
-	Open() error
+// Serializer is an interface for serializing pilosa types to bytes and back.
+type Serializer interface {
+	Marshal(Message) ([]byte, error)
+	Unmarshal([]byte, Message) error
 }
 
-// StaticNodeSet represents a basic NodeSet for testing.
-type StaticNodeSet struct {
-	nodes []*Node
+// NopSerializer represents a Serializer that doesn't do anything.
+var NopSerializer Serializer = &nopSerializer{}
+
+type nopSerializer struct{}
+
+// Marshal is a no-op implementation of Serializer Marshal method.
+func (*nopSerializer) Marshal(Message) ([]byte, error) { return nil, nil }
+
+// Unmarshal is a no-op implementation of Serializer Unmarshal method.
+func (*nopSerializer) Unmarshal([]byte, Message) error { return nil }
+
+// broadcaster is an interface for broadcasting messages.
+type broadcaster interface {
+	SendSync(Message) error
+	SendAsync(Message) error
+	SendTo(*disco.Node, Message) error
 }
 
-// NewStaticNodeSet creates a statically defined NodeSet.
-func NewStaticNodeSet() *StaticNodeSet {
-	return &StaticNodeSet{}
-}
-
-// Nodes implements the NodeSet interface and returns a list of nodes in the cluster.
-func (s *StaticNodeSet) Nodes() []*Node {
-	return s.nodes
-}
-
-// Open implements the NodeSet interface to start network activity, but for a static NodeSet it does nothing.
-func (s *StaticNodeSet) Open() error {
-	return nil
-}
-
-// Join sets the NodeSet nodes to the slice of Nodes passed in.
-func (s *StaticNodeSet) Join(nodes []*Node) error {
-	s.nodes = nodes
-	return nil
-}
-
-// Broadcaster is an interface for broadcasting messages.
-type Broadcaster interface {
-	SendSync(pb proto.Message) error
-	SendAsync(pb proto.Message) error
-}
-
-func init() {
-	NopBroadcaster = &nopBroadcaster{}
-}
+// Message is the interface implemented by all core pilosa types which can be serialized to messages.
+// TODO add at least a single "isMessage()" method.
+type Message interface{}
 
 // NopBroadcaster represents a Broadcaster that doesn't do anything.
-var NopBroadcaster Broadcaster
+var NopBroadcaster broadcaster = &nopBroadcaster{}
 
 type nopBroadcaster struct{}
 
-// SendSync A no-op implemenetation of Broadcaster SendSync method.
-func (c *nopBroadcaster) SendSync(pb proto.Message) error {
-	return nil
-}
+// SendSync A no-op implementation of Broadcaster SendSync method.
+func (nopBroadcaster) SendSync(Message) error { return nil }
 
-// SendAsync A no-op implemenetation of Broadcaster SendAsync method.
-func (c *nopBroadcaster) SendAsync(pb proto.Message) error {
-	return nil
-}
+// SendAsync A no-op implementation of Broadcaster SendAsync method.
+func (nopBroadcaster) SendAsync(Message) error { return nil }
 
-// BroadcastHandler is the interface for the pilosa object which knows how to
-// handle broadcast messages. (Hint: this is implemented by pilosa.Server)
-type BroadcastHandler interface {
-	ReceiveMessage(pb proto.Message) error
-}
-
-// BroadcastReceiver is the interface for the object which will listen for and
-// decode broadcast messages before passing them to pilosa to handle. The
-// implementation of this could be an http server which listens for messages,
-// gets the protobuf payload, and then passes it to
-// BroadcastHandler.ReceiveMessage.
-type BroadcastReceiver interface {
-	// Start starts listening for broadcast messages - it should return
-	// immediately, spawning a goroutine if necessary.
-	Start(BroadcastHandler) error
-}
-
-type nopBroadcastReceiver struct{}
-
-func (n *nopBroadcastReceiver) Start(b BroadcastHandler) error { return nil }
-
-// NopBroadcastReceiver is a no-op implementation of the BroadcastReceiver.
-var NopBroadcastReceiver = &nopBroadcastReceiver{}
+// SendTo is a no-op implementation of Broadcaster SendTo method.
+func (nopBroadcaster) SendTo(*disco.Node, Message) error { return nil }
 
 // Broadcast message types.
 const (
-	MessageTypeCreateSlice           = 1
-	MessageTypeCreateIndex           = 2
-	MessageTypeDeleteIndex           = 3
-	MessageTypeCreateFrame           = 4
-	MessageTypeDeleteFrame           = 5
-	MessageTypeCreateInputDefinition = 6
-	MessageTypeDeleteInputDefinition = 7
-	MessageTypeDeleteView            = 8
+	messageTypeCreateShard = iota
+	messageTypeCreateIndex
+	messageTypeDeleteIndex
+	messageTypeCreateField
+	messageTypeDeleteField
+	messageTypeCreateView
+	messageTypeDeleteView
+	messageTypeClusterStatus
+	messageTypeUNUSED0 // used to be ResizeInstruction
+	messageTypeUNUSED1 // used to be ResizeInstructionComplete
+	messageTypeNodeState
+	messageTypeRecalculateCaches
+	messageTypeLoadSchemaMessage
+	messageTypeNodeEvent
+	messageTypeNodeStatus
+	messageTypeTransaction
+	messageTypeUNUSED2 // used to be ResizeNodeMessage
+	messageTypeUNUSED3 // used to be ResizeAbortMessage
+	messageTypeUpdateField
+	messageTypeDeleteDataframe
 )
 
-// MarshalMessage encodes the protobuf message into a byte slice.
-func MarshalMessage(m proto.Message) ([]byte, error) {
-	var typ uint8
-	switch obj := m.(type) {
-	case *internal.CreateSliceMessage:
-		typ = MessageTypeCreateSlice
-	case *internal.CreateIndexMessage:
-		typ = MessageTypeCreateIndex
-	case *internal.DeleteIndexMessage:
-		typ = MessageTypeDeleteIndex
-	case *internal.CreateFrameMessage:
-		typ = MessageTypeCreateFrame
-	case *internal.DeleteFrameMessage:
-		typ = MessageTypeDeleteFrame
-	case *internal.CreateInputDefinitionMessage:
-		typ = MessageTypeCreateInputDefinition
-	case *internal.DeleteInputDefinitionMessage:
-		typ = MessageTypeDeleteInputDefinition
-	case *internal.DeleteViewMessage:
-		typ = MessageTypeDeleteView
-	default:
-		return nil, fmt.Errorf("message type not implemented for marshalling: %s", reflect.TypeOf(obj))
-	}
-	buf, err := proto.Marshal(m)
+// MarshalInternalMessage serializes the pilosa message and adds pilosa internal
+// type info which is used by the internal messaging stuff.
+func MarshalInternalMessage(m Message, s Serializer) ([]byte, error) {
+	typ := getMessageType(m)
+	buf, err := s.Marshal(m)
 	if err != nil {
-		return nil, err
+		return nil, errors.Wrap(err, "marshaling")
 	}
 	return append([]byte{typ}, buf...), nil
 }
 
-// UnmarshalMessage decodes the byte slice into a protobuf message.
-func UnmarshalMessage(buf []byte) (proto.Message, error) {
-	typ, buf := buf[0], buf[1:]
-
-	var m proto.Message
+func getMessage(typ byte) Message {
 	switch typ {
-	case MessageTypeCreateSlice:
-		m = &internal.CreateSliceMessage{}
-	case MessageTypeCreateIndex:
-		m = &internal.CreateIndexMessage{}
-	case MessageTypeDeleteIndex:
-		m = &internal.DeleteIndexMessage{}
-	case MessageTypeCreateFrame:
-		m = &internal.CreateFrameMessage{}
-	case MessageTypeDeleteFrame:
-		m = &internal.DeleteFrameMessage{}
-	case MessageTypeCreateInputDefinition:
-		m = &internal.CreateInputDefinitionMessage{}
-	case MessageTypeDeleteInputDefinition:
-		m = &internal.DeleteInputDefinitionMessage{}
-	case MessageTypeDeleteView:
-		m = &internal.DeleteViewMessage{}
+	case messageTypeCreateShard:
+		return &CreateShardMessage{}
+	case messageTypeCreateIndex:
+		return &CreateIndexMessage{}
+	case messageTypeDeleteIndex:
+		return &DeleteIndexMessage{}
+	case messageTypeCreateField:
+		return &CreateFieldMessage{}
+	case messageTypeDeleteField:
+		return &DeleteFieldMessage{}
+	case messageTypeCreateView:
+		return &CreateViewMessage{}
+	case messageTypeDeleteView:
+		return &DeleteViewMessage{}
+	case messageTypeClusterStatus:
+		return &ClusterStatus{}
+	case messageTypeNodeState:
+		return &NodeStateMessage{}
+	case messageTypeRecalculateCaches:
+		return &RecalculateCaches{}
+	case messageTypeLoadSchemaMessage:
+		return &LoadSchemaMessage{}
+	case messageTypeNodeEvent:
+		return &NodeEvent{}
+	case messageTypeNodeStatus:
+		return &NodeStatus{}
+	case messageTypeTransaction:
+		return &TransactionMessage{}
+	case messageTypeUpdateField:
+		return &UpdateFieldMessage{}
+	case messageTypeDeleteDataframe:
+		return &DeleteDataframeMessage{}
 	default:
-		return nil, fmt.Errorf("invalid message type: %d", typ)
+		panic(fmt.Sprintf("unknown message type %d", typ))
+	}
+}
+
+func getMessageType(m Message) byte {
+	switch m.(type) {
+	case *CreateShardMessage:
+		return messageTypeCreateShard
+	case *CreateIndexMessage:
+		return messageTypeCreateIndex
+	case *DeleteIndexMessage:
+		return messageTypeDeleteIndex
+	case *CreateFieldMessage:
+		return messageTypeCreateField
+	case *DeleteFieldMessage:
+		return messageTypeDeleteField
+	case *CreateViewMessage:
+		return messageTypeCreateView
+	case *DeleteViewMessage:
+		return messageTypeDeleteView
+	case *ClusterStatus:
+		return messageTypeClusterStatus
+	case *NodeStateMessage:
+		return messageTypeNodeState
+	case *RecalculateCaches:
+		return messageTypeRecalculateCaches
+	case *LoadSchemaMessage:
+		return messageTypeLoadSchemaMessage
+	case *NodeEvent:
+		return messageTypeNodeEvent
+	case *NodeStatus:
+		return messageTypeNodeStatus
+	case *TransactionMessage:
+		return messageTypeTransaction
+	case *UpdateFieldMessage:
+		return messageTypeUpdateField
+	case *DeleteDataframeMessage:
+		return messageTypeDeleteDataframe
+	default:
+		panic(fmt.Sprintf("don't have type for message %#v", m))
 	}
-
-	if err := proto.Unmarshal(buf, m); err != nil {
-		return nil, err
-	}
-	return m, nil
 }

broadcast_test.go

@@ -1,105 +0,0 @@
-// Copyright 2017 Pilosa Corp.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package pilosa_test
-
-import (
-	"reflect"
-	"testing"
-
-	"github.com/gogo/protobuf/proto"
-	"github.com/pilosa/pilosa"
-	"github.com/pilosa/pilosa/internal"
-)
-
-// Ensure a message can be marshaled and unmarshaled.
-func TestMessage_Marshal(t *testing.T) {
-
-	testMessageMarshal(t, &internal.CreateSliceMessage{
-		Index: "i",
-		Slice: 8,
-	})
-
-	testMessageMarshal(t, &internal.DeleteIndexMessage{
-		Index: "i",
-	})
-}
-
-func testMessageMarshal(t *testing.T, m proto.Message) {
-	marshalled, err := pilosa.MarshalMessage(m)
-	if err != nil {
-		t.Fatal(err)
-	}
-	unmarshalled, err := pilosa.UnmarshalMessage(marshalled)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if !reflect.DeepEqual(unmarshalled, m) {
-		t.Fatalf("unexpected message marshalling: %s", unmarshalled)
-	}
-}
-
-// Ensure that BroadcastReceiver can register a BroadcastHandler.
-func TestBroadcast_BroadcastReceiver(t *testing.T) {
-
-	s := pilosa.NewServer()
-
-	sbr := NewSimpleBroadcastReceiver()
-	sbh := NewSimpleBroadcastHandler()
-
-	s.BroadcastReceiver = sbr
-	s.BroadcastReceiver.Start(sbh)
-
-	msg := &internal.DeleteIndexMessage{
-		Index: "i",
-	}
-
-	s.BroadcastReceiver.(*SimpleBroadcastReceiver).Receive(msg)
-
-	// Make sure the message received is what was sentd
-	if !reflect.DeepEqual(sbh.receivedMessage, msg) {
-		t.Fatalf("unexpected message: %s", sbh.receivedMessage)
-	}
-}
-
-type SimpleBroadcastReceiver struct {
-	broadcastHandler pilosa.BroadcastHandler
-}
-
-func NewSimpleBroadcastReceiver() *SimpleBroadcastReceiver {
-	return &SimpleBroadcastReceiver{}
-}
-
-func (r *SimpleBroadcastReceiver) Start(h pilosa.BroadcastHandler) error {
-	r.broadcastHandler = h
-	return nil
-}
-
-func (r *SimpleBroadcastReceiver) Receive(pb proto.Message) error {
-	r.broadcastHandler.ReceiveMessage(pb)
-	return nil
-}
-
-type SimpleBroadcastHandler struct {
-	receivedMessage proto.Message
-}
-
-func NewSimpleBroadcastHandler() *SimpleBroadcastHandler {
-	return &SimpleBroadcastHandler{}
-}
-
-func (h *SimpleBroadcastHandler) ReceiveMessage(pb proto.Message) error {
-	h.receivedMessage = pb.(proto.Message)
-	return nil
-}

bsi.go

@@ -0,0 +1,284 @@
+// Copyright 2022 Molecula Corp. (DBA FeatureBase).
+// SPDX-License-Identifier: Apache-2.0
+package pilosa
+
+import (
+	"math/bits"
+
+	"github.com/featurebasedb/featurebase/v3/roaring"
+)
+
+// BSIData contains BSI-structured data.
+type BSIData []*Row
+
+// PivotDescending loops over nonzero BSI values in descending order.
+// For each value, the provided function is called with the value and a slice of the associated columns.
+// If limit or offset are not-nil, they will be applied.
+// Applying a limit or offset may modify the pointed-to value.
+func (bsi BSIData) PivotDescending(filter *Row, branch uint64, limit, offset *uint64, fn func(uint64, ...uint64)) {
+	// This "pivot" algorithm works by treating the BSI data as a tree.
+	// Each branch of this tree corresponds to a power-of-2-sized range of BSI values.
+	// Each range is subdivided into 2 ranges of half size, which form lower branches.
+	// Eventually, a range of width 1 cannot be subdivided and forms a leaf.
+	// At each branch and leaf, there is a bitmap of all columns within the corresponding range.
+	// The lower branches are formed as a difference or intersect of the upper branch's bitmap with the BSI bit that subdivides the range.
+	// This function uses a depth-first search over this virtual tree.
+
+	switch {
+	case !filter.Any():
+		// There are no remaining data.
+
+	case offset != nil && *offset >= filter.Count():
+		// Skip this entire branch.
+		*offset -= filter.Count()
+
+	case limit != nil && *limit == 0:
+		// The limit has been reached.
+		// No more data is necessary.
+
+	case len(bsi) == 0:
+		// This is a leaf node.
+		cols := filter.Columns()
+		if offset != nil {
+			cols = cols[*offset:]
+			*offset = 0
+		}
+		if limit != nil {
+			if *limit < uint64(len(cols)) {
+				cols = cols[:*limit]
+			}
+			*limit -= uint64(len(cols))
+		}
+		fn(branch, cols...)
+
+	default:
+		// Pivot over the highest bit.
+		upperBranch, lowerBranch := branch|(1<<uint(len(bsi)-1)), branch
+		splitBit := bsi[len(bsi)-1]
+		lowerBits := bsi[:len(bsi)-1]
+		lowerBits.PivotDescending(filter.Intersect(splitBit), upperBranch, limit, offset, fn)
+		lowerBits.PivotDescending(filter.Difference(splitBit), lowerBranch, limit, offset, fn)
+	}
+}
+
+/*
+// distribution generates a BSI histogram for the input.
+// TODO: I forgot what I was going to use this for.
+// Could probbably use this for:
+// - quartile queries
+// - TopN on int
+func (bsi bsiData) distribution(filter *Row) bsiData {
+	var dist bsiData
+	bsi.PivotDescending(filter, 0, nil, nil, func(count uint64, values ...uint64) {
+		dist.insert(count, uint64(len(values)))
+	})
+	return dist
+}
+*/
+
+var placeholderBitmap = roaring.NewBitmap()
+
+// AddBSI adds two BSI bitmaps together.
+// It does not handle sign and has no concept of overflow.
+func AddBSI(x, y BSIData) BSIData {
+	// Accumulate row segments.
+	segments := make([][]RowSegment, len(x)+len(y))
+	xsegs, ysegs := segments[:len(x)], segments[len(x):]
+	for i, r := range x {
+		xsegs[i] = r.Segments
+	}
+	for i, r := range y {
+		ysegs[i] = r.Segments
+	}
+
+	var dst BSIData
+	var xbitmaps, ybitmaps []*roaring.Bitmap
+	for {
+		// Find the next shard.
+		next := ^uint64(0)
+		for _, s := range segments {
+			if len(s) == 0 {
+				continue
+			}
+			shard := s[0].shard
+			if shard < next {
+				next = shard
+			}
+		}
+		if next == ^uint64(0) {
+			// There are no remaining shards.
+			break
+		}
+
+		// Accumulate bitmaps for this shard.
+		xbitmaps, ybitmaps = xbitmaps[:0], ybitmaps[:0]
+		for i, segs := range xsegs {
+			if len(segs) == 0 || segs[0].shard != next {
+				continue
+			}
+			xsegs[i] = segs[1:]
+			bm := segs[0].data
+			if !bm.Any() {
+				continue
+			}
+			for len(xbitmaps) < i {
+				xbitmaps = append(xbitmaps, placeholderBitmap)
+			}
+			xbitmaps = append(xbitmaps, bm)
+		}
+		for i, segs := range ysegs {
+			if len(segs) == 0 || segs[0].shard != next {
+				continue
+			}
+			ysegs[i] = segs[1:]
+			bm := segs[0].data
+			if !bm.Any() {
+				continue
+			}
+			for len(ybitmaps) < i {
+				ybitmaps = append(ybitmaps, placeholderBitmap)
+			}
+			ybitmaps = append(ybitmaps, bm)
+		}
+
+		// Add the shard values together.
+		var out []*roaring.Bitmap
+		switch {
+		case len(xbitmaps) == 0:
+			// There are no values in x.
+			out = ybitmaps
+		case len(ybitmaps) == 0:
+			// There are no values in y.
+			out = xbitmaps
+		default:
+			out = roaring.Add(xbitmaps, ybitmaps)
+		}
+
+		// Convert the bitmaps to output segments.
+		for i, b := range out {
+			if !b.Any() {
+				continue
+			}
+			for len(dst) <= i {
+				dst = append(dst, NewRow())
+			}
+			dst[i].Segments = append(dst[i].Segments, RowSegment{
+				shard:    next,
+				writable: true,
+				data:     b,
+				n:        b.Count(),
+			})
+		}
+	}
+
+	return dst
+}
+
+// rowBuilder builds a row quickly from individual values.
+// It is optimized for the case in which values are generated sequentially.
+type rowBuilder struct {
+	bm    *roaring.Bitmap
+	mask  *[1024]uint64
+	array []uint16
+	key   uint64
+	n     int32
+}
+
+// flushKey flushes the data at the current key to the bitmap.
+func (b *rowBuilder) flushKey() {
+	var c *roaring.Container
+	switch {
+	case b.mask != nil:
+		c = roaring.NewContainerBitmapN(b.mask[:], b.n)
+		b.mask = nil
+	case len(b.array) > 0:
+		c = roaring.NewContainerArrayCopy(b.array)
+		b.array = b.array[:0]
+	default:
+		return
+	}
+
+	if b.bm == nil {
+		b.bm = roaring.NewBitmap()
+	}
+	if old := b.bm.Containers.Get(b.key); old != nil {
+		c = roaring.Union(c, old)
+	}
+	b.bm.Containers.Put(b.key, c)
+}
+
+// Add a value to the bitmap.
+// Values must be added sequentially.
+func (b *rowBuilder) Add(v uint64) {
+	vkey := v / (1 << 16)
+	if b.key != vkey {
+		// This is a new key, so flush the old one.
+		b.flushKey()
+		b.key = vkey
+	}
+
+	if b.mask != nil {
+		// Add to the mask.
+		b.n += int32(1 &^ (b.mask[uint16(v)/64] >> (v % 64)))
+		b.mask[uint16(v)/64] |= 1 << (v % 64)
+		return
+	}
+
+	// Add to an array.
+	b.array = append(b.array, uint16(v))
+	if len(b.array) >= roaring.ArrayMaxSize {
+		// The array is too big.
+		// Convert it to a bitmask.
+		m := [1024]uint64{}
+		for _, v := range b.array {
+			m[v/64] |= 1 << (v % 64)
+		}
+		b.n = int32(len(b.array))
+		b.array = b.array[:0]
+		b.mask = &m
+	}
+}
+
+// Build a Row from stored data.
+// This resets the builder.
+func (b *rowBuilder) Build() *Row {
+	// Flush the active key to the bitmap.
+	b.flushKey()
+
+	// Remove the bitmap and convert it to a Row.
+	bm := b.bm
+	b.bm = nil
+	if bm == nil {
+		return NewRow()
+	}
+	return NewRowFromBitmap(bm)
+}
+
+// bsiBuilder assembles BSI data.
+// It is optimized for the case in which values are generated sequentially.
+type bsiBuilder []rowBuilder
+
+// Insert a value into the BSI data.
+// Columns must be inserted sequentially, and duplicates are not allowed.
+func (b *bsiBuilder) Insert(col, val uint64) {
+	for val != 0 {
+		i := bits.TrailingZeros64(val)
+		val &^= 1 << i
+		for len(*b) <= i {
+			*b = append(*b, rowBuilder{})
+		}
+		(*b)[i].Add(col)
+	}
+}
+
+// Build BSI data.
+// This resets the builder.
+func (b *bsiBuilder) Build() BSIData {
+	builders := *b
+	*b = builders[:0]
+	rows := make(BSIData, len(builders))
+	for i := range builders {
+		rows[i] = builders[i].Build()
+	}
+	return rows
+}

bsi_test.go

@@ -0,0 +1,160 @@
+// Copyright 2022 Molecula Corp. (DBA FeatureBase).
+// SPDX-License-Identifier: Apache-2.0
+package pilosa
+
+import (
+	"fmt"
+	"math/rand"
+	"sort"
+	"testing"
+)
+
+// TestBSIAdd does a number of iterations. For each iteration, it
+// generates a random number of ids, and two random values for each id
+// to add together.
+func TestBSIAdd(t *testing.T) {
+	// TODO wouldn't it be cool if our test suite had a randomized
+	// burn-in mode where you could run any test which supported it
+	// with a random seed and way more iterations?
+	rnd := rand.New(rand.NewSource(99))
+	//numZipf := rand.NewZipf(rnd, 1.5, 2, ShardWidth-1)
+	idZipf := rand.NewZipf(rnd, 1.8, 4, ShardWidth)
+	var builderA, builderB bsiBuilder
+	// a and b are generated slices of numbers to add together
+	var a, b []uint64
+	// idToIndex maps record ids to indexes in a and b
+	idToIndex := make(map[int]int)
+	// indexToID has the record id for each value in a and b
+	indexToID := []uint64{}
+
+	min := 999999999
+	max := 0
+
+	for iteration := 0; iteration < 1; iteration++ {
+		t.Run(fmt.Sprintf("%d", iteration), func(t *testing.T) {
+			// reset generated data
+			a, b = a[:0], b[:0]
+			indexToID = indexToID[:0]
+			for k := range idToIndex {
+				delete(idToIndex, k)
+			}
+
+			// z generates the values, they can be fairly large, but are usually small
+			z := rand.NewZipf(rnd, 1.3, 7, 1<<44)
+			id := -1
+			for i := 0; true; i++ {
+				// get the next id, skipping a random amount
+				id = id + int(idZipf.Uint64()+1)
+				if id >= ShardWidth {
+					if i < min {
+						min = i
+					}
+					if max < i {
+						max = i
+					}
+					break
+				}
+				idToIndex[id] = int(i)
+				indexToID = append(indexToID, uint64(id))
+
+				// append a random value to each data slice
+				a = append(a, z.Uint64())
+				b = append(b, z.Uint64())
+			}
+
+			// build the BSIs based on the data slices and generated IDs
+			for index, id := range indexToID {
+				va, vb := a[index], b[index]
+				builderA.Insert(uint64(id), va)
+				builderB.Insert(uint64(id), vb)
+			}
+			dataA, dataB := builderA.Build(), builderB.Build()
+			dataC := AddBSI(dataA, dataB)
+
+			// build results from added bsiData; results[i] should hold a[i]+b[i]
+			results := make([]uint64, len(a))
+			dataC.PivotDescending(NewRow().Union(dataC...), 0, nil, nil, func(count uint64, ids ...uint64) {
+				for _, id := range ids {
+					results[idToIndex[int(id)]] = count
+				}
+			})
+
+			for i, res := range results {
+				if res != a[i]+b[i] {
+					t.Errorf("Mismatch at %d\na: %v\nb: %v\nr: %v", i, a, b, results)
+				}
+			}
+		})
+	}
+}
+
+type bsiAddCase struct {
+	positions []uint64
+	a         []uint64
+	b         []uint64
+}
+
+func (b bsiAddCase) Len() int {
+	return len(b.positions)
+}
+
+// Less reports whether the element with
+// index i should sort before the element with index j.
+func (b bsiAddCase) Less(i, j int) bool {
+	return b.positions[i] < b.positions[j]
+}
+
+// Swap swaps the elements with indexes i and j.
+func (b bsiAddCase) Swap(i, j int) {
+	b.positions[i], b.positions[j] = b.positions[j], b.positions[i]
+	b.a[i], b.a[j] = b.a[j], b.a[i]
+	b.b[i], b.b[j] = b.b[j], b.b[i]
+}
+
+// TestBSIAddCases tests specific cases of bsiAdd (would generally be
+// pulled from randomly generated ones from TestBSIAdd upon failure).
+func TestBSIAddCases(t *testing.T) {
+	tests := []bsiAddCase{
+		{
+			positions: []uint64{161311, 611110, 82544, 996022, 836077, 64964, 480737, 156534, 240525, 580896, 239236, 54607, 1019438, 894260, 17570, 884645, 936658, 682651, 987695, 390274},
+			a:         []uint64{17, 1, 2846, 45437619, 23781, 36, 88, 168691, 13417, 1301, 10, 71, 0, 176, 1010, 21, 1, 509, 17, 4},
+			b:         []uint64{24, 288, 12737, 14, 150, 21, 24, 354, 0, 19, 5, 150, 3940, 121, 25, 621, 7, 9023592401, 6033, 7},
+		},
+		{
+			positions: []uint64{17570, 54607},
+			a:         []uint64{1010, 71},
+			b:         []uint64{25, 150},
+		},
+	}
+
+	var builderA, builderB bsiBuilder
+	for i, tst := range tests {
+		t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
+			if len(tst.a) != len(tst.b) || len(tst.a) != len(tst.positions) {
+				t.Fatalf("Malformed test, a is %d, but b is %d", len(tst.a), len(tst.b))
+			}
+			sort.Sort(tst)
+
+			for i := 0; i < len(tst.a); i++ {
+				builderA.Insert(tst.positions[i], tst.a[i])
+				builderB.Insert(tst.positions[i], tst.b[i])
+			}
+
+			dataA, dataB := builderA.Build(), builderB.Build()
+			dataC := AddBSI(dataA, dataB)
+			// maps id to count
+			results := make(map[uint64]uint64)
+			dataC.PivotDescending(NewRow().Union(dataC...), 0, nil, nil, func(count uint64, ids ...uint64) {
+				for _, id := range ids {
+					results[id] = count
+				}
+			})
+
+			for i, id := range tst.positions {
+				if results[id] != tst.a[i]+tst.b[i] {
+					t.Fatalf("value %d mismatch, id: %d. got %d, want %d", i, id, results[id], tst.a[i]+tst.b[i])
+				}
+			}
+		})
+	}
+}

buffer/filebuffer.go

@@ -0,0 +1,110 @@
+package buffer
+
+import (
+	"bytes"
+	"io"
+	"io/ioutil"
+	"os"
+	"sync"
+)
+
+// NewFileBuffer returns a file buffer which will use an in-memory buffer, until `max` bytes have been written, at which point it will write the contents of memory to a file, and continue writing future data to the file.
+// The file will be written to `temp` directory. The buffer fulfills the io.Reader and io.Writer interface
+func NewFileBuffer(max int, temp string) *FileBuffer {
+	return &FileBuffer{max: max, tempDir: temp}
+}
+
+type FileBuffer struct {
+	max     int
+	buf     bytes.Buffer
+	file    *os.File
+	tempDir string
+	reading bool
+	files   []*os.File
+	mu      sync.Mutex
+}
+
+func (fb *FileBuffer) Write(p []byte) (n int, err error) {
+	if fb.reading {
+		panic("cannot write after read")
+	}
+	if fb.file != nil {
+		return fb.file.Write(p)
+	}
+	n, err = fb.buf.Write(p)
+	if err != nil {
+		return
+	}
+	if fb.buf.Len() > fb.max {
+		fb.file, err = ioutil.TempFile(fb.tempDir, "filebuffer-")
+		if err != nil {
+			return
+		}
+		_, err = io.Copy(fb.file, &fb.buf)
+		fb.buf.Reset()
+	}
+	return
+}
+
+func (fb *FileBuffer) Len() (int64, error) {
+	if fb.file == nil {
+		return int64(fb.buf.Len()), nil
+	}
+	fi, err := fb.file.Stat()
+	if err != nil {
+		return 0, err
+	}
+
+	return fi.Size(), nil
+}
+
+func (fb *FileBuffer) Read(p []byte) (n int, err error) {
+	if fb.file != nil {
+		if !fb.reading {
+			fb.reading = true
+			_, err = fb.file.Seek(0, 0)
+			if err != nil {
+				return
+			}
+		}
+		return fb.file.Read(p)
+	}
+	fb.reading = true
+	return fb.buf.Read(p)
+}
+
+func (fb *FileBuffer) Close() error {
+	if fb.file != nil {
+		name := fb.file.Name()
+		if err := fb.file.Close(); err != nil {
+			return err
+		}
+		for _, f := range fb.files {
+			f.Close()
+		}
+		fb.files = fb.files[:0]
+		fb.file = nil
+		return os.Remove(name)
+	}
+	return nil
+}
+
+func (fb *FileBuffer) Reset() error {
+	fb.mu.Lock()
+	defer fb.mu.Unlock()
+	fb.reading = false
+	fb.buf.Reset()
+	return fb.Close()
+}
+
+func (fb *FileBuffer) NewReader() (io.Reader, error) {
+	fb.mu.Lock()
+	defer fb.mu.Unlock()
+	fb.reading = true
+	if fb.file == nil {
+		return bytes.NewReader(fb.buf.Bytes()), nil
+	}
+	f, err := os.OpenFile(fb.file.Name(), os.O_RDONLY, 0)
+	fb.files = append(fb.files, f)
+	return f, err
+}

bufferpool/bufferpool.go

@@ -0,0 +1,262 @@
+package bufferpool
+
+import (
+	"errors"
+	"fmt"
+	"sync"
+)
+
+// FrameID is the type for frame id
+type FrameID int
+
+// PageID is the type for page id
+type PageID int
+
+var pageSyncPool = sync.Pool{
+	New: func() any {
+		pg := new(Page)
+		pg.id = PageID(INVALID_PAGE)
+		pg.isDirty = false
+		pg.pinCount = 0
+		return pg
+	},
+}
+
+// BufferPool represents a buffer pool of pages
+type BufferPool struct {
+	// the underlying storage
+	diskManager DiskManager
+	// the actual pages in the buffer pool
+	pages []*Page
+	// the replacer that will elect replacements when buffer pool is full
+	replacer *ClockReplacer
+	// the list of free frames
+	freeList []FrameID
+	// the map of frames to page ids to frame ids
+	// frame ids are the offset into pages
+	// if you ask the pool for page 673, this will know at
+	// what offset in pages page 673 will exist
+	pageTable map[PageID]FrameID
+}
+
+// TODO(pok) implement a lazy writer
+//  * if free list is 'low' then
+//		* increase size of cache if there is physical memory available
+//		* write out old pages and boot them from the cache to increase free list
+
+// TODO(pok) implement a checkpoint that scans the pool and writes out dirty pages every
+// minute or so
+
+// NewBufferPool returns a buffer pool
+func NewBufferPool(maxSize int, diskManager DiskManager) *BufferPool {
+	freeList := make([]FrameID, 0)
+	pages := make([]*Page, maxSize)
+	for i := 0; i < maxSize; i++ {
+		frameNumber := FrameID(i)
+		freeList = append(freeList, frameNumber)
+	}
+	clockReplacer := NewClockReplacer(maxSize)
+	return &BufferPool{
+		diskManager: diskManager,
+		pages:       pages,
+		replacer:    clockReplacer,
+		freeList:    freeList,
+		pageTable:   make(map[PageID]FrameID),
+	}
+}
+
+// Dumps all the pages in the buffer pool
+func (b *BufferPool) Dump() {
+	fmt.Println()
+	fmt.Printf("------------------------------------------------------------------------------------------\n")
+	fmt.Printf("BUFFER POOL\n")
+	for _, p := range b.pages {
+		if p != nil {
+			p.Dump("")
+		}
+	}
+	fmt.Printf("------------------------------------------------------------------------------------------\n")
+	fmt.Println()
+}
+
+// FetchPage fetches the requested page from the buffer pool.
+func (b *BufferPool) FetchPage(pageID PageID) (*Page, error) {
+	// if it is in buffer pool already then just return it
+	if frameID, ok := b.pageTable[pageID]; ok {
+		page := b.pages[frameID]
+		page.pinCount++
+		b.replacer.Pin(frameID)
+		return page, nil
+	}
+
+	// not in the buffer pool so try the free list or
+	// the replacer will vote a page off the island
+	frameID, isFromFreeList, err := b.getFrameID()
+	if err != nil {
+		return nil, err
+	}
+
+	if !isFromFreeList {
+		// if it didn't come from the freelist then
+		// remove page from current frame, writing it out if dirty
+		currentPage := b.pages[frameID]
+		if currentPage != nil {
+			if currentPage.isDirty {
+				b.diskManager.WritePage(currentPage)
+			}
+
+			delete(b.pageTable, currentPage.id)
+		}
+	}
+
+	// if we got to here, sorry, have to do an I/O
+	page, err := b.diskManager.ReadPage(pageID)
+	if err != nil {
+		return nil, err
+	}
+	page.pinCount = 1
+	b.pageTable[pageID] = frameID
+	pageSyncPool.Put(b.pages[frameID])
+	b.pages[frameID] = page
+	b.replacer.Pin(frameID)
+
+	return page, nil
+}
+
+// UnpinPage unpins the target page from the buffer pool
+func (b *BufferPool) UnpinPage(pageID PageID) error {
+	if frameID, ok := b.pageTable[pageID]; ok {
+		page := b.pages[frameID]
+		page.DecPinCount()
+
+		if page.pinCount <= 0 {
+			b.replacer.Unpin(frameID)
+		}
+		return nil
+	}
+
+	return errors.New("could not find page")
+}
+
+// FlushPage Flushes the target page to disk
+func (b *BufferPool) FlushPage(pageID PageID) bool {
+	if frameID, ok := b.pageTable[pageID]; ok {
+		page := b.pages[frameID]
+		page.DecPinCount()
+
+		b.diskManager.WritePage(page)
+		page.isDirty = false
+
+		return true
+	}
+	return false
+}
+
+// NewPage allocates a new page in the buffer pool with the disk manager help
+func (b *BufferPool) NewPage() (*Page, error) {
+	// get a free frame
+	frameID, isFromFreeList, err := b.getFrameID()
+	if err != nil {
+		return nil, err
+	}
+
+	if !isFromFreeList {
+		// remove page from current frame
+		currentPage := b.pages[frameID]
+		if currentPage != nil {
+			if currentPage.isDirty {
+				b.diskManager.WritePage(currentPage)
+			}
+
+			delete(b.pageTable, currentPage.id)
+		}
+	}
+
+	// allocates new page
+	pageID, err := b.diskManager.AllocatePage()
+	if err != nil {
+		return nil, err
+	}
+	page := &Page{pageID, 1, false, [PAGE_SIZE]byte{}}
+	page.WritePageNumber(int32(pageID))
+	page.WriteFreeSpaceOffset(int16(PAGE_SIZE))
+	page.WriteNextPointer(int32(INVALID_PAGE))
+	page.WritePrevPointer(int32(INVALID_PAGE))
+
+	// update the frame table
+	b.pageTable[pageID] = frameID
+	pageSyncPool.Put(b.pages[frameID])
+	b.pages[frameID] = page
+
+	return page, nil
+}
+
+// ScratchPage returns a page outside the buffer pool - do not use if you intend the page
+// to be in the buffer pool (use NewPage() for that)
+// ScratchPage is intended to be used in cases where you need the Page primitives
+// and will copy the scratch page back over a real page later
+func (b *BufferPool) ScratchPage() *Page {
+	page := &Page{
+		id:       PageID(INVALID_PAGE),
+		pinCount: 0,
+		isDirty:  false,
+		data:     [PAGE_SIZE]byte{},
+	}
+	page.WritePageNumber(int32(INVALID_PAGE))
+	page.WriteFreeSpaceOffset(int16(PAGE_SIZE))
+	page.WriteNextPointer(int32(INVALID_PAGE))
+	page.WritePrevPointer(int32(INVALID_PAGE))
+	return page
+}
+
+// DeletePage deletes a page from the buffer pool
+func (b *BufferPool) DeletePage(pageID PageID) error {
+	var frameID FrameID
+	var ok bool
+	if frameID, ok = b.pageTable[pageID]; !ok {
+		return nil
+	}
+
+	page := b.pages[frameID]
+
+	if page.pinCount > 0 {
+		return errors.New("pin count greater than 0")
+	}
+	delete(b.pageTable, page.id)
+	b.replacer.Pin(frameID)
+	b.diskManager.DeallocatePage(pageID)
+
+	b.freeList = append(b.freeList, frameID)
+
+	return nil
+}
+
+// FlushAllpages flushes all the pages in the buffer pool to disk
+// Yeah, never call this unless you know what you are doing
+func (b *BufferPool) FlushAllpages() {
+	for pageID := range b.pageTable {
+		b.FlushPage(pageID)
+	}
+}
+
+func (b *BufferPool) getFrameID() (FrameID, bool, error) {
+	if len(b.freeList) > 0 {
+		frameID, newFreeList := b.freeList[0], b.freeList[1:]
+		b.freeList = newFreeList
+		return frameID, true, nil
+	}
+
+	victim, err := b.replacer.Victim()
+	return victim, false, err
+}
+
+// OnDiskSize exposes the on disk size of the backing store
+// behind this buffer pool
+func (b *BufferPool) OnDiskSize() int64 {
+	return b.diskManager.FileSize()
+}
+
+// Close closes the buffer pool
+func (b *BufferPool) Close() {
+	b.diskManager.Close()
+}

bufferpool/circularlist.go

@@ -0,0 +1,93 @@
+package bufferpool
+
+import (
+	"errors"
+)
+
+type circularListNode struct {
+	key   interface{}
+	value interface{}
+	next  *circularListNode
+	prev  *circularListNode
+}
+
+type circularList struct {
+	head     *circularListNode
+	tail     *circularListNode
+	size     int
+	capacity int
+}
+
+func newCircularList(maxSize int) *circularList {
+	return &circularList{nil, nil, 0, maxSize}
+}
+
+func (c *circularList) find(key interface{}) *circularListNode {
+	ptr := c.head
+	for i := 0; i < c.size; i++ {
+		if ptr.key == key {
+			return ptr
+		}
+		ptr = ptr.next
+	}
+	return nil
+}
+
+func (c *circularList) hasKey(key interface{}) bool {
+	return c.find(key) != nil
+}
+
+func (c *circularList) insert(key interface{}, value interface{}) error {
+	if c.size == c.capacity {
+		return errors.New("list is full")
+	}
+	newNode := &circularListNode{key, value, nil, nil}
+	if c.size == 0 {
+		newNode.next = newNode
+		newNode.prev = newNode
+		c.head = newNode
+		c.tail = newNode
+		c.size++
+		return nil
+	}
+
+	node := c.find(key)
+	if node != nil {
+		node.value = value
+		return nil
+	}
+
+	newNode.next = c.head
+	newNode.prev = c.tail
+	c.tail.next = newNode
+	if c.head == c.tail {
+		c.head.next = newNode
+	}
+	c.tail = newNode
+	c.head.prev = c.tail
+
+	c.size++
+	return nil
+}
+
+func (c *circularList) remove(key interface{}) {
+	node := c.find(key)
+	if node == nil {
+		return
+	}
+	if c.size == 1 {
+		c.head = nil
+		c.tail = nil
+		c.size--
+		return
+	}
+	if node == c.head {
+		c.head = c.head.next
+	}
+	if node == c.tail {
+		c.tail = c.tail.prev
+	}
+	node.next.prev = node.prev
+	node.prev.next = node.next
+	c.size--
+}

bufferpool/clockreplacer.go

@@ -0,0 +1,64 @@
+package bufferpool
+
+import "errors"
+
+// ClockReplacer implements a clock replacer algorithm
+type ClockReplacer struct {
+	cList     *circularList
+	clockHand **circularListNode
+}
+
+// NewClockReplacer instantiates a new clock replacer
+func NewClockReplacer(poolSize int) *ClockReplacer {
+	cList := newCircularList(poolSize)
+	return &ClockReplacer{cList, &cList.head}
+}
+
+// Victim removes the victim frame as defined by the replacement policy
+func (c *ClockReplacer) Victim() (FrameID, error) {
+	if c.cList.size == 0 {
+		return FrameID(INVALID_PAGE), errors.New("no victims available")
+	}
+	var victimFrameID FrameID
+	currentNode := (*c.clockHand)
+	for {
+
+		if currentNode.value.(bool) {
+			currentNode.value = false
+			c.clockHand = &currentNode.next
+		} else {
+			frameID := currentNode.key.(FrameID)
+			victimFrameID = frameID
+			c.clockHand = &currentNode.next
+			c.cList.remove(currentNode.key)
+			return victimFrameID, nil
+		}
+	}
+}
+
+// Unpin unpins a frame, indicating that it can now be victimized
+func (c *ClockReplacer) Unpin(id FrameID) {
+	if !c.cList.hasKey(id) {
+		c.cList.insert(id, true)
+		if c.cList.size == 1 {
+			c.clockHand = &c.cList.head
+		}
+	}
+}
+
+// Pin pins a frame, indicating that it should not be victimized until it is unpinned
+func (c *ClockReplacer) Pin(id FrameID) {
+	node := c.cList.find(id)
+	if node == nil {
+		return
+	}
+	if (*c.clockHand) == node {
+		c.clockHand = &(*c.clockHand).next
+	}
+	c.cList.remove(id)
+}
+
+// Size returns the size of the clock
+func (c *ClockReplacer) Size() int {
+	return c.cList.size
+}

bufferpool/diskmanager.go

@@ -0,0 +1,21 @@
+package bufferpool
+
+// DiskManager is responsible for interacting with disk
+type DiskManager interface {
+	// reads a page from the disk
+	ReadPage(PageID) (*Page, error)
+	// writes a page to the disk
+	WritePage(*Page) error
+
+	// allocates a page
+	AllocatePage() (PageID, error)
+
+	// deallocates a page
+	DeallocatePage(PageID) error
+
+	// returns on disk file size
+	FileSize() int64
+
+	// closes and does any clean up
+	Close()
+}

bufferpool/inmemdiskmanager.go

@@ -0,0 +1,162 @@
+package bufferpool
+
+import (
+	"errors"
+	"fmt"
+	"os"
+
+	uuid "github.com/satori/go.uuid"
+)
+
+// InMemDiskSpillingDiskManager is a memory implementation for a DiskManager interface
+// that can spill to disk when a threshold is reached
+type InMemDiskSpillingDiskManager struct {
+	// tracks the number of pages
+	numPages int
+
+	onDiskPages int
+
+	// tracks the number of pages we can consume before spilling
+	thresholdPages int
+	hasSpilled     *struct{}
+	fd             *os.File
+
+	// the data buffer
+	data []byte
+}
+
+// NewInMemDiskSpillingDiskManager returns a in-memory version of disk manager
+func NewInMemDiskSpillingDiskManager(thresholdPages int) *InMemDiskSpillingDiskManager {
+	dm := &InMemDiskSpillingDiskManager{
+		numPages:       0,
+		thresholdPages: thresholdPages,
+		data:           make([]byte, 0),
+	}
+	return dm
+}
+
+// ReadPage reads a page from pages
+func (d *InMemDiskSpillingDiskManager) ReadPage(pageID PageID) (*Page, error) {
+	// check we're not asking for page out of range
+	if pageID < 0 || int(pageID) >= d.numPages {
+		return nil, errors.New("page not found")
+	}
+	// check that the offset is within range
+	offset := int(pageID) * PAGE_SIZE
+
+	var page = pageSyncPool.Get().(*Page)
+	// we have to do this stupid check because if -cpuprofile is set for go test, this
+	// the previous line return a weird nil-ish thing...
+	if page == (*Page)(nil) {
+		page = pageSyncPool.New().(*Page)
+	}
+	page.id = pageID
+
+	// do the read
+	if d.hasSpilled == nil {
+		if offset+PAGE_SIZE > len(d.data) {
+			return nil, errors.New("offset out of range")
+		}
+		b := copy(page.data[:], d.data[offset:offset+PAGE_SIZE])
+		fmt.Printf("bytes read: %d", b)
+	} else {
+		var err error
+		if offset+PAGE_SIZE > d.numPages*PAGE_SIZE {
+			return nil, errors.New("offset out of range")
+		}
+		_, err = d.fd.ReadAt(page.data[:], int64(offset))
+		if err != nil {
+			return nil, err
+		}
+	}
+	return page, nil
+}
+
+// WritePage writes a page in memory to pages
+func (d *InMemDiskSpillingDiskManager) WritePage(page *Page) error {
+	// make sure the offset is sensible
+	offset := int(page.ID()) * PAGE_SIZE
+	// do the write
+	if d.hasSpilled == nil {
+		if offset+PAGE_SIZE > len(d.data) {
+			return errors.New("offset out of range")
+		}
+		copy(d.data[offset:], page.data[:])
+	} else {
+		var err error
+		if offset+PAGE_SIZE > d.numPages*PAGE_SIZE {
+			return errors.New("offset out of range")
+		}
+		_, err = d.fd.WriteAt(page.data[:], int64(offset))
+		if err != nil {
+			return err
+		}
+		// err = d.fd.Sync()
+		// if err != nil {
+		// 	return err
+		// }
+	}
+	return nil
+}
+
+// AllocatePage allocates a page and returns the page number
+func (d *InMemDiskSpillingDiskManager) AllocatePage() (PageID, error) {
+	d.numPages = d.numPages + 1
+	pageID := PageID(d.numPages - 1)
+
+	if d.hasSpilled == nil {
+		// we have not spilled (yet), so make storage bigger
+		newData := make([]byte, PAGE_SIZE)
+		d.data = append(d.data, newData...)
+
+		// check to see if we need to spill
+		if d.numPages > d.thresholdPages {
+			fileUUID, err := uuid.NewV4()
+			if err != nil {
+				return PageID(INVALID_PAGE), err
+			}
+			// TODO(pok) we should try to tell the OS not to cache this file
+			d.fd, err = os.CreateTemp("", fmt.Sprintf("fb-ehash-%s", fileUUID.String()))
+			if err != nil {
+				return PageID(INVALID_PAGE), err
+			}
+			_, err = d.fd.WriteAt(d.data, 0)
+			if err != nil {
+				return PageID(INVALID_PAGE), err
+			}
+			d.data = []byte{}
+			d.hasSpilled = &struct{}{}
+		}
+	} else {
+		if d.numPages >= d.onDiskPages {
+			// grow the file by a chunk - 512 pages
+			d.onDiskPages += 512
+			var err error
+			size := int64(d.onDiskPages * PAGE_SIZE)
+			_, err = d.fd.WriteAt([]byte{0}, size-1)
+			if err != nil {
+				return PageID(INVALID_PAGE), err
+			}
+		}
+	}
+
+	return pageID, nil
+}
+
+// DeallocatePage removes page from disk
+func (d *InMemDiskSpillingDiskManager) DeallocatePage(pageID PageID) error {
+	// nothing to do right now
+	return nil
+}
+
+func (d *InMemDiskSpillingDiskManager) FileSize() int64 {
+	return int64(len(d.data))
+}
+
+func (d *InMemDiskSpillingDiskManager) Close() {
+	// close and delete the file if we spilled
+	if d.fd != nil {
+		_ = d.fd.Close()
+		os.Remove(d.fd.Name())
+	}
+}

bufferpool/page.go

@@ -0,0 +1,371 @@
+package bufferpool
+
+import (
+	"encoding/binary"
+	"errors"
+	"fmt"
+)
+
+const PAGE_SIZE int = 8192
+
+const INVALID_PAGE int = -1
+
+const PAGE_TYPE_BTREE_INTERNAL = 10
+const PAGE_TYPE_BTREE_LEAF = 11
+const PAGE_TYPE_HASH_TABLE = 12
+
+// PAGE
+// page size 8192 bytes
+// byte aligned, big endian
+
+// |====================================================|
+// | offset | length        |                           |
+// |----------------------------------------------------|
+// | header                                             |
+// |====================================================|
+// | 0      | 4             |  pageNumber (int32)       |
+// | 4      | 2             |  pageType (int16)         |
+// | 6      | 2             |  slotCount (int16)        |
+// | 8      | 2             |  localDepth (int16)       |
+// | 10     | 2             |  freeSpaceOffset (int16)  |
+// | 12     | 4             |  prevPointer (int32)      |
+// | 16     | 4             |  nextPointer (int32)      |
+// |====================================================|
+// | <start of slot array 1..slotCount>                 |
+// |----------------------------------------------------|
+// | 20     | slotcount     |  slot entry is 2 int16    |
+// |        |  * slotwidth  |   values (payloadOffset,  |
+// |        |  * #slots     |   payloadLength)          |
+// |----------------------------------------------------|
+// | <free space>                                       |
+// |----------------------------------------------------|
+// | <payload starting at freeSpaceOffset>              |
+// |  payload entries are keylength (int16), key bytes, |
+// |  payload length (int32), payload bytes             |
+// |====================================================|
+
+const PAGE_NUMBER_OFFSET = 0        // offset 0, length 4, end 4
+const PAGE_TYPE_OFFSET = 4          // offset 4, length 2, end 6
+const PAGE_SLOT_COUNT_OFFSET = 6    // offset 6, length 2, end 8
+const PAGE_LOCAL_DEPTH_OFFSET = 8   // offset 8, length 2, end 10
+const PAGE_FREE_SPACE_OFFSET = 10   // offset 10, length 2, end 12
+const PAGE_PREV_POINTER_OFFSET = 12 // offset 12, length 4, end 16
+const PAGE_NEXT_POINTER_OFFSET = 16 // offset 16, length 4, end 20
+const PAGE_SLOTS_START_OFFSET = 20  // offset 20
+
+// PAGE_SLOT_LENGTH is the size of the page slot key/value.
+//
+// key offset int16    	//offset 0, length 2, end 2
+// value offset int16	    //offset 2, length 2, end 4
+const PAGE_SLOT_LENGTH = 4
+
+// Page represents a page on disk
+type Page struct {
+	id       PageID
+	pinCount int
+	isDirty  bool
+	data     [PAGE_SIZE]byte
+}
+
+type PageSlot struct {
+	KeyOffset   int16
+	ValueOffset int16
+}
+
+func (s *PageSlot) KeyBytes(page *Page) []byte {
+	offset := s.KeyOffset
+	keyLen := int16(binary.BigEndian.Uint16(page.data[offset:]))
+	offset += 2
+	result := make([]byte, keyLen)
+	copy(result, page.data[offset:offset+keyLen])
+	return result
+}
+
+func (s *PageSlot) KeyAsInt(page *Page) int32 {
+	return int32(binary.BigEndian.Uint32(page.data[s.KeyOffset+2:]))
+}
+
+func (s *PageSlot) ValueBytes(page *Page) []byte {
+	offset := s.ValueOffset
+	valueLen := int32(binary.BigEndian.Uint32(page.data[offset:]))
+	offset += 4
+	result := make([]byte, valueLen)
+	copy(result, page.data[offset:int32(offset)+valueLen])
+	return result
+}
+
+func (s *PageSlot) ValueAsPagePointer(page *Page) int32 {
+	return int32(binary.BigEndian.Uint32(page.data[s.ValueOffset+4:]))
+}
+
+type PageChunk struct {
+	KeyLength int16
+	KeyBytes  []byte
+	// TODO(pok) ValueBytes can be up to int32 long
+	// this requires an overflow page mechanism, that is not implemented
+	// yet, so be aware of this when storing stuff...
+	ValueLength int32
+	ValueBytes  []byte
+}
+
+func (pc *PageChunk) Length() int {
+	return 2 + len(pc.KeyBytes) + 4 + len(pc.ValueBytes)
+}
+
+func (pc *PageChunk) ComputeKeyOffset(pageOffset int) int {
+	return pageOffset
+}
+
+func (pc *PageChunk) ComputeValueOffset(pageOffset int) int {
+	return pageOffset + 2 + len(pc.KeyBytes)
+}
+
+func (p *Page) WritePageNumber(pageNumber int32) {
+	p.id = PageID(pageNumber)
+	binary.BigEndian.PutUint32(p.data[PAGE_NUMBER_OFFSET:], uint32(pageNumber))
+	p.isDirty = true
+}
+
+func (p *Page) ReadPageNumber() int {
+	return int(binary.BigEndian.Uint32(p.data[PAGE_NUMBER_OFFSET:]))
+}
+
+func (p *Page) WritePageType(pageType int16) {
+	binary.BigEndian.PutUint16(p.data[PAGE_TYPE_OFFSET:], uint16(pageType))
+	p.isDirty = true
+}
+
+func (p *Page) ReadPageType() int16 {
+	return int16(binary.BigEndian.Uint16(p.data[PAGE_TYPE_OFFSET:]))
+}
+
+func (p *Page) WriteSlotCount(slotCount int16) {
+	binary.BigEndian.PutUint16(p.data[PAGE_SLOT_COUNT_OFFSET:], uint16(slotCount))
+	p.isDirty = true
+}
+
+func (p *Page) ReadSlotCount() int16 {
+	return int16(binary.BigEndian.Uint16(p.data[PAGE_SLOT_COUNT_OFFSET:]))
+}
+
+func (p *Page) WriteLocalDepth(localDepth int16) {
+	binary.BigEndian.PutUint16(p.data[PAGE_LOCAL_DEPTH_OFFSET:], uint16(localDepth))
+	p.isDirty = true
+}
+
+func (p *Page) ReadLocalDepth() int16 {
+	return int16(binary.BigEndian.Uint16(p.data[PAGE_LOCAL_DEPTH_OFFSET:]))
+}
+
+func (p *Page) ReadSlot(slot int16) PageSlot {
+	offset := PAGE_SLOTS_START_OFFSET + PAGE_SLOT_LENGTH*slot
+	keyOffset := int16(binary.BigEndian.Uint16(p.data[offset:]))
+	offset += 2
+	valueOffset := int16(binary.BigEndian.Uint16(p.data[offset:]))
+	return PageSlot{
+		KeyOffset:   keyOffset,
+		ValueOffset: valueOffset,
+	}
+}
+
+func (p *Page) WriteSlot(slot int16, value PageSlot) {
+	offset := PAGE_SLOTS_START_OFFSET + PAGE_SLOT_LENGTH*slot
+	binary.BigEndian.PutUint16(p.data[offset:], uint16(value.KeyOffset))
+	offset += 2
+	binary.BigEndian.PutUint16(p.data[offset:], uint16(value.ValueOffset))
+}
+
+func (p *Page) WriteFreeSpaceOffset(offset int16) {
+	binary.BigEndian.PutUint16(p.data[PAGE_FREE_SPACE_OFFSET:], uint16(offset))
+	p.isDirty = true
+}
+
+func (p *Page) ReadFreeSpaceOffset() int16 {
+	return int16(binary.BigEndian.Uint16(p.data[PAGE_FREE_SPACE_OFFSET:]))
+}
+
+func (p *Page) WritePrevPointer(prevPointer int32) {
+	binary.BigEndian.PutUint32(p.data[PAGE_PREV_POINTER_OFFSET:], uint32(prevPointer))
+	p.isDirty = true
+}
+
+func (p *Page) ReadPrevPointer() int {
+	return int(binary.BigEndian.Uint32(p.data[PAGE_PREV_POINTER_OFFSET:]))
+}
+
+func (p *Page) WriteNextPointer(nextPointer int32) {
+	binary.BigEndian.PutUint32(p.data[PAGE_NEXT_POINTER_OFFSET:], uint32(nextPointer))
+	p.isDirty = true
+}
+
+func (p *Page) ReadNextPointer() int {
+	return int(binary.BigEndian.Uint32(p.data[PAGE_NEXT_POINTER_OFFSET:]))
+}
+
+func (p *Page) WriteChunk(offset int16, chunk PageChunk) {
+	binary.BigEndian.PutUint16(p.data[offset:], uint16(chunk.KeyLength))
+	offset += 2
+	copy(p.data[offset:], chunk.KeyBytes)
+	offset += int16(len(chunk.KeyBytes))
+	binary.BigEndian.PutUint32(p.data[offset:], uint32(chunk.ValueLength))
+	offset += 4
+	copy(p.data[offset:], chunk.ValueBytes)
+	p.isDirty = true
+}
+
+func (p *Page) ReadChunk(offset int16) PageChunk {
+	keyLen := int16(binary.BigEndian.Uint16(p.data[offset:]))
+	offset += 2
+	keyBytes := make([]byte, keyLen)
+	copy(keyBytes, p.data[offset:offset+keyLen])
+	offset += keyLen
+	valueLen := int32(binary.BigEndian.Uint32(p.data[offset:]))
+	offset += 4
+	valueBytes := make([]byte, valueLen)
+	copy(valueBytes, p.data[offset:int32(offset)+valueLen])
+	return PageChunk{
+		KeyLength:   keyLen,
+		KeyBytes:    keyBytes,
+		ValueLength: valueLen,
+		ValueBytes:  valueBytes,
+	}
+}
+
+func (p *Page) FreeSpace() int16 {
+	freeSpaceOffset := p.ReadFreeSpaceOffset()
+	freespace := freeSpaceOffset - (p.ReadSlotCount()*PAGE_SLOT_LENGTH + PAGE_SLOT_LENGTH + PAGE_SLOTS_START_OFFSET)
+	return freespace
+}
+
+func (p *Page) WriteKeyValueInSlot(slotNumber int16, key []byte, value []byte) error {
+	freeSpaceOffset := p.ReadFreeSpaceOffset()
+
+	// build a chunk
+	chunk := PageChunk{
+		KeyLength:   int16(len(key)),
+		KeyBytes:    key,
+		ValueLength: int32(len(value)),
+		ValueBytes:  value,
+	}
+
+	// compute the new free space offset
+	freeSpaceOffset -= int16(chunk.Length())
+
+	// check we won't blow free space on page
+	slotCount := p.ReadSlotCount()
+	slotEndOffset := slotCount*PAGE_SLOT_LENGTH + PAGE_SLOT_LENGTH + PAGE_SLOTS_START_OFFSET
+
+	// DEBUG!!
+	//fmt.Printf("freeSpaceOffset: %d, slotCount: %d, slotCount*4 + 4 + 20: %d, freeSpace: %d\n", freeSpaceOffset, slotCount, slotEndOffset, freeSpaceOffset-slotEndOffset)
+
+	if freeSpaceOffset-slotEndOffset <= 0 {
+		return errors.New("page is full")
+	}
+
+	keyOffset := chunk.ComputeKeyOffset(int(freeSpaceOffset))
+	valueOffset := chunk.ComputeValueOffset(int(freeSpaceOffset))
+
+	p.WriteChunk(freeSpaceOffset, chunk)
+
+	// update the free space offset
+	p.WriteFreeSpaceOffset(int16(freeSpaceOffset))
+
+	// make a slot
+	slot := PageSlot{
+		KeyOffset:   int16(keyOffset),
+		ValueOffset: int16(valueOffset),
+	}
+	// write the slot
+	p.WriteSlot(slotNumber, slot)
+
+	return nil
+}
+
+func (p *Page) WritePage(page *Page) {
+	// copy everything but pageNumber & pageType
+	offset := PAGE_SLOT_COUNT_OFFSET
+	copy(page.data[offset:], p.data[offset:offset+PAGE_SIZE-offset])
+}
+
+func (p *Page) PinCount() int {
+	return p.pinCount
+}
+
+func (p *Page) ID() PageID {
+	return p.id
+}
+
+func (p *Page) DecPinCount() {
+	if p.pinCount > 0 {
+		p.pinCount--
+	}
+}
+
+type PageSlotIterator struct {
+	page      *Page
+	slotCount int16
+	cursor    int16
+}
+
+func NewPageSlotIterator(page *Page, fromSlot int16) *PageSlotIterator {
+	i := &PageSlotIterator{
+		page:      page,
+		slotCount: page.ReadSlotCount(),
+		cursor:    fromSlot,
+	}
+	return i
+}
+
+func (i *PageSlotIterator) Next() *PageSlot {
+	if i.cursor < i.slotCount {
+		s := i.page.ReadSlot(i.cursor)
+		i.cursor++
+		return &s
+	}
+	return nil
+}
+
+func (i *PageSlotIterator) Cursor() int16 {
+	return i.cursor
+}
+
+func (pg *Page) Dump(label string) {
+	indent := 0
+	if len(label) > 0 {
+		fmt.Printf("%s%s:\n", fmt.Sprintf("%*s", indent, ""), label)
+		indent += 4
+	}
+	pageType := pg.ReadPageType()
+	fmt.Printf("%sPAGE(%d) pageType: %d slotCount: %d, prevPtr: %d, nextPtr: %d\n", fmt.Sprintf("%*s", indent, ""), pg.ID(), pageType, pg.ReadSlotCount(), pg.ReadPrevPointer(), pg.ReadNextPointer())
+	fmt.Printf("%sKEYS: -->\n", fmt.Sprintf("%*s", indent, ""))
+	indent += 4
+
+	// get the keys off the page
+	keys := make([]int, 0)
+	pointers := make([]int, 0)
+	iter := NewPageSlotIterator(pg, 0)
+	for {
+		ps := iter.Next()
+		if ps == nil {
+			break
+		}
+		keys = append(keys, int(ps.KeyAsInt(pg)))
+		if pageType == /*nodeTypeInternal*/ 10 {
+			pointers = append(pointers, int(ps.ValueAsPagePointer(pg)))
+		}
+	}
+
+	if pageType == /*nodeTypeLeaf*/ 11 {
+		for _, key := range keys {
+			fmt.Printf("%s(%d)\n", fmt.Sprintf("%*s", indent, ""), key)
+		}
+	} else {
+		for idx, key := range keys {
+			ptr := pointers[idx]
+			fmt.Printf("%s(%d, %d)\n", fmt.Sprintf("%*s", indent, ""), key, ptr)
+		}
+		ptr := pg.ReadNextPointer()
+		fmt.Printf("%s(-->, %d)\n", fmt.Sprintf("%*s", indent, ""), ptr)
+	}
+
+}

cache.go

@@ -1,38 +1,28 @@
-// Copyright 2017 Pilosa Corp.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
+// Copyright 2022 Molecula Corp. (DBA FeatureBase).
+// SPDX-License-Identifier: Apache-2.0
 package pilosa
 
 import (
 	"bytes"
+	"encoding/json"
 	"fmt"
 	"io"
 	"sort"
 	"sync"
 	"time"
 
-	"github.com/golang/groupcache/lru"
-	"github.com/pilosa/pilosa/internal"
+	"github.com/featurebasedb/featurebase/v3/lru"
+	pb "github.com/featurebasedb/featurebase/v3/proto"
+	"github.com/pkg/errors"
 )
 
 const (
-	// ThresholdFactor is used to calculate the threshold for new items entering the cache
-	ThresholdFactor = 1.1
+	// thresholdFactor is used to calculate the threshold for new items entering the cache
+	thresholdFactor = 1.1
 )
 
-// Cache represents a cache of counts.
-type Cache interface {
+// cache represents a cache of counts.
+type cache interface {
 	Add(id uint64, n uint64)
 	BulkAdd(id uint64, n uint64)
 	Get(id uint64) uint64
@@ -41,66 +31,67 @@ type Cache interface {
 	// Returns a list of all IDs.
 	IDs() []uint64
 
-	// Updates the cache, if necessary.
+	// Soft ask for the cache to be rebuilt - may not if it has been done recently.
 	Invalidate()
 
-	// Rebuilds the cache
+	// Rebuilds the cache.
 	Recalculate()
 
 	// Returns an ordered list of the top ranked bitmaps.
-	Top() []BitmapPair
+	Top() []bitmapPair
 
-	// SetStats defines the stats client used in the cache.
-	SetStats(s StatsClient)
+	// Clear removes everything from the cache. If possible it should leave allocated structures in place to be reused.
+	Clear()
 }
 
-// LRUCache represents a least recently used Cache implementation.
-type LRUCache struct {
+// lruCache represents a least recently used Cache implementation.
+type lruCache struct {
 	cache  *lru.Cache
 	counts map[uint64]uint64
-	stats  StatsClient
+	// maxEntries is saved to support Clear which recreates the cache.
+	maxEntries uint32
 }
 
-// NewLRUCache returns a new instance of LRUCache.
-func NewLRUCache(maxEntries uint32) *LRUCache {
-	c := &LRUCache{
-		cache:  lru.New(int(maxEntries)),
-		counts: make(map[uint64]uint64),
-		stats:  NopStatsClient,
+// newLRUCache returns a new instance of LRUCache.
+func newLRUCache(maxEntries uint32) *lruCache {
+	c := &lruCache{
+		cache:      lru.New(int(maxEntries)),
+		counts:     make(map[uint64]uint64),
+		maxEntries: maxEntries,
 	}
 	c.cache.OnEvicted = c.onEvicted
 	return c
 }
 
 // BulkAdd adds a count to the cache unsorted. You should Invalidate after completion.
-func (c *LRUCache) BulkAdd(id, n uint64) {
+func (c *lruCache) BulkAdd(id, n uint64) {
 	c.Add(id, n)
 }
 
 // Add adds a count to the cache.
-func (c *LRUCache) Add(id, n uint64) {
+func (c *lruCache) Add(id, n uint64) {
 	c.cache.Add(id, n)
 	c.counts[id] = n
 }
 
 // Get returns a count for a given id.
-func (c *LRUCache) Get(id uint64) uint64 {
+func (c *lruCache) Get(id uint64) uint64 {
 	n, _ := c.cache.Get(id)
 	nn, _ := n.(uint64)
 	return nn
 }
 
 // Len returns the number of items in the cache.
-func (c *LRUCache) Len() int { return c.cache.Len() }
+func (c *lruCache) Len() int { return c.cache.Len() }
 
 // Invalidate is a no-op.
-func (c *LRUCache) Invalidate() {}
+func (c *lruCache) Invalidate() {}
 
 // Recalculate is a no-op.
-func (c *LRUCache) Recalculate() {}
+func (c *lruCache) Recalculate() {}
 
 // IDs returns a list of all IDs in the cache.
-func (c *LRUCache) IDs() []uint64 {
+func (c *lruCache) IDs() []uint64 {
 	a := make([]uint64, 0, len(c.counts))
 	for id := range c.counts {
 		a = append(a, id)
@@ -110,33 +101,39 @@ func (c *LRUCache) IDs() []uint64 {
 }
 
 // Top returns all counts in the cache.
-func (c *LRUCache) Top() []BitmapPair {
-	a := make([]BitmapPair, 0, len(c.counts))
+func (c *lruCache) Top() []bitmapPair {
+	a := make([]bitmapPair, 0, len(c.counts))
 	for id, n := range c.counts {
-		a = append(a, BitmapPair{
+		a = append(a, bitmapPair{
 			ID:    id,
-			Count: uint64(n),
+			Count: n,
 		})
 	}
-	sort.Sort(BitmapPairs(a))
+	pairs := bitmapPairs(a)
+	sort.Sort(&pairs)
 	return a
 }
 
-// SetStats defines the stats client used in the cache.
-func (c *LRUCache) SetStats(s StatsClient) {
-	c.stats = s
+func (c *lruCache) Clear() {
+	for k := range c.counts {
+		delete(c.counts, k)
+	}
+	c.cache = lru.New(int(c.maxEntries))
 }
 
-func (c *LRUCache) onEvicted(key lru.Key, _ interface{}) { delete(c.counts, key.(uint64)) }
+func (c *lruCache) onEvicted(key lru.Key, _ interface{}) { delete(c.counts, key.(uint64)) }
 
 // Ensure LRUCache implements Cache.
-var _ Cache = &LRUCache{}
+var _ cache = &lruCache{}
 
-// RankCache represents a cache with sorted entries.
-type RankCache struct {
-	mu       sync.Mutex
-	entries  map[uint64]uint64
-	rankings []BitmapPair // cached, ordered list
+// rankCache represents a cache with sorted entries.
+type rankCache struct {
+	// TODO why does this have a lock and lruCache doesn't?
+	mu           sync.Mutex
+	entries      map[uint64]uint64
+	rankings     bitmapPairs // cached, ordered list
+	rankingsRead bool
+	dirty        bool
 
 	updateN    int
 	updateTime time.Time
@@ -150,26 +147,46 @@ type RankCache struct {
 
 	// thresholdValue is the value of the last item in the cache
 	thresholdValue uint64
-
-	stats StatsClient
 }
 
 // NewRankCache returns a new instance of RankCache.
-func NewRankCache(maxEntries uint32) *RankCache {
-	return &RankCache{
+func NewRankCache(maxEntries uint32) *rankCache {
+	return &rankCache{
 		maxEntries:      maxEntries,
-		thresholdBuffer: int(ThresholdFactor * float64(maxEntries)),
+		thresholdBuffer: int(thresholdFactor * float64(maxEntries)),
 		entries:         make(map[uint64]uint64),
-		stats:           NopStatsClient,
 	}
 }
 
-// Add adds a count to the cache.
-func (c *RankCache) Add(id uint64, n uint64) {
+func (c *rankCache) Clear() {
 	c.mu.Lock()
 	defer c.mu.Unlock()
-	// Ignore if the bit count is below the threshold.
-	if n < c.thresholdValue {
+	for k := range c.entries {
+		delete(c.entries, k)
+	}
+	c.rankings = c.rankings[:0]
+	c.rankingsRead = false
+	c.dirty = false
+
+	c.updateN = 0
+	c.updateTime = time.Time{}
+	c.thresholdValue = 0
+}
+
+// Add adds a count to the cache.
+func (c *rankCache) Add(id uint64, n uint64) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	// Flag the cache as dirty.
+	// This forces recalculation if top is called before the cache is recalculated.
+	c.dirty = true
+
+	// Ignore if the column count is below the threshold,
+	// unless the count is 0, which is effectively used
+	// to clear the cache value.
+	if n < c.thresholdValue && n > 0 {
+		delete(c.entries, id)
 		return
 	}
 
@@ -179,84 +196,116 @@ func (c *RankCache) Add(id uint64, n uint64) {
 }
 
 // BulkAdd adds a count to the cache unsorted. You should Invalidate after completion.
-func (c *RankCache) BulkAdd(id uint64, n uint64) {
+func (c *rankCache) BulkAdd(id uint64, n uint64) {
 	c.mu.Lock()
 	defer c.mu.Unlock()
+
+	// Flag the cache as dirty.
+	// This forces recalculation if top is called before the cache is recalculated.
+	c.dirty = true
+
 	if n < c.thresholdValue {
+		delete(c.entries, id)
 		return
 	}
 
 	c.entries[id] = n
+
+	// FB-1206: Periodically invalidate the cache when we are bulk loading
+	// as this can take up an upbounded amount of memory. This is especially
+	// true when restoring shards as all rows will be added.
+	if len(c.entries) > int(2*c.maxEntries) {
+		CounterRecalculateCache.Inc()
+		c.recalculate()
+	}
 }
 
 // Get returns a count for a given id.
-func (c *RankCache) Get(id uint64) uint64 {
+func (c *rankCache) Get(id uint64) uint64 {
 	c.mu.Lock()
 	defer c.mu.Unlock()
 	return c.entries[id]
 }
 
 // Len returns the number of items in the cache.
-func (c *RankCache) Len() int {
+func (c *rankCache) Len() int {
 	c.mu.Lock()
 	defer c.mu.Unlock()
 	return len(c.entries)
 }
 
 // IDs returns a list of all IDs in the cache.
-func (c *RankCache) IDs() []uint64 {
+func (c *rankCache) IDs() []uint64 {
 	c.mu.Lock()
 	defer c.mu.Unlock()
-	a := make([]uint64, 0, len(c.entries))
-	for id := range c.entries {
-		a = append(a, id)
+	if len(c.entries) == 0 {
+		return nil
 	}
-	sort.Sort(uint64Slice(a))
-	return a
+	ids := make([]uint64, 0, len(c.entries))
+	for id := range c.entries {
+		ids = append(ids, id)
+	}
+	sort.Sort(uint64Slice(ids))
+	return ids
 }
 
 // Invalidate recalculates the entries by rank.
-func (c *RankCache) Invalidate() {
+func (c *rankCache) Invalidate() {
 	c.mu.Lock()
 	defer c.mu.Unlock()
 	c.invalidate()
 }
 
 // Recalculate rebuilds the cache.
-func (c *RankCache) Recalculate() {
+func (c *rankCache) Recalculate() {
 	c.mu.Lock()
 	defer c.mu.Unlock()
-	c.stats.Count("cache.recalculate", 1, 1.0)
+	CounterRecalculateCache.Inc()
 	c.recalculate()
 }
 
-func (c *RankCache) invalidate() {
+func (c *rankCache) invalidate() {
 	// Don't invalidate more than once every X seconds.
 	// TODO: consider making this configurable.
-	if time.Now().Sub(c.updateTime).Seconds() < 10 {
+	if time.Since(c.updateTime).Seconds() < 10 {
+		// Skipping recalculation means that the ranked cache's growth is unbounded.
+		// This is somewhat necessary for now since recalculation is not cheap.
+		// The cache will remain flagged as dirty and will be recalculated if Top is called.
+		// This may cause unexpected memory growth, so record it in metrics for debugging purposes.
+		CounterInvalidateCacheSkipped.Inc()
+		// Ensure that we're marked as dirty even if we weren't otherwise.
+		c.dirty = true
 		return
 	}
-	c.stats.Count("cache.invalidate", 1, 1.0)
+	CounterInvalidateCache.Inc()
 	c.recalculate()
 }
 
-func (c *RankCache) recalculate() {
+func (c *rankCache) recalculate() {
+	if c.rankingsRead {
+		c.rankings = nil
+		c.rankingsRead = false
+	}
+
 	// Convert cache to a sorted list.
-	rankings := make([]BitmapPair, 0, len(c.entries))
+	rankings := c.rankings[:0]
+	if cap(rankings) < len(c.entries) {
+		rankings = make([]bitmapPair, 0, len(c.entries))
+	}
 	for id, cnt := range c.entries {
-		rankings = append(rankings, BitmapPair{
+		rankings = append(rankings, bitmapPair{
 			ID:    id,
 			Count: cnt,
 		})
 	}
-	sort.Sort(BitmapPairs(rankings))
+	c.rankings = rankings
+	sort.Sort(&c.rankings)
 
 	// Store the count of the item at the threshold index.
-	c.rankings = rankings
 	length := len(c.rankings)
-	c.stats.Gauge("RankCache", float64(length), 1.0)
+	GaugeRankCacheLength.Set(float64(length))
 
-	var removeItems []BitmapPair // cached, ordered list
+	var removeItems []bitmapPair // cached, ordered list
 	if length > int(c.maxEntries) {
 		c.thresholdValue = rankings[c.maxEntries].Count
 		removeItems = c.rankings[c.maxEntries:]
@@ -270,67 +319,115 @@ func (c *RankCache) recalculate() {
 
 	// If size is larger than the threshold then trim it.
 	if len(c.entries) > c.thresholdBuffer {
-		c.stats.Count("cache.threshold", 1, 1.0)
+		CounterCacheThresholdReached.Inc()
 		for _, pair := range removeItems {
 			delete(c.entries, pair.ID)
 		}
 	}
-}
 
-// SetStats defines the stats client used in the cache.
-func (c *RankCache) SetStats(s StatsClient) {
-	c.stats = s
+	// The cache is no longer dirty.
+	c.dirty = false
 }
 
 // Top returns an ordered list of pairs.
-func (c *RankCache) Top() []BitmapPair { return c.rankings }
+func (c *rankCache) Top() []bitmapPair {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	if c.dirty {
+		// The cache is dirty, so we need to recalculate it to get a consistent view.
+		CounterReadDirtyCache.Inc()
+		c.recalculate()
+	}
+
+	c.rankingsRead = true
+	return c.rankings
+}
 
 // WriteTo writes the cache to w.
-func (c *RankCache) WriteTo(w io.Writer) (n int64, err error) {
+func (c *rankCache) WriteTo(w io.Writer) (n int64, err error) {
 	panic("FIXME: TODO")
 }
 
 // ReadFrom read from r into the cache.
-func (c *RankCache) ReadFrom(r io.Reader) (n int64, err error) {
+func (c *rankCache) ReadFrom(r io.Reader) (n int64, err error) {
 	panic("FIXME: TODO")
 }
 
 // Ensure RankCache implements Cache.
-var _ Cache = &RankCache{}
+var _ cache = &rankCache{}
 
-// BitmapPair represents a id/count pair with an associated identifier.
-type BitmapPair struct {
+// bitmapPair represents a id/count pair with an associated identifier.
+type bitmapPair struct {
 	ID    uint64
 	Count uint64
 }
 
-// BitmapPairs is a sortable list of BitmapPair objects.
-type BitmapPairs []BitmapPair
+// bitmapPairs is a sortable list of BitmapPair objects.
+type bitmapPairs []bitmapPair
 
-func (p BitmapPairs) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }
-func (p BitmapPairs) Len() int           { return len(p) }
-func (p BitmapPairs) Less(i, j int) bool { return p[i].Count > p[j].Count }
+func (p *bitmapPairs) Swap(i, j int)      { (*p)[i], (*p)[j] = (*p)[j], (*p)[i] }
+func (p *bitmapPairs) Len() int           { return len(*p) }
+func (p *bitmapPairs) Less(i, j int) bool { return (*p)[i].Count > (*p)[j].Count }
 
 // Pair holds an id/count pair.
 type Pair struct {
 	ID    uint64 `json:"id"`
+	Key   string `json:"key"`
 	Count uint64 `json:"count"`
 }
 
-func encodePair(p Pair) *internal.Pair {
-	return &internal.Pair{
-		Key:   p.ID,
-		Count: p.Count,
+// PairField is a Pair with its associated field.
+type PairField struct {
+	Pair  Pair
+	Field string
+}
+
+func (p PairField) Clone() (r PairField) {
+	return PairField{
+		Pair:  p.Pair,
+		Field: p.Field,
 	}
 }
 
-func decodePair(pb *internal.Pair) Pair {
-	return Pair{
-		ID:    pb.Key,
-		Count: pb.Count,
+// ToTable implements the ToTabler interface.
+func (p PairField) ToTable() (*pb.TableResponse, error) {
+	return pb.RowsToTable(p, 1)
+}
+
+// ToRows implements the ToRowser interface.
+func (p PairField) ToRows(callback func(*pb.RowResponse) error) error {
+	if p.Pair.Key != "" {
+		return callback(&pb.RowResponse{
+			Headers: []*pb.ColumnInfo{
+				{Name: p.Field, Datatype: "string"},
+				{Name: "count", Datatype: "uint64"},
+			},
+			Columns: []*pb.ColumnResponse{
+				{ColumnVal: &pb.ColumnResponse_StringVal{StringVal: p.Pair.Key}},
+				{ColumnVal: &pb.ColumnResponse_Uint64Val{Uint64Val: p.Pair.Count}},
+			},
+		})
+	} else {
+		return callback(&pb.RowResponse{
+			Headers: []*pb.ColumnInfo{
+				{Name: p.Field, Datatype: "uint64"},
+				{Name: "count", Datatype: "uint64"},
+			},
+			Columns: []*pb.ColumnResponse{
+				{ColumnVal: &pb.ColumnResponse_Uint64Val{Uint64Val: p.Pair.ID}},
+				{ColumnVal: &pb.ColumnResponse_Uint64Val{Uint64Val: p.Pair.Count}},
+			},
+		})
 	}
 }
 
+// MarshalJSON marshals PairField into a JSON-encoded byte slice,
+// excluding `Field`.
+func (p PairField) MarshalJSON() ([]byte, error) {
+	return json.Marshal(p.Pair)
+}
+
 // Pairs is a sortable slice of Pair objects.
 type Pairs []Pair
 
@@ -338,14 +435,14 @@ func (p Pairs) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }
 func (p Pairs) Len() int           { return len(p) }
 func (p Pairs) Less(i, j int) bool { return p[i].Count > p[j].Count }
 
-// PairHeap is a heap implementation over a group of Pairs.
-type PairHeap struct {
+// pairHeap is a heap implementation over a group of Pairs.
+type pairHeap struct {
 	Pairs
 }
 
 // Less implemets the Sort interface.
 // reports whether the element with index i should sort before the element with index j.
-func (p PairHeap) Less(i, j int) bool { return p.Pairs[i].Count < p.Pairs[j].Count }
+func (p pairHeap) Less(i, j int) bool { return p.Pairs[i].Count < p.Pairs[j].Count }
 
 // Push appends the element onto the Pair slice.
 func (p *Pairs) Push(x interface{}) {
@@ -406,22 +503,82 @@ func (p Pairs) String() string {
 	return buf.String()
 }
 
-func encodePairs(a Pairs) []*internal.Pair {
-	other := make([]*internal.Pair, len(a))
-	for i := range a {
-		other[i] = encodePair(a[i])
-	}
-	return other
+// PairsField is a Pairs object with its associated field.
+type PairsField struct {
+	Pairs []Pair
+	Field string
 }
 
-func decodePairs(a []*internal.Pair) []Pair {
-	other := make([]Pair, len(a))
-	for i := range a {
-		other[i] = decodePair(a[i])
+func (p *PairsField) Clone() (r *PairsField) {
+	r = &PairsField{
+		Pairs: make([]Pair, len(p.Pairs)),
+		Field: p.Field,
 	}
-	return other
+	copy(r.Pairs, p.Pairs)
+	return
 }
 
+// ToTable implements the ToTabler interface.
+func (p *PairsField) ToTable() (*pb.TableResponse, error) {
+	return pb.RowsToTable(p, len(p.Pairs))
+}
+
+// ToRows implements the ToRowser interface.
+func (p *PairsField) ToRows(callback func(*pb.RowResponse) error) error {
+	// Determine if the ID has string keys.
+	var stringKeys bool
+	if len(p.Pairs) > 0 {
+		if p.Pairs[0].Key != "" {
+			stringKeys = true
+		}
+	}
+
+	dtype := "uint64"
+	if stringKeys {
+		dtype = "string"
+	}
+	ci := []*pb.ColumnInfo{
+		{Name: p.Field, Datatype: dtype},
+		{Name: "count", Datatype: "uint64"},
+	}
+	for _, pair := range p.Pairs {
+		if stringKeys {
+			if err := callback(&pb.RowResponse{
+				Headers: ci,
+				Columns: []*pb.ColumnResponse{
+					{ColumnVal: &pb.ColumnResponse_StringVal{StringVal: pair.Key}},
+					{ColumnVal: &pb.ColumnResponse_Uint64Val{Uint64Val: uint64(pair.Count)}},
+				}}); err != nil {
+				return errors.Wrap(err, "calling callback")
+			}
+		} else {
+			if err := callback(&pb.RowResponse{
+				Headers: ci,
+				Columns: []*pb.ColumnResponse{
+					{ColumnVal: &pb.ColumnResponse_Uint64Val{Uint64Val: uint64(pair.ID)}},
+					{ColumnVal: &pb.ColumnResponse_Uint64Val{Uint64Val: uint64(pair.Count)}},
+				}}); err != nil {
+				return errors.Wrap(err, "calling callback")
+			}
+		}
+		ci = nil //only send on the first
+	}
+	return nil
+}
+
+// MarshalJSON marshals PairsField into a JSON-encoded byte slice,
+// excluding `Field`.
+func (p PairsField) MarshalJSON() ([]byte, error) {
+	return json.Marshal(p.Pairs)
+}
+
+// int64Slice represents a sortable slice of int64 numbers.
+type int64Slice []int64
+
+func (p int64Slice) Len() int           { return len(p) }
+func (p int64Slice) Less(i, j int) bool { return p[i] < p[j] }
+func (p int64Slice) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }
+
 // uint64Slice represents a sortable slice of uint64 numbers.
 type uint64Slice []uint64
 
@@ -429,89 +586,23 @@ func (p uint64Slice) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }
 func (p uint64Slice) Len() int           { return len(p) }
 func (p uint64Slice) Less(i, j int) bool { return p[i] < p[j] }
 
-// merge combines p and other to a unique sorted set of values.
-// p and other must both have unique sets and be sorted.
-func (p uint64Slice) merge(other []uint64) []uint64 {
-	ret := make([]uint64, 0, len(p))
-
-	i, j := 0, 0
-	for i < len(p) && j < len(other) {
-		a, b := p[i], other[j]
-		if a == b {
-			ret = append(ret, a)
-			i, j = i+1, j+1
-		} else if a < b {
-			ret = append(ret, a)
-			i++
-		} else {
-			ret = append(ret, b)
-			j++
-		}
-	}
-
-	if i < len(p) {
-		ret = append(ret, p[i:]...)
-	} else if j < len(other) {
-		ret = append(ret, other[j:]...)
-	}
-
-	return ret
-}
-
-// BitmapCache provides an interface for caching full bitmaps.
-type BitmapCache interface {
-	Fetch(id uint64) (*Bitmap, bool)
-	Add(id uint64, b *Bitmap)
-}
-
-// SimpleCache implements BitmapCache
-// it is meant to be a short-lived cache for cases where writes are continuing to access
-// the same bit within a short time frame (i.e. good for write-heavy loads)
-// A read-heavy use case would cause the cache to get bigger, potentially causing the
-// node to run out of memory.
-type SimpleCache struct {
-	cache map[uint64]*Bitmap
-}
-
-// Fetch retrieves the bitmap at the id in the cache.
-func (s *SimpleCache) Fetch(id uint64) (*Bitmap, bool) {
-	m, ok := s.cache[id]
-	return m, ok
-}
-
-// Add adds the bitmap to the cache, keyed on the id.
-func (s *SimpleCache) Add(id uint64, b *Bitmap) {
-	s.cache[id] = b
-}
-
-// NopCache represents a no-op Cache implementation.
-type NopCache struct {
-	stats StatsClient
-}
+// nopCache represents a no-op Cache implementation.
+type nopCache struct{}
 
 // Ensure NopCache implements Cache.
-var _ Cache = &NopCache{}
+var globalNopCache cache = nopCache{}
 
-// NewNopCache returns a new instance of NopCache.
-func NewNopCache() *NopCache {
-	return &NopCache{
-		stats: NopStatsClient,
-	}
-}
+func (c nopCache) Add(uint64, uint64)     {}
+func (c nopCache) BulkAdd(uint64, uint64) {}
+func (c nopCache) Get(uint64) uint64      { return 0 }
+func (c nopCache) IDs() []uint64          { return []uint64{} }
 
-func (c *NopCache) Add(id uint64, n uint64)     {}
-func (c *NopCache) BulkAdd(id uint64, n uint64) {}
-func (c *NopCache) Get(id uint64) uint64        { return 0 }
-func (c *NopCache) IDs() []uint64               { return make([]uint64, 0, 0) }
+func (c nopCache) Invalidate()  {}
+func (c nopCache) Len() int     { return 0 }
+func (c nopCache) Recalculate() {}
 
-func (c *NopCache) Invalidate() {}
-func (c *NopCache) Len() int    { return 0 }
-func (c *NopCache) Recalculate() {
-}
-func (c *NopCache) SetStats(s StatsClient) {
-	c.stats = s
-}
+func (c nopCache) Clear() {}
 
-func (c *NopCache) Top() []BitmapPair {
-	return []BitmapPair{}
+func (c nopCache) Top() []bitmapPair {
+	return []bitmapPair{}
 }

cache_test.go

@@ -1,27 +1,16 @@
-// Copyright 2017 Pilosa Corp.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
+// Copyright 2022 Molecula Corp. (DBA FeatureBase).
+// SPDX-License-Identifier: Apache-2.0
 package pilosa_test
 
 import (
+	"reflect"
 	"testing"
 
-	"github.com/pilosa/pilosa"
+	pilosa "github.com/featurebasedb/featurebase/v3"
 )
 
-// Ensure a bitmap query can be executed.
-func TestCache_Rank(t *testing.T) {
+// Ensure cache stays constrained to its configured size.
+func TestCache_Rank_Size(t *testing.T) {
 	cacheSize := uint32(3)
 	cache := pilosa.NewRankCache(cacheSize)
 	for i := 1; i < int(2*cacheSize); i++ {
@@ -31,5 +20,66 @@ func TestCache_Rank(t *testing.T) {
 	if cache.Len() != int(cacheSize) {
 		t.Fatalf("unexpected cache Size: %d!=%d expected\n", cache.Len(), cacheSize)
 	}
-
+}
+
+// Ensure cache entries set below threshold are handled appropriately.
+func TestCache_Rank_Threshold(t *testing.T) {
+	cacheSize := uint32(5)
+	cache := pilosa.NewRankCache(cacheSize)
+	for i := 1; i < int(2*cacheSize); i++ {
+		cache.Add(uint64(i), 3)
+	}
+
+	// Set the cache value for rows 4 and 5 to a number below the threshold
+	// value (which is 3), and ensure that they gets zeroed out.
+	cache.Add(4, 1)
+	cache.BulkAdd(5, 1)
+	cache.Recalculate()
+
+	if cache.Get(4) != 0 {
+		t.Fatalf("unexpected cache value after Add: %d!=%d expected\n", cache.Get(4), 0)
+	}
+	if cache.Get(5) != 0 {
+		t.Fatalf("unexpected cache value after BulkAdd: %d!=%d expected\n", cache.Get(5), 0)
+	}
+}
+
+// Test that consecutive writes show up in Top.
+// On later writes, the cache skips recalculation to save CPU time.
+// This used to mean that the later writes would not show up in Top.
+// Now, the cache is flagged as dirty and recalculated during the call to Top.
+func TestCache_Rank_Dirty(t *testing.T) {
+	cacheSize := uint32(5)
+	cache := pilosa.NewRankCache(cacheSize)
+
+	type pair struct{ ID, Count uint64 }
+	expect := []pair{
+		{5, 2},
+		{4, 1},
+	}
+
+	for _, v := range expect {
+		cache.Add(v.ID, v.Count)
+	}
+
+	var got []pair //nolint:prealloc
+	for _, p := range cache.Top() {
+		got = append(got, pair(p))
+	}
+
+	if !reflect.DeepEqual(expect, got) {
+		t.Fatalf("wrote %v but got %v", expect, got)
+	}
+}
+
+func TestCache_Rank_BulkAdd(t *testing.T) {
+	const cacheSize = 10
+	cache := pilosa.NewRankCache(uint32(cacheSize))
+
+	for i := uint64(0); i < 1000; i++ {
+		cache.BulkAdd(i, i)
+		if n := cache.Len(); n > cacheSize*2 {
+			t.Fatalf("entry count exceed 2x cache size: %d", n)
+		}
+	}
 }

catcher.go

@@ -0,0 +1,233 @@
+// Copyright 2022 Molecula Corp. (DBA FeatureBase).
+// SPDX-License-Identifier: Apache-2.0
+package pilosa
+
+import (
+	"github.com/featurebasedb/featurebase/v3/roaring"
+	txkey "github.com/featurebasedb/featurebase/v3/short_txkey"
+	"github.com/featurebasedb/featurebase/v3/vprint"
+)
+
+// catcher is useful to report error locations with a
+// Stack dump before the complexity
+// of the executor_test swallows up
+// the location of a PanicOn.
+type catcherTx struct {
+	b Tx
+}
+
+func newCatcherTx(b Tx) *catcherTx {
+	return &catcherTx{b: b}
+}
+
+func init() {
+	// keep golangci-lint happy
+	_ = newCatcherTx
+}
+
+var _ Tx = (*catcherTx)(nil)
+
+func (c *catcherTx) ImportRoaringBits(index, field, view string, shard uint64, rit roaring.RoaringIterator, clear bool, log bool, rowSize uint64) (changed int, rowSet map[uint64]int, err error) {
+	defer func() {
+		if r := recover(); r != nil {
+			vprint.AlwaysPrintf("see ImportRoaringBits() PanicOn '%v' at '%v'", r, vprint.Stack())
+			vprint.PanicOn(r)
+		}
+	}()
+	return c.b.ImportRoaringBits(index, field, view, shard, rit, clear, log, rowSize)
+}
+
+func (c *catcherTx) Rollback() {
+	defer func() {
+		if r := recover(); r != nil {
+			vprint.AlwaysPrintf("see Rollback() PanicOn '%v' at '%v'", r, vprint.Stack())
+			vprint.PanicOn(r)
+		}
+	}()
+	c.b.Rollback()
+}
+
+func (c *catcherTx) Commit() error {
+
+	defer func() {
+		if r := recover(); r != nil {
+			vprint.AlwaysPrintf("see Commit() PanicOn '%v' at '%v'", r, vprint.Stack())
+			vprint.PanicOn(r)
+		}
+	}()
+	return c.b.Commit()
+}
+
+func (c *catcherTx) RoaringBitmap(index, field, view string, shard uint64) (*roaring.Bitmap, error) {
+
+	defer func() {
+		if r := recover(); r != nil {
+			vprint.AlwaysPrintf("see RoaringBitmap() PanicOn '%v' at '%v'", r, vprint.Stack())
+			vprint.PanicOn(r)
+		}
+	}()
+	return c.b.RoaringBitmap(index, field, view, shard)
+}
+
+func (c *catcherTx) Container(index, field, view string, shard uint64, key uint64) (ct *roaring.Container, err error) {
+
+	defer func() {
+		if r := recover(); r != nil {
+			vprint.AlwaysPrintf("see Container() PanicOn '%v' at '%v'", r, vprint.Stack())
+			vprint.PanicOn(r)
+		}
+	}()
+	return c.b.Container(index, field, view, shard, key)
+}
+
+func (c *catcherTx) PutContainer(index, field, view string, shard uint64, key uint64, rc *roaring.Container) error {
+
+	defer func() {
+		if r := recover(); r != nil {
+			vprint.AlwaysPrintf("see PutContainer() PanicOn '%v' at '%v'", r, vprint.Stack())
+			vprint.PanicOn(r)
+		}
+	}()
+	return c.b.PutContainer(index, field, view, shard, key, rc)
+}
+
+func (c *catcherTx) RemoveContainer(index, field, view string, shard uint64, key uint64) error {
+
+	defer func() {
+		if r := recover(); r != nil {
+			vprint.AlwaysPrintf("see RemoveContainer() PanicOn '%v' at '%v'", r, vprint.Stack())
+			vprint.PanicOn(r)
+		}
+	}()
+	return c.b.RemoveContainer(index, field, view, shard, key)
+}
+
+func (c *catcherTx) Add(index, field, view string, shard uint64, a ...uint64) (changeCount int, err error) {
+
+	defer func() {
+		if r := recover(); r != nil {
+			vprint.AlwaysPrintf("see Add() PanicOn '%v' at '%v'", r, vprint.Stack())
+			vprint.PanicOn(r)
+		}
+	}()
+	return c.b.Add(index, field, view, shard, a...)
+}
+
+func (c *catcherTx) Remove(index, field, view string, shard uint64, a ...uint64) (changeCount int, err error) {
+
+	defer func() {
+		if r := recover(); r != nil {
+			vprint.AlwaysPrintf("see Remove() PanicOn '%v' at '%v'", r, vprint.Stack())
+			vprint.PanicOn(r)
+		}
+	}()
+	return c.b.Remove(index, field, view, shard, a...)
+}
+
+func (c *catcherTx) Removed(index, field, view string, shard uint64, a ...uint64) (changed []uint64, err error) {
+
+	defer func() {
+		if r := recover(); r != nil {
+			vprint.AlwaysPrintf("see Removed() PanicOn '%v' at '%v'", r, vprint.Stack())
+			vprint.PanicOn(r)
+		}
+	}()
+	return c.b.Removed(index, field, view, shard, a...)
+}
+
+func (c *catcherTx) Contains(index, field, view string, shard uint64, key uint64) (exists bool, err error) {
+
+	defer func() {
+		if r := recover(); r != nil {
+			vprint.AlwaysPrintf("see Contains() PanicOn '%v' at '%v'", r, vprint.Stack())
+			vprint.PanicOn(r)
+		}
+	}()
+	return c.b.Contains(index, field, view, shard, key)
+}
+
+func (c *catcherTx) ContainerIterator(index, field, view string, shard uint64, firstRoaringContainerKey uint64) (citer roaring.ContainerIterator, found bool, err error) {
+
+	defer func() {
+		if r := recover(); r != nil {
+			vprint.AlwaysPrintf("see ContainerIterator() PanicOn '%v' at '%v'", r, vprint.Stack())
+			vprint.PanicOn(r)
+		}
+	}()
+	return c.b.ContainerIterator(index, field, view, shard, firstRoaringContainerKey)
+}
+
+func (c *catcherTx) Count(index, field, view string, shard uint64) (uint64, error) {
+
+	defer func() {
+		if r := recover(); r != nil {
+			vprint.AlwaysPrintf("see Count() PanicOn '%v' at '%v'", r, vprint.Stack())
+			vprint.PanicOn(r)
+		}
+	}()
+	return c.b.Count(index, field, view, shard)
+}
+
+func (c *catcherTx) Max(index, field, view string, shard uint64) (uint64, error) {
+
+	defer func() {
+		if r := recover(); r != nil {
+			vprint.AlwaysPrintf("see Max() PanicOn '%v' at '%v'", r, vprint.Stack())
+			vprint.PanicOn(r)
+		}
+	}()
+	return c.b.Max(index, field, view, shard)
+}
+
+func (c *catcherTx) Min(index, field, view string, shard uint64) (uint64, bool, error) {
+
+	defer func() {
+		if r := recover(); r != nil {
+			vprint.AlwaysPrintf("see Min() PanicOn '%v' at '%v'", r, vprint.Stack())
+			vprint.PanicOn(r)
+		}
+	}()
+	return c.b.Min(index, field, view, shard)
+}
+
+func (c *catcherTx) CountRange(index, field, view string, shard uint64, start, end uint64) (n uint64, err error) {
+
+	defer func() {
+		if r := recover(); r != nil {
+			vprint.AlwaysPrintf("see CountRange() PanicOn '%v' at '%v'", r, vprint.Stack())
+			vprint.PanicOn(r)
+		}
+	}()
+	return c.b.CountRange(index, field, view, shard, start, end)
+}
+
+func (c *catcherTx) OffsetRange(index, field, view string, shard, offset, start, end uint64) (other *roaring.Bitmap, err error) {
+
+	defer func() {
+		if r := recover(); r != nil {
+			vprint.AlwaysPrintf("see OffsetRange() PanicOn '%v' at '%v'", r, vprint.Stack())
+			vprint.PanicOn(r)
+		}
+	}()
+	return c.b.OffsetRange(index, field, view, shard, offset, start, end)
+}
+
+func (c *catcherTx) Type() string {
+	return c.b.Type()
+}
+
+func (c *catcherTx) ApplyFilter(index, field, view string, shard uint64, ckey uint64, filter roaring.BitmapFilter) (err error) {
+	return GenericApplyFilter(c, index, field, view, shard, ckey, filter)
+}
+
+func (c *catcherTx) ApplyRewriter(index, field, view string, shard uint64, ckey uint64, filter roaring.BitmapRewriter) (err error) {
+	return c.b.ApplyRewriter(index, field, view, shard, ckey, filter)
+}
+
+func (c *catcherTx) GetSortedFieldViewList(idx *Index, shard uint64) (fvs []txkey.FieldView, err error) {
+	return c.b.GetSortedFieldViewList(idx, shard)
+}
+
+func (c *catcherTx) GetFieldSizeBytes(index, field string) (uint64, error) {
+	return 0, nil
+}

cli/Makefile

@@ -0,0 +1,15 @@
+.PHONY: test testv test-integration testv-integration
+
+GO=go
+
+test:
+	$(GO) test ./... -short
+
+testv:
+	$(GO) test -v ./... -short
+
+test-integration:
+	$(GO) test . -count 1 -timeout 20m -run TestCLIIntegration/$(RUN)
+
+testv-integration:
+	$(GO) test -v . -count 1 -timeout 20m -run TestCLIIntegration/$(RUN)

cli/batch/inserter.go

@@ -0,0 +1,8 @@
+package batch
+
+// Inserter can be implemented by anything which can handle a SQL statement
+// representing a write operation. An example is `BULK INSERT`. The Insert()
+// method on this interface does not return any results other than an error.
+type Inserter interface {
+	Insert(sql string) error
+}

cli/batch/sql.go

@@ -0,0 +1,215 @@
+package batch
+
+import (
+	"encoding/json"
+	"fmt"
+	"strings"
+	"time"
+
+	fbbatch "github.com/featurebasedb/featurebase/v3/batch"
+	"github.com/featurebasedb/featurebase/v3/dax"
+	"github.com/featurebasedb/featurebase/v3/errors"
+	"github.com/featurebasedb/featurebase/v3/pql"
+)
+
+// Ensure type implements interface.
+var _ fbbatch.Batcher = (*sqlBatcher)(nil)
+
+type sqlBatcher struct {
+	inserter Inserter
+	fields   []*dax.Field
+}
+
+func NewSQLBatcher(i Inserter, flds []*dax.Field) *sqlBatcher {
+	return &sqlBatcher{
+		inserter: i,
+		fields:   flds,
+	}
+}
+
+func (b *sqlBatcher) NewBatch(cfg fbbatch.Config, tbl *dax.Table, flds []*dax.Field) (fbbatch.RecordBatch, error) {
+	fields := flds
+	if b.fields != nil {
+		fields = b.fields
+	}
+	return &sqlBatch{
+		table:        tbl,
+		fields:       fields,
+		size:         cfg.Size,
+		maxStaleness: cfg.MaxStaleness,
+		ids:          make([]interface{}, 0, cfg.Size),
+		rows:         make([][]interface{}, 0, cfg.Size),
+		inserter:     b.inserter,
+	}, nil
+}
+
+// Ensure type implements interface.
+var _ fbbatch.RecordBatch = (*sqlBatch)(nil)
+
+type sqlBatch struct {
+	table  *dax.Table
+	fields []*dax.Field
+	size   int
+
+	ids  []interface{}
+	rows [][]interface{}
+
+	// staleTime tracks the time the first record of the batch was inserted
+	// plus the maxStaleness, in order to raise ErrBatchNowStale if the
+	// maxStaleness has elapsed
+	staleTime    time.Time
+	maxStaleness time.Duration
+
+	// inserter handles SQL INSERT statements generated for each batch.
+	inserter Inserter
+}
+
+func (b *sqlBatch) Add(rec fbbatch.Row) error {
+	// Clear rec.Values and rec.Clears upon return.
+	defer func() {
+		for i := range rec.Values {
+			rec.Values[i] = nil
+		}
+		for k := range rec.Clears {
+			delete(rec.Clears, k)
+		}
+	}()
+
+	if len(b.ids) == cap(b.ids) {
+		return fbbatch.ErrBatchAlreadyFull
+	}
+	if len(rec.Values) != len(b.fields) {
+		return errors.Errorf("record needs to match up with batch fields, got %d fields and %d record", len(b.fields), len(rec.Values))
+	}
+
+	// Append the ID to b.ids.
+	b.ids = append(b.ids, rec.ID)
+
+	// Convert decimal fields (which come in as int64, along with the scale in
+	// field) to pql.Decimal.
+	for i, fld := range b.fields {
+		switch b.fields[i].Type {
+		case dax.BaseTypeDecimal:
+			if val, ok := rec.Values[i].(int64); ok {
+				rec.Values[i] = pql.NewDecimal(val, fld.Options.Scale)
+			}
+		case dax.BaseTypeTimestamp:
+			if val, ok := rec.Values[i].(int64); ok {
+				ts := time.Unix(val, 0)
+				rec.Values[i] = ts.Format(time.RFC3339)
+			}
+		}
+	}
+
+	// Append the record to b.rows.
+	vals := make([]interface{}, 0, len(rec.Values))
+	vals = append(vals, rec.Values...)
+	b.rows = append(b.rows, vals)
+
+	// Check for batch full or stale.
+	if len(b.ids) == cap(b.ids) {
+		return fbbatch.ErrBatchNowFull
+	}
+	if b.maxStaleness != time.Duration(0) { // set maxStaleness to 0 to disable staleness checking
+		if len(b.ids) == 1 {
+			b.staleTime = time.Now().Add(b.maxStaleness)
+		} else if time.Now().After(b.staleTime) {
+			return fbbatch.ErrBatchNowStale
+		}
+	}
+	return nil
+}
+
+func (b *sqlBatch) Import() error {
+	if len(b.rows) == 0 {
+		return nil
+	}
+
+	// Construct the BULK INSERT statement based on the table and fields.
+	sql, err := buildBulkInsert(b.table, b.fields, b.ids, b.rows)
+	if err != nil {
+		return errors.Wrap(err, "building bulk insert statement")
+	}
+
+	// Reset batch data.
+	b.reset()
+
+	// Submit the SQL statement.
+	return b.inserter.Insert(sql)
+}
+
+func (b *sqlBatch) reset() {
+	b.ids = b.ids[:0]
+	b.rows = b.rows[:0]
+}
+
+func (b *sqlBatch) Len() int {
+	return len(b.rows)
+}
+
+func (b *sqlBatch) Flush() error {
+	return nil
+}
+
+func buildBulkInsert(tbl *dax.Table, fields []*dax.Field, ids []interface{}, rows [][]interface{}) (string, error) {
+	// Validation.
+	if tbl.Name == "" {
+		return "", errors.New(errors.ErrUncoded, "table name is required")
+	} else if len(fields) == 0 {
+		return "", errors.New(errors.ErrUncoded, "at least one field is required")
+	}
+
+	var sb strings.Builder
+
+	sb.WriteString(`BULK INSERT INTO `)
+	sb.WriteString(string(tbl.Name))
+	sb.WriteString(` (_id,`)
+
+	flds := make([]string, 0, len(fields))
+	maps := make([]string, 0, len(fields))
+	for i := range fields {
+		flds = append(flds, string(fields[i].Name))
+		maps = append(maps, fmt.Sprintf("'$.col_%d' %s", i, fields[i].FullType()))
+	}
+	// Fields
+	sb.WriteString(strings.Join(flds, ","))
+
+	// MAP
+	keyType := dax.BaseTypeID
+	if tbl.StringKeys() {
+		keyType = dax.BaseTypeString
+	}
+	sb.WriteString(`) MAP ('$._id' `)
+	sb.WriteString(keyType)
+	sb.WriteString(`,`)
+	sb.WriteString(strings.Join(maps, ","))
+	sb.WriteString(`) FROM x'`)
+
+	// Row values.
+
+	// m is a map representing a single row to be marshalled and added to the
+	// bulk insert as one line in the NDJSON payload. We re-use the map for each
+	// row.
+	m := make(map[string]interface{})
+	for i := range rows {
+		// Write the ID value.
+		m[string(dax.PrimaryKeyFieldName)] = ids[i]
+		// Write the rest of the data values.
+		for col := range rows[i] {
+			m[fmt.Sprintf("col_%d", col)] = rows[i][col]
+		}
+
+		// Marshal the map to json and add to the sql statement.
+		if j, err := json.Marshal(m); err != nil {
+			return "", errors.Wrap(err, "marshalling row to json")
+		} else {
+			sb.Write(j)
+			sb.WriteString("\n")
+		}
+	}
+
+	// WITH
+	sb.WriteString(fmt.Sprintf(`' WITH BATCHSIZE %d FORMAT 'NDJSON' INPUT 'STREAM'`, len(rows)))
+
+	return sb.String(), nil
+}

cli/batch/sql_test.go

@@ -0,0 +1,47 @@
+package batch
+
+import (
+	"testing"
+
+	"github.com/featurebasedb/featurebase/v3/dax"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestBatchSQL(t *testing.T) {
+	tbl := &dax.Table{
+		Name: "foo",
+	}
+	fields := []*dax.Field{
+		{
+			Name: "name",
+			Type: dax.BaseTypeString,
+		},
+		{
+			Name: "age",
+			Type: dax.BaseTypeInt,
+		},
+	}
+	ids := []interface{}{
+		0, 1, 2,
+	}
+	rows := [][]interface{}{
+		{
+			[]interface{}{"Alice", int64(11)},
+		},
+		{
+			[]interface{}{"Bob", int64(22)},
+		},
+		{
+			[]interface{}{"Carl,Comma", int64(33)},
+		},
+	}
+
+	s, err := buildBulkInsert(tbl, fields, ids, rows)
+	assert.NoError(t, err)
+
+	exp := `BULK INSERT INTO foo (_id,name,age) MAP ('$._id' id,'$.col_0' string,'$.col_1' int) FROM x'{"_id":0,"col_0":["Alice",11]}
+{"_id":1,"col_0":["Bob",22]}
+{"_id":2,"col_0":["Carl,Comma",33]}
+' WITH BATCHSIZE 3 FORMAT 'NDJSON' INPUT 'STREAM'`
+	assert.Equal(t, exp, s)
+}

cli/buffer.go

@@ -0,0 +1,91 @@
+package cli
+
+import (
+	"io"
+	"strings"
+
+	"github.com/featurebasedb/featurebase/v3/errors"
+)
+
+// buffer is a query buffer for SQL statements. Note that this is not a query
+// buffer as you would find on a database server (buffering query results).
+// Rather, this buffers the working SQL statement. The buffer has two
+// components: the buffer of query parts making up the working, incomplete SQL
+// statement, and the last completed SQL statement submitted to the Queryer.
+type buffer struct {
+	parts     []queryPart
+	lastQuery query
+
+	hasBatchFile bool
+}
+
+func newBuffer() *buffer {
+	return &buffer{}
+}
+
+// addPart adds the given queryPart to the buffer. If the part is of type
+// `partTerminator` (which is generally singified in the CLI by a ";"), the
+// buffer will finalize the query and return it. In all other cases, the
+// returned query is nil.
+func (b *buffer) addPart(part queryPart) (query, error) {
+	// Check for part type compatibility. For example, multiple batchFile parts
+	// are not allowed in the same query.
+	switch part.(type) {
+	case *partBatchFile:
+		if b.hasBatchFile {
+			return nil, errors.Errorf("multiple batch files in one query is not supported")
+		}
+		b.hasBatchFile = true
+	case *partTerminator:
+		return b.finalize(), nil
+	}
+
+	b.parts = append(b.parts, part)
+	return nil, nil
+}
+
+// finalize copies the contents (queryParts) of buffer to lastQuery and then
+// resets the buffer. It returns the query that was finalized.
+func (b *buffer) finalize() query {
+	q := make(query, len(b.parts))
+	copy(q, b.parts)
+	b.lastQuery = q
+	b.reset()
+	return q
+
+}
+
+// print returns the contents of the buffer as a string. This is generally used
+// to visually inspect the state of the buffer (for example, when a user issues
+// a `\p` meta-command in the CLI).
+func (b *buffer) print() string {
+	if len(b.parts) > 0 {
+		return query(b.parts).String()
+	} else if b.lastQuery != nil {
+		return b.lastQuery.String() + ";"
+	}
+	return "Query buffer is empty."
+}
+
+// reset clears the buffer. It returns a message which may optionally be used to
+// display to a user.
+func (b *buffer) reset() string {
+	b.parts = b.parts[:0]
+	b.hasBatchFile = false
+	return "Query buffer reset (cleared)."
+}
+
+func (b *buffer) Reader() io.Reader {
+	if len(b.parts) > 0 {
+		return query(b.parts).Reader()
+	} else if b.lastQuery != nil {
+		r := b.lastQuery.Reader()
+		// TODO(tlt): terminating the query here results in a line feed just
+		// before the semi-colon (for example, when you print out the query
+		// buffer using `\w [FILE]`). The removal and re-introduction of line
+		// feeds is kind of a mess.
+		term := strings.NewReader(";")
+		return io.MultiReader(r, term)
+	}
+	return strings.NewReader("")
+}

cli/cli.go

@@ -0,0 +1,816 @@
+// Package cli contains a FeatureBase command line interface.
+package cli
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"github.com/chzyer/readline"
+	featurebase "github.com/featurebasedb/featurebase/v3"
+	"github.com/featurebasedb/featurebase/v3/cli/batch"
+	"github.com/featurebasedb/featurebase/v3/cli/fbcloud"
+	"github.com/featurebasedb/featurebase/v3/errors"
+	"github.com/featurebasedb/featurebase/v3/logger"
+)
+
+const (
+	defaultHost     string = "localhost"
+	defaultClientID string = "6i2gs7mu215ab23cnvmshdoq6t" // production Cognito client ID
+	defaultRegion   string = "us-east-2"
+	terminationChar string = ";"
+	nullValue       string = "NULL"
+)
+
+var (
+	Stdin  io.ReadCloser = os.Stdin
+	Stdout io.Writer     = os.Stdout
+	Stderr io.Writer     = os.Stderr
+)
+
+var splash string = fmt.Sprintf(`FeatureBase CLI (%s)
+Type "\q" to quit.
+`, featurebase.Version)
+
+// Ensure type implments interfaces.
+var _ printer = (*Command)(nil)
+var _ batch.Inserter = (*Command)(nil)
+
+type Command struct {
+	host string
+	port string
+
+	splitter   *splitter
+	buffer     *buffer
+	workingDir *workingDir
+
+	organizationID string
+	database       string
+	databaseID     string
+	databaseName   string
+
+	Queryer Queryer `json:"-"`
+
+	stdin  io.ReadCloser `json:"-"`
+	stdout io.Writer     `json:"-"`
+	stderr io.Writer     `json:"-"`
+
+	// output is where actual results are written. This might point to stdout,
+	// or to a file, based on the current configuration.
+	output       io.Writer `json:"-"`
+	writeOptions *writeOptions
+
+	Config *Config `json:"config"`
+
+	historyPath string
+
+	// Commands contains optional commands provided via one or more `-c` (or
+	// `--command`) flags. If this is non-empty, the cli will run in
+	// non-interactive mode; i.e. it will quit after the command is complete.
+	Commands []string `json:"commands"`
+
+	// Files contains optional files provided via one or more `-f` (or `--file`)
+	// flags. If this is non-empty, the cli will run in non-interactive mode;
+	// i.e. it will quit after the command is complete.
+	Files []string `json:"files"`
+
+	// variables holds the variables created with the \set meta-command.
+	variables map[string]string
+
+	// nonInteractiveMode is set to true when fbsql is running in
+	// non-ineracative mode. And example of this is when the user has provided a
+	// `-c` flag in the command line.
+	nonInteractiveMode bool
+
+	// quit gets closed when Run should stop listening for input.
+	quit chan struct{}
+}
+
+func NewCommand(logdest logger.Logger) *Command {
+	variables := make(map[string]string)
+
+	return &Command{
+		Config: &Config{
+			Host: defaultHost,
+			Port: "",
+
+			OrganizationID: "",
+			Database:       "",
+
+			CloudAuth: CloudAuthConfig{
+				ClientID: defaultClientID,
+				Region:   defaultRegion,
+				Email:    "",
+				Password: "",
+			},
+
+			HistoryPath: "",
+
+			CSV: false,
+		},
+
+		buffer:     newBuffer(),
+		splitter:   newSplitter(newReplacer(variables)),
+		workingDir: newWorkingDir(),
+
+		stdin:  Stdin,
+		stdout: Stdout,
+		stderr: Stderr,
+
+		output:       Stdout,
+		writeOptions: defaultWriteOptions(),
+
+		variables: variables,
+
+		quit: make(chan struct{}),
+	}
+}
+
+// SetStdin sets stdin. This is useful for initial configuration in tests.
+func (cmd *Command) SetStdin(rc io.ReadCloser) {
+	cmd.stdin = rc
+}
+
+// SetStdout sets both stdout and output to the value provided. This is useful
+// for initial configuration in tests.
+func (cmd *Command) SetStdout(w io.Writer) {
+	cmd.stdout = w
+	cmd.output = w
+}
+
+// SetStderr sets stderr. This is useful for initial configuration in tests.
+func (cmd *Command) SetStderr(w io.Writer) {
+	cmd.stderr = w
+}
+
+// Run is the main entry-point to the CLI.
+func (cmd *Command) Run(ctx context.Context) error {
+	if err := cmd.run(ctx); err != nil {
+		cmd.Errorf(err.Error() + "\n")
+		return err
+	}
+	return nil
+}
+
+// run is effectively wrapped by the Run() method, but it's split out this way
+// so that run() can simply return errors, rather than worrying about how errors
+// should be printed; printing errors returned by run() is left up to the Run()
+// method.
+func (cmd *Command) run(ctx context.Context) error {
+	if err := cmd.setupConfig(); err != nil {
+		return errors.Wrap(err, "setting up config")
+	}
+
+	// Check to see if Command needs to run in non-interactive mode.
+	if len(cmd.Commands) > 0 ||
+		len(cmd.Files) > 0 ||
+		cmd.Config.KafkaConfig != "" ||
+		cmd.Config.CSV {
+		cmd.nonInteractiveMode = true
+	}
+
+	// Print the splash message.
+	if !cmd.nonInteractiveMode {
+		cmd.Printf(splash)
+	}
+
+	if err := cmd.setupClient(); err != nil {
+		return errors.Wrap(err, "setting up client")
+	}
+
+	// Print the connection info.
+	if !cmd.nonInteractiveMode {
+		cmd.printConnInfo()
+	}
+
+	if err := cmd.connectToDatabase(cmd.database); err != nil {
+		cmd.Errorf(errors.Wrap(err, "connecting to database").Error() + "\n")
+		// We intentionally do not return err here.
+	}
+
+	// Run in non-interactive mode based on flags and configuration.
+	// This includes either handling `-c` and/or `-f` flags, or handling a
+	// `--kafka-config` flag.
+	if len(cmd.Commands) > 0 || len(cmd.Files) > 0 {
+		// Run Commands.
+		for _, line := range cmd.Commands {
+			if err := cmd.handleLine(line); err != nil {
+				return errors.Wrapf(err, "handling line: %s", line)
+			}
+		}
+
+		// Run Files.
+		for _, fname := range cmd.Files {
+			if _, err := executeFile(cmd, fname); err != nil {
+				return errors.Wrapf(err, "executing file: %s", fname)
+			}
+		}
+
+		return nil
+	} else if cmd.Config.KafkaConfig != "" {
+		runner, err := cmd.newKafkaRunner(cmd.Config.KafkaConfig)
+		if err != nil {
+			return errors.Wrap(err, "getting new kafka runner")
+		}
+		if err := runner.Main.Run(); err != nil {
+			return errors.Wrap(err, "running kafka")
+		}
+		return nil
+	}
+
+	// From this point on, we should be in interactive mode.
+
+	// Set up history for saving user input.
+	cmd.setupHistory()
+
+	rl, err := readline.NewEx(&readline.Config{
+		Prompt:                 cmd.prompt(false),
+		HistoryFile:            cmd.historyPath,
+		HistoryLimit:           100000,
+		DisableAutoSaveHistory: true,
+
+		Stdin:  cmd.stdin,
+		Stdout: cmd.stdout,
+		Stderr: cmd.stderr,
+	})
+	if err != nil {
+		return errors.Wrap(err, "getting readline")
+	}
+	defer rl.Close()
+
+	// inMidCommand indicates whether a partial command has been received and
+	// we're still waiting for a termination character.
+	var inMidCommand bool
+
+	for {
+		rl.SetPrompt(cmd.prompt(inMidCommand))
+
+		// Read user provided input.
+		line, err := rl.Readline()
+		if err == readline.ErrInterrupt {
+			inMidCommand = false
+			cmd.buffer.reset()
+			continue
+		} else if err != nil {
+			return errors.Wrap(err, "reading line")
+		}
+
+		// We append a line feed at the end of each line because at this point
+		// we have effectively stripped any intentional line feeds (since we are
+		// reading a line at a time), and we don't want to do that. An example
+		// of an intentional line feed is in a BULK INSERT CSV STREAM like this
+		// example:
+		//
+		// bulk replace
+		// 	 into foo (_id, age)
+		// 	 map (0 id, 1 int)
+		// from
+		// 	 x'3,33
+		// 	 4,44
+		// 	 5,55'
+		// with
+		// 	 format 'CSV'
+		// 	 input 'STREAM';
+		//
+		// We want to preserve the line feeds that are contained in the x''
+		// block; those are intentional as they demarc records within the csv.
+		qps, mcs, err := cmd.splitter.split(line + "\n")
+		if err != nil {
+			cmd.Errorf("error splitting line: %s\n", err)
+			continue
+		}
+
+		// Save line in the history.
+		if err := rl.SaveHistory(line); err != nil {
+			cmd.Errorf("Couldn't save history: %v\n", err)
+		}
+
+		// This is wrapped in an anonymous function so we can capture any
+		// errors, ignore the rest of the line, and return back to a prompt.
+		if err := func() error {
+			for i := range qps {
+				if qry, err := cmd.buffer.addPart(qps[i]); err != nil {
+					return errors.Wrap(err, "adding part to buffer")
+				} else if qry != nil {
+					if err := cmd.executeAndWriteQuery(qry); err != nil {
+						return errors.Wrap(err, "executing query")
+					}
+					// In addition to saving each line in the history, we also
+					// save each successful query.
+					if err := rl.SaveHistory(qry.String() + ";"); err != nil {
+						cmd.Errorf("Couldn't save query in history: %v\n", err)
+					}
+					inMidCommand = false
+				} else {
+					inMidCommand = true
+				}
+			}
+			return nil
+		}(); err != nil {
+			cmd.Errorf(err.Error() + "\n")
+			inMidCommand = false
+			continue
+		}
+
+		// This is wrapped in an anonymous function so we can capture any
+		// errors, ignore the rest of the line, and return back to a prompt.
+		if err := func() error {
+			for i := range mcs {
+				action, err := mcs[i].execute(cmd)
+				if err != nil {
+					return errors.Wrap(err, "executing meta command")
+				}
+				switch action {
+				case actionQuit:
+					close(cmd.quit)
+					return nil
+				case actionReset:
+					inMidCommand = false
+				}
+			}
+			return nil
+		}(); err != nil {
+			cmd.Errorf(err.Error() + "\n")
+			inMidCommand = false
+			continue
+		}
+
+		select {
+		case <-cmd.quit:
+			if err := cmd.close(); err != nil {
+				cmd.Errorf("closing: %s\n", err)
+			}
+			return nil
+		default:
+			// pass
+		}
+	}
+}
+
+// prompt constructs the prompt that the user sees based on the currently
+// connected database and whether the user is in the middle of a sql statement.
+func (cmd *Command) prompt(mid bool) string {
+	db := "fbsql" // default prompt when a database is not set.
+	if cmd.databaseName != "" {
+		db = cmd.databaseName
+	}
+
+	if mid {
+		return strings.Repeat(" ", len(db)) + "-# "
+	}
+	return db + "=# "
+}
+
+// close is called upon quitting. It should close any remaining open file
+// handles used by the CLICommand.
+func (cmd *Command) close() error {
+	return cmd.closeOutput()
+}
+
+// setupConfig sets up private struct members based on values provided via the
+// configuration flags.
+func (cmd *Command) setupConfig() error {
+	if cmd.Config == nil {
+		return nil
+	}
+
+	cmd.host = cmd.Config.Host
+	cmd.port = cmd.Config.Port
+
+	cmd.organizationID = cmd.Config.OrganizationID
+	cmd.database = cmd.Config.Database
+
+	cmd.historyPath = cmd.Config.HistoryPath
+
+	// Apply any pset flag arguments.
+	for _, pset := range cmd.Config.PSets {
+		if err := cmd.applyPSet(pset); err != nil {
+			return errors.Wrapf(err, "applying pset: %s", pset)
+		}
+	}
+
+	// If running with the `--csv` flag, configure things to ensure the output
+	// is correct (i.e. that it's just the csv).
+	if cmd.Config.CSV {
+		cmd.writeOptions.format = formatCSV
+	}
+
+	return nil
+}
+
+// applyPSet takes a pset string of the form `arg` or `arg=val` and applies it
+// as if the user had run `\pset arg val`. The only difference is that applying
+// pset here suppresses any output to stdout.
+func (cmd *Command) applyPSet(pset string) error {
+	// We expect arg to be one of the folowing formats:
+	// arg
+	// arg=val
+	args := strings.SplitN(pset, "=", 2)
+
+	// This is kind of hacky, but until we re-think the metaCommand interface to
+	// take a printer interface somewhere (so we can pass in the nopPrinter
+	// here), we're just going to discard stdout for the duration of this apply,
+	// and then set stdout back to its previous writer after the apply.
+	hold := cmd.stdout
+	cmd.stdout = io.Discard
+	defer func() {
+		cmd.stdout = hold
+	}()
+
+	_, err := newMetaPSet(args).execute(cmd)
+	return err
+}
+
+func (cmd *Command) executeAndWriteQuery(qry query) error {
+	queryResponse, err := cmd.executeQuery(qry)
+	if err != nil {
+		if errors.Is(err, ErrOrganizationRequired) {
+			// Print an error message and return nil, effectively aborting any
+			// further writes for this query.
+			cmd.Errorf("Organization required. Use \\org to set an organization.\n")
+			return nil
+		}
+		return errors.Wrap(err, "making query")
+	}
+	if err := writeOutput(queryResponse, cmd.writeOptions, cmd.output, cmd.stdout, cmd.stderr); err != nil {
+		return errors.Wrap(err, "writing out response")
+	}
+
+	return nil
+}
+
+func (cmd *Command) executeQuery(qry query) (*featurebase.WireQueryResponse, error) {
+	wqr, err := cmd.Queryer.Query(cmd.organizationID, cmd.databaseID, qry.Reader())
+	if err != nil {
+		return nil, errors.Wrap(err, "executing query")
+	}
+
+	// If we're running in non-interactive mode, we need to check the error that
+	// comes back in the WireQueryResponse. If there's an error, we want to
+	// return it now (rather than just printing it later) so that we immediately
+	// stop any further execution of commands.
+	if cmd.nonInteractiveMode && wqr.Error != "" {
+		return nil, errors.Errorf(wqr.Error)
+	}
+
+	return wqr, nil
+}
+
+// printer is an interface which encapsulates the methods used to print output
+// to the various io.Writers.
+type printer interface {
+	Printf(format string, a ...any)
+	Outputf(format string, a ...any)
+	Errorf(format string, a ...any)
+}
+
+type nopPrinter struct{}
+
+func newNopPrinter() *nopPrinter {
+	return &nopPrinter{}
+}
+
+func (n *nopPrinter) Printf(format string, a ...any)  {}
+func (n *nopPrinter) Outputf(format string, a ...any) {}
+func (n *nopPrinter) Errorf(format string, a ...any)  {}
+
+// Printf is a helper method which sends the given payload to stdout.
+func (cmd *Command) Printf(format string, a ...any) {
+	out := fmt.Sprintf(format, a...)
+	cmd.stdout.Write([]byte(out))
+}
+
+// Outputf is a helper method which sends the given payload to output.
+func (cmd *Command) Outputf(format string, a ...any) {
+	out := fmt.Sprintf(format, a...)
+	cmd.output.Write([]byte(out))
+}
+
+// Errorf is a helper method which sends the given payload to stderr.
+func (cmd *Command) Errorf(format string, a ...any) {
+	out := fmt.Sprintf(format, a...)
+	cmd.stderr.Write([]byte(out))
+}
+
+func (cmd *Command) setupHistory() {
+	// If HistoryPath has already been configured (i.e. with a command flag),
+	// don't bother setting up the default in the home directory.
+	if cmd.historyPath != "" {
+		return
+	}
+
+	historyPath := ""
+	if home, err := os.UserHomeDir(); err != nil {
+		cmd.Errorf("Error getting home directory, command history persistence will be disabled: %v\n", err)
+	} else {
+		historyDir := filepath.Join(home, ".featurebase")
+		err := os.MkdirAll(historyDir, 0o750)
+		if err != nil {
+			cmd.Errorf("Creating directory for history: %v\n", err)
+		} else {
+			historyPath = filepath.Join(historyDir, "fbsql_history")
+		}
+	}
+	cmd.historyPath = historyPath
+}
+
+// printConnInfo displays the currently set host.
+// TODO(tlt): extend this to be the output of the /conninfo meta-command.
+func (cmd *Command) printConnInfo() {
+	cmd.Printf("Host: %s\n", hostPort(cmd.host, cmd.port))
+}
+
+func (cmd *Command) connectToDatabase(dbName string) error {
+	var p printer = cmd
+	if cmd.nonInteractiveMode {
+		p = newNopPrinter()
+	}
+
+	// Providing a blank ("") or hyphen ("-") dbName is the equivalent of
+	// disconnecting from the current database. We support the hyphen option
+	// because calling the `\c` meta-command without an argument is how you
+	// print the current connection.
+	switch dbName {
+	case "-", "":
+		cmd.databaseID = ""
+		cmd.databaseName = ""
+		p.Printf(cmd.connectionMessage())
+		return nil
+	}
+
+	// Look up dbID based on dbName.
+	wqr, err := cmd.executeQuery(newRawQuery("SHOW DATABASES"))
+	if err != nil {
+		return errors.Wrap(err, "executing query")
+	}
+
+	for _, db := range wqr.Data {
+		// 0: _id
+		// 1: name
+		if db[1] == dbName {
+			cmd.databaseName = dbName
+			cmd.databaseID = db[0].(string)
+			p.Printf(cmd.connectionMessage())
+			return nil
+		}
+	}
+	return errors.Errorf("invalid database: %s", dbName)
+}
+
+func (cmd *Command) orgMessage() string {
+	if cmd.organizationID == "" {
+		return "You have not set an organization.\n"
+	}
+	return fmt.Sprintf("You have set organization \"%s\".\n", cmd.organizationID)
+}
+
+func (cmd *Command) connectionMessage() string {
+	if cmd.databaseName == "" {
+		return "You are not connected to a database.\n"
+	}
+	return fmt.Sprintf("You are now connected to database \"%s\" (%s).\n", cmd.databaseName, cmd.databaseID)
+}
+
+func (cmd *Command) setupClient() error {
+	// If the Queryer has already been set (in tests for example), don't bother
+	// trying to detect it.
+	if cmd.Queryer != nil {
+		return nil
+	}
+
+	var p printer = cmd
+	if cmd.nonInteractiveMode {
+		p = newNopPrinter()
+	}
+
+	if strings.TrimSpace(cmd.host) == "" {
+		return errors.Errorf("no host provided\n")
+	}
+
+	if !strings.HasPrefix(cmd.host, "http") {
+		cmd.host = "http://" + cmd.host
+	}
+
+	typ, err := cmd.detectFBType()
+	if err != nil {
+		return errors.Wrap(err, "detecting FeatureBase deployment type")
+	}
+
+	switch typ {
+	case featurebaseTypeOnPremClassic:
+		p.Printf("Detected on-prem, classic deployment.\n")
+		cmd.Queryer = &standardQueryer{
+			Host: cmd.host,
+			Port: cmd.port,
+		}
+	case featurebaseTypeOnPremServerless:
+		p.Printf("Detected on-prem, serverless deployment.\n")
+		cmd.Queryer = &serverlessQueryer{
+			Host: cmd.host,
+			Port: cmd.port,
+		}
+	case featurebaseTypeCloud:
+		p.Printf("Detected cloud deployment.\n")
+
+		cmd.Queryer = &fbcloud.Queryer{
+			Host: hostPort(cmd.host, cmd.port),
+
+			ClientID: cmd.Config.CloudAuth.ClientID,
+			Region:   cmd.Config.CloudAuth.Region,
+			Email:    cmd.Config.CloudAuth.Email,
+			Password: cmd.Config.CloudAuth.Password,
+		}
+	case featurebaseTypeUnknown:
+		p.Printf("Could not detect deployment\n")
+		// cmd.Queryer = &nopQueryer{}
+		// Instead of using a no-op queryer when the type can't be detected, we
+		// default to using a cloud queryer.
+		cmd.Queryer = &fbcloud.Queryer{
+			Host: hostPort(cmd.host, cmd.port),
+
+			ClientID: cmd.Config.CloudAuth.ClientID,
+			Region:   cmd.Config.CloudAuth.Region,
+			Email:    cmd.Config.CloudAuth.Email,
+			Password: cmd.Config.CloudAuth.Password,
+		}
+	default:
+		return errors.Errorf("unknown type: %s", typ)
+	}
+	return nil
+}
+
+type featurebaseType string
+
+const (
+	featurebaseTypeUnknown          featurebaseType = "unknown"            // unknown
+	featurebaseTypeOnPremClassic    featurebaseType = "on-prem-standard"   // on-prem, classic
+	featurebaseTypeOnPremServerless featurebaseType = "on-prem-serverless" // on-prem, serverless
+	featurebaseTypeCloud            featurebaseType = "cloud"              // cloud, (both classic and serverless)?
+)
+
+func hostPort(host, port string) string {
+	if port == "" {
+		return host
+	}
+	return host + ":" + port
+}
+
+// detectFBType determines if we're talking to standalone FeatureBase
+// or FeatureBase Cloud
+func (cmd *Command) detectFBType() (featurebaseType, error) {
+	type trial struct {
+		port   string
+		health string
+		typ    featurebaseType
+	}
+
+	// trials is populated with the url/endpoints to try in order to detect if a
+	// process is running there which can support the cli requests.
+	trials := []trial{}
+
+	var clientTimeout time.Duration
+	if cmd.port != "" {
+		clientTimeout = 100 * time.Millisecond
+		trials = append(trials,
+			// on-prem, serverless
+			trial{
+				port:   cmd.port,
+				health: "/queryer/health",
+				typ:    featurebaseTypeOnPremServerless,
+			},
+			// on-prem, classic
+			trial{
+				port:   cmd.port,
+				health: "/status",
+				typ:    featurebaseTypeOnPremClassic,
+			},
+		)
+	} else if strings.HasPrefix(cmd.host, "https") {
+		// https suggesting we might be connecting to a cloud host
+		clientTimeout = 1 * time.Second
+		trials = append(trials,
+			// cloud
+			trial{
+				port:   "",
+				health: "/health",
+				typ:    featurebaseTypeCloud,
+			},
+		)
+	} else {
+		// Try default ports just in case.
+		clientTimeout = 100 * time.Millisecond
+		trials = append(trials,
+			// on-prem, serverless
+			trial{
+				port:   "8080",
+				health: "/queryer/health",
+				typ:    featurebaseTypeOnPremServerless,
+			},
+			// on-prem, classic
+			trial{
+				port:   "10101",
+				health: "/status",
+				typ:    featurebaseTypeOnPremClassic,
+			},
+		)
+	}
+
+	client := http.Client{
+		Timeout: clientTimeout,
+	}
+	for _, trial := range trials {
+		url := hostPort(cmd.host, trial.port) + trial.health
+		if resp, err := client.Get(url); err != nil {
+			continue
+		} else if resp.StatusCode/100 == 2 {
+			cmd.port = trial.port
+			return trial.typ, nil
+		}
+	}
+
+	return featurebaseTypeUnknown, nil
+}
+
+func (cmd *Command) closeOutput() error {
+	if cmd.output == nil {
+		return nil
+	}
+
+	if closer, ok := cmd.output.(io.Closer); ok {
+		return closer.Close()
+	}
+
+	return nil
+}
+
+func (cmd *Command) handleLine(line string) error {
+	// For single-line command handling, we handle either a meta-command, or
+	// query parts, but not both. The logic is that any line which begins with
+	// "\" will be handled as a meta-command, otherwise it will be handled as a
+	// query.
+	if len(line) == 0 {
+		return nil
+	} else if line[0] == byte('\\') {
+		return cmd.handleLineAsMetaCommand(line)
+	} else {
+		return cmd.handleLineAsQueryParts(line)
+	}
+}
+
+func (cmd *Command) handleLineAsMetaCommand(line string) error {
+	_, mcs, err := cmd.splitter.split(line)
+	if err != nil {
+		return errors.Wrapf(err, "splitting line")
+	}
+
+	for i := range mcs {
+		_, err := mcs[i].execute(cmd)
+		if err != nil {
+			return errors.Wrap(err, "executing meta command")
+		}
+	}
+
+	return nil
+}
+
+func (cmd *Command) handleLineAsQueryParts(line string) error {
+	qps, mcs, err := cmd.splitter.split(line)
+	if err != nil {
+		return errors.Wrapf(err, "splitting line")
+	} else if len(mcs) > 0 {
+		return errors.Errorf("--command does not support meta-commands")
+	}
+
+	// Add a termintor part to the end of []queryPart. We do this because the
+	// command is coming in from the --command flag, it may not end with a
+	// semi-colon, but we still want to execute it.
+	if len(qps) > 0 {
+		if _, ok := qps[len(qps)-1].(*partTerminator); !ok {
+			qps = append(qps, newPartTerminator())
+		}
+	}
+
+	for i := range qps {
+		if qry, err := cmd.buffer.addPart(qps[i]); err != nil {
+			return errors.Wrap(err, "adding part to buffer")
+		} else if qry != nil {
+			if err := cmd.executeAndWriteQuery(qry); err != nil {
+				return errors.Wrap(err, "executing query")
+			}
+		}
+	}
+	return nil
+}
+
+func (cmd *Command) Insert(sql string) error {
+	wqr, err := cmd.executeQuery(newRawQuery(sql))
+	if wqr.Error != "" {
+		return errors.Errorf(wqr.Error)
+	}
+	return err
+}

cli/cli_integration_test.go

@@ -0,0 +1,257 @@
+package cli_test
+
+import (
+	"bufio"
+	"context"
+	"fmt"
+	"os"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/featurebasedb/featurebase/v3/cli"
+	"github.com/featurebasedb/featurebase/v3/dax/server/test"
+	"github.com/featurebasedb/featurebase/v3/logger"
+	"github.com/stretchr/testify/require"
+)
+
+func TestCLIIntegration(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test")
+	}
+
+	ctx := context.Background()
+
+	t.Run("Stubbed Framework", func(t *testing.T) {
+		mc := test.MustRunManagedCommand(t)
+		defer mc.Close()
+
+		addr := mc.Address()
+
+		capture := newCapture(t)
+
+		comparer := newComparer(t)
+		comparer.run()
+
+		fbsql := cli.NewCommand(logger.StderrLogger)
+		fbsql.SetStdin(capture)
+		fbsql.SetStdout(comparer)
+		fbsql.SetStderr(comparer)
+
+		fbsql.Config = &cli.Config{
+			Host: addr.Host(),
+			Port: fmt.Sprintf("%d", addr.Port()),
+		}
+
+		// Run fbsql in a goroutine so we can continue to send it commands
+		// below.
+		didQuit := make(chan struct{})
+		go func() {
+			require.NoError(t, fbsql.Run(ctx))
+			close(didQuit)
+		}()
+
+		// testFiles reference files located in the cli/testdata directory. All
+		// tests should be placed there; other than adding another test file to
+		// this list, you probably shouldn't be editing this file unless you are
+		// trying to modify the way the test framework itself works.
+		testFiles := []string{
+			"setup",
+			"database",
+			"table",
+			// the tests below may be dependent on the previous tests, which do
+			// setup and some shared database and table creation.
+			"query_buffer",
+			// meta commands
+			"meta_bang",
+			"meta_cd",
+			"meta_echo",
+			"meta_describe",
+			"meta_file",
+			"meta_pset_border",
+			"meta_pset_expanded",
+			"meta_pset_format_csv",
+			"meta_pset_tuples_only",
+			"meta_include",
+			"meta_output",
+			"meta_set",
+			"meta_timing",
+			"meta_write",
+		}
+
+		for _, testFile := range testFiles {
+			t.Run(testFile, func(t *testing.T) {
+				f, err := os.Open("testdata/" + testFile)
+				require.NoError(t, err)
+
+				scanner := bufio.NewScanner(f)
+				var lineNo int
+				for scanner.Scan() {
+					line := scanner.Text()
+					lineNo++
+
+					// Empty lines and comments (//) are ignored.
+					if line == "" {
+						continue
+					} else if strings.HasPrefix(line, "//") {
+						continue
+					}
+
+					parts := strings.SplitN(line, ":", 2)
+
+					switch parts[0] {
+					case "SEND":
+						v := ""
+						if len(parts) == 2 {
+							v = parts[1]
+						}
+						capture.sendLine(v)
+					case "EXPECT":
+						v := ""
+						if len(parts) == 2 {
+							v = parts[1]
+						}
+						comparer.expectLine(v, testFile, lineNo)
+					case "EXPECTCOMP":
+						if len(parts) == 2 {
+							comps := strings.SplitN(parts[1], ":", 2)
+							v := ""
+							if len(comps) == 2 {
+								v = comps[1]
+							}
+							comparer.expectLineComp(comparator(comps[0]), v, testFile, lineNo)
+						} else {
+							t.Errorf("unexpected line: %s[%d]:%s", testFile, lineNo, line)
+						}
+					default:
+						t.Errorf("unexpected line: %s[%d]:%s", testFile, lineNo, line)
+					}
+				}
+				require.NoError(t, scanner.Err())
+			})
+		}
+
+		// End with quit to ensure that fbsql closes without error.
+		capture.sendLine(`\q`)
+
+		// Ensure fbsql quits cleanly.
+		select {
+		case <-didQuit:
+		case <-time.After(time.Second):
+			t.Fatalf("expected fbsql to quit")
+		}
+	})
+}
+
+// compare is used to compare fbsql output written to its Stdout with expected
+// lines.
+type comparer struct {
+	t       *testing.T
+	out     chan byte
+	outline chan []byte
+	exp     chan []byte
+}
+
+func newComparer(t *testing.T) *comparer {
+	return &comparer{
+		t:       t,
+		out:     make(chan byte, 1024),
+		outline: make(chan []byte, 128),
+		exp:     make(chan []byte, 1024),
+	}
+}
+
+func (c *comparer) run() {
+	// Read bytes off output, and for every line (designated by a line feed "\n"),
+	// push the line onto the outline channel.
+	go func() {
+		var line []byte
+		for {
+			b := <-c.out
+			if b == byte('\n') {
+				c.outline <- line
+				line = []byte{}
+				continue
+			}
+			line = append(line, b)
+		}
+	}()
+}
+
+type comparator string
+
+const (
+	compEquals     = "Equals"
+	compHasPrefix  = "HasPrefix"
+	compWithFormat = "WithFormat"
+)
+
+// expectLine is a convenience method which calls expectLineComp with the compEq
+// comparator and the given line.
+func (c *comparer) expectLine(line string, fileName string, lineNo int) {
+	c.expectLineComp(compEquals, line, fileName, lineNo)
+}
+
+// expectLineComp reads the next line from the outline channel and compares it
+// with the given `line`. A comparator can be provided to inform how the lines
+// should be compared (for example, the compHasPrefix comparator will just
+// compare the beginning part of the outline).
+func (c *comparer) expectLineComp(comp comparator, line string, fileName string, lineNo int) {
+	var outline []byte
+	select {
+	case outline = <-c.outline:
+	case <-time.After(10 * time.Second):
+		// TODO(tlt): this is 10 seconds to account for the fb_views creation on
+		// a local mac. This should really be something like 2 seconds. Put this
+		// back to 2 once fb_views issue is addressed.
+		c.t.Fatalf("expected output line %s[%d]: >%s<", fileName, lineNo, line)
+	}
+
+	// msg is included in any require which fails.
+	msg := []interface{}{"exp: %s[%d], got: >%s<", fileName, lineNo, outline}
+
+	switch comp {
+	case compEquals:
+		require.Equal(c.t, []byte(line), outline, msg...)
+	case compHasPrefix:
+		require.True(c.t, strings.HasPrefix(string(outline), line), msg...)
+	case compWithFormat:
+		require.True(c.t, compareByteSlices(outline, []byte(line)), msg...)
+	default:
+		c.t.Fatalf("invalid comparator: %s", comp)
+	}
+}
+
+func (c *comparer) Write(b []byte) (n int, err error) {
+	for i := range b {
+		c.out <- b[i]
+	}
+	return len(b), err
+}
+
+// compareByteSlices compares a byte slice s with another byte slice format and
+// returns true if they are the same. It will accept underscore as a
+// single-character wildcard anywhere in slice format.
+func compareByteSlices(s, format []byte) bool {
+	// Replace some helpers in format before comparing.
+	f := string(format)
+	f = strings.ReplaceAll(f, `{uuid}`, `________-____-____-____-____________`)
+	f = strings.ReplaceAll(f, `{timestamp}`, `____-__-__T__:__:__Z`)
+	format = []byte(f)
+
+	if len(s) != len(format) {
+		return false
+	}
+
+	for i := range s {
+		if format[i] == '_' {
+			continue
+		}
+		if s[i] != format[i] {
+			// log.Printf("DEBUG: characters differ: (%d): '%v' != '%v'", i, s[i], format[i])
+			return false
+		}
+	}
+
+	return true
+}

cli/cli_test.go

@@ -0,0 +1,199 @@
+package cli_test
+
+import (
+	"context"
+	"io"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	featurebase "github.com/featurebasedb/featurebase/v3"
+	"github.com/featurebasedb/featurebase/v3/cli"
+	"github.com/featurebasedb/featurebase/v3/logger"
+	"github.com/pkg/errors"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestCLI(t *testing.T) {
+	t.Run("Input", func(t *testing.T) {
+		ctx := context.Background()
+
+		capture := newCapture(t)
+
+		cli := cli.NewCommand(logger.StderrLogger)
+		cli.SetStdin(capture)
+		cli.SetStdout(capture)
+		cli.Queryer = capture
+
+		go func() {
+			assert.NoError(t, cli.Run(ctx))
+		}()
+
+		none := []string{}
+
+		// One statement, one line.
+		capture.Assert("one;", []string{"one\n"})
+
+		// One statement, multiple lines.
+		capture.Assert("one", none)
+		capture.Assert(" two ", none)
+		capture.Assert("three;", []string{"one\ntwo\nthree\n"})
+
+		// Multiple statements, one line.
+		capture.Assert("foo; bar;", []string{"foo\n", "bar\n"})
+
+		// Multiple statements, multiple lines.
+		capture.Assert("a1", none)
+		capture.Assert("a2; b1", []string{"a1\na2\n"})
+		capture.Assert("b2;", []string{"b1\nb2\n"})
+
+		// Blank lines.
+		capture.Assert("one", none)
+		capture.Assert("", none)
+		capture.Assert("three;", []string{"one\nthree\n"})
+
+		// Just a semi-colon.
+		capture.Assert(";", []string{""})
+
+		// Multi-line with just a semi-colon.
+		capture.Assert("one", none)
+		capture.Assert(";", []string{"one\n"})
+
+		// Ensure a clean exit with no errors.
+		assert.NoError(t, capture.Exit())
+	})
+}
+
+////////////////////////////////////////////////////////
+
+// Ensure type implementes interface.
+var _ io.ReadCloser = (*capture)(nil)
+var _ io.Writer = (*capture)(nil)
+var _ cli.Queryer = (*capture)(nil)
+
+// capture implements the various CLI interfaces in order to capture test input
+// and submit it as though that input were being read from the command line. It
+// also captures calls made to the Queryer.Query method and ensures the sql they
+// contain is expected.
+type capture struct {
+	t *testing.T
+
+	// ch is a channel of strings (one line at a time) of CLI input.
+	ch chan string
+
+	mu   sync.RWMutex
+	sqls []string
+
+	// queryDone will receive an event any time the Query method is called and
+	// has completed. This is to tell the Assert method that it's safe to
+	// compare the sqls slice.
+	queryDone chan struct{}
+
+	asserting chan struct{}
+
+	err error
+}
+
+func newCapture(t *testing.T) *capture {
+	return &capture{
+		t:         t,
+		ch:        make(chan string),
+		sqls:      make([]string, 0),
+		queryDone: make(chan struct{}),
+	}
+}
+
+func (c *capture) Exit() error {
+	c.sendLine(`\q`)
+	c.mu.RLock()
+	defer c.mu.RUnlock()
+	return c.err
+}
+
+func (c *capture) Assert(in string, out []string) {
+	c.asserting = make(chan struct{})
+
+	c.sendLine(in)
+
+	// Wait for the CLI command to complete processing the input and send the
+	// sql to Query() by blocking on the queryDone channel. Because Query gets
+	// called for every sql statement in the input, an input resulting in
+	// multiple sql statements needs to wait for all expected queries to
+	// complete. A timeout is included to this so it doesn't deadlock in the
+	// case where Query is expected to be called, but isn't; after the timeout,
+	// the test should fail completely. In summary: we wait on queryDone the
+	// number of sql statements we expect. If we receive fewer than expected,
+	// the timeout will occur. If we receive more than expected, the Query()
+	// method will effectively deadlock, reach its own timout, then write to
+	// capture.err, which will be reported upon Exit().
+	for range out {
+		select {
+		case <-c.queryDone:
+		case <-time.After(2 * time.Second):
+			c.t.Fatalf("expected Query() to be called")
+		}
+	}
+
+	close(c.asserting)
+
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	assert.Equal(c.t, out, c.sqls)
+
+	// Reset the slice.
+	c.sqls = c.sqls[:0]
+
+}
+
+// sendLine sends the given string as a line input to the CLI command. It
+// appends a line feed to the end of string in order to mimic the user hitting
+// the return key.
+func (c *capture) sendLine(s string) {
+	// Add a line feed before putting s on the channel in order to mimic the
+	// user hitting the return key.
+	c.ch <- s + "\n"
+}
+
+// Read is read by the CLI in place of user input. It effectively sends lines of
+// input to the CLI, getting each line to be sent off the channel.
+func (c *capture) Read(b []byte) (n int, err error) {
+	s := <-c.ch
+	return strings.NewReader(s).Read(b)
+}
+
+func (c *capture) Close() error {
+	close(c.ch)
+	return nil
+}
+
+// Write is called with anything written to output. This would included results
+// from calling Query() under normal, non-testing conditions, as well as other
+// informational text sent to output, such as the splash message.
+func (c *capture) Write(b []byte) (n int, err error) {
+	return 0, nil
+}
+
+// Query is called by the CLI command once a full SQL statement is received
+// (signified by the terminator: `;`).
+func (c *capture) Query(org string, db string, sql io.Reader) (*featurebase.WireQueryResponse, error) {
+	tmpBuf := new(strings.Builder)
+	_, err := io.Copy(tmpBuf, sql)
+	if err != nil {
+		return nil, err
+	}
+
+	c.mu.Lock()
+	c.sqls = append(c.sqls, tmpBuf.String())
+	c.mu.Unlock()
+
+	select {
+	case c.queryDone <- struct{}{}:
+	case <-c.asserting:
+		c.mu.Lock()
+		c.err = errors.Errorf("unexpected query: %s", sql)
+		c.mu.Unlock()
+	}
+	return &featurebase.WireQueryResponse{}, nil
+}

cli/config.go

@@ -0,0 +1,31 @@
+package cli
+
+// Config represents the configuration for the command.
+type Config struct {
+	Host string `json:"host"`
+	Port string `json:"port"`
+
+	OrganizationID string `json:"org-id"`
+	Database       string `json:"db"`
+
+	// CloudAuth
+	CloudAuth CloudAuthConfig `json:"cloud-auth"`
+
+	// Kafka
+	KafkaConfig string `json:"kafka-config"`
+
+	HistoryPath string `json:"history-path"`
+
+	// CSV (Comma-Separated Values) table output mode.
+	CSV bool `json:"csv"`
+
+	// PSet takes one or more pset arguments of the form: `--pset=VAR[=ARG]`.
+	PSets []string `json:"pset"`
+}
+
+type CloudAuthConfig struct {
+	ClientID string `json:"client-id"`
+	Region   string `json:"region"`
+	Email    string `json:"email"`
+	Password string `json:"password"`
+}

cli/errors.go

@@ -0,0 +1,16 @@
+package cli
+
+import (
+	"github.com/featurebasedb/featurebase/v3/errors"
+)
+
+const (
+	ErrOrganizationRequired errors.Code = "OrganizationRequired"
+)
+
+func NewErrOrganizationRequired() error {
+	return errors.New(
+		ErrOrganizationRequired,
+		"organization required",
+	)
+}

cli/fbcloud/auth.go

@@ -0,0 +1,79 @@
+package fbcloud
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+
+	"github.com/pkg/errors"
+)
+
+const (
+	authFlow           = "USER_PASSWORD_AUTH"
+	cognitoURLTemplate = "https://cognito-idp.%s.amazonaws.com"
+)
+
+type cognitoParameters struct {
+	Email    string `json:"USERNAME"`
+	Password string `json:"PASSWORD"`
+}
+
+type cognitoAuthRequest struct {
+	AuthParameters cognitoParameters `json:"AuthParameters"`
+	AuthFlow       string            `json:"AuthFlow"`
+	AppClientID    string            `json:"ClientId"`
+}
+
+type cognitoAuthResult struct {
+	IDToken string `json:"IdToken"`
+}
+
+type cognitoAuthResponse struct {
+	Result cognitoAuthResult `json:"AuthenticationResult"`
+}
+
+func authenticate(clientID, region, email, password string) (string, error) {
+	authPayload := cognitoAuthRequest{
+		AuthParameters: cognitoParameters{
+			Email:    email,
+			Password: password,
+		},
+		AuthFlow:    authFlow,
+		AppClientID: clientID,
+	}
+
+	data, err := json.Marshal(authPayload)
+	if err != nil {
+		return "", errors.Wrap(err, "marshaling json")
+	}
+
+	url := fmt.Sprintf(cognitoURLTemplate, region)
+	req, err := http.NewRequest(http.MethodPost, url, bytes.NewBuffer(data))
+	if err != nil {
+		return "", errors.Wrap(err, "creating authentication request object")
+	}
+	req.Header.Add("Content-Type", "application/x-amz-json-1.1")
+	req.Header.Add("X-Amz-Target", "AWSCognitoIdentityProviderService.InitiateAuth")
+
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return "", errors.Wrap(err, "making request")
+	}
+	defer resp.Body.Close()
+
+	fullbod, err := io.ReadAll(resp.Body)
+
+	if resp.StatusCode != http.StatusOK || err != nil {
+		return "", errors.Errorf("HTTP status code=%d from Cognito authentication response. reading body: %v, body: '%s'", resp.StatusCode, err, fullbod)
+	}
+
+	var auth cognitoAuthResponse
+	err = json.Unmarshal(fullbod, &auth)
+	if err != nil {
+		return "", errors.Wrap(err, "decoding cognito auth response")
+	}
+
+	return auth.Result.IDToken, nil
+}

cli/fbcloud/client.go

@@ -0,0 +1,131 @@
+package fbcloud
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"strings"
+	"time"
+
+	featurebase "github.com/featurebasedb/featurebase/v3"
+	"github.com/pkg/errors"
+)
+
+// TokenRefreshTimeout is currently hardcoded to be just under the
+// Cognito token timeout for cloud which is 15 minutes (I think I
+// heard that somewhere anyway). It seems to work.
+const TokenRefreshTimeout = time.Minute * 13
+
+type Queryer struct {
+	Host string
+
+	ClientID string
+	Region   string
+	Email    string
+	Password string
+
+	token       string
+	lastRefresh time.Time
+}
+
+func (cq *Queryer) tokenRefresh() error {
+	token, err := authenticate(cq.ClientID, cq.Region, cq.Email, cq.Password)
+	if err != nil {
+		return errors.Wrap(err, "getting token")
+	}
+	cq.token = token
+	cq.lastRefresh = time.Now()
+	return nil
+}
+
+// Query issues a SQL query formatted for the FeatureBase cloud query endpoint.
+func (cq *Queryer) Query(org string, db string, sql io.Reader) (*featurebase.WireQueryResponse, error) {
+	if time.Since(cq.lastRefresh) > TokenRefreshTimeout {
+		if err := cq.tokenRefresh(); err != nil {
+			return nil, errors.Wrap(err, "refreshing token")
+		}
+	}
+	url := fmt.Sprintf("%s/databases/%s/sql", cq.Host, db)
+	if db == "" {
+		url = fmt.Sprintf("%s/sql", cq.Host)
+	}
+
+	client := &http.Client{
+		Timeout: time.Second * 30,
+	}
+	req, err := http.NewRequest(http.MethodPost, url, sql)
+	if err != nil {
+		return nil, errors.Wrap(err, "creating new post request")
+	}
+	req.Header.Add("Content-Type", "text/plain")
+	req.Header.Add("Authorization", cq.token)
+
+	var resp *http.Response
+	if resp, err = client.Do(req); err != nil {
+		return nil, errors.Wrap(err, "executing post request")
+	}
+
+	fullbod, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return nil, errors.Wrap(err, "reading cloud response")
+	}
+	if resp.StatusCode/100 != 2 {
+		return nil, errors.Errorf("unexpected status: %s, full body: '%s'", resp.Status, fullbod)
+	}
+
+	var sqlResponse featurebase.WireQueryResponse
+	if err := json.Unmarshal(fullbod, &sqlResponse); err != nil {
+		return nil, errors.Wrapf(err, "decoding cloud response, body:\n%s", fullbod)
+	}
+	return &sqlResponse, nil
+}
+
+// HTTPRequest can make an arbitrary http request to the host and
+// tries to json unmarshal the response body into v if v is
+// non-nil. This is handy for hitting cloud endpoints other than the
+// query endpoint which is handled by Query. I don't think this is
+// currently used, but I'd like to keep it around for debugging.
+func (cq *Queryer) HTTPRequest(method, path, body string, v interface{}) ([]byte, error) {
+	if time.Since(cq.lastRefresh) > TokenRefreshTimeout {
+		if err := cq.tokenRefresh(); err != nil {
+			return nil, errors.Wrap(err, "refreshing token")
+		}
+	}
+	var bod io.Reader
+	if body == "" {
+		bod = nil
+	} else {
+		bod = strings.NewReader(body)
+	}
+	req, err := http.NewRequest(method, fmt.Sprintf("%s%s", cq.Host, path), bod)
+	if err != nil {
+		return nil, errors.Errorf("creating request: %v", err)
+	}
+	// fmt.Printf("%+v\n", req)
+
+	req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", cq.token))
+	if bod != nil {
+		req.Header.Add("Content-Type", "application/json")
+	}
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return nil, errors.Errorf("doing request: %v", err)
+	}
+	bodbytes, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return nil, errors.Errorf("reading response body: %v", err)
+	}
+	if resp.StatusCode/100 != 2 {
+		return nil, errors.Errorf("bad status: %s. body: '%s'", resp.Status, bodbytes)
+	}
+
+	if v != nil {
+		err = json.Unmarshal(bodbytes, v)
+		if err != nil {
+			return nil, errors.Errorf("unmarshaling: %v", err)
+		}
+	}
+
+	return bodbytes, nil
+}

cli/kafka.go

@@ -0,0 +1,70 @@
+package cli
+
+import (
+	"fmt"
+
+	"github.com/featurebasedb/featurebase/v3/cli/batch"
+	"github.com/featurebasedb/featurebase/v3/cli/kafka"
+	"github.com/featurebasedb/featurebase/v3/errors"
+	"github.com/spf13/viper"
+)
+
+func (cmd *Command) newKafkaRunner(cfgFile string) (*kafka.Runner, error) {
+	// Read the kafka config file.
+	v := viper.New()
+	v.SetConfigFile(cfgFile)
+	v.SetConfigType("toml")
+	err := v.ReadInConfig()
+	if err != nil {
+		return nil, fmt.Errorf("error reading configuration file '%s': %v", cfgFile, err)
+	}
+
+	cfg := kafka.Config{}
+	if err := v.Unmarshal(&cfg); err != nil {
+		return nil, errors.Wrap(err, "unmarshalling config")
+	}
+
+	if err := kafka.ValidateConfig(cfg); err != nil {
+		return nil, errors.Wrap(err, "validating config")
+	}
+
+	// Create a new config with defaults.
+
+	// Look up fields based on table provided in the config.
+	wqr, err := cmd.executeQuery(newRawQuery("SHOW COLUMNS FROM " + cfg.Table))
+	if err != nil {
+		return nil, errors.Wrap(err, "executing query")
+	}
+
+	scr, err := wqr.ShowColumnsResponse()
+	if err != nil {
+		return nil, errors.Wrap(err, "getting show columns from wire query response")
+	}
+
+	// If no fields were provided in the config, use the fields defined on the
+	// table and assume a 1-to-1 mapping of source to destination.
+	if len(cfg.Fields) == 0 {
+		cfg.Fields = kafka.FieldsToConfig(scr.Fields)
+	} else {
+		cfg.Fields, err = kafka.CheckFieldCompatibility(cfg.Fields, scr)
+		if err != nil {
+			return nil, errors.Wrap(err, "validating config fields")
+		}
+	}
+
+	idkCfg, err := kafka.ConvertConfig(cfg)
+	if err != nil {
+		return nil, errors.Wrap(err, "cleaning config")
+	}
+
+	flds, err := kafka.ConfigToFields(cfg)
+	if err != nil {
+		return nil, errors.Wrap(err, "getting fields from config")
+	}
+
+	return kafka.NewRunner(
+		idkCfg,
+		batch.NewSQLBatcher(cmd, flds),
+		cmd.stderr,
+	), nil
+}

cli/kafka/config.go

@@ -0,0 +1,251 @@
+package kafka
+
+import (
+	"fmt"
+	"time"
+
+	featurebase "github.com/featurebasedb/featurebase/v3"
+	"github.com/featurebasedb/featurebase/v3/dax"
+	"github.com/featurebasedb/featurebase/v3/idk"
+	"github.com/pkg/errors"
+)
+
+// Config is the user-facing configuration for kafka support in the CLI. This is
+// unmarshalled from the the toml config file supplied by the user.
+type Config struct {
+	Hosts  []string `mapstructure:"hosts" help:"Kafka hosts."`
+	Group  string   `mapstructure:"group" help:"Kafka group."`
+	Topics []string `mapstructure:"topics" help:"Kafka topics to read from."`
+
+	BatchSize         int           `mapstructure:"batch-size" help:"Batch size."`
+	BatchMaxStaleness time.Duration `mapstructure:"batch-max-staleness" help:"Maximum length of time that the oldest record in a batch can exist before flushing the batch. Note that this can potentially stack with timeouts waiting for the source."`
+	Timeout           time.Duration `mapstructure:"timeout" help:"Time to wait for more records from Kafka before flushing a batch. 0 to disable."`
+
+	Table  string  `mapstructure:"table" help:"Destination table name."`
+	Fields []Field `mapstructure:"fields"`
+}
+
+// Field is a user-facing configuration field.
+type Field struct {
+	Name       string   `mapstructure:"name"`
+	SourceType string   `mapstructure:"source-type"`
+	SourcePath []string `mapstructure:"source-path"`
+	PrimaryKey bool     `mapstructure:"primary-key"`
+}
+
+// ConfigForIDK represents Config converted to values suitable for IDK. In
+// particular, the idk.RawField is used in parsing the schema in IDK.
+type ConfigForIDK struct {
+	Hosts  []string
+	Group  string
+	Topics []string
+
+	BatchSize         int
+	BatchMaxStaleness time.Duration
+	Timeout           time.Duration
+
+	Table   string
+	IDField string
+	Fields  []idk.RawField
+}
+
+// ValidateConfig validates the config is usable.
+func ValidateConfig(c Config) error {
+	if c.Table == "" {
+		return errors.Errorf("table is required")
+	} else if len(c.Topics) == 0 {
+		return errors.Errorf("at least one topic is required")
+	} else if len(c.Fields) > 0 {
+		// We only need to do these checks if any fields are specified at all.
+		// If no fields are specified, that's ok because then we default to
+		// using fields based off the existing table.
+		if len(c.Fields) < 2 {
+			return errors.Errorf("at least two fields are required (one should be a primary key)")
+		} else {
+			var found int
+			for i := range c.Fields {
+				if c.Fields[i].PrimaryKey {
+					found++
+				}
+				if c.Fields[i].Name == "" {
+					return errors.Errorf("a name attribute (which isn't equal to \"\") should exist for all fields")
+				}
+			}
+			if found != 1 {
+				return errors.Errorf("exactly one primary key field is required")
+			}
+		}
+	}
+	return nil
+}
+
+// ConvertConfig converts a Config to one that suitable for IDK.
+func ConvertConfig(c Config) (ConfigForIDK, error) {
+	// Set a default kafka host in case one isn't provided.
+	hosts := []string{"localhost:9092"}
+	if len(c.Hosts) > 0 {
+		hosts = c.Hosts
+	}
+
+	// Copy all the shared members from Config to ConfigForIDK.
+	out := ConfigForIDK{
+		Hosts:             hosts,
+		Group:             c.Group,
+		Topics:            c.Topics,
+		BatchSize:         c.BatchSize,
+		BatchMaxStaleness: c.BatchMaxStaleness,
+		Timeout:           c.Timeout,
+		Table:             c.Table,
+	}
+
+	if len(c.Fields) == 0 {
+		return out, errors.New("fields cannot be empty")
+	}
+
+	// rawFields wil be the same as c.Fields, but possibly enhanced.
+	rawFields := make([]idk.RawField, 0, len(c.Fields))
+
+	var foundPK bool
+	for _, fld := range c.Fields {
+		if fld.PrimaryKey {
+			out.IDField = fld.Name
+			foundPK = true
+		}
+
+		typ, quals, err := dax.SplitFieldType(fld.SourceType)
+		if err != nil {
+			return out, errors.Wrap(err, "getting base type")
+		}
+
+		rawFld := idk.RawField{
+			Name: fld.Name,
+			Type: string(typ),
+			Path: fld.SourcePath,
+		}
+		// If a SourcePath wasn't provided, default to using the field name.
+		if len(rawFld.Path) == 0 {
+			rawFld.Path = []string{fld.Name}
+		}
+
+		switch typ {
+		case dax.BaseTypeInt:
+			// We don't have to handle min/max because we don't create the table.
+		case dax.BaseTypeDecimal:
+			if len(quals) != 1 {
+				return out, errors.Errorf("expected decimal scale")
+			}
+			rawFld.Config = []byte(fmt.Sprintf(`{"scale":%d}`, quals[0]))
+		case dax.BaseTypeID:
+			rawFld.Config = []byte("{\"mutex\":true}")
+		case dax.BaseTypeIDSet:
+			rawFld.Type = "ids"
+		case dax.BaseTypeString:
+			rawFld.Config = []byte("{\"mutex\":true}")
+		case dax.BaseTypeStringSet:
+			rawFld.Type = "strings"
+		case dax.BaseTypeTimestamp:
+			// No timestamp options are handled for now.
+		}
+
+		rawFields = append(rawFields, rawFld)
+	}
+	if !foundPK {
+		return out, errors.New("primary-key not found in fields")
+	}
+
+	out.Fields = rawFields
+
+	return out, nil
+}
+
+// ConfigToFields returns a list of *dax.Field based on the IDField and Fields
+// in the Config.
+func ConfigToFields(c Config) ([]*dax.Field, error) {
+	// We don't know if a primary key will be found, so we can't set the
+	// capacity to `len(c.Fields)-1`.
+	out := make([]*dax.Field, 0, len(c.Fields))
+
+	for _, fld := range c.Fields {
+		if fld.PrimaryKey {
+			continue
+		}
+		typ, quals, err := dax.SplitFieldType(fld.SourceType)
+		if err != nil {
+			return nil, errors.Wrap(err, "splitting field type")
+		}
+		dfld := &dax.Field{
+			Name: dax.FieldName(fld.Name),
+			Type: typ,
+		}
+		switch typ {
+		case dax.BaseTypeDecimal:
+			if len(quals) != 1 {
+				return nil, errors.Errorf("expected decimal scale")
+			}
+			scale, ok := quals[0].(int64)
+			if !ok {
+				return nil, errors.Errorf("invalid decimal scale: %v", quals[0])
+			}
+			dfld.Options.Scale = scale
+		}
+		out = append(out, dfld)
+	}
+
+	return out, nil
+}
+
+// FieldsToConfig returns a Config.Fields based on a list of *dax.Field.
+func FieldsToConfig(flds []*dax.Field) []Field {
+	out := make([]Field, 0, len(flds))
+	for _, fld := range flds {
+		out = append(out, Field{
+			Name:       string(fld.Name),
+			SourceType: fld.FullType(),
+			PrimaryKey: fld.IsPrimaryKey(),
+		})
+	}
+	return out
+}
+
+// CheckFieldCompatibility ensures that the fields provided in the kafka config
+// are compatible with the fields in the existing table. It returns a copy of
+// the kafka config fields with empty values defaulted to the table field
+// configuration.
+func CheckFieldCompatibility(cflds []Field, scr *featurebase.ShowColumnsResponse) ([]Field, error) {
+	out := make([]Field, len(cflds))
+	for i, cfld := range cflds {
+		out[i] = cfld
+		cfldName := dax.FieldName(cfld.Name)
+
+		// Primary key field.
+		if cfld.PrimaryKey {
+			f := scr.Field(dax.PrimaryKeyFieldName)
+			if f == nil {
+				return nil, dax.NewErrFieldDoesNotExist(dax.PrimaryKeyFieldName) // It should be impossible to hit this.
+			}
+			if out[i].SourceType == "" {
+				if f.StringKeys() {
+					out[i].SourceType = dax.BaseTypeString
+				} else {
+					out[i].SourceType = dax.BaseTypeID
+				}
+			}
+			continue
+		}
+
+		// Non primary key fields.
+		if cfldName == dax.PrimaryKeyFieldName {
+			return nil, errors.Errorf("field named '%s' must be a primary key", dax.PrimaryKeyFieldName)
+		}
+
+		f := scr.Field(cfldName)
+		if f == nil {
+			return nil, dax.NewErrFieldDoesNotExist(cfldName)
+		}
+
+		if out[i].SourceType == "" {
+			out[i].SourceType = f.FullType()
+		}
+	}
+	return out, nil
+}

cli/kafka/runner.go

@@ -0,0 +1,67 @@
+package kafka
+
+import (
+	"io"
+	"time"
+
+	fbbatch "github.com/featurebasedb/featurebase/v3/batch"
+	"github.com/featurebasedb/featurebase/v3/errors"
+	"github.com/featurebasedb/featurebase/v3/idk"
+	"github.com/featurebasedb/featurebase/v3/idk/kafka_static"
+	"github.com/featurebasedb/featurebase/v3/logger"
+)
+
+// Runner is a CLI-specific kafka consumer. It's similar to
+// idk.kafka_static.Main in that it embeds idk.Main and contains additional
+// functionality specific to its use case.
+type Runner struct {
+	idk.Main   `flag:"!embed"`
+	KafkaHosts []string       `help:"Comma separated list of host:port pairs for Kafka."`
+	Group      string         `help:"Kafka group."`
+	Topics     []string       `help:"Kafka topics to read from."`
+	Timeout    time.Duration  `help:"Time to wait for more records from Kafka before flushing a batch. 0 to disable."`
+	Header     []idk.RawField `help:"Header configuration."`
+}
+
+func NewRunner(cfg ConfigForIDK, batcher fbbatch.Batcher, logWriter io.Writer) *Runner {
+	idkMain := idk.NewMain()
+	idkMain.IDField = cfg.IDField
+	idkMain.Index = cfg.Table
+	idkMain.Batcher = batcher
+	idkMain.BatchSize = cfg.BatchSize
+	idkMain.BatchMaxStaleness = cfg.BatchMaxStaleness
+	idkMain.SetBasic()
+	idkMain.SetLog(logger.NewStandardLogger(logWriter))
+
+	kr := &Runner{
+		Main:       *idkMain,
+		KafkaHosts: cfg.Hosts,
+		Group:      cfg.Group,
+		Topics:     cfg.Topics,
+		Header:     cfg.Fields,
+		Timeout:    cfg.Timeout,
+	}
+	kr.OffsetMode = true
+	kr.Main.Namespace = "cli_kafka_runner"
+	kr.Main.Pprof = "" // don't initialize pprof until we actually use it in tests
+	kr.NewSource = func() (idk.Source, error) {
+		source := kafka_static.NewSource()
+		source.Hosts = kr.KafkaHosts
+		source.Group = kr.Group
+		source.Topics = kr.Topics
+		source.Log = kr.Main.Log()
+		// source.TLS = m.KafkaTLS
+		source.Timeout = kr.Timeout
+		// source.SkipOld = m.SkipOld
+		source.HeaderFields = kr.Header
+		// source.S3Region = m.S3Region
+		// source.AllowMissingFields = m.AllowMissingFields
+
+		err := source.Open()
+		if err != nil {
+			return nil, errors.Wrap(err, "opening source")
+		}
+		return source, nil
+	}
+	return kr
+}

cli/parts.go

@@ -0,0 +1,136 @@
+package cli
+
+import (
+	"fmt"
+	"io"
+	"os"
+	"strings"
+)
+
+// query is a collection of queryParts which, when applied together, make up an
+// executable SQL query.
+type query []queryPart
+
+func (q query) String() string {
+	var sb strings.Builder
+	for i := range q {
+		sb.WriteString(q[i].String())
+		if i < len(q)-1 {
+			sb.WriteRune('\n')
+		}
+	}
+	return sb.String()
+}
+
+// Reader returns the query as an io.Reader so that it can be passed to, for
+// example, http.Post().
+func (q query) Reader() io.Reader {
+	readers := make([]io.Reader, 0, len(q))
+	for i := range q {
+		readers = append(readers, q[i].Reader())
+	}
+	return io.MultiReader(readers...)
+}
+
+// queryPart is an interface representing anything which can use to build up a
+// query.
+type queryPart interface {
+	fmt.Stringer
+	Reader() io.Reader
+}
+
+func newRawQuery(s string) query {
+	return []queryPart{
+		newPartRaw(s),
+	}
+}
+
+// ////////////////////////////////////////////////////////////////////////////
+// raw
+// ////////////////////////////////////////////////////////////////////////////
+
+// Ensure type implements interface.
+var _ queryPart = (*partRaw)(nil)
+
+type partRaw struct {
+	raw string
+}
+
+func newPartRaw(s string) *partRaw {
+	return &partRaw{
+		raw: s,
+	}
+}
+
+func (p *partRaw) Reader() io.Reader {
+	return strings.NewReader(p.raw + "\n")
+}
+
+func (p *partRaw) String() string {
+	return p.raw
+}
+
+// ////////////////////////////////////////////////////////////////////////////
+// file
+// ////////////////////////////////////////////////////////////////////////////
+
+// Ensure type implements interface.
+var _ queryPart = (*partFile)(nil)
+
+type partFile struct {
+	file *os.File
+}
+
+func newPartFile(f *os.File) *partFile {
+	return &partFile{
+		file: f,
+	}
+}
+
+func (p *partFile) Reader() io.Reader {
+	return p.file
+}
+
+func (p *partFile) String() string {
+	return fmt.Sprintf("[file: %s]", p.file.Name())
+}
+
+// ////////////////////////////////////////////////////////////////////////////
+// batch file
+// ////////////////////////////////////////////////////////////////////////////
+
+// Ensure type implements interface.
+var _ queryPart = (*partBatchFile)(nil)
+
+type partBatchFile struct {
+	file *os.File
+}
+
+func (p *partBatchFile) Reader() io.Reader {
+	return p.file
+}
+
+func (p *partBatchFile) String() string {
+	return p.file.Name()
+}
+
+// ////////////////////////////////////////////////////////////////////////////
+// terminator (i.e. ";")
+// ////////////////////////////////////////////////////////////////////////////
+
+// Ensure type implements interface.
+var _ queryPart = (*partTerminator)(nil)
+
+type partTerminator struct{}
+
+func newPartTerminator() *partTerminator {
+	return &partTerminator{}
+}
+
+func (p *partTerminator) Reader() io.Reader {
+	return nil
+}
+
+func (p *partTerminator) String() string {
+	return terminationChar
+}