2026-10-11 05:17:54 +00:00
733 changed files with 39695 additions and 106385 deletions
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@ -22,22 +22,6 @@ jobs:
          github-token: ${{ secrets.GITHUB_TOKEN }}
          dry: true
  # golangci-lint must be run separately from "validate" as there are go mod issues if you run it after the vet
  golangci:
    name: lint
    runs-on: ubuntu-latest
    steps:
      - uses: actions/setup-go@v3
        with:
          go-version: 1.19
      - uses: actions/checkout@v3
      - name: golangci-lint
        uses: golangci/golangci-lint-action@v3
        with:
          # Optional: version of golangci-lint to use in form of v1.2 or v1.2.3 or `latest` to use the latest version
          # version: v1.29
          args: --timeout=8m
  validate:
    name: Code Checks
    runs-on: ubuntu-latest
@ -55,3 +39,11 @@ jobs:
      - name: go vet
        run: go vet ./...
      - name: golangci-lint
        uses: golangci/golangci-lint-action@v3
        with:
          args: --timeout=5m
      - name: test
        run: go test ./...
--- a/.gitignore
+++ b/.gitignore
@ -24,8 +24,6 @@ builds/
 *.tfstate.backup
 .vscode
 batch/testdata/batch*.out
 idk/testdata/idk*.out
 idk/testenv/certs/*
@ -75,13 +73,3 @@ tags.dot
 # SQL3
 /sql3/sql3.html
 staticcheck.conf
 .quick
 dax/dax-data
 coverage-from-docker
 *.client_id.txt
--- a/.gitlab/.gitlab-ci.yml
+++ b/.gitlab/.gitlab-ci.yml
@ -1,747 +0,0 @@
 # You will see a couple of instances of:
 # PKG_LIST=$(go list ./... | grep -Ev 'internal/clustertests|simulacraData|batch|idk|v3/dax/test/dax' | paste -s -d, -)
 # This gets us a package list, comma-separated, which excludes the batch
 # and IDK tests, and a couple of subdirs with specialized stuff. We can
 # then use this with -coverpkg, or we can use ${PKG_LIST//,/ } to get a
 # space-separated list for use with `go test` to run the tests for those
 # directories only.
 include:
  - local: /.gitlab/batch-ci.yml
  - template: Security/SAST.gitlab-ci.yml
  - template: Security/License-Scanning.gitlab-ci.yml
  - template: Security/Dependency-Scanning.gitlab-ci.yml
 default:
  retry:
    max: 2 # This is confusing but this means "3 runs at max".
    when:
      - unknown_failure
      - api_failure
      - runner_system_failure
      - job_execution_timeout
      - stuck_or_timeout_failure
 variables:
  GOVERSION: "1.19.3"
  GOFUTURE: "latest"
  CI_IMAGE: "${CI_REGISTRY_IMAGE}/ci-builder:0.0.1"
  CI_PRE_CLONE_SCRIPT: |
    set -x
 stages:
  - ci_image_build
  - lint
  - test
  - build
  - post build
  - integration
  - gauntlet
  - performance
  - nonblocking
  - cleanup_build
 gosec-sast:
  allow_failure: false
  before_script:
    - export GOPRIVATE=github.com/molecula/*
    - apk add openssh-client
    - eval $(ssh-agent -s)
    - echo "$FB_SSH_PRIVATE_KEY" | tr -d '\r' | ssh-add -
    - mkdir -p ~/.ssh
    - chmod 700 ~/.ssh
    - git config --global --add url."ssh://git@github.com/".insteadOf "https://github.com/"
    - ssh-keygen -F github.com || echo "$SSH_KNOWN_HOSTS_HASHED" >> ~/.ssh/known_hosts
    - chmod 644 ~/.ssh/known_hosts
  script:
    ## securego/gosec works for scanning, but not converting to the gitlab report format.
    - go install github.com/securego/gosec/v2/cmd/gosec@v2.12.0
    - gosec -fmt=json -out=gosec.json -tests ./... || true
    ## gitlab's wrapper for gosec works for converting, but not for scanning.
    - go install 'gitlab.com/gitlab-org/security-products/analyzers/gosec@v1.4.0'
    - gosec convert gosec.json > gl-sast-report.json
 .go-cache:
  variables:
    GOPATH: $CI_PROJECT_DIR/.go
  before_script:
    - mkdir -p .go
  cache:
    # this caching strategy makes it so each branch uses the same cache
    key: "$CI_COMMIT_REF_SLUG"
    paths:
      - .go/pkg/mod/
 smoke build:
  image: golang:$GOVERSION
  extends: .go-cache
  stage: lint
  allow_failure: false
  rules:
    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
  script:
    - echo "Let's just see if it compiles... (sometimes the linter gives unclear errors if it doesn't)"
    - go build ./...
 golangci-lint:
  image: golangci/golangci-lint:v1.46.2
  extends: .go-cache
  stage: lint
  allow_failure: true
  rules:
    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
  script:
    - echo "Checking for issues in new code"
    - golangci-lint run -v --timeout=8m
 go mod tidy:
  stage: lint
  image: golang:$GOVERSION
  extends: .go-cache
  rules:
    - if: '$CI_COMMIT_TAG == null && ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web")'
  script:
    - go mod tidy
    - git diff --exit-code -- go.mod go.sum
 build lattice:
  stage: test
  image: node:14
  variables:
    AWS_PROFILE: "service-fb-ci"
    AWS_ACCESS_KEY_ID: $AWS_FBCI_ACCESS_KEY_ID
    AWS_SECRET_ACCESS_KEY: $AWS_FBCI_SECRET_ACCESS_KEY
    CI: "false"
  rules:
    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
  before_script:
    - curl -sS "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
    - unzip -qq awscliv2.zip
    - ./aws/install
    - aws --version
    - aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID
    - aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY
    - aws configure set region "us-east-2"
    - aws configure set aws_profile $AWS_PROFILE
    - aws sts get-caller-identity # ensure we have a valid AWS login
  script:
    - cd lattice
    - cache=$(find . -type f -print0 | sort -z | xargs -0 sha1sum | sha1sum | cut -d ' ' -f 1)
    - echo "'$cache'"
    - echo "looking for s3://molecula-artifact-storage/lattice/$cache/build.tar.gz"
    # if is for if we had a cache object in S3
    # else is for if we didn't have a cache object (and have to build).
    - |
      if aws s3api head-object --bucket molecula-artifact-storage --key "lattice/$cache/build.tar.gz"; then
        # download object, extract, name the folder `build`
        aws s3 cp "s3://molecula-artifact-storage/lattice/$cache/build.tar.gz" build.tar.gz
        tar -xf build.tar.gz
      else # cache file not found
        yarn install --frozen-lockfile # CI needs to enforce that the lockfile doesn't need to be updated
        yarn build
        tar -czvf "$cache.tar.gz" build/
        aws s3 mv "$cache.tar.gz" "s3://molecula-artifact-storage/lattice/$cache/build.tar.gz"
        touch "$CI_COMMIT_SHA"
        aws s3 mv "$CI_COMMIT_SHA" "s3://molecula-artifact-storage/lattice/$cache/$CI_COMMIT_SHA"
      fi
    - | # Ensure that we have build directory after the caching step
      if [ ! -d build ]; then
        echo "no build directory, erroring out" || exit 1
      fi
    - mv build ../
    - cd ../
    - rm -r lattice
    - mv build lattice
    - tar -czvf lattice.tar.gz lattice
  artifacts:
    paths:
      - lattice.tar.gz
 build featurebase:
  stage: test
  image: golang:$GOVERSION
  extends: .go-cache
  rules:
    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
  script:
    - rm -r lattice
    - tar -xvf lattice.tar.gz
    - go install github.com/rakyll/statik@v0.1.7
    - $GOPATH/bin/statik -src=lattice
    - export SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct)
    - |
      for goos in "darwin" "linux"; do
        for goarch in "amd64" "arm64"; do
          GOOS="${goos}" GOARCH="${goarch}" make build FLAGS="-o featurebase_${goos}_${goarch}"
        done
      done
  artifacts:
    paths:
      - featurebase_*
  needs:
    - job: build lattice
 build fbsql amd64:
  stage: test
  variables:
    BUILD_NAME: build_${CI_COMMIT_SHA}_${CI_CONCURRENT_ID}
  tags:
    - shell
  rules:
    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
  script:
    - export SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct)
    - date
    - GOOS="linux" GOARCH="amd64" make docker-build-fbsql BUILD_CGO=1
    - GOOS="darwin" GOARCH="amd64" make docker-build-fbsql
  artifacts:
    paths:
      - ./build/fbsql_*
 build fbsql arm64:
  stage: test
  variables:
    BUILD_NAME: build_${CI_COMMIT_SHA}_${CI_CONCURRENT_ID}
  tags:
    - shell-arm64
  rules:
    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
  script:
    - export SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct)
    - date
    - GOOS="linux" GOARCH="arm64" make docker-build-fbsql BUILD_CGO=1
    - GOOS="darwin" GOARCH="arm64" make docker-build-fbsql
  artifacts:
    paths:
      - ./build/fbsql_*
 build amd container fb:
  stage: test
  tags:
    - shell
  rules:
    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
  before_script:
    - echo "${DOCKER_DEPLOY_TOKEN}" | docker login -u ${DOCKER_DEPLOY_USER} --password-stdin ${CI_REGISTRY}
  script:
    - tag=${CI_REGISTRY_IMAGE}/featurebase:linux-amd64-${CI_COMMIT_REF_SLUG}
    - export SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct)
    - docker build --build-arg GO_VERSION=$GOVERSION --build-arg ARCH=amd64 --build-arg SOURCE_DATE_EPOCH=$SOURCE_DATE_EPOCH -t $tag -f .gitlab/Dockerfile .
    - docker push $tag
    - echo Created docker featurebase image with tag "$tag"
  needs:
    - job: build featurebase
 run jest tests:
  stage: test
  image: node:14
  variables:
    CI: "true"
  rules:
    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
  script:
    - echo "Testing lattice..."
    - cd lattice
    - npm install --force
    - npm test -- --coverage --testResultsProcessor=jest-sonar-reporter
  artifacts:
    paths:
      - lattice/coverage/lcov.info
 # We run go test -race on all the standard packages, skipping the ones that have their
 # own separate tests. we spin this off as nonblocking because it used to take a really
 # long time and even now it's pretty slow.
 run go tests race:
  stage: nonblocking # don't let this job block any other jobs because it takes much longer than the other tests.
  image: golang:$GOVERSION
  extends: .go-cache
  variables:
    FEATUREBASE_CONTROLLER_CONFIG_SQLDB_DATABASE: run_go_tests_race
    POSTGRES_DB: run_go_tests_race
    FEATUREBASE_CONTROLLER_CONFIG_SQLDB_USER: postgres
    POSTGRES_USER: postgres
    FEATUREBASE_CONTROLLER_CONFIG_SQLDB_PASSWORD: $POSTGRES_PASSWORD
    POSTGRES_PASSWORD: $POSTGRES_PASSWORD
    FEATUREBASE_CONTROLLER_CONFIG_SQLDB_HOST: postgres
  services:
    - postgres:14.7
  rules:
    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
  needs: ["smoke build"] # we do block on smoke build though bc it's pretty dumb to test stuff if it doesn't build
  script:
    - echo "Running featurebase race tests..."
    - PKG_LIST=$(go list ./... | grep -Ev 'internal/clustertests|simulacraData|batch|idk|v3/dax/test/dax' | paste -s -d, -)
    - export TMPDIR=/mnt/ramdisk/test-$CI_JOB_ID
    - mkdir -p $TMPDIR
    - go test -race -v -timeout=10m ${PKG_LIST//,/ }
  after_script:
    - rm -rf /mnt/ramdisk/test-$CI_JOB_ID
  tags:
    - docker
 # We run our base tests against $GOVERSION (a reasonably current version that we trust)
 # and use shardwidth22 for them. This gives us a canary for things breaking for
 # unusual shard widths.
 run go tests:
  stage: test
  image: golang:$GOVERSION
  extends: .go-cache
  variables:
    FEATUREBASE_CONTROLLER_CONFIG_SQLDB_DATABASE: run_go_tests
    POSTGRES_DB: run_go_tests
    FEATUREBASE_CONTROLLER_CONFIG_SQLDB_USER: postgres
    POSTGRES_USER: postgres
    FEATUREBASE_CONTROLLER_CONFIG_SQLDB_PASSWORD: $POSTGRES_PASSWORD
    POSTGRES_PASSWORD: $POSTGRES_PASSWORD
    FEATUREBASE_CONTROLLER_CONFIG_SQLDB_HOST: postgres
  services:
    - postgres:14.7
  rules:
    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
  script:
    - echo "Running featurebase unit tests..."
    - PKG_LIST=$(go list ./... | grep -Ev 'internal/clustertests|simulacraData|batch|idk|v3/dax/test/dax' | paste -s -d, -)
    - export TMPDIR=/mnt/ramdisk/test-$CI_JOB_ID
    - mkdir -p $TMPDIR
    - go test -tags=shardwidth22 -timeout=10m -coverprofile=coverage.out -covermode=atomic -coverpkg=${PKG_LIST} ${PKG_LIST//,/ }
  after_script:
    - rm -rf /mnt/ramdisk/test-$CI_JOB_ID
  artifacts:
    paths:
      - coverage.out
  tags:
    - docker
 run go tests dax/test/dax:
  stage: test
  image: golang:$GOVERSION
  extends: .go-cache
  tags:
    - docker
  variables:
    FEATUREBASE_CONTROLLER_CONFIG_SQLDB_DATABASE: run_go_tests_dax
    POSTGRES_DB: run_go_tests_dax
    FEATUREBASE_CONTROLLER_CONFIG_SQLDB_USER: postgres
    POSTGRES_USER: postgres
    FEATUREBASE_CONTROLLER_CONFIG_SQLDB_PASSWORD: $POSTGRES_PASSWORD
    POSTGRES_PASSWORD: $POSTGRES_PASSWORD
    FEATUREBASE_CONTROLLER_CONFIG_SQLDB_HOST: postgres
  services:
    - postgres:14.7
  rules:
    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
  script:
    - echo "Building FB and Datagen docker images for DAX tests"
    - PKG_LIST=$(go list ./... | grep -Ev 'internal/clustertests|simulacraData' | paste -s -d, -)
    - export TMPDIR=/mnt/ramdisk/test-$CI_JOB_ID
    - mkdir -p $TMPDIR
    - go test -coverprofile=coverage-dax-integration.out -covermode=atomic -coverpkg=${PKG_LIST} -timeout=20m ./dax/test/dax
  after_script:
    - rm -rf /mnt/ramdisk/test-$CI_JOB_ID
  artifacts:
    paths:
      - coverage-dax-integration.out
 # idk tests
 run go tests idk race:
  variables:
    PROJECT: race_${CI_CONCURRENT_ID}
  stage: nonblocking
  retry: 1
  script:
    - echo "Running test-all-race"
    - cd ./idk/
    - echo $PROJECT
    - echo $CI_COMMIT_REF_SLUG
    - BRANCH_NAME=${CI_COMMIT_REF_SLUG} IDK_FEATUREBASE_TAG=${CI_COMMIT_TAG} IDK_FEATUREBASE_HASH=${CI_COMMIT_SHA} make test-all-race
  after_script:
    - cd ./idk/
    - make save-pilosa-logs
    - make shutdown
  artifacts:
    paths:
      - ./idk/testdata/*_coverage.out
  tags:
    - shell
    - aws
  needs:
    - job: build amd container fb
 run go tests idk shard transactional:
  variables:
    IDK_DEFAULT_SHARD_TRANSACTIONAL: 1
    PROJECT: shardttrans_${CI_CONCURRENT_ID}
  stage: nonblocking
  retry: 1
  script:
    - echo "Running shard transactional tests"
    - cd ./idk/
    - echo $PROJECT
    - echo $CI_COMMIT_REF_SLUG
    - BRANCH_NAME=${CI_COMMIT_REF_SLUG} IDK_FEATUREBASE_TAG=${CI_COMMIT_TAG} IDK_FEATUREBASE_HASH=${CI_COMMIT_SHA} make test-all
  after_script:
    - cd ./idk/
    - make save-pilosa-logs
    - make shutdown
  artifacts:
    paths:
      - ./idk/testdata/*_coverage.out
      - ./idk/testdata/*_logs.txt
  tags:
    - shell
    - aws
  needs:
    - job: build amd container fb
 run go tests idk 533:
  variables:
    USERNAME: fb-idk-access
    PROJECT: test533_${CI_CONCURRENT_ID}
  stage: nonblocking
  retry: 1
  script:
    - echo "Running confluent 5.3.3 test-all"
    - cd ./idk/
    - echo $PROJECT
    - echo $CI_COMMIT_REF_SLUG
    - CONFLUENT_VERSION=5.3.3 BRANCH_NAME=${CI_COMMIT_REF_SLUG} IDK_FEATUREBASE_TAG=${CI_COMMIT_TAG} IDK_FEATUREBASE_HASH=${CI_COMMIT_SHA} make test-all
  after_script:
    - cd ./idk/
    - make save-pilosa-logs
    - make shutdown
  tags:
    - shell
    - aws
  artifacts:
    paths:
      - ./idk/testdata/*_coverage.out
  needs:
    - job: build amd container fb
 run go tests idk sasl:
  variables:
    PROJECT: sasl_${CI_CONCURRENT_ID}
  stage: nonblocking
  retry: 1
  script:
    - echo "Running test-all-kafka-sasl"
    - cd ./idk/
    - echo $PROJECT
    - echo $CI_COMMIT_REF_SLUG
    - BRANCH_NAME=${CI_COMMIT_REF_SLUG} IDK_FEATUREBASE_TAG=${CI_COMMIT_TAG} IDK_FEATUREBASE_HASH=${CI_COMMIT_SHA} make test-all-kafka-sasl
  after_script:
    - cd ./idk/
    - make save-pilosa-logs
    - make shutdown
  tags:
    - shell
    - aws
  artifacts:
    paths:
      - ./idk/testdata/*_coverage.out
  needs:
    - job: build amd container fb
 upload to sonarcloud:
  stage: nonblocking
  image: sonarsource/sonar-scanner-cli:4.7
  variables:
    SONAR_TOKEN: $SONAR_TOKEN
  rules:
    - if: '$CI_COMMIT_TAG == null && ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web")'
  script:
    - sonar-scanner -Dsonar.projectKey=molecula_featurebase -Dsonar.organization=molecula -Dsonar.sources=. -Dsonar.host.url=https://sonarcloud.io -Dsonar.go.coverage.reportPaths=coverage*.out,results/coverage*out,idk/testdata/*coverage.out,batch/testdata/*coverage.out,coverage-from-docker/*.out -Dsonar.javascript.lcov.reportPaths=lattice/coverage/lcov.info
  needs:
    - job: run go tests
    - job: run jest tests
    - job: external lookup tests
    - job: run go tests idk race
      optional: true
    - job: run go tests idk shard transactional
      optional: true
    - job: run go tests idk sasl
      optional: true
    - job: run go tests idk 533
      optional: true
    - job: run go tests batch
      optional: true
    - job: run go tests dax/test/dax
      optional: true
 package for linux amd64:
  stage: build
  image: golang:$GOVERSION
  extends: .go-cache
  rules:
    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
  variables:
    GOOS: "linux"
    GOARCH: "amd64"
  script:
    - echo 'deb [trusted=yes] https://repo.goreleaser.com/apt/ /' | tee /etc/apt/sources.list.d/goreleaser.list
    - apt update && apt install nfpm=2.11.3
    - make package
  artifacts:
    paths:
      - "*.deb"
      - "*.rpm"
 upload_artifacts_to_nexus:
  stage: post build
  image: golang:$GOVERSION
  rules:
    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
  script:
    - curl -v --user "$NEXUS_YUM_CREDS" --upload-file *.arm64.rpm https://nexus.molecula.com/repository/molecula-yum/release/
    - curl -v --user "$NEXUS_YUM_CREDS" --upload-file *.amd64.rpm https://nexus.molecula.com/repository/molecula-yum/release/
  dependencies:
    - package for linux amd64
    - package for linux arm64
 trigger_m-cloud-images:
  stage: post build
  rules:
    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
  needs:
    - upload_artifacts_to_nexus
  trigger: molecula/m-cloud-images
 package for linux arm64:
  stage: build
  image: golang:$GOVERSION
  extends: .go-cache
  rules:
    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
  variables:
    GOOS: "linux"
    GOARCH: "arm64"
  script:
    - echo 'deb [trusted=yes] https://repo.goreleaser.com/apt/ /' | tee /etc/apt/sources.list.d/goreleaser.list
    - apt update && apt install nfpm=2.11.3
    - make package
  artifacts:
    paths:
      - "*.deb"
      - "*.rpm"
 build arm container fb:
  stage: build
  needs:
    - "build featurebase"
  tags:
    - shell
  rules:
    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
  before_script:
    - echo "${DOCKER_DEPLOY_TOKEN}" | docker login -u ${DOCKER_DEPLOY_USER} --password-stdin ${CI_REGISTRY}
  script:
    - echo $CI_COMMIT_REF_SLUG
    - tag=${CI_REGISTRY_IMAGE}/featurebase:linux-arm64-${CI_COMMIT_REF_SLUG}
    - export SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct)
    - docker build --build-arg GO_VERSION=$GOVERSION --build-arg ARCH=arm64 --build-arg SOURCE_DATE_EPOCH=$SOURCE_DATE_EPOCH -t $tag -f .gitlab/Dockerfile .
    - docker push $tag
    - echo Created docker featurebase image with tag "$tag"
 ### start idk builds ###
 # building them all serially because otherwise you get container name conflicts.
 idk build_amd64:
  stage: build
  variables:
    BUILD_NAME: build_${CI_COMMIT_SHA}_${CI_CONCURRENT_ID}
  tags:
    - shell
  rules:
    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
  script:
    - export SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct)
    - cd ./idk/
    - date
    - make docker-build GOOS="linux" GOARCH="amd64" BUILD_CGO=1
    - date
    - make docker-build GOOS="darwin" GOARCH="amd64"
    - date
  artifacts:
    paths:
      - ./idk/build/*
  needs:
    # doesn't actually need this... just want it to start executing before *all* the tests finish
    - job: run go tests
 idk build_arm64:
  stage: build
  tags:
    - shell-arm64
  rules:
    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
  script:
    - export SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct)
    - cd ./idk/
    - date
    - make docker-build GOOS="linux" GOARCH="arm64" BUILD_CGO=1 BUILD_NAME="linux-arm64"
    - date
    - make docker-build GOOS="darwin" GOARCH="arm64"
    - date
  artifacts:
    paths:
      - ./idk/build/*
  needs:
    # doesn't actually need this... just want it to start executing before *all* the tests finish
    - job: run go tests
 # building them all serially because otherwise you get container name conflicts.
 # only do containers on default branch
 idk package_docker_all:
  stage: build
  tags:
    - shell
  rules:
    - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web")'
  script:
    - export SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct)
    - make docker-idk GOOS="linux" GOARCH="amd64"
    - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
    - make docker-idk-tag-push GOOS="linux" GOARCH="amd64"
  needs:
    - job: idk build_amd64
    - job: idk build_arm64
 idk s3 dump:
  stage: post build
  allow_failure: false
  variables:
    PROFILE: "service-fb-ci"
    AWS_SSH_PRIVATE_KEY: $AWS_FBCI_SSH_KEY
    AWS_ACCESS_KEY_ID: $AWS_FBCI_ACCESS_KEY_ID
    AWS_SECRET_ACCESS_KEY: $AWS_FBCI_SECRET_ACCESS_KEY
  tags:
    - shell
  rules:
    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
  script:
    - aws configure set aws_access_key_id $AWS_FBCI_ACCESS_KEY_ID
    - aws configure set aws_secret_access_key $AWS_FBCI_SECRET_ACCESS_KEY
    - aws configure set region "us-east-2"
    - aws configure set aws_profile $PROFILE
    - aws s3 cp ./idk/build/ s3://molecula-artifact-storage/idk/${CI_COMMIT_BRANCH}/${CI_COMMIT_SHORT_SHA}/ --recursive
    - aws s3 cp ./idk/build/ s3://molecula-artifact-storage/idk/${CI_COMMIT_BRANCH}/_latest/ --recursive
  needs:
    - job: idk build_amd64
    - job: idk build_arm64
 idk s3 dump tag:
  stage: post build
  variables:
    PROFILE: "service-fb-ci"
    AWS_SSH_PRIVATE_KEY: $AWS_FBCI_SSH_KEY
    AWS_ACCESS_KEY_ID: $AWS_FBCI_ACCESS_KEY_ID
    AWS_SECRET_ACCESS_KEY: $AWS_FBCI_SECRET_ACCESS_KEY
    LOCATION: molecula-artifact-storage/idk/_tags
  tags:
    - shell
  rules:
    - if: '$CI_COMMIT_TAG != null && ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web")'
  script:
    - aws configure set aws_access_key_id $AWS_FBCI_ACCESS_KEY_ID
    - aws configure set aws_secret_access_key $AWS_FBCI_SECRET_ACCESS_KEY
    - aws configure set region "us-east-2"
    - aws configure set aws_profile $PROFILE
    - |
      for goos in "darwin" "linux"; do
        for goarch in "amd64" "arm64"; do
          dir=idk-${CI_COMMIT_TAG}-${goos}-${goarch}
          echo "Directory ${dir}"
          mkdir ${dir}
          mv ./idk/build/idk-${goos}-${goarch}/molecula-consumer-* ${dir}/
          tar cvzf ${dir}.tar.gz ${dir}
          aws s3 cp ${dir} s3://${LOCATION}/${CI_COMMIT_TAG}/${dir}/ --recursive
          aws s3 cp ${dir}.tar.gz s3://${LOCATION}/${CI_COMMIT_TAG}/
        done
      done
  needs:
    - job: idk build_amd64
    - job: idk build_arm64
 ### end idk builds ###
 external lookup tests:
  stage: integration
  image: golang:$GOVERSION
  extends: .go-cache
  # TODO: no rules here, do we need to add the rules line?
  variables:
    POSTGRES_DB: $POSTGRES_DB
    POSTGRES_USER: $POSTGRES_USER
    POSTGRES_PASSWORD: $POSTGRES_PASSWORD
    POSTGRES_HOST_AUTH_METHOD: trust
  services:
    - postgres:13.5
  script:
    - apt-get update --allow-releaseinfo-change -y
    - apt-get install -y postgresql-client
    - go test . -run "^TestExternalLookup" -externalLookupDSN postgresql://$POSTGRES_USER:$POSTGRES_PASSWORD@postgres/$POSTGRES_DB?sslmode=disable
 s3 dump:
  stage: post build
  variables:
    PROFILE: "service-fb-ci"
    AWS_SSH_PRIVATE_KEY: $AWS_FBCI_SSH_KEY
    AWS_ACCESS_KEY_ID: $AWS_FBCI_ACCESS_KEY_ID
    AWS_SECRET_ACCESS_KEY: $AWS_FBCI_SECRET_ACCESS_KEY
  tags:
    - shell
  rules:
    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "web"'
  script:
    - aws configure set aws_access_key_id $AWS_FBCI_ACCESS_KEY_ID
    - aws configure set aws_secret_access_key $AWS_FBCI_SECRET_ACCESS_KEY
    - aws configure set region "us-east-2"
    - aws configure set aws_profile $PROFILE
    - |
      for goos in "darwin" "linux"; do
        for goarch in "amd64" "arm64"; do
          aws s3 cp featurebase_${goos}_${goarch} s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/${CI_COMMIT_SHORT_SHA}/featurebase_${goos}_${goarch}
          aws s3 cp featurebase_${goos}_${goarch} s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/_latest/featurebase_${goos}_${goarch}
          aws s3 cp ./build/fbsql_${goos}_${goarch} s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/${CI_COMMIT_SHORT_SHA}/fbsql_${goos}_${goarch}
          aws s3 cp ./build/fbsql_${goos}_${goarch} s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/_latest/fbsql_${goos}_${goarch}
        done
      done
  needs:
    - job: build featurebase
    - job: build fbsql amd64
    - job: build fbsql arm64
 s3 dump tag:
  stage: post build
  variables:
    PROFILE: "service-fb-ci"
    AWS_SSH_PRIVATE_KEY: $AWS_FBCI_SSH_KEY
    AWS_ACCESS_KEY_ID: $AWS_FBCI_ACCESS_KEY_ID
    AWS_SECRET_ACCESS_KEY: $AWS_FBCI_SECRET_ACCESS_KEY
    LOCATION: molecula-artifact-storage/featurebase/_tags
  tags:
    - shell
  rules:
    - if: '$CI_COMMIT_TAG != null && ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "web")'
  script:
    - aws configure set aws_access_key_id $AWS_FBCI_ACCESS_KEY_ID
    - aws configure set aws_secret_access_key $AWS_FBCI_SECRET_ACCESS_KEY
    - aws configure set region "us-east-2"
    - aws configure set aws_profile $PROFILE
    - |
      for goos in "darwin" "linux"; do
        for goarch in "amd64" "arm64"; do
          dir=featurebase-${CI_COMMIT_TAG}-${goos}-${goarch}
          echo "Directory ${dir}"
          mkdir $dir
          mv featurebase_${goos}_${goarch} ${dir}/featurebase
          mv ./build/fbsql_${goos}_${goarch} ${dir}/fbsql
          cp NOTICE install/featurebase.conf install/featurebase.*.service ${dir}/
          tar cvzf ${dir}.tar.gz ${dir}
          aws s3 cp ${dir} s3://${LOCATION}/${CI_COMMIT_TAG}/${dir}/ --recursive
          aws s3 cp ${dir}.tar.gz s3://${LOCATION}/${CI_COMMIT_TAG}/
        done
      done
  needs:
    - job: build featurebase
    - job: build fbsql amd64
    - job: build fbsql arm64
--- a/.gitlab/Dockerfile
+++ b/.gitlab/Dockerfile
@ -1,24 +0,0 @@
 FROM alpine:3.14.2
 LABEL maintainer "dev@molecula.com"
 LABEL org.opencontainers.image.authors="dev@molecula.com"
 ARG ARCH
 WORKDIR /
 RUN apk add --no-cache curl jq
 COPY NOTICE .
 COPY featurebase_linux_$ARCH featurebase
 RUN chmod ugo+x .
 EXPOSE 10101
 VOLUME /data
 ENV PILOSA_DATA_DIR /data
 ENV PILOSA_BIND 0.0.0.0:10101
 ENV PILOSA_BIND_GRPC 0.0.0.0:20101
 ENTRYPOINT ["/featurebase"]
 CMD ["server"]
--- a/.gitlab/batch-ci.yml
+++ b/.gitlab/batch-ci.yml
@ -1,28 +0,0 @@
 run go tests batch:
  variables:
    PROJECT: batch_${CI_CONCURRENT_ID}
  # this test relies on stuff that happens after build-lattice, which
  # makes it pause the entire CI run waiting for this. we accept the
  # small risk of wasting a build against the near certainty of spending
  # five minutes running only one job.
  stage: nonblocking
  retry: 1
  script:
    - echo "Running test-all"
    - cd ./batch/
    - echo $PROJECT
    - make build-featurebase
    - make test-all
  after_script:
    - cd ./batch/
    - make save-featurebase-logs
    - make shutdown
  artifacts:
    paths:
      - ./batch/testdata/*_coverage.out
      - ./batch/testdata/*_logs.txt
  tags:
    - shell
    - aws
  needs:
    - job: build amd container fb
--- a/.golangci.yml
+++ b/.golangci.yml
--- a/5
+++ b/5
@ -20,8 +20,6 @@ RUN yarn build
 FROM golang:${GO_VERSION} as pilosa-builder
 ARG MAKE_FLAGS
 ARG SOURCE_DATE_EPOCH
 WORKDIR /pilosa
 RUN go install github.com/rakyll/statik@v0.1.7
@ -30,7 +28,6 @@ COPY . ./
 COPY --from=lattice-builder /lattice/build /lattice
 RUN /go/bin/statik -src=/lattice -dest=/pilosa
 ENV SOURCE_DATE_EPOCH=${SOURCE_DATE_EPOCH}
 RUN make build FLAGS="-o build/featurebase" ${MAKE_FLAGS}
 #####################
@ -41,7 +38,7 @@ FROM alpine:3.13.2 as runner
 LABEL maintainer "dev@molecula.com"
-RUN apk add --no-cache curl jq tree
+RUN apk add --no-cache curl jq
 COPY --from=pilosa-builder /pilosa/build/featurebase /
--- a/13
+++ b/13
@ -20,19 +20,16 @@ RUN apt install -y docker.io
 ADD https://github.com/docker/compose/releases/latest/download/docker-compose-Linux-x86_64 /usr/local/bin/docker-compose
 RUN chmod +x /usr/local/bin/docker-compose
 WORKDIR /go/src/github.com/featurebasedb/featurebase/cmd/featurebase
 # generate an instrumented binary to allow for calculating code coverage for clustertests
 # the entrypoint for the binary is TestRunMain, which is wrapper for main
-RUN go test -covermode=atomic -coverpkg=../../... -c -tags testrunmain -o featurebase
+RUN cd /go/src/github.com/featurebasedb/featurebase/cmd/featurebase && \
-RUN cp /go/src/github.com/featurebasedb/featurebase/cmd/featurebase/featurebase /featurebase
+    go test -covermode=atomic -coverpkg=../../... -c -tags testrunmain -o featurebase && \
    cp /go/src/github.com/featurebasedb/featurebase/cmd/featurebase/featurebase /featurebase
 COPY NOTICE /NOTICE
 EXPOSE 10101
 VOLUME /data
-
+ENTRYPOINT ["bash", "-c"]
-# use e.g. "-test.coverprofile=/results/coverage.out"
+CMD ["/featurebase", "-test.run=TestRunMain", "-test.coverprofile=/results/coverage.out", "server", "--data-dir", "/data", "--bind", "http://0.0.0.0:10101"]
 CMD ["/featurebase", "-test.run=TestRunMain", "server"]
--- a/10
+++ b/10
@ -19,10 +19,9 @@ RUN apt install -y docker.io
 ADD https://github.com/docker/compose/releases/latest/download/docker-compose-Linux-x86_64 /usr/local/bin/docker-compose
 RUN chmod +x /usr/local/bin/docker-compose
-WORKDIR /go/src/github.com/featurebasedb/featurebase/cmd/featurebase
+RUN cd /go/src/github.com/featurebasedb/featurebase/cmd/featurebase && \
-
+    go test -covermode=atomic -coverpkg=../../... -c -tags testrunmain -o featurebase && \
-RUN go test -covermode=atomic -coverpkg=../../... -c -tags testrunmain -o featurebase
+    cp /go/src/github.com/featurebasedb/featurebase/cmd/featurebase/featurebase /featurebase
 RUN cp /go/src/github.com/featurebasedb/featurebase/cmd/featurebase/featurebase /featurebase
 COPY NOTICE /NOTICE
@ -32,6 +31,5 @@ COPY ./internal/clustertests /go/src/github.com/featurebasedb/featurebase/intern
 EXPOSE 10101
 VOLUME /data
-WORKDIR /go/src/github.com/featurebasedb/featurebase
+ENTRYPOINT ["bash", "-c"]
 CMD ["/featurebase", "-test.run=TestRunMain", "-test.coverprofile=/results/coverage.out", "server", "--data-dir", "/data", "--bind", "http://0.0.0.0:10101"]
--- a/47
+++ b/47
@ -1,47 +0,0 @@
 # syntax=docker/dockerfile:1
 ##########################
 ### datagen builder    ###
 ##########################
 FROM golang:alpine as builder
 WORKDIR /featurebase
 COPY . ./
 RUN apk add --no-cache build-base bash git make librdkafka pkgconfig
 # install librdkafka
 RUN git clone https://github.com/edenhill/librdkafka.git
 RUN cd librdkafka && ./configure --prefix /usr && make && make install
 ENV PKG_CONFIG_PATH=/usr/lib/pkgconfig/
 RUN cd idk && make build-datagen
 # ENTRYPOINT ["tail", "-f", "/dev/null"]
 #########################
 ### datagen runner    ###
 #########################
 FROM alpine:3.15.3 as runner
 WORKDIR /
 LABEL maintainer "dev@molecula.com"
 RUN apk add --no-cache curl jq
 COPY --from=builder /featurebase/idk/build/datagen /bin/
 COPY --from=builder /usr/lib/librdkafka* /usr/lib/
 COPY idk/datagen/testdata/* /testdata/
 EXPOSE 8080
 # VOLUME /data
 # ENV ADDR 0.0.0.0:8080
 #ENTRYPOINT ["sleep", "infinity"]
 ENTRYPOINT ["datagen"]
--- a/32
+++ b/32
@ -1,32 +0,0 @@
 ARG GO_VERSION=latest
 ###########################
 ### FeatureBase Builder ###
 ###########################
 FROM golang:${GO_VERSION} as featurebase-builder
 ARG MAKE_FLAGS
 WORKDIR /fb
 COPY . ./
 RUN make build FLAGS="-o build/featurebase" ${MAKE_FLAGS}
 ##########################
 ### FeatureBase runner ###
 ##########################
 FROM golang:alpine as runner
 LABEL maintainer "dev@featurebase.com"
 RUN apk add --no-cache curl jq tree
 COPY --from=featurebase-builder /fb/build/featurebase /
 COPY NOTICE /NOTICE
 EXPOSE 8080
 ENTRYPOINT ["/featurebase"]
 CMD ["dax"]
--- a/18
+++ b/18
@ -1,18 +0,0 @@
 ARG GO_VERSION=latest
 ##########################
 ### FeatureBase runner ###
 ##########################
 FROM alpine:3.13.2 as runner
 LABEL maintainer "dev@featurebase.com"
 RUN apk add --no-cache curl jq tree
 COPY ./fb_linux /featurebase
 EXPOSE 8080
 ENTRYPOINT ["/featurebase"]
 CMD ["dax"]
--- a/48
+++ b/48
@ -1,48 +0,0 @@
 ARG GO_VERSION=1.19
 FROM golang:1.19-buster as builder
 WORKDIR /
 RUN apt-get update -y -qq && apt-get install -y -qq \
  build-essential \
  git \
  musl-tools \
  netcat \
  unixodbc \
  unixodbc-dev \
  && rm -rf /var/lib/apt/lists/*
 RUN ["git", "clone", "https://github.com/edenhill/librdkafka.git"]
 WORKDIR /librdkafka
 RUN ./configure --prefix /usr && \
  make && \
  make install
 WORKDIR /featurebase
 COPY . .
 ARG MAKE_FLAGS
 ARG GO_BUILD_FLAGS
 ARG SOURCE_DATE_EPOCH
 WORKDIR /featurebase/
 ENV SOURCE_DATE_EPOCH=${SOURCE_DATE_EPOCH}
 RUN make build-fbsql GO_BUILD_FLAGS="-mod=vendor ${GO_BUILD_FLAGS}" ${MAKE_FLAGS}
 FROM ubuntu:20.04 as runner
 RUN apt-get update -y -qq && apt-get install -y -qq \
  ca-certificates \
  musl-tools \
  netcat \
  unixodbc-dev \
  && rm -rf /var/lib/apt/lists/*
 COPY --from=builder /featurebase/fbsql /usr/local/bin/
 # Verify that the linker can find everything.
 FROM runner as linkcheck
 RUN if [ -e /usr/local/bin/fbsql ] ; then ldd /usr/local/bin/fbsql; fi
 FROM runner
--- a/201
+++ b/201
@ -1,201 +0,0 @@
                                 Apache License
                           Version 2.0, January 2004
                        http://www.apache.org/licenses/
   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
   1. Definitions.
      "License" shall mean the terms and conditions for use, reproduction,
      and distribution as defined by Sections 1 through 9 of this document.
      "Licensor" shall mean the copyright owner or entity authorized by
      the copyright owner that is granting the License.
      "Legal Entity" shall mean the union of the acting entity and all
      other entities that control, are controlled by, or are under common
      control with that entity. For the purposes of this definition,
      "control" means (i) the power, direct or indirect, to cause the
      direction or management of such entity, whether by contract or
      otherwise, or (ii) ownership of fifty percent (50%) or more of the
      outstanding shares, or (iii) beneficial ownership of such entity.
      "You" (or "Your") shall mean an individual or Legal Entity
      exercising permissions granted by this License.
      "Source" form shall mean the preferred form for making modifications,
      including but not limited to software source code, documentation
      source, and configuration files.
      "Object" form shall mean any form resulting from mechanical
      transformation or translation of a Source form, including but
      not limited to compiled object code, generated documentation,
      and conversions to other media types.
      "Work" shall mean the work of authorship, whether in Source or
      Object form, made available under the License, as indicated by a
      copyright notice that is included in or attached to the work
      (an example is provided in the Appendix below).
      "Derivative Works" shall mean any work, whether in Source or Object
      form, that is based on (or derived from) the Work and for which the
      editorial revisions, annotations, elaborations, or other modifications
      represent, as a whole, an original work of authorship. For the purposes
      of this License, Derivative Works shall not include works that remain
      separable from, or merely link (or bind by name) to the interfaces of,
      the Work and Derivative Works thereof.
      "Contribution" shall mean any work of authorship, including
      the original version of the Work and any modifications or additions
      to that Work or Derivative Works thereof, that is intentionally
      submitted to Licensor for inclusion in the Work by the copyright owner
      or by an individual or Legal Entity authorized to submit on behalf of
      the copyright owner. For the purposes of this definition, "submitted"
      means any form of electronic, verbal, or written communication sent
      to the Licensor or its representatives, including but not limited to
      communication on electronic mailing lists, source code control systems,
      and issue tracking systems that are managed by, or on behalf of, the
      Licensor for the purpose of discussing and improving the Work, but
      excluding communication that is conspicuously marked or otherwise
      designated in writing by the copyright owner as "Not a Contribution."
      "Contributor" shall mean Licensor and any individual or Legal Entity
      on behalf of whom a Contribution has been received by Licensor and
      subsequently incorporated within the Work.
   2. Grant of Copyright License. Subject to the terms and conditions of
      this License, each Contributor hereby grants to You a perpetual,
      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
      copyright license to reproduce, prepare Derivative Works of,
      publicly display, publicly perform, sublicense, and distribute the
      Work and such Derivative Works in Source or Object form.
   3. Grant of Patent License. Subject to the terms and conditions of
      this License, each Contributor hereby grants to You a perpetual,
      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
      (except as stated in this section) patent license to make, have made,
      use, offer to sell, sell, import, and otherwise transfer the Work,
      where such license applies only to those patent claims licensable
      by such Contributor that are necessarily infringed by their
      Contribution(s) alone or by combination of their Contribution(s)
      with the Work to which such Contribution(s) was submitted. If You
      institute patent litigation against any entity (including a
      cross-claim or counterclaim in a lawsuit) alleging that the Work
      or a Contribution incorporated within the Work constitutes direct
      or contributory patent infringement, then any patent licenses
      granted to You under this License for that Work shall terminate
      as of the date such litigation is filed.
   4. Redistribution. You may reproduce and distribute copies of the
      Work or Derivative Works thereof in any medium, with or without
      modifications, and in Source or Object form, provided that You
      meet the following conditions:
      (a) You must give any other recipients of the Work or
          Derivative Works a copy of this License; and
      (b) You must cause any modified files to carry prominent notices
          stating that You changed the files; and
      (c) You must retain, in the Source form of any Derivative Works
          that You distribute, all copyright, patent, trademark, and
          attribution notices from the Source form of the Work,
          excluding those notices that do not pertain to any part of
          the Derivative Works; and
      (d) If the Work includes a "NOTICE" text file as part of its
          distribution, then any Derivative Works that You distribute must
          include a readable copy of the attribution notices contained
          within such NOTICE file, excluding those notices that do not
          pertain to any part of the Derivative Works, in at least one
          of the following places: within a NOTICE text file distributed
          as part of the Derivative Works; within the Source form or
          documentation, if provided along with the Derivative Works; or,
          within a display generated by the Derivative Works, if and
          wherever such third-party notices normally appear. The contents
          of the NOTICE file are for informational purposes only and
          do not modify the License. You may add Your own attribution
          notices within Derivative Works that You distribute, alongside
          or as an addendum to the NOTICE text from the Work, provided
          that such additional attribution notices cannot be construed
          as modifying the License.
      You may add Your own copyright statement to Your modifications and
      may provide additional or different license terms and conditions
      for use, reproduction, or distribution of Your modifications, or
      for any such Derivative Works as a whole, provided Your use,
      reproduction, and distribution of the Work otherwise complies with
      the conditions stated in this License.
   5. Submission of Contributions. Unless You explicitly state otherwise,
      any Contribution intentionally submitted for inclusion in the Work
      by You to the Licensor shall be under the terms and conditions of
      this License, without any additional terms or conditions.
      Notwithstanding the above, nothing herein shall supersede or modify
      the terms of any separate license agreement you may have executed
      with Licensor regarding such Contributions.
   6. Trademarks. This License does not grant permission to use the trade
      names, trademarks, service marks, or product names of the Licensor,
      except as required for reasonable and customary use in describing the
      origin of the Work and reproducing the content of the NOTICE file.
   7. Disclaimer of Warranty. Unless required by applicable law or
      agreed to in writing, Licensor provides the Work (and each
      Contributor provides its Contributions) on an "AS IS" BASIS,
      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
      implied, including, without limitation, any warranties or conditions
      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
      PARTICULAR PURPOSE. You are solely responsible for determining the
      appropriateness of using or redistributing the Work and assume any
      risks associated with Your exercise of permissions under this License.
   8. Limitation of Liability. In no event and under no legal theory,
      whether in tort (including negligence), contract, or otherwise,
      unless required by applicable law (such as deliberate and grossly
      negligent acts) or agreed to in writing, shall any Contributor be
      liable to You for damages, including any direct, indirect, special,
      incidental, or consequential damages of any character arising as a
      result of this License or out of the use or inability to use the
      Work (including but not limited to damages for loss of goodwill,
      work stoppage, computer failure or malfunction, or any and all
      other commercial damages or losses), even if such Contributor
      has been advised of the possibility of such damages.
   9. Accepting Warranty or Additional Liability. While redistributing
      the Work or Derivative Works thereof, You may choose to offer,
      and charge a fee for, acceptance of support, warranty, indemnity,
      or other liability obligations and/or rights consistent with this
      License. However, in accepting such obligations, You may act only
      on Your own behalf and on Your sole responsibility, not on behalf
      of any other Contributor, and only if You agree to indemnify,
      defend, and hold each Contributor harmless for any liability
      incurred by, or claims asserted against, such Contributor by reason
      of your accepting any such warranty or additional liability.
   END OF TERMS AND CONDITIONS
   APPENDIX: How to apply the Apache License to your work.
      To apply the Apache License to your work, attach the following
      boilerplate notice, with the fields enclosed by brackets "[]"
      replaced with your own identifying information. (Don't include
      the brackets!)  The text should be enclosed in the appropriate
      comment syntax for the file format. We also recommend that a
      file or class name and description of purpose be included on the
      same "printed page" as the copyright notice for easier
      identification within third-party archives.
   Copyright 2023 Molecula Corp. All rights reserved.
   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at
       http://www.apache.org/licenses/LICENSE-2.0
   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
--- a/286
+++ b/286
@ -1,38 +1,33 @@
-.PHONY: build clean build-lattice cover cover-viz default docker docker-build docker-build-fbsql docker-tag-push generate generate-protoc generate-pql generate-statik generate-stringer install install-protoc-gen-gofast install-protoc install-statik install-peg test docker-login
+.PHONY: build check-clean clean build-lattice cover cover-viz default docker docker-build docker-test docker-tag-push generate generate-protoc generate-pql generate-statik gometalinter install install-build-deps install-golangci-lint install-gometalinter install-protoc install-protoc-gen-gofast install-peg install-statik release release-build test testv testv-race testvsub testvsub-race test-txstore-rbf
-SHELL := /bin/bash
+CLONE_URL=github.com/pilosa/pilosa
 VERSION := $(shell git describe --tags 2> /dev/null || echo unknown)
 VARIANT = Molecula
 GO=go
 GOOS=$(shell $(GO) env GOOS)
 GOARCH=$(shell $(GO) env GOARCH)
 VERSION_ID=$(if $(TRIAL_DEADLINE),trial-$(TRIAL_DEADLINE)-,)$(VERSION)-$(GOOS)-$(GOARCH)
-DATE_FMT="+%FT%T%z"
+BRANCH := $(if $(CIRCLE_BRANCH),$(CIRCLE_BRANCH),$(shell git rev-parse --abbrev-ref HEAD))
-# set SOURCE_DATE_EPOCH like this to use the last git commit timestamp
+BRANCH_ID := $(BRANCH)-$(GOOS)-$(GOARCH)
-# export SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct) instead of the current time from running `date`
+BUILD_TIME := $(shell date -u +%FT%T%z)
 ifdef SOURCE_DATE_EPOCH
 	BUILD_TIME ?= $(shell date -u -d "@$(SOURCE_DATE_EPOCH)" "$(DATE_FMT)" 2>/dev/null || date -u -r "$(SOURCE_DATE_EPOCH)" "$(DATE_FMT)" 2>/dev/null || date -u "$(DATE_FMT)")
 else
 	BUILD_TIME ?= $(shell date -u "$(DATE_FMT)")
 endif
 SHARD_WIDTH = 20
 COMMIT := $(shell git describe --exact-match >/dev/null 2>&1 || git rev-parse --short HEAD)
 LDFLAGS="-X github.com/featurebasedb/featurebase/v3.Version=$(VERSION) -X github.com/featurebasedb/featurebase/v3.BuildTime=$(BUILD_TIME) -X github.com/featurebasedb/featurebase/v3.Variant=$(VARIANT) -X github.com/featurebasedb/featurebase/v3.Commit=$(COMMIT) -X github.com/featurebasedb/featurebase/v3.TrialDeadline=$(TRIAL_DEADLINE)"
 GO_VERSION=1.19
 GO_BUILD_FLAGS=
 DOCKER_BUILD= # set to 1 to use `docker-build` instead of `build` when creating a release
 BUILD_TAGS += 
 TEST_TAGS = roaringparanoia
 UNAME := $(shell uname -s)
 TEST_TIMEOUT=10m
 RACE_TEST_TIMEOUT=10m
-# size in GB to use for ramdisk, ?= so you can override it with env
+ifeq ($(UNAME), Darwin)
-# 4GB is not enough for `make test`, 8GB usually is.
+    IS_MACOS:=1
-RAMDISK_SIZE ?= 8
+else
    IS_MACOS:=0
 endif
 export GO111MODULE=on
 export GOPRIVATE=github.com/molecula
 export CGO_ENABLED=0
 AWS_ACCOUNTID ?= undefined
 # Run tests and compile Pilosa
 default: test build
@ -41,7 +36,7 @@ default: test build
 clean:
 	rm -rf vendor build
 	rm -f *.rpm *.deb
-
+	
 # Set up vendor directory using `go mod vendor`
 vendor: go.mod
 	$(GO) mod vendor
@ -49,13 +44,13 @@ vendor: go.mod
 version:
 	@echo $(VERSION)
-# We build a list of packages that omits the IDK and batch packages because
+# We build a list of packages that omits the IDK packages because the IDK
-# those packages require fancy environment setup.
+# packages require fancy environment setup.
-GOPACKAGES := $(shell $(GO) list ./... | grep -v "/v3/idk" | grep -v "/v3/batch")
+GOPACKAGES := $(shell $(GO) list ./... | grep -v "/idk")
 # Run test suite
 test:
-	$(GO) test $(GOPACKAGES) -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) -v -timeout $(TEST_TIMEOUT) -count=1
+	$(GO) test $(GOPACKAGES) -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) -v -timeout $(TEST_TIMEOUT)
 # Run test suite with race flag
 test-race:
@ -81,21 +76,6 @@ testvsub:
 			echo; echo "999 done testing subpkg $$pkg"; \
 		done
 # make a $(RAMDISK_SIZE)GB RAMDisk. Speed up tests by running
 # them with TMPDIR=/mnt/ramdisk.
 ramdisk-linux:
 	mount -o size=$(RAMDISK__SIZE)G -t tmpfs none /mnt/ramdisk
 # make a $(RAMDISK_SIZE)GB RAMDisk. Speed up tests by running
 # them with TMPDIR=/Volumes/RAMDisk. This is more important on
 # OS X than it is on Linux, because there's performance issues
 # with fsync on OS X that can make the SSD slow down to moving-platters
 # drive speeds. Oops.
 ramdisk-osx:
 	diskutil erasevolume HFS+ 'RAMDisk' $$(hdiutil attach -nobrowse -nomount ram://$$(expr 2097152 \* $(RAMDISK_SIZE)))
 detach-ramdisk-osx:
 	hdiutil detach /Volumes/RAMDisk
 testvsub-race:
 	@set -e; for pkg in $(GOPACKAGES); do \
@ -104,7 +84,6 @@ testvsub-race:
           echo; echo "999 done testing subpkg $$pkg"; \
        done
 bench:
 	$(GO) test $(GOPACKAGES) -bench=. -run=NoneZ -timeout=127m $(TESTFLAGS)
@ -117,16 +96,52 @@ cover:
 cover-viz: cover
 	$(GO) tool cover -html=build/coverage.out
-# Build featurebase
+# Compile Pilosa
 build:
 	$(GO) build -tags='$(BUILD_TAGS)' -ldflags $(LDFLAGS) $(FLAGS) ./cmd/featurebase
 # Create a single release build under the build directory
 release-build:
 	$(MAKE) $(if $(DOCKER_BUILD),docker-)build FLAGS="-o build/featurebase-$(VERSION_ID)/featurebase"
 	cp NOTICE install/featurebase.conf install/featurebase*.service build/featurebase-$(VERSION_ID)
 	tar -cvz -C build -f build/featurebase-$(VERSION_ID).tar.gz featurebase-$(VERSION_ID)/
 	@echo Created release build: build/featurebase-$(VERSION_ID).tar.gz
 test-release-build: docker-build
 	mv build/featurebase-$(VERSION_ID).tar.gz install/
 	cd install && docker build -t featurebase:test_installation \
 		-f test_installation.Dockerfile \
 		--build-arg release_tarball=featurebase-$(VERSION_ID).tar.gz .
 	mv install/featurebase-$(VERSION_ID).tar.gz build/
 	docker run -it -v /sys/fs/cgroup:/sys/fs/cgroup:ro \
 		featurebase:test_installation
 # Error out if there are untracked changes in Git
 check-clean:
 ifndef SKIP_CHECK_CLEAN
 	$(if $(shell git status --porcelain),$(error Git status is not clean! Please commit or checkout/reset changes.))
 endif
 # Create release build tarballs for all supported platforms. DEPRECATED: Use `docker-release`
 release: check-clean generate-statik-docker
 	$(MAKE) release-build GOOS=darwin GOARCH=amd64
 	$(MAKE) release-build GOOS=darwin GOARCH=arm64
 	$(MAKE) release-build GOOS=linux GOARCH=amd64
 	$(MAKE) release-build GOOS=linux GOARCH=arm64
 # Create release build tarballs for all supported platforms. Same as `release`, but without embedded Lattice UI.
 release-sans-ui: check-clean
 	rm -f statik/statik.go
 	$(MAKE) release-build GOOS=darwin GOARCH=amd64
 	$(MAKE) release-build GOOS=darwin GOARCH=arm64
 	$(MAKE) release-build GOOS=linux GOARCH=amd64
 	$(MAKE) release-build GOOS=linux GOARCH=arm64
 package:
-	GOOS=$(GOOS) GOARCH=$(GOARCH) $(MAKE) build
+	go build -o featurebase ./cmd/featurebase
 	GOOS=$(GOOS) GOARCH=$(GOARCH) $(MAKE) build-fbsql
 	GOARCH=$(GOARCH) VERSION=$(VERSION) nfpm package --packager deb --target featurebase.$(VERSION).$(GOARCH).deb
 	GOARCH=$(GOARCH) VERSION=$(VERSION) nfpm package --packager rpm --target featurebase.$(VERSION).$(GOARCH).rpm
-
+	
 # We allow setting a custom docker-compose "project". Multiple of the
 # same docker-compose environment can exist simultaneously as long as
 # they use different projects (the project name is prepended to
@ -153,17 +168,16 @@ authclustertests: vendor
 	PROJECT=$(PROJECT) ENABLE_AUTH=1 $(DOCKER_COMPOSE) -f internal/clustertests/docker-compose.yml run client1
 	CLUSTERTESTS_FB_ARGS=$(AUTH_ARGS) $(DOCKER_COMPOSE) -f internal/clustertests/docker-compose.yml down
-# Install FeatureBase and IDK
+# Install Pilosa
-install: install-featurebase install-idk install-fbsql
+install:
 install-featurebase:
 	$(GO) install -tags='$(BUILD_TAGS)' -ldflags $(LDFLAGS) $(FLAGS) ./cmd/featurebase
-install-idk:
+# Install the single-node PLG version of FeatureBase
-	$(MAKE) -C ./idk install
+plg:
 	$(GO) build -tags='plg $(BUILD_TAGS)' -ldflags $(LDFLAGS) $(FLAGS) ./cmd/featurebase
-install-fbsql:
+install-bench:
-	CGO_ENABLED=1 $(GO) install ./cmd/fbsql
+	$(GO) install -tags='$(BUILD_TAGS)' -ldflags $(LDFLAGS) $(FLAGS) ./cmd/pilosa-bench
 # Build the lattice assets
 build-lattice:
@ -209,7 +223,6 @@ docker-build: vendor
 	docker build \
 	    --build-arg GO_VERSION=$(GO_VERSION) \
 	    --build-arg MAKE_FLAGS="TRIAL_DEADLINE=$(TRIAL_DEADLINE) GOOS=$(GOOS) GOARCH=$(GOARCH)" \
 	    --build-arg SOURCE_DATE_EPOCH=$(SOURCE_DATE_EPOCH) \
 	    --target pilosa-builder \
 	    --tag featurebase:build .
 	docker create --name featurebase-build featurebase:build
@ -227,61 +240,6 @@ docker-image: vendor
 	    --tag featurebase:$(VERSION) .
 	@echo Created docker image: featurebase:$(VERSION)
 docker-image-featurebase: vendor
 	docker build \
 	    --build-arg GO_VERSION=$(GO_VERSION) \
 		--file Dockerfile-dax \
 		--tag dax/featurebase .
 docker-image-featurebase-linux-amd64: vendor
 	docker build \
 	    --build-arg GO_VERSION=$(GO_VERSION) \
 	    --platform linux/amd64 \
 	    --file Dockerfile-dax \
 	    --tag dax/featurebase .
 docker-image-featurebase-test: vendor
 	docker build \
 	    --build-arg GO_VERSION=$(GO_VERSION) \
 		--file Dockerfile-clustertests \
 		--tag dax/featurebase-test .
 # build-for-quick builds a linux featurebase binary outside of docker
 # (which is much faster for some reason), and places it in the .quick
 # subdirectory.
 build-for-quick:
 	GOOS=linux $(MAKE) build FLAGS="-o .quick/fb_linux"
 # docker-image-featurebase-quick uses a pre-built featurebase binary
 # to quickly create a fresh docker image without needing to send the
 # context of the featurebase top level directory.
 docker-image-featurebase-quick: build-for-quick
 	docker build \
 	    --build-arg GO_VERSION=$(GO_VERSION) \
 	    --file Dockerfile-dax-quick \
 	    --tag dax/featurebase ./.quick/
 docker-image-datagen: vendor
 	docker build --tag dax/datagen --file Dockerfile-datagen .
 get-account-id:
 	$(eval AWS_ACCOUNTID := $(shell aws sts get-caller-identity --output=json | jq -r .Account))
 ecr-push-featurebase: docker-login
 	echo "Pushing to account $(AWS_ACCOUNTID), profile $(AWS_PROFILE)"
 	docker tag dax/featurebase:latest $(AWS_ACCOUNTID).dkr.ecr.us-east-2.amazonaws.com/dax/featurebase:latest
 	docker push $(AWS_ACCOUNTID).dkr.ecr.us-east-2.amazonaws.com/dax/featurebase:latest
 ecr-push-datagen: docker-login
 	docker tag dax/datagen:latest $(AWS_ACCOUNTID).dkr.ecr.us-east-2.amazonaws.com/dax/datagen:latest
 	docker push $(AWS_ACCOUNTID).dkr.ecr.us-east-2.amazonaws.com/dax/datagen:latest
 docker-login: get-account-id
 	aws ecr get-login-password --region us-east-2 | docker login --username AWS --password-stdin $(AWS_ACCOUNTID).dkr.ecr.us-east-2.amazonaws.com
 # Create docker image (alias)
 docker: docker-image # alias
@ -307,6 +265,36 @@ docker-idk-tag-push:
 	docker push registry.gitlab.com/molecula/featurebase/idk:$(VERSION_ID)
 	@echo Pushed docker image: registry.gitlab.com/molecula/featurebase/idk:$(VERSION_ID)
 # Install diagnostic pilosa-keydump tool. Allows viewing the keys in a transaction-engine directory.
 pilosa-keydump:
 	$(GO) install -tags='$(BUILD_TAGS)' -ldflags $(LDFLAGS) $(FLAGS) ./cmd/pilosa-keydump
 # Install diagnostic pilosa-chk tool for string translations and fragment checksums.
 pilosa-chk:
 	$(GO) install -tags='$(BUILD_TAGS)' -ldflags $(LDFLAGS) $(FLAGS) ./cmd/pilosa-chk
 pilosa-fsck:
 	cd ./cmd/pilosa-fsck && make install && make release
 # Run Pilosa tests inside Docker container
 docker-test:
 	docker run --rm -v $(PWD):/go/src/$(CLONE_URL) -w /go/src/$(CLONE_URL) golang:$(GO_VERSION) go test -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) -timeout $(TEST_TIMEOUT) $(GOPACKAGES)
 # Must use bash in order to -o pipefail; otherwise the tee will hide red tests.
 # run top tests, not subdirs. print summary red/green after.
 # The \-\-\- FAIL avoids counting the extra two FAIL strings at then bottom of log.topt.
 topt:
 	mv log.topt.roar log.topt.roar.prev || true
 	$(eval SHELL:=/bin/bash) set -o pipefail; $(GO) test -v -timeout $(RACE_TEST_TIMEOUT) -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) 2>&1 | tee log.topt.roar
 	@echo "   log.topt.roar green: \c"; cat log.topt.roar | grep PASS |wc -l
 	@echo "   log.topt.roar   red: \c"; cat log.topt.roar | grep '\-\-\- FAIL' | wc -l
 topt-race:
 	mv log.topt.race log.topt.race.prev || true
 	$(eval SHELL:=/bin/bash) set -o pipefail; CGO_ENABLED=1 $(GO) test -race -timeout $(RACE_TEST_TIMEOUT) -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) 2>&1 | tee log.topt.race
 	@echo "   log.topt.race green: \c"; cat log.topt.race | grep PASS |wc -l
 	@echo "   log.topt.race   red: \c"; cat log.topt.race | grep '\-\-\- FAIL' | wc -l
 # Run golangci-lint
 golangci-lint: require-golangci-lint
 	golangci-lint run --timeout 3m --skip-files '.*\.peg\.go'
@ -317,6 +305,31 @@ linter: golangci-lint
 # Better alias
 ocd: golangci-lint
 # Run gometalinter with custom flags
 # Note the "./..." in gometalinter is still allowed, because we do want
 # linting to reach IDK pagkages.
 gometalinter: require-gometalinter vendor
 	GO111MODULE=off gometalinter --vendor --disable-all \
 	    --deadline=300s \
 	    --enable=deadcode \
 	    --enable=gochecknoinits \
 	    --enable=gofmt \
 	    --enable=goimports \
 	    --enable=gotype \
 	    --enable=gotypex \
 	    --enable=ineffassign \
 	    --enable=interfacer \
 	    --enable=maligned \
 	    --enable=misspell \
 	    --enable=nakedret \
 	    --enable=staticcheck \
 	    --enable=unconvert \
 	    --enable=unparam \
 	    --enable=vet \
 	    --exclude "^internal/.*\.pb\.go" \
 	    --exclude "^pql/pql.peg.go" \
 	    ./...
 ######################
 # Build dependencies #
 ######################
@ -327,14 +340,20 @@ require-%:
 		$(info Verified build dependency "$*" is installed.),\
 		$(error Build dependency "$*" not installed. To install, try `make install-$*`))
-install-build-deps: install-protoc-gen-gofast install-protoc install-statik install-peg
+install-build-deps: install-protoc-gen-gofast install-protoc install-statik install-stringer install-peg
 install-statik:
 	go install github.com/rakyll/statik@latest
 install-stringer:
 	GO111MODULE=off $(GO) get -u golang.org/x/tools/cmd/stringer
 install-protoc-gen-gofast:
 	GO111MODULE=off $(GO) get -u github.com/gogo/protobuf/protoc-gen-gofast
 install-protoc-gen-go:
 	GO111MODULE=off $(GO) get -u github.com/golang/protobuf/protoc-gen-go
 install-protoc:
 	@echo This tool cannot automatically install protoc. Please download and install protoc from https://google.github.io/proto-lens/installing-protoc.html
 	@echo On mac, brew install protobuf seems to work.
@ -348,58 +367,13 @@ install-peg:
 install-golangci-lint:
 	GO111MODULE=off $(GO) get github.com/golangci/golangci-lint/cmd/golangci-lint
 install-gometalinter:
 	GO111MODULE=off $(GO) get -u github.com/alecthomas/gometalinter
 	GO111MODULE=off gometalinter --install
 	GO111MODULE=off $(GO) get github.com/remyoudompheng/go-misc/deadcode
 test-external-lookup:
 	$(GO) test . -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) -run ^TestExternalLookup$$ -externalLookupDSN $(EXTERNAL_LOOKUP_DSN)
 bnf:
 	ebnf2railroad --no-overview-diagram  --no-optimizations ./sql3/sql3.ebnf
 #################################
 # fbsql builds in docker
 #################################
 # This allows multiple concurrent builds to happen in CI without
 # creating container name conflicts and such. (different BUILD_NAMEs
 # are passed in from gitlab-ci.yml)
 BUILD_NAME ?= fbsql-build
 LDFLAGS_STATIC="-linkmode external -extldflags \"-static\" -X 'github.com/featurebasedb/featurebase/v3/fbsql.Version=$(VERSION)' -X 'github.com/featurebasedb/featurebase/v3/fbsql.BuildTime=$(BUILD_TIME)' "
 UNAME_P := $(shell uname -p)
 BUILD_CGO ?= 0
 # Build fbsql
 build-fbsql:
 	@echo GOOS=$(GOOS) GOARCH=$(GOARCH) uname -p=$(UNAME_P) build_cgo=$(BUILD_CGO)
 ifeq ($(BUILD_CGO), 0)
 	make build-fbsql-non-cgo
 endif
 ifeq ($(BUILD_CGO), 1)
 	make build-fbsql-cgo
 endif
 build-fbsql-non-cgo:
 	CGO_ENABLED=0 $(GO) build -ldflags $(LDFLAGS) $(GO_BUILD_FLAGS) -o fbsql ./cmd/fbsql
 build-fbsql-cgo:
 ifeq ($(GOARCH), arm64)
 	CGO_ENABLED=1 $(GO) build -tags dynamic $(GO_BUILD_FLAGS) -o fbsql ./cmd/fbsql
 endif
 ifeq ($(GOARCH), amd64)
 	CC=/usr/bin/musl-gcc CGO_ENABLED=1 $(GO) build -tags "musl static" -ldflags $(LDFLAGS_STATIC) $(GO_BUILD_FLAGS) -o fbsql ./cmd/fbsql
 endif
 docker-build-fbsql: vendor
 	DOCKER_BUILDKIT=0 docker build \
 		--file Dockerfile-fbsql \
 	    --build-arg GO_VERSION=$(GO_VERSION) \
 	    --build-arg MAKE_FLAGS="GOOS=$(GOOS) GOARCH=$(GOARCH) BUILD_CGO=$(BUILD_CGO)" \
 	    --build-arg GO_BUILD_FLAGS=$(GO_BUILD_FLAGS) \
 		--build-arg SOURCE_DATE_EPOCH=$(SOURCE_DATE_EPOCH) \
 	    --target builder \
 	    --tag fbsql:$(BUILD_NAME) .
 	mkdir -p build
 	docker create --name $(BUILD_NAME) fbsql:$(BUILD_NAME)
 	docker cp $(BUILD_NAME):/featurebase/fbsql ./build/fbsql_$(GOOS)_$(GOARCH)
 	docker rm $(BUILD_NAME)
--- a/OPENSOURCE.md
+++ b/OPENSOURCE.md
@ -1,45 +0,0 @@
 ## User Contribution Guidelines for FeatureBase
 Thank you for your interest in contributing to FeatureBase! We appreciate your support in making this open-source project even better. Here are some guidelines to help you get started with contributing to FeatureBase:
 1. Familiarize Yourself with the Project:
   - Visit the FeatureBase website at www.featurebase.com to understand the project's goals, capabilities, and features.
   - Read the documentation available on the website, including the installation guide, configuration options, and data modeling concepts.
   - Explore the codebase by cloning the repository and reviewing the source code.
 2. Join the Community:
   - Visit the FeatureBase community page at https://www.featurebase.com/community to learn more about the project's community and how to get involved.
   - Join the Discord server at https://discord.gg/FBn2vEp7Na to chat with other contributors and users, ask questions, and share your ideas.
 3. Set Up Your Development Environment:
   - Ensure you have Go installed on your machine. Make sure your shell's search path includes the go/bin directory.
   - Clone the FeatureBase repository or download it as a zip file from the repository's page.
   - Follow the "Build FeatureBase Server from source" instructions in the README file to compile the server binary and the ingester binaries.
 4. Choose a Contribution Area:
   - Identify the area you'd like to contribute to, such as bug fixes, new features, performance improvements, documentation updates, or community support.
   - Check the issue tracker on the repository or the FeatureBase community for open issues or feature requests that align with your interests and skills. Alternatively, propose your own idea by creating a new issue.
 5. Create a New Branch:
   - Before making any changes, create a new branch in the repository's Git repository. This branch will contain your contributions.
   - Give your branch a descriptive name that reflects the nature of your contribution.
 6. Make Your Changes:
   - Follow the coding style and conventions used in the existing codebase.
   - Write clear and concise commit messages for each logical change.
   - If you're introducing new features or modifying existing behavior, make sure to update the documentation to reflect the changes.
 7. Test Your Changes:
   - Run the existing test suite to ensure that your modifications do not introduce any regressions.
   - If applicable, write additional tests to cover the changes you made.
   - Document any new testing procedures required for your contribution.
 8. Submitting Your Contribution:
   - Push your branch to the main repository or create a fork and submit a pull request to the main repository.
   - Provide a detailed description of your changes, including the problem you solved and the approach you took.
   - Be responsive to any feedback or suggestions provided by the project maintainers or other contributors.
   - Once your contribution is approved, it will be reviewed and merged into the main codebase.
 Please note that by contributing to FeatureBase, you agree that your contributions will be licensed under the Apache 2.0 license, which governs the project.
 Thank you for considering contributing to FeatureBase! Your contributions are valuable and help improve the project for everyone.
--- a/README.md
+++ b/README.md
@ -1,10 +1,4 @@
-# FeatureBase Community
+# FeatureBase
 FeatureBase Community is now archived and no longer maintained.
 * [FeatureBase Community Help](https://github.com/FeatureBaseDB/FB-community-help)
 ## Pilosa is now FeatureBase
@ -16,8 +10,6 @@ For more information about FeatureBase, please visit [www.featurebase.com][HomeP
 ## Getting Started
 * [Learn how to install FeatureBase Community](https://github.com/FeatureBaseDB/FB-community-help/blob/main/docs/community/com-getstart/com-getstart-home.md)
 ### Build FeatureBase Server from source
 0. Install go. Ensure that your shell's search path includes the go/bin directory.
@ -27,20 +19,42 @@ For more information about FeatureBase, please visit [www.featurebase.com][HomeP
 4. Run `featurebase server --handler.allowed-origins=http://localhost:3000` to run FeatureBase server with default settings (learn more about configuring FeatureBase at the link below). The `--handler.allowed-origins` parameter allows the standalone web UI to talk to the server; this can be omitted if the web UI is not needed.
 5. Run `curl localhost:10101/status` to verify the server is running and accessible.
 ### Ingest Data and Query
 1. Run 
 ```
 molecula-consumer-csv \
    --index repository \
    --header "language__ID_F,project_id__ID_F" \
    --id-field project_id \
    --batch-size 1000 \
    --files example.csv
 ```
 This will ingest the `example.csv` file into a FeatureBase table called `repository`. If the table does not exist, it will be automatically created. Learn more about ingesting into FeatureBase: [https://docs.featurebase.com/data-ingestion/enterprise/ingesters][Ingest]
 2. Query your data. 
 ```
 curl localhost:10101/index/repository/query \
     -X POST \
     -d 'Row(example=5)'
 ```
 Learn about supported [SQL][SQL], native [Pilosa Query Language (PQL)][PQL].
 ### Data Model
 Because FeatureBase is built on bitmaps, there is bit of a learning curve to grasp how your data is represented. 
 Data Model Guide: [https://docs.featurebase.com/data-modeling-guide/data-modeling][DataModel]
-* [Learn about Data Modeling](https://github.com/FeatureBaseDB/FB-community-help/blob/main/docs/concepts/concepts-home.md)
+### More Information
 Installation:[https://docs.featurebase.com/setting-up-featurebase/enterprise/installing-featurebase][Install]
-### Ingest Data and Query
+Configuration: [https://docs.featurebase.com/setting-up-featurebase/enterprise/featurebase-configuration][Config]
 * [Learn how to ingest data from multiple data sources](https://github.com/FeatureBaseDB/FB-community-help/blob/main/docs/community/com-ingest/com-ingest-manage.md)
 ## Community
-You can email us at community@featurebase.com and [learn more about contributing](https://github.com/FeatureBaseDB/featurebase/blob/master/OPENSOURCE.md).
+You can email us at community@featurebase.com or learn more about contributing at [https://www.featurebase.com/community][Community].
 Chat with us: [https://discord.gg/FBn2vEp7Na][Discord]
@ -59,14 +73,13 @@ A lot has changed since the days of Pilosa. This list highlights some new capabi
 FeatureBase is licensed under the [Apache License, Version 2.0][License]
-[Community]: https://github.com/FeatureBaseDB/FB-community-help/tree/main
+[Community]: http://www.featurebase.com/community?utm_campaign=Open%20Source&utm_source=GitHub
-[Install]:https://github.com/FeatureBaseDB/FB-community-help/blob/main/docs/community/com-getstart/com-getstart-home.md
+[Config]: https://docs.featurebase.com/setting-up-featurebase/enterprise/featurebase-configuration?utm_campaign=Open%20Source&utm_source=GitHub
-[Config]: https://github.com/FeatureBaseDB/FB-community-help/tree/main/docs/community/com-config
+[DataModel]: https://docs.featurebase.com/data-modeling-guide/data-modeling?utm_campaign=Open%20Source&utm_source=GitHub
 [DataModel]: https://github.com/FeatureBaseDB/FB-community-help/blob/main/docs/concepts/concepts-home.md
 [Discord]: https://discord.gg/FBn2vEp7Na
 [HomePage]: http://featurebase.com?utm_campaign=Open%20Source&utm_source=GitHub
-[Ingest]: https://github.com/FeatureBaseDB/FB-community-help/blob/main/docs/community/com-ingest/com-ingest-manage.md
+[Ingest]: https://docs.featurebase.com/data-ingestion/enterprise/ingesters?utm_campaign=Open%20Source&utm_source=GitHub
- 
+[Install]: http://docs.featurebase.com/setting-up-featurebase/enterprise/installing-featurebase?utm_campaign=Open%20Source&utm_source=GitHub
 [License]: http://www.apache.org/licenses/LICENSE-2.0
-[PQL]: https://docs.featurebase.com/docs/pql-guide/pql-home/?utm_campaign=Open%20Source&utm_source=GitHub
+[PQL]: http://docs.featurebase.com/data-querying/pql/introduction?utm_campaign=Open%20Source&utm_source=GitHub
-[SQL]: https://docs.featurebase.com/docs/sql-guide/sql-guide-home/?utm_campaign=Open%20Source&utm_source=GitHub
+[SQL]: http://docs.featurebase.com/data-querying/sql?utm_campaign=Open%20Source&utm_source=GitHub
--- a/api.go
+++ b/api.go
--- a/api/client/grpc.go
+++ b/api/client/grpc.go
@ -13,7 +13,6 @@ import (
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/connectivity"
 	"google.golang.org/grpc/credentials"
 	"google.golang.org/grpc/credentials/insecure"
 )
 const maxMsgSize = 1024 * 1024 * 100 // 100 megs ought to be enough for anybody!
@ -65,7 +64,7 @@ func (c *GRPCClient) resetConn() error {
 		creds := credentials.NewTLS(c.tlsConfig)
 		opts = append(opts, grpc.WithTransportCredentials(creds))
 	} else {
-		opts = append(opts, grpc.WithTransportCredentials(insecure.NewCredentials()))
+		opts = append(opts, grpc.WithInsecure())
 	}
 	opts = append(opts, grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(maxMsgSize)))
--- a/api_directive.go
+++ b/api_directive.go
@ -1,987 +0,0 @@
 // Copyright 2021 Molecula Corp. All rights reserved.
 package pilosa
 import (
 	"context"
 	"io"
 	"log"
 	"sync"
 	"github.com/featurebasedb/featurebase/v3/dax"
 	"github.com/featurebasedb/featurebase/v3/dax/computer"
 	"github.com/featurebasedb/featurebase/v3/dax/storage"
 	"github.com/featurebasedb/featurebase/v3/disco"
 	"github.com/pkg/errors"
 )
 // ApplyDirective applies a Directive received, from the Controller, at the
 // /directive endpoint.
 func (api *API) ApplyDirective(ctx context.Context, d *dax.Directive) error {
 	// Get the current directive for comparison.
 	previousDirective := api.holder.Directive()
 	// Check that incoming version is newer.
 	// Note: 0 is an invalid Directive version. This decision was made because
 	// previousDirective is not a pointer to a directive, but a concrete
 	// Directive. Which means we can't check for nil, and by default it has a
 	// version of 0. So in order to ensure the version has increased, we need to
 	// require that incoming directive versions are greater than 0.
 	if d.Version == 0 {
 		return errors.Errorf("directive version cannot be 0")
 	} else if previousDirective.Version >= d.Version {
 		return errors.Errorf("directive version mismatch, got %d, but already have %d", d.Version, previousDirective.Version)
 	}
 	// Handle the operations based on the directive method.
 	switch d.Method {
 	case dax.DirectiveMethodDiff:
 		// In order to prevent adding too much code specific to handling a diff
 		// directive (e.g. adding something like an `enactDirectiveDiff()`
 		// method), we are instead going to build a full Directive based on the
 		// diff, and then proceed normally as if we had received a full
 		// Directive. We do that by copying the previous Directive and then
 		// applying the diffs to the copy.
 		newD := previousDirective.Copy()
 		// Apply the diffs from the incoming Directive to the new, copied
 		// Directive.
 		newD.ApplyDiff(d)
 		// Now proceed with the new diff as if we had received it as a full diff.
 		d = newD
 	case dax.DirectiveMethodFull:
 		// pass: normal operation
 	case dax.DirectiveMethodReset:
 		// Delete all tables.
 		if err := api.deleteAllIndexes(ctx); err != nil {
 			return errors.Wrap(err, "deleting all indexes")
 		}
 		// Set previousDirective to empty so the diff handles everything as new.
 		previousDirective = dax.Directive{}
 	case dax.DirectiveMethodSnapshot:
 		// TODO(tlt): this was the existing logic, but we should really diff the
 		// directive and ensure that overwriting the value in the cache doesn't
 		// have a negative effect.
 		api.holder.SetDirective(d)
 		return nil
 	default:
 		return errors.Errorf("invalid directive method: %s", d.Method)
 	}
 	// Cache this directive as the latest applied. There is functionality within
 	// the "enactDirective" stage of ApplyDirective which validates against this
 	// cached Directive, so it's important that it be set before calling
 	// enactDirective(). An example: when loading partition data from the
 	// Writelogger, there are validations to ensure that the partition being
 	// loaded is meant to be handled by this node; that validation is done
 	// against the cached Directive.
 	// TODO(tlt): despite what this comment says, this logic is not sound; we
 	// shouldn't be setting the directive until enactiveDirective() succeeds.
 	api.holder.SetDirective(d)
 	defer api.holder.SetDirectiveApplied(true)
 	return api.enactDirective(ctx, &previousDirective, d)
 }
 // deleteAllIndexes deletes all indexes handled by this node.
 func (api *API) deleteAllIndexes(ctx context.Context) error {
 	indexes, err := api.Schema(ctx, false)
 	if err != nil {
 		return errors.Wrap(err, "getting schema")
 	}
 	for i := range indexes {
 		if err := api.DeleteIndex(ctx, indexes[i].Name); err != nil {
 			return errors.Wrapf(err, "deleting index: %s", indexes[i].Name)
 		}
 	}
 	return nil
 }
 // directiveJobType allows us to switch on jobType in the directiveWorker in
 // order to use a single worker pool for all job types (as opposed to having a
 // separate worker pool for each job type).
 type directiveJobType interface {
 	// We have this method just to prevent *any* struct from implementing this
 	// interface automatically. But, interestingly enough, we don't actually
 	// have to have this method on the implementation because we embed the
 	// interface.
 	isJobType() bool
 }
 type directiveJobTableKeys struct {
 	directiveJobType
 	idx       *Index
 	tkey      dax.TableKey
 	partition dax.PartitionNum
 }
 type directiveJobFieldKeys struct {
 	directiveJobType
 	tkey  dax.TableKey
 	field dax.FieldName
 }
 type directiveJobShards struct {
 	directiveJobType
 	tkey  dax.TableKey
 	shard dax.ShardNum
 }
 // directiveWorker is a worker in a worker pool which handles portions of a
 // directive. Multiple instances of directiveWorker run in goroutines in order
 // to load data from snapshotter and writelogger concurrently. Note: unlike the
 // api.ingestWorkerPool, of which one pool is always running, the
 // directiveWorker pool is only running during the life of the
 // api.ApplyDirective call. Technically, this means that multiple
 // directiveWorker pools could be active at the same time, but we should never
 // be running more than once instance of ApplyDirective concurrently.
 func (api *API) directiveWorker(ctx context.Context, jobs <-chan directiveJobType, errs chan<- error) {
 	for j := range jobs {
 		switch job := j.(type) {
 		case directiveJobTableKeys:
 			if err := api.loadTableKeys(ctx, job.idx, job.tkey, job.partition); err != nil {
 				errs <- errors.Wrapf(err, "loading table keys: %s, %s", job.tkey, job.partition)
 			}
 		case directiveJobFieldKeys:
 			if err := api.loadFieldKeys(ctx, job.tkey, job.field); err != nil {
 				errs <- errors.Wrapf(err, "loading field keys: %s, %s", job.tkey, job.field)
 			}
 		case directiveJobShards:
 			if err := api.loadShard(ctx, job.tkey, job.shard); err != nil {
 				errs <- errors.Wrapf(err, "loading shard: %s, %s", job.tkey, job.shard)
 			}
 		default:
 			errs <- errors.Errorf("unsupported job type: %T %[1]v", job)
 		}
 		select {
 		case <-ctx.Done():
 			return
 		default:
 			// continue pulling jobs off the channel
 		}
 	}
 }
 func (api *API) enactDirective(ctx context.Context, fromD, toD *dax.Directive) error {
 	// enactTables is called before the jobs that run in the worker pool because
 	// it probably makes sense to apply the schema before trying to load data
 	// concurrently.
 	if err := api.enactTables(ctx, fromD, toD); err != nil {
 		return errors.Wrap(err, "enactTables")
 	}
 	// The following types use a shared pool of workers to run each
 	// directiveJobType.
 	var wg sync.WaitGroup
 	// open job channel
 	jobs := make(chan directiveJobType, api.directiveWorkerPoolSize)
 	errs := make(chan error)
 	done := make(chan struct{})
 	// Spin up n workers in goroutines that pull jobs from the jobs channel.
 	for i := 0; i < api.directiveWorkerPoolSize; i++ {
 		wg.Add(1)
 		go func() {
 			api.directiveWorker(ctx, jobs, errs)
 			defer wg.Done()
 		}()
 	}
 	// Wait for the WaitGroup counter to reach 0. When it has, indicate that
 	// we're done processing all jobs by closing the done channel.
 	go func() {
 		wg.Wait()
 		close(done)
 	}()
 	// Run through all the "enact" methods. These push jobs onto the jobs
 	// channel. Once all the jobs have been queued to the channel, we close the
 	// jobs channel. This allows the directiveWorkers to exit out of the
 	// function, which will then decrement the WaitGroup counter.
 	go func() {
 		api.pushJobsTableKeys(ctx, jobs, fromD, toD)
 		api.pushJobsFieldKeys(ctx, jobs, fromD, toD)
 		api.pushJobsShards(ctx, jobs, fromD, toD)
 		close(jobs)
 	}()
 	// Keep running until we get an error or until the done channel is closed.
 	// Note: the code is written such that only non-nil errors are pushed to the
 	// errs channel.
 	for {
 		select {
 		case err := <-errs:
 			return err
 		case <-done:
 			return nil
 		}
 	}
 }
 func (api *API) enactTables(ctx context.Context, fromD, toD *dax.Directive) error {
 	currentIndexes := api.holder.Indexes()
 	// Make a list of indexes that currently exist (from).
 	from := make(dax.TableKeys, 0, len(currentIndexes))
 	for _, idx := range currentIndexes {
 		qtid, err := dax.QualifiedTableIDFromKey(idx.Name())
 		if err != nil {
 			return errors.Wrap(err, "converting index name to qualified table id")
 		}
 		from = append(from, qtid.Key())
 	}
 	// TODO sanity check holder against fromD. We're getting existing
 	// indexes from holder, but in theory fromD should be
 	// identical. If we have an error in our directive-caching logic
 	// (it has happened before (just now, in fact!) and we'd be
 	// foolish to think it won't happen again), or we have schema
 	// mutations that are not going through the directive path, we
 	// could potentially catch them here.
 	// Make a list of tables that are in the directive (to) along with a map of
 	// tableKey to table (m).
 	m := make(map[dax.TableKey]*dax.QualifiedTable, len(toD.Tables))
 	to := make(dax.TableKeys, 0, len(toD.Tables))
 	for _, t := range toD.Tables {
 		m[t.Key()] = t
 		to = append(to, t.Key())
 	}
 	sc := newSliceComparer(from, to)
 	// Remove all indexes that are no longer part of the directive.
 	for _, tkey := range sc.removed() {
 		idx := string(tkey)
 		if err := api.DeleteIndex(ctx, idx); err != nil {
 			return errors.Wrapf(err, "deleting index: %s", tkey)
 		}
 	}
 	// Put partitions into a map by table.
 	partitionMap := toD.TranslatePartitionsMap()
 	// Add all indexes that weren't previously (but now are) a part of the
 	// directive.
 	for _, tkey := range sc.added() {
 		if qtbl, found := m[tkey]; !found {
 			return errors.Errorf("table '%s' was not in map", tkey)
 		} else if err := api.createTableAndFields(qtbl, partitionMap[tkey]); err != nil {
 			return err
 		}
 	}
 	// Check fields on all indexes present in both from and to.
 	for _, tkey := range sc.same() {
 		if err := api.enactFieldsForTable(ctx, tkey, fromD, toD); err != nil {
 			return errors.Wrapf(err, "enacting fields for table: '%s'", tkey)
 		}
 	}
 	return nil
 }
 func (api *API) enactFieldsForTable(ctx context.Context, tkey dax.TableKey, fromD, toD *dax.Directive) error {
 	qtid := tkey.QualifiedTableID()
 	fromT, err := fromD.Table(qtid)
 	if err != nil {
 		return errors.Wrap(err, "getting from table")
 	}
 	toT, err := toD.Table(qtid)
 	if err != nil {
 		return errors.Wrap(err, "getting to table")
 	}
 	// Get the index for tkey.
 	idx := api.holder.Index(string(tkey))
 	if idx == nil {
 		return errors.Errorf("index not found: %s", tkey)
 	}
 	sc := newSliceComparer(fromT.FieldNames(), toT.FieldNames())
 	// Add fields new to toT.
 	for _, fldName := range sc.added() {
 		if field, found := toT.Field(fldName); !found {
 			return dax.NewErrFieldDoesNotExist(fldName)
 		} else if err := createField(idx, field); err != nil {
 			return errors.Wrapf(err, "creating field: %s/%s", tkey, fldName)
 		}
 	}
 	// Remove fields which don't exist in toT.
 	for _, fldName := range sc.removed() {
 		if err := api.DeleteField(ctx, string(tkey), string(fldName)); err != nil {
 			return errors.Wrapf(err, "deleting field: %s/%s", tkey, fldName)
 		}
 	}
 	// // Update any field options which have changed for existing fields.
 	// for _, fldName := range sc.same() {
 	// 	// handle changed field options??
 	// }
 	return nil
 }
 func (api *API) pushJobsTableKeys(ctx context.Context, jobs chan<- directiveJobType, fromD, toD *dax.Directive) {
 	toPartitionsMap := toD.TranslatePartitionsMap()
 	// Get the diff between from/to directive.partitions.
 	partComp := newPartitionsComparer(fromD.TranslatePartitionsMap(), toPartitionsMap)
 	// Remove any partitions which are no longer assigned to this worker.
 	// TODO(tlt): currently, this is just removing the file lock on the
 	// resource; it's not actually removing the resource from the local
 	// computer. We should do that.
 	for tkey, partitions := range partComp.removed() {
 		qtid := tkey.QualifiedTableID()
 		for _, partition := range partitions {
 			api.serverlessStorage.RemoveTableKeyResource(qtid, partition)
 		}
 	}
 	// Loop over the partition map and load from Writelogger.
 	for tkey, partitions := range partComp.added() {
 		// Get index in order to find the translate stores (by partition) for
 		// the table.
 		idx := api.holder.Index(string(tkey))
 		if idx == nil {
 			log.Printf("index not found in holder: %s", tkey)
 			continue
 		}
 		// Update the cached version of translate partitions that we keep on the
 		// Index.
 		idx.SetTranslatePartitions(toPartitionsMap[tkey])
 		for _, partition := range partitions {
 			jobs <- directiveJobTableKeys{
 				idx:       idx,
 				tkey:      tkey,
 				partition: partition,
 			}
 		}
 	}
 }
 func (api *API) loadTableKeys(ctx context.Context, idx *Index, tkey dax.TableKey, partition dax.PartitionNum) error {
 	qtid := tkey.QualifiedTableID()
 	resource := api.serverlessStorage.GetTableKeyResource(qtid, partition)
 	if resource.IsLocked() {
 		api.logger().Warnf("skipping loadTableKeys (already held) %s %d", tkey, partition)
 		return nil
 	}
 	// load latest snapshot
 	if rc, err := resource.LoadLatestSnapshot(); err != nil {
 		return errors.Wrap(err, "loading table key snapshot")
 	} else if rc != nil {
 		defer rc.Close()
 		if err := api.TranslateIndexDB(ctx, string(tkey), int(partition), rc); err != nil {
 			return errors.Wrap(err, "restoring table keys")
 		}
 	}
 	// define write log loading in a function since we have to do it
 	// before and after locking
 	loadWriteLog := func() error {
 		writelog, err := resource.LoadWriteLog()
 		if err != nil {
 			return errors.Wrap(err, "getting write log reader for table keys")
 		}
 		if writelog == nil {
 			return nil
 		}
 		reader := storage.NewTableKeyReader(qtid, partition, writelog)
 		defer reader.Close()
 		store := idx.TranslateStore(int(partition))
 		for msg, err := reader.Read(); err != io.EOF; msg, err = reader.Read() {
 			if err != nil {
 				return errors.Wrap(err, "reading from log reader")
 			}
 			for key, id := range msg.StringToID {
 				if err := store.ForceSet(id, key); err != nil {
 					return errors.Wrapf(err, "forcing set id, key: %d, %s", id, key)
 				}
 			}
 		}
 		return nil
 	}
 	// 1st write log load
 	if err := loadWriteLog(); err != nil {
 		return err
 	}
 	// acquire lock on this partition's keys
 	if err := resource.Lock(); err != nil {
 		return errors.Wrap(err, "locking table key partition")
 	}
 	// reload writelog in case of changes between last load and
 	// lock. The resource object takes care of only loading new data.
 	return loadWriteLog()
 }
 func (api *API) pushJobsFieldKeys(ctx context.Context, jobs chan<- directiveJobType, fromD, toD *dax.Directive) {
 	// Get the diff between from/to directive.fields.
 	fieldComp := newFieldsComparer(fromD.TranslateFieldsMap(), toD.TranslateFieldsMap())
 	// Remove any field keys which are no longer assigned to this worker.
 	// TODO(tlt): currently, this is just removing the file lock on the
 	// resource; it's not actually removing the resource from the local
 	// computer. We should do that.
 	for tkey, fields := range fieldComp.removed() {
 		qtid := tkey.QualifiedTableID()
 		for _, field := range fields {
 			api.serverlessStorage.RemoveFieldKeyResource(qtid, field)
 		}
 	}
 	// Loop over the field map and load from Writelogger.
 	for tkey, fields := range fieldComp.added() {
 		for _, field := range fields {
 			jobs <- directiveJobFieldKeys{
 				tkey:  tkey,
 				field: field,
 			}
 		}
 	}
 }
 func (api *API) loadFieldKeys(ctx context.Context, tkey dax.TableKey, field dax.FieldName) error {
 	qtid := tkey.QualifiedTableID()
 	resource := api.serverlessStorage.GetFieldKeyResource(qtid, field)
 	if resource.IsLocked() {
 		api.logger().Warnf("skipping loadFieldKeys (already held) %s %s", tkey, field)
 		return nil
 	}
 	// load latest snapshot
 	if rc, err := resource.LoadLatestSnapshot(); err != nil {
 		return errors.Wrap(err, "loading field key snapshot")
 	} else if rc != nil {
 		defer rc.Close()
 		if err := api.TranslateFieldDB(ctx, string(tkey), string(field), rc); err != nil {
 			return errors.Wrap(err, "restoring field keys")
 		}
 	}
 	// define write log loading in a function since we have to do it
 	// before and after locking
 	loadWriteLog := func() error {
 		writelog, err := resource.LoadWriteLog()
 		if err != nil {
 			return errors.Wrap(err, "getting write log reader for field keys")
 		}
 		if writelog == nil {
 			return nil
 		}
 		reader := storage.NewFieldKeyReader(qtid, field, writelog)
 		defer reader.Close()
 		// Get field in order to find the translate store.
 		fld := api.holder.Field(string(tkey), string(field))
 		if fld == nil {
 			log.Printf("field not found in holder: %s", field)
 			return nil
 		}
 		store := fld.TranslateStore()
 		for msg, err := reader.Read(); err != io.EOF; msg, err = reader.Read() {
 			if err != nil {
 				return errors.Wrap(err, "reading from log reader")
 			}
 			for key, id := range msg.StringToID {
 				if err := store.ForceSet(id, key); err != nil {
 					return errors.Wrapf(err, "forcing set id, key: %d, %s", id, key)
 				}
 			}
 		}
 		return nil
 	}
 	// 1st write log load
 	if err := loadWriteLog(); err != nil {
 		return err
 	}
 	// acquire lock on this partition's keys
 	if err := resource.Lock(); err != nil {
 		return errors.Wrap(err, "locking field key partition")
 	}
 	// reload writelog in case of changes between last load and
 	// lock. The resource object takes care of only loading new data.
 	return loadWriteLog()
 }
 func (api *API) pushJobsShards(ctx context.Context, jobs chan<- directiveJobType, fromD, toD *dax.Directive) {
 	// Put shards into a map by table.
 	shardMap := toD.ComputeShardsMap()
 	// Get the diff between from/to directive shards.
 	shardComp := newShardsComparer(fromD.ComputeShardsMap(), shardMap)
 	// Remove any shards which are no longer assigned to this worker.
 	// TODO(tlt): currently, this is just removing the file lock on the
 	// resource; it's not actually removing the resource from the local
 	// computer. We should do that.
 	for tkey, shards := range shardComp.removed() {
 		qtid := tkey.QualifiedTableID()
 		for _, shard := range shards {
 			partition := dax.PartitionNum(disco.ShardToShardPartition(string(tkey), uint64(shard), disco.DefaultPartitionN))
 			api.serverlessStorage.RemoveShardResource(qtid, partition, shard)
 		}
 	}
 	// Loop over the shard map and load from Writelogger.
 	for tkey, shards := range shardComp.added() {
 		for _, shard := range shards {
 			jobs <- directiveJobShards{
 				tkey:  tkey,
 				shard: shard,
 			}
 		}
 	}
 }
 func (api *API) loadShard(ctx context.Context, tkey dax.TableKey, shard dax.ShardNum) error {
 	qtid := tkey.QualifiedTableID()
 	partition := dax.PartitionNum(disco.ShardToShardPartition(string(tkey), uint64(shard), disco.DefaultPartitionN))
 	resource := api.serverlessStorage.GetShardResource(qtid, partition, shard)
 	if resource.IsLocked() {
 		api.logger().Warnf("skipping loadShard (already held) %s %d", tkey, shard)
 		return nil
 	}
 	if rc, err := resource.LoadLatestSnapshot(); err != nil {
 		return errors.Wrap(err, "reading latest snapshot for shard")
 	} else if rc != nil {
 		defer rc.Close()
 		if err := api.RestoreShard(ctx, string(tkey), uint64(shard), rc); err != nil {
 			return errors.Wrap(err, "restoring shard data")
 		}
 	}
 	// define write log loading in a func because we do it twice.
 	loadWriteLog := func() error {
 		writelog, err := resource.LoadWriteLog()
 		if err != nil {
 			return errors.Wrap(err, "")
 		}
 		if writelog == nil {
 			return nil
 		}
 		reader := storage.NewShardReader(qtid, partition, shard, writelog)
 		defer reader.Close()
 		for logMsg, err := reader.Read(); err != io.EOF; logMsg, err = reader.Read() {
 			if err != nil {
 				return errors.Wrap(err, "reading from log reader")
 			}
 			switch msg := logMsg.(type) {
 			case *computer.ImportRoaringMessage:
 				req := &ImportRoaringRequest{
 					Clear:           msg.Clear,
 					Action:          msg.Action,
 					Block:           msg.Block,
 					Views:           msg.Views,
 					UpdateExistence: msg.UpdateExistence,
 					SuppressLog:     true,
 				}
 				if err := api.ImportRoaring(ctx, msg.Table, msg.Field, msg.Shard, true, req); err != nil {
 					return errors.Wrapf(err, "import roaring, table: %s, field: %s, shard: %d", msg.Table, msg.Field, msg.Shard)
 				}
 			case *computer.ImportMessage:
 				req := &ImportRequest{
 					Index:      msg.Table,
 					Field:      msg.Field,
 					Shard:      msg.Shard,
 					RowIDs:     msg.RowIDs,
 					ColumnIDs:  msg.ColumnIDs,
 					RowKeys:    msg.RowKeys,
 					ColumnKeys: msg.ColumnKeys,
 					Timestamps: msg.Timestamps,
 					Clear:      msg.Clear,
 				}
 				qcx := api.Txf().NewQcx()
 				defer qcx.Abort()
 				opts := []ImportOption{
 					OptImportOptionsClear(msg.Clear),
 					OptImportOptionsIgnoreKeyCheck(msg.IgnoreKeyCheck),
 					OptImportOptionsPresorted(msg.Presorted),
 					OptImportOptionsSuppressLog(true),
 				}
 				if err := api.Import(ctx, qcx, req, opts...); err != nil {
 					return errors.Wrapf(err, "import, table: %s, field: %s, shard: %d", msg.Table, msg.Field, msg.Shard)
 				}
 			case *computer.ImportValueMessage:
 				req := &ImportValueRequest{
 					Index:           msg.Table,
 					Field:           msg.Field,
 					Shard:           msg.Shard,
 					ColumnIDs:       msg.ColumnIDs,
 					ColumnKeys:      msg.ColumnKeys,
 					Values:          msg.Values,
 					FloatValues:     msg.FloatValues,
 					TimestampValues: msg.TimestampValues,
 					StringValues:    msg.StringValues,
 					Clear:           msg.Clear,
 				}
 				qcx := api.Txf().NewQcx()
 				defer qcx.Abort()
 				opts := []ImportOption{
 					OptImportOptionsClear(msg.Clear),
 					OptImportOptionsIgnoreKeyCheck(msg.IgnoreKeyCheck),
 					OptImportOptionsPresorted(msg.Presorted),
 					OptImportOptionsSuppressLog(true),
 				}
 				if err := api.ImportValue(ctx, qcx, req, opts...); err != nil {
 					return errors.Wrapf(err, "import value, table: %s, field: %s, shard: %d", msg.Table, msg.Field, msg.Shard)
 				}
 			case *computer.ImportRoaringShardMessage:
 				req := &ImportRoaringShardRequest{
 					Remote:      true,
 					Views:       make([]RoaringUpdate, len(msg.Views)),
 					SuppressLog: true,
 				}
 				for i, view := range msg.Views {
 					req.Views[i] = RoaringUpdate{
 						Field:        view.Field,
 						View:         view.View,
 						Clear:        view.Clear,
 						Set:          view.Set,
 						ClearRecords: view.ClearRecords,
 					}
 				}
 				if err := api.ImportRoaringShard(ctx, msg.Table, msg.Shard, req); err != nil {
 					return errors.Wrapf(err, "import roaring shard table: %s, shard: %d", msg.Table, msg.Shard)
 				}
 			}
 		}
 		return nil
 	}
 	// 1st write log load
 	if err := loadWriteLog(); err != nil {
 		return err
 	}
 	// acquire lock on this partition's keys
 	if err := resource.Lock(); err != nil {
 		return errors.Wrap(err, "locking field key partition")
 	}
 	// reload writelog in case of changes between last load and
 	// lock. The resource object takes care of only loading new data.
 	return loadWriteLog()
 }
 //////////////////////////////////////////////////////////////
 // sliceComparer is used to compare the differences between two slices of comparables.
 type sliceComparer[K comparable] struct {
 	from []K
 	to   []K
 }
 func newSliceComparer[K comparable](from []K, to []K) *sliceComparer[K] {
 	return &sliceComparer[K]{
 		from: from,
 		to:   to,
 	}
 }
 // added returns the items which are present in `to` but not in `from`.
 func (s *sliceComparer[K]) added() []K {
 	return thingsAdded(s.from, s.to)
 }
 // removed returns the items which are present in `from` but not in `to`.
 func (s *sliceComparer[K]) removed() []K {
 	return thingsAdded(s.to, s.from)
 }
 // same returns the items which are in both `to` and `from`.
 func (s *sliceComparer[K]) same() []K {
 	var same []K
 	for _, fromThing := range s.from {
 		for _, toThing := range s.to {
 			if fromThing == toThing {
 				same = append(same, fromThing)
 				break
 			}
 		}
 	}
 	return same
 }
 // thingsAdded returns the comparable things which are present in `to` but not
 // in `from`.
 func thingsAdded[K comparable](from []K, to []K) []K {
 	var added []K
 	for i := range to {
 		var found bool
 		for j := range from {
 			if from[j] == to[i] {
 				found = true
 				break
 			}
 		}
 		if !found {
 			added = append(added, to[i])
 		}
 	}
 	return added
 }
 // partitionsComparer is used to compare the differences between two maps of
 // table:[]partition.
 type partitionsComparer struct {
 	from map[dax.TableKey]dax.PartitionNums
 	to   map[dax.TableKey]dax.PartitionNums
 }
 func newPartitionsComparer(from map[dax.TableKey]dax.PartitionNums, to map[dax.TableKey]dax.PartitionNums) *partitionsComparer {
 	return &partitionsComparer{
 		from: from,
 		to:   to,
 	}
 }
 // added returns the partitions which are present in `to` but not in `from`. The
 // results remain in the format of a map of table:[]partition.
 func (p *partitionsComparer) added() map[dax.TableKey]dax.PartitionNums {
 	return partitionsAdded(p.from, p.to)
 }
 // removed returns the partitions which are present in `from` but not in `to`.
 // The results remain in the format of a map of table:[]partition.
 func (p *partitionsComparer) removed() map[dax.TableKey]dax.PartitionNums {
 	return partitionsAdded(p.to, p.from)
 }
 // partitionsAdded returns the partitions which are present in `to` but not in `from`.
 func partitionsAdded(from map[dax.TableKey]dax.PartitionNums, to map[dax.TableKey]dax.PartitionNums) map[dax.TableKey]dax.PartitionNums {
 	if from == nil {
 		return to
 	}
 	added := make(map[dax.TableKey]dax.PartitionNums)
 	for tt, tps := range to {
 		fps, found := from[tt]
 		if !found {
 			added[tt] = tps
 			continue
 		}
 		addedPartitions := dax.PartitionNums{}
 		for i := range tps {
 			var found bool
 			for j := range fps {
 				if fps[j] == tps[i] {
 					found = true
 					break
 				}
 			}
 			if !found {
 				addedPartitions = append(addedPartitions, tps[i])
 			}
 		}
 		if len(addedPartitions) > 0 {
 			added[tt] = addedPartitions
 		}
 	}
 	return added
 }
 // fieldsComparer is used to compare the differences between two maps of
 // table:[]fieldVersion.
 type fieldsComparer struct {
 	from map[dax.TableKey][]dax.FieldName
 	to   map[dax.TableKey][]dax.FieldName
 }
 func newFieldsComparer(from map[dax.TableKey][]dax.FieldName, to map[dax.TableKey][]dax.FieldName) *fieldsComparer {
 	return &fieldsComparer{
 		from: from,
 		to:   to,
 	}
 }
 // added returns the fields which are present in `to` but not in `from`. The
 // results remain in the format of a map of table:[]field.
 func (f *fieldsComparer) added() map[dax.TableKey][]dax.FieldName {
 	return fieldsAdded(f.from, f.to)
 }
 // removed returns the fields which are present in `from` but not in `to`.
 // The results remain in the format of a map of table:[]field.
 func (f *fieldsComparer) removed() map[dax.TableKey][]dax.FieldName {
 	return fieldsAdded(f.to, f.from)
 }
 // fieldsAdded returns the fields which are present in `to` but not in `from`.
 func fieldsAdded(from map[dax.TableKey][]dax.FieldName, to map[dax.TableKey][]dax.FieldName) map[dax.TableKey][]dax.FieldName {
 	if from == nil {
 		return to
 	}
 	added := make(map[dax.TableKey][]dax.FieldName)
 	for tt, tps := range to {
 		fps, found := from[tt]
 		if !found {
 			added[tt] = tps
 			continue
 		}
 		addedFieldVersions := []dax.FieldName{}
 		for i := range tps {
 			var found bool
 			for j := range fps {
 				if fps[j] == tps[i] {
 					found = true
 					break
 				}
 			}
 			if !found {
 				addedFieldVersions = append(addedFieldVersions, tps[i])
 			}
 		}
 		if len(addedFieldVersions) > 0 {
 			added[tt] = addedFieldVersions
 		}
 	}
 	return added
 }
 // shardsComparer is used to compare the differences between two maps of
 // table:[]shardV.
 type shardsComparer struct {
 	from map[dax.TableKey]dax.ShardNums
 	to   map[dax.TableKey]dax.ShardNums
 }
 func newShardsComparer(from map[dax.TableKey]dax.ShardNums, to map[dax.TableKey]dax.ShardNums) *shardsComparer {
 	return &shardsComparer{
 		from: from,
 		to:   to,
 	}
 }
 // added returns the shards which are present in `to` but not in `from`. The
 // results remain in the format of a map of table:[]shard.
 func (s *shardsComparer) added() map[dax.TableKey]dax.ShardNums {
 	return shardsAdded(s.from, s.to)
 }
 // removed returns the shards which are present in `from` but not in `to`. The
 // results remain in the format of a map of table:[]shard.
 func (s *shardsComparer) removed() map[dax.TableKey]dax.ShardNums {
 	return shardsAdded(s.to, s.from)
 }
 // shardsAdded returns the shards which are present in `to` but not in `from`.
 func shardsAdded(from map[dax.TableKey]dax.ShardNums, to map[dax.TableKey]dax.ShardNums) map[dax.TableKey]dax.ShardNums {
 	if from == nil {
 		return to
 	}
 	added := make(map[dax.TableKey]dax.ShardNums)
 	for tt, tss := range to {
 		fss, found := from[tt]
 		if !found {
 			added[tt] = tss
 			continue
 		}
 		addedShards := dax.ShardNums{}
 		for i := range tss {
 			var found bool
 			for j := range fss {
 				if fss[j] == tss[i] {
 					found = true
 					break
 				}
 			}
 			if !found {
 				addedShards = append(addedShards, tss[i])
 			}
 		}
 		if len(addedShards) > 0 {
 			added[tt] = addedShards
 		}
 	}
 	return added
 }
 // createTableAndFields creates the FeatureBase Tables and Fields provided in
 // the dax.Directive format.
 func (api *API) createTableAndFields(tbl *dax.QualifiedTable, partitions dax.PartitionNums) error {
 	cim := &CreateIndexMessage{
 		Index:     string(tbl.Key()),
 		CreatedAt: 0,
 		Meta: IndexOptions{
 			Keys:           tbl.StringKeys(),
 			TrackExistence: true,
 		},
 	}
 	// Create the index in etcd as the system of record.
 	if err := api.holder.persistIndex(context.Background(), cim); err != nil {
 		return errors.Wrap(err, "persisting index")
 	}
 	idx, err := api.holder.createIndexWithPartitions(cim, partitions)
 	if err != nil {
 		return errors.Wrapf(err, "adding index: %s", tbl.Name)
 	}
 	// Add the fields
 	for _, fld := range tbl.Fields {
 		if fld.IsPrimaryKey() {
 			continue
 		}
 		if err := createField(idx, fld); err != nil {
 			return errors.Wrapf(err, "creating field: %s", fld.Name)
 		}
 	}
 	return nil
 }
 // createField creates a FeatureBase Field in the provided FeatureBase Index
 // based on the provided field's type.
 func createField(idx *Index, fld *dax.Field) error {
 	opts, err := FieldOptionsFromField(fld)
 	if err != nil {
 		return errors.Wrapf(err, "creating field options from field: %s", fld.Name)
 	}
 	if _, err := idx.createNullableField(string(fld.Name), "", opts...); err != nil {
 		return errors.Wrapf(err, "creating field on index: %s", fld.Name)
 	}
 	return nil
 }
--- a/api_directive_internal_test.go
+++ b/api_directive_internal_test.go
@ -1,25 +0,0 @@
 package pilosa
 import (
 	"testing"
 	"github.com/stretchr/testify/assert"
 )
 func TestThingsAddedGeneric(t *testing.T) {
 	from := []string{"a", "b", "c"}
 	to := []string{"b", "c", "d"}
 	added := thingsAdded(from, to)
 	assert.Equal(t, added, []string{"d"})
 }
 func TestSliceComparer(t *testing.T) {
 	from := []string{"a", "b", "c"}
 	to := []string{"b", "c", "d"}
 	sc := newSliceComparer(from, to)
 	added := sc.added()
 	assert.Equal(t, added, []string{"d"})
 }
--- a/api_directive_test.go
+++ b/api_directive_test.go
@ -1,97 +0,0 @@
 package pilosa_test
 import (
 	"context"
 	"testing"
 	pilosa "github.com/featurebasedb/featurebase/v3"
 	"github.com/featurebasedb/featurebase/v3/dax"
 	daxtest "github.com/featurebasedb/featurebase/v3/dax/test"
 	"github.com/featurebasedb/featurebase/v3/test"
 	"github.com/stretchr/testify/assert"
 )
 // Ensure holder can handle an incoming directive.
 func TestAPI_Directive(t *testing.T) {
 	c := test.MustRunCluster(t, 1)
 	defer c.Close()
 	api := c.GetPrimary().API
 	ctx := context.Background()
 	qdbid := dax.NewQualifiedDatabaseID("acme", "db1")
 	tbl1 := daxtest.TestQualifiedTableWithID(t, qdbid, "1", "tbl1", 12, false)
 	tbl2 := daxtest.TestQualifiedTableWithID(t, qdbid, "2", "tbl2", 12, false)
 	tbl3 := daxtest.TestQualifiedTableWithID(t, qdbid, "3", "tbl3", 12, false)
 	t.Run("Schema", func(t *testing.T) {
 		// Empty directive (and empty holder).
 		{
 			d := &dax.Directive{
 				Method:  dax.DirectiveMethodFull,
 				Version: 1,
 			}
 			err := api.ApplyDirective(ctx, d)
 			assert.NoError(t, err)
 			assertTablesMatch(t, []string{}, api.Holder().Indexes())
 		}
 		// Add a new table.
 		{
 			d := &dax.Directive{
 				Method: dax.DirectiveMethodFull,
 				Tables: []*dax.QualifiedTable{
 					tbl1,
 				},
 				Version: 2,
 			}
 			err := api.ApplyDirective(ctx, d)
 			assert.NoError(t, err)
 			assertTablesMatch(t, []string{"tbl__acme__db1__1"}, api.Holder().Indexes())
 		}
 		// Add a new table, and keep the existing table.
 		{
 			d := &dax.Directive{
 				Method: dax.DirectiveMethodFull,
 				Tables: []*dax.QualifiedTable{
 					tbl1,
 					tbl2,
 				},
 				Version: 3,
 			}
 			err := api.ApplyDirective(ctx, d)
 			assert.NoError(t, err)
 			assertTablesMatch(t, []string{"tbl__acme__db1__1", "tbl__acme__db1__2"}, api.Holder().Indexes())
 		}
 		// Add a new table and remove one of the existing tables.
 		{
 			d := &dax.Directive{
 				Method: dax.DirectiveMethodFull,
 				Tables: []*dax.QualifiedTable{
 					tbl2,
 					tbl3,
 				},
 				Version: 4,
 			}
 			err := api.ApplyDirective(ctx, d)
 			assert.NoError(t, err)
 			assertTablesMatch(t, []string{"tbl__acme__db1__2", "tbl__acme__db1__3"}, api.Holder().Indexes())
 		}
 	})
 }
 // assertTablesMatch is a helper function which asserts that the list of index
 // names in `actual` match those provided in `expected`.
 func assertTablesMatch(t *testing.T, expected []string, actual []*pilosa.Index) {
 	t.Helper()
 	act := make([]string, len(actual))
 	for i := range actual {
 		act[i] = actual[i].Name()
 	}
 	assert.ElementsMatch(t, expected, act)
 }
--- a/api_test.go
+++ b/api_test.go
@ -522,6 +522,72 @@ func TestAPI_Ingest(t *testing.T) {
 		t.Fatalf("creating field: %v", err)
 	}
 	t.Run("IngestAPI", func(t *testing.T) {
 		sampleJson := []byte(`
 	[
 	  {
 	    "action": "set",
 	    "records": {
 	      "2": {
 		"set": [2],
 		"tq": { "time": "2006-01-02T15:04:05.999999999Z", "values": [6] }
 	      },
 	      "5": { "set": [3] },
 	      "8": { "set": [3] },
 	      "1": {
 		"set": [2],
 		"tq": { "time": "2006-01-02T15:04:05.999999999Z", "values": [3, 4] }
 	      },
 	      "4": { "set": [3, 7] }
 	    }
 	  },
 	  {
 	    "action": "clear",
 	    "record_ids": [ 5, 6, 7 ],
 	    "fields": [ "tq", "set" ]
 	  },
 	  {
 	    "action": "write",
 	    "records": {
 	      "8": { "tq": { "time": "2006-01-02T15:04:05.999999999Z", "values": [3, 4] } },
 	      "9": { "set": [7, 3] }
 	    }
 	  },
 	  {
 	    "action": "delete",
 	    "record_ids": [ 9 ]
 	  }
 	]
 	`)
 		// just for set row 3:
 		// first operation should set it for 4, 5, and 8.
 		// clear operation should clear it for 5, 6, and 7, leaving it still set for 4 and 8.
 		// the write operation should clear set for record 8, even though record 8 doesn't
 		// contain that field in that op, because set is present in record 9, which also
 		// gets row 3 set. but then we delete 9.
 		// so after all that we expect Row(set=3) to be 4...
 		sampleBuf := bytes.NewBuffer(sampleJson)
 		qcx := coord.API.Txf().NewQcx()
 		defer func() {
 			if err := qcx.Finish(); err != nil {
 				t.Fatalf("finishing qcx: %v", err)
 			}
 		}()
 		err = coord.API.IngestOperations(ctx, qcx, index, sampleBuf)
 		if err != nil {
 			t.Fatalf("importing data: %v", err)
 		}
 		query := "Row(set=3)"
 		res, err := coord.API.Query(context.Background(), &pilosa.QueryRequest{Index: index, Query: query})
 		if err != nil {
 			t.Errorf("query: %v", err)
 		}
 		r := res.Results[0].(*pilosa.Row).Columns()
 		if len(r) != 1 || r[0] != 4 {
 			t.Fatalf("expected row with 4 set, got %d", r)
 		}
 	})
 	t.Run("ImportRoaringShard", func(t *testing.T) {
 		setBuf := &bytes.Buffer{}
 		setBits := roaring.NewBitmap(7, pilosa.ShardWidth+7)
@ -614,6 +680,73 @@ func TestAPI_Ingest(t *testing.T) {
 	})
 }
 // ingestBenchmarkHelper makes it easier to exclude this from benchmark computations
 // and profiles.
 func ingestBenchmarkHelper() []byte {
 	buf := &bytes.Buffer{}
 	buf.WriteString(`[{"action": "write", "records": {`)
 	comma := ""
 	now := time.Now().Add(-3840000 * time.Second)
 	for i := 0; i < 1000000; i++ {
 		then := now.Add(time.Duration(rand.Int63n(1234567)) * time.Second)
 		fmt.Fprintf(buf, `%s"%d": { "set": [%d, %d], "int": %d, "tq": { "time": "%s", "values": %d } }`, comma, i, i%2, (i%4)+2, rand.Int63n(163840),
 			then.Format(time.RFC3339), rand.Int63n(25))
 		comma = ", "
 	}
 	buf.WriteString(`}}]`)
 	data := buf.Bytes()
 	return data
 }
 func BenchmarkIngest(b *testing.B) {
 	b.StopTimer()
 	data := ingestBenchmarkHelper()
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 	c := test.MustRunCluster(b, 1)
 	defer c.Close()
 	coord := c.GetPrimary()
 	m0 := c.GetNode(0)
 	// m1 := c.GetNode(1)
 	// m2 := c.GetNode(2)
 	index := c.Idx()
 	setField := "set"
 	intField := "int"
 	tqField := "tq"
 	_, err := coord.API.CreateIndex(ctx, index, pilosa.IndexOptions{Keys: false})
 	if err != nil {
 		b.Fatalf("creating index: %v", err)
 	}
 	_, err = coord.API.CreateField(ctx, index, setField, pilosa.OptFieldTypeSet("none", 0))
 	if err != nil {
 		b.Fatalf("creating field: %v", err)
 	}
 	_, err = coord.API.CreateField(ctx, index, intField, pilosa.OptFieldTypeInt(0, 163840))
 	if err != nil {
 		b.Fatalf("creating field: %v", err)
 	}
 	_, err = coord.API.CreateField(ctx, index, tqField, pilosa.OptFieldTypeTime("YMDH", "0"))
 	if err != nil {
 		b.Fatalf("creating field: %v", err)
 	}
 	b.ReportAllocs()
 	b.StartTimer()
 	for i := 0; i < b.N; i++ {
 		qcx := m0.API.Txf().NewQcx()
 		defer qcx.Abort()
 		err = coord.API.IngestOperations(ctx, qcx, index, bytes.NewBuffer(data))
 		if err != nil {
 			b.Fatalf("ingest: %v", err)
 		}
 		err = qcx.Finish()
 		if err != nil {
 			b.Fatalf("finish: %v", err)
 		}
 	}
 }
 func TestAPI_ClearFlagForImportAndImportValues(t *testing.T) {
 	c := test.MustRunCluster(t, 1)
 	defer c.Close()
@ -837,7 +970,7 @@ func TestAPI_IDAlloc(t *testing.T) {
 			t.Fatalf("obtaining random bytes: %v", err)
 		}
 		ids3, err := primary.ReserveIDs(key, session, 0, 2)
-		var esync pilosa.IDOffsetDesyncError
+		var esync pilosa.ErrIDOffsetDesync
 		if errors.As(err, &esync) {
 			if esync.Requested != 0 {
 				t.Errorf("incorrect requested offset in error: provided %d but got %d", 0, esync.Requested)
@ -1265,8 +1398,8 @@ func TestVariousApiTranslateCalls(t *testing.T) {
 		t.Run("translateIndexDbOnNilIndex",
 			func(t *testing.T) {
 				err := api.TranslateIndexDB(context.Background(), "nonExistentIndex", 0, r)
-				expected := fmt.Sprintf("index %q not found", "nonExistentIndex")
+				expected := fmt.Errorf("index %q not found", "nonExistentIndex")
-				if err == nil || err.Error() != expected {
+				if !reflect.DeepEqual(err, expected) {
 					t.Fatalf("expected '%#v', got '%#v'", expected, err)
 				}
 			})
@ -1274,8 +1407,8 @@ func TestVariousApiTranslateCalls(t *testing.T) {
 		t.Run("translateIndexDbOnNilTranslateStore",
 			func(t *testing.T) {
 				err := api.TranslateIndexDB(context.Background(), c.Idx(), 0, r)
-				expected := fmt.Sprintf("index %q has no translate store", c.Idx())
+				expected := fmt.Errorf("index %q has no translate store", c.Idx())
-				if err == nil || err.Error() != expected {
+				if !reflect.DeepEqual(err, expected) {
 					t.Fatalf("expected '%#v', got '%#v'", expected, err)
 				}
 			})
@ -1283,8 +1416,8 @@ func TestVariousApiTranslateCalls(t *testing.T) {
 		t.Run("translateFieldDbOnNilIndex",
 			func(t *testing.T) {
 				err := api.TranslateFieldDB(context.Background(), "nonExistentIndex", "field", r)
-				expected := fmt.Sprintf("index %q not found", "nonExistentIndex")
+				expected := fmt.Errorf("index %q not found", "nonExistentIndex")
-				if err == nil || err.Error() != expected {
+				if !reflect.DeepEqual(err, expected) {
 					t.Fatalf("expected '%#v', got '%#v'", expected, err)
 				}
 			})
@ -1292,8 +1425,8 @@ func TestVariousApiTranslateCalls(t *testing.T) {
 		t.Run("translateFieldDbOnNilField",
 			func(t *testing.T) {
 				err := api.TranslateFieldDB(context.Background(), c.Idx(), "nonExistentField", r)
-				expected := fmt.Sprintf("field %q/%q not found", c.Idx(), "nonExistentField")
+				expected := fmt.Errorf("field %q/%q not found", c.Idx(), "nonExistentField")
-				if err == nil || err.Error() != expected {
+				if !reflect.DeepEqual(err, expected) {
 					t.Fatalf("expected '%#v', got '%#v'", expected, err)
 				}
 			})
@ -1403,11 +1536,23 @@ func TestAuth_MultiNode(t *testing.T) {
    "test": "write"
 admin: "ac97c9e2-346b-42a2-b6da-18bcb61a32fe"`
 	adminUser := makeUser(t, []authn.Group{{GroupID: "ac97c9e2-346b-42a2-b6da-18bcb61a32fe", GroupName: "adminGroup"}}, "admin", "DEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEF")
-	adminCtx := authn.WithUserInfo(context.Background(), adminUser)
+	adminCtx := context.WithValue(
 		context.Background(),
 		"userinfo",
 		adminUser,
 	)
 	readUser := makeUser(t, []authn.Group{{GroupID: "dca35310-ecda-4f23-86cd-876aee55906b", GroupName: "readGroup"}}, "reader", "DEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEF")
-	readCtx := authn.WithUserInfo(context.Background(), readUser)
+	readCtx := context.WithValue(
 		context.Background(),
 		"userinfo",
 		readUser,
 	)
 	writeUser := makeUser(t, []authn.Group{{GroupID: "dca35310-ecda-4f23-86cd-876aee55906f", GroupName: "writeGroup"}}, "writer", "DEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEED")
-	writeCtx := authn.WithUserInfo(context.Background(), writeUser)
+	writeCtx := context.WithValue(
 		context.Background(),
 		"userinfo",
 		writeUser,
 	)
 	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		token, ok := r.Header["Authorization"]
 		if !ok || len(token) == 0 {
--- a/apimethod_string.go
+++ b/apimethod_string.go
@ -42,12 +42,14 @@ func _() {
 	_ = x[apiIDCommit-31]
 	_ = x[apiIDReset-32]
 	_ = x[apiPartitionNodes-33]
-	_ = x[apiMutexCheck-34]
+	_ = x[apiIngestOperations-34]
 	_ = x[apiIngestNodeOperations-35]
 	_ = x[apiMutexCheck-36]
 }
-const _apiMethod_name = "apiClusterMessageapiCreateFieldapiCreateIndexapiDeleteFieldapiDeleteAvailableShardapiDeleteIndexapiDeleteViewapiExportCSVapiFragmentBlockDataapiFragmentBlocksapiFragmentDataapiTranslateDataapiFieldTranslateDataapiFieldapiImportapiImportValueapiIndexapiQueryapiRecalculateCachesapiSchemaapiShardNodesapiStateapiViewsapiApplySchemaapiStartTransactionapiFinishTransactionapiTransactionsapiGetTransactionapiActiveQueriesapiPastQueriesapiIDReserveapiIDCommitapiIDResetapiPartitionNodesapiMutexCheck"
+const _apiMethod_name = "apiClusterMessageapiCreateFieldapiCreateIndexapiDeleteFieldapiDeleteAvailableShardapiDeleteIndexapiDeleteViewapiExportCSVapiFragmentBlockDataapiFragmentBlocksapiFragmentDataapiTranslateDataapiFieldTranslateDataapiFieldapiImportapiImportValueapiIndexapiQueryapiRecalculateCachesapiSchemaapiShardNodesapiStateapiViewsapiApplySchemaapiStartTransactionapiFinishTransactionapiTransactionsapiGetTransactionapiActiveQueriesapiPastQueriesapiIDReserveapiIDCommitapiIDResetapiPartitionNodesapiIngestOperationsapiIngestNodeOperationsapiMutexCheck"
-var _apiMethod_index = [...]uint16{0, 17, 31, 45, 59, 82, 96, 109, 121, 141, 158, 173, 189, 210, 218, 227, 241, 249, 257, 277, 286, 299, 307, 315, 329, 348, 368, 383, 400, 416, 430, 442, 453, 463, 480, 493}
+var _apiMethod_index = [...]uint16{0, 17, 31, 45, 59, 82, 96, 109, 121, 141, 158, 173, 189, 210, 218, 227, 241, 249, 257, 277, 286, 299, 307, 315, 329, 348, 368, 383, 400, 416, 430, 442, 453, 463, 480, 499, 522, 535}
 func (i apiMethod) String() string {
 	if i < 0 || i >= apiMethod(len(_apiMethod_index)-1) {
--- a/apply.go
+++ b/apply.go
@ -1,686 +0,0 @@
 // Copyright 2021 Molecula Corp. All rights reserved.
 package pilosa
 import (
 	"context"
 	"fmt"
 	"io"
 	"os"
 	"path/filepath"
 	"sort"
 	"strings"
 	"sync"
 	"github.com/apache/arrow/go/v10/arrow"
 	"github.com/apache/arrow/go/v10/arrow/array"
 	"github.com/apache/arrow/go/v10/arrow/memory"
 	"github.com/featurebasedb/featurebase/v3/pql"
 	"github.com/featurebasedb/featurebase/v3/tracing"
 	"github.com/featurebasedb/featurebase/v3/vprint"
 	"github.com/gomem/gomem/pkg/dataframe"
 	"github.com/pkg/errors"
 	ivy "robpike.io/ivy/arrow"
 	config "robpike.io/ivy/config"
 	"robpike.io/ivy/exec"
 	"robpike.io/ivy/parse"
 	"robpike.io/ivy/run"
 	"robpike.io/ivy/scan"
 	"robpike.io/ivy/value"
 )
 type (
 	ApplyResult *arrow.Column
 )
 func runIvyString(context value.Context, str string) (ok bool, err error) {
 	defer func() {
 		if r := recover(); r != nil {
 			err = r.(value.Error)
 		}
 	}()
 	scanner := scan.New(context, "<args>", strings.NewReader(str))
 	parser := parse.NewParser("<args>", scanner, context)
 	ok = run.Run(parser, context, false)
 	return
 }
 // Possibly combine all arrays together then apply some interesting
 // computation at the end?
 func IvyReduce(reduceCode string, opCode string, opt *ExecOptions) (func(ctx context.Context, prev, v interface{}) interface{}, func() (*dataframe.DataFrame, error)) {
 	var accumulator value.Value
 	mu := &sync.Mutex{}
 	concat := value.BinaryOps[opCode]
 	conf := getDefaultConfig()
 	ctxIvy := exec.NewContext(&conf)
 	// concat returned results at coordinating node.
 	reduceFn := func(ctx context.Context, prev, v interface{}) interface{} {
 		if v == nil {
 			return prev
 		}
 		if accumulator == nil {
 			switch val := v.(type) {
 			case *dataframe.DataFrame:
 				col := val.ColumnAt(0)
 				resolver := dataframe.NewChunkResolver(col)
 				accumulator = value.NewArrowVector(col, &conf, &resolver)
 			case value.Value:
 				accumulator = v.(value.Value)
 			default:
 				return errors.New(fmt.Sprintf("ivy reduction failed first unexpected type %T", v))
 			}
 			return nil
 		}
 		switch val := v.(type) {
 		case *dataframe.DataFrame:
 			col := val.ColumnAt(0)
 			resolver := dataframe.NewChunkResolver(col)
 			x := value.NewArrowVector(col, &conf, &resolver)
 			mu.Lock() // i'm being overyerly cautious..need to confirm this can be concurrent
 			accumulator = concat.EvalBinary(ctxIvy, accumulator, x)
 			mu.Unlock()
 		case value.Value:
 			mu.Lock()
 			accumulator = concat.EvalBinary(ctxIvy, accumulator, val)
 			mu.Unlock()
 		default:
 			return errors.New(fmt.Sprintf("ivy reduction failed unexpected type %T", v))
 		}
 		return nil
 	}
 	tablerFn := func() (*dataframe.DataFrame, error) {
 		pool := memory.NewGoAllocator() // TODO(twg) 2022/09/01 singledton?
 		if opt.Remote {
 			col := value.ToArrowColumn(accumulator, pool)
 			return dataframe.NewDataFrameFromColumns(pool, []arrow.Column{*col})
 		}
 		// only actually reduce on the initiating node i hate the network
 		// over head but oh well
 		ctxIvy.AssignGlobal("_", accumulator)
 		ok, err := runIvyString(ctxIvy, reduceCode)
 		if err != nil {
 			return nil, err
 		}
 		if ok {
 			v := ctxIvy.Global("_")
 			if v == nil {
 				return nil, errors.New("ivy reduction no result ")
 			}
 			col := value.ToArrowColumn(ctxIvy.Global("_"), pool)
 			return dataframe.NewDataFrameFromColumns(pool, []arrow.Column{*col})
 		}
 		return nil, errors.New("ivy reduction failed ")
 	}
 	return reduceFn, tablerFn
 }
 // executeApply executes a Apply() call.
 func (e *executor) executeApply(ctx context.Context, qcx *Qcx, index string, c *pql.Call, shards []uint64, opt *ExecOptions) (*dataframe.DataFrame, error) {
 	if !e.dataframeEnabled {
 		return nil, errors.New("Dataframe support not enabled")
 	}
 	span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeMax")
 	defer span.Finish()
 	if _, err := c.FirstStringArg("_ivy"); err != nil {
 		return nil, errors.Wrap(err, " no ivy program supplied")
 	}
 	if len(c.Children) > 1 {
 		return nil, errors.New("Apply() only accepts a single bitmap input filter")
 	}
 	// Execute calls in bulk on each remote node and merge.
 	mapFn := func(ctx context.Context, shard uint64, mopt *mapOptions) (_ interface{}, err error) {
 		return e.executeApplyShard(ctx, qcx, index, c, shard)
 	}
 	ivyReduce, ok, err := c.StringArg("_ivyReduce")
 	if err != nil {
 		return nil, err
 	}
 	reduceFn, tablerFn := IvyReduce("_", ",", opt)
 	if ok {
 		reduceFn, tablerFn = IvyReduce(ivyReduce, ",", opt)
 	}
 	_, err = e.mapReduce(ctx, index, shards, c, opt, mapFn, reduceFn)
 	if err != nil {
 		return nil, err
 	}
 	return tablerFn()
 }
 func getDefaultConfig() config.Config {
 	maxbits := uint(1e9)   // "maximum size of an integer, in bits; 0 means no limit")
 	maxdigits := uint(1e4) // "above this many `digits`, integers print as floating point; 0 disables")
 	maxstack := uint(100000)
 	origin := 1  // "set index origin to `n` (must be 0 or 1)")
 	prompt := "" // flag.String("prompt", "", "command `prompt`")
 	format := ""
 	//      debugFlag := "" // flag.String("debug", "", "comma-separated `names` of debug settings to enable")
 	conf := config.Config{}
 	conf.SetFormat(format)
 	conf.SetMaxBits(maxbits)
 	conf.SetMaxDigits(maxdigits)
 	conf.SetMaxStack(maxstack)
 	conf.SetOrigin(origin)
 	conf.SetPrompt(prompt)
 	conf.SetOutput(io.Discard)
 	conf.SetErrOutput(io.Discard)
 	conf.SetEmbedded(true) // needed to propagate panic
 	return conf
 }
 func filterDataframe(resolver dataframe.Resolver, pool memory.Allocator, filter []int64) (*dataframe.IndexResolver, error) {
 	if resolver.NumRows() == 0 {
 		return nil, errors.New("No data")
 	}
 	indexResolver := dataframe.NewIndexResolver(len(filter), uint32(ShardWidth-1))
 	for i, id := range filter {
 		if int(id) >= resolver.NumRows() {
 			continue
 		}
 		c, o := resolver.Resolve(int(id))
 		indexResolver.Set(i, c, o)
 	}
 	return indexResolver, nil
 }
 func (e *executor) executeApplyShard(ctx context.Context, qcx *Qcx, index string, c *pql.Call, shard uint64) (value.Value, error) {
 	span, _ := tracing.StartSpanFromContext(ctx, "Executor.executeApplyShard")
 	defer span.Finish()
 	ivyProgram, ok, err := c.StringArg("_ivy")
 	if err != nil || !ok {
 		return nil, errors.Wrap(err, "finding ivy program")
 	}
 	var filter *Row
 	if len(c.Children) == 1 {
 		row, err := e.executeBitmapCallShard(ctx, qcx, index, c.Children[0], shard)
 		if err != nil {
 			return nil, err
 		}
 		filter = row
 		if !filter.Any() {
 			// no need to actuall run the query for its not operating against any values
 			return value.NewVector([]value.Value{}), nil
 		}
 	}
 	//
 	pool := memory.NewGoAllocator() // TODO(twg) 2022/09/01 singledton?
 	ids := filter.ShardColumns() // needs to be shard columns
 	// Fetch index.
 	idx := e.Holder.Index(index)
 	if idx == nil {
 		return nil, newNotFoundError(ErrIndexNotFound, index)
 	}
 	fname := idx.GetDataFramePath(shard)
 	if !e.dataFrameExists(fname) {
 		return value.NewVector([]value.Value{}), nil
 	}
 	table, err := e.getDataTable(ctx, fname, pool)
 	if err != nil {
 		return nil, err
 	}
 	defer table.Release()
 	df, err := dataframe.NewDataFrameFromTable(pool, table)
 	if err != nil {
 		return nil, err
 	}
 	p := dataframe.NewChunkResolver(df.ColumnAt(0))
 	var resolver dataframe.Resolver
 	resolver = &p
 	if filter != nil {
 		if len(ids) == 0 {
 			return value.NewVector([]value.Value{}), nil
 		}
 		resolver, err = filterDataframe(resolver, pool, ids)
 		if err != nil {
 			return nil, err
 		}
 	}
 	conf := getDefaultConfig()
 	context, err := ivy.RunArrow(dataframe.NewTableFacade(df), ivyProgram, conf, resolver)
 	if err != nil {
 		return nil, fmt.Errorf("ivy map error: %w", err)
 	}
 	return context.Global("_"), nil
 }
 // ///////////////////////////////////////////////////////
 // all the ingest supporting functions
 // ///////////////////////////////////////////////////////
 func NewShardFile(ctx context.Context, name string, mem memory.Allocator, e *executor) (*ShardFile, error) {
 	if !e.dataFrameExists(name) {
 		return &ShardFile{dest: name, executor: e, strings: make(map[key][]string)}, nil
 	}
 	// else read in existing
 	table, err := e.getDataTable(ctx, name, mem)
 	if err != nil {
 		return nil, err
 	}
 	return &ShardFile{table: table, schema: table.Schema(), dest: name, executor: e, strings: make(map[key][]string)}, nil
 }
 type NameType struct {
 	Name     string
 	DataType arrow.DataType
 }
 type ChangesetRequest struct {
 	ShardIds     []int64 // only shardwidth bits to provide 0 indexing inside shard file
 	Columns      []interface{}
 	SimpleSchema []NameType
 }
 // TODO(twg) 2022/09/30 Needs a refactor
 func cast(v interface{}) arrow.DataType {
 	switch v.(type) {
 	case *arrow.Int64Type:
 		return arrow.PrimitiveTypes.Int64
 	case int64:
 		return arrow.PrimitiveTypes.Int64
 	case *arrow.Float64Type:
 		return arrow.PrimitiveTypes.Float64
 	case float64:
 		return arrow.PrimitiveTypes.Float64
 	case *arrow.StringType:
 		return arrow.BinaryTypes.String
 	default:
 		vprint.VV("%T .... %v", v, v)
 	}
 	return arrow.PrimitiveTypes.Int64
 }
 func (cr *ChangesetRequest) ArrowSchema() *arrow.Schema {
 	fields := make([]arrow.Field, len(cr.SimpleSchema))
 	for i := range cr.SimpleSchema {
 		fields[i] = arrow.Field{Name: cr.SimpleSchema[i].Name, Type: cast(cr.SimpleSchema[i].DataType)}
 	}
 	return arrow.NewSchema(fields, nil)
 }
 type key struct {
 	col   int
 	chunk int
 }
 type ShardFile struct {
 	table      arrow.Table
 	schema     *arrow.Schema
 	beforeRows int64
 	added      int64
 	columns    []interface{}
 	dest       string
 	executor   *executor
 	strings    map[key][]string
 }
 func compareSchema(s1, s2 *arrow.Schema) bool {
 	if s1 == nil || s2 == nil {
 		return false
 	}
 	if len(s1.Fields()) != len(s2.Fields()) {
 		return false
 	}
 	for i := 0; i < len(s1.Fields()); i++ {
 		f1 := s1.Field(i)
 		f2 := s2.Field(i)
 		if f1.Name != f2.Name {
 			return false
 		}
 		if f1.Type != f2.Type {
 			return false
 		}
 	}
 	return true
 }
 func (sf *ShardFile) EnsureSchema(cs *ChangesetRequest) error {
 	schema := cs.ArrowSchema()
 	if sf.schema == nil {
 		sf.schema = schema
 	} else {
 		if !compareSchema(sf.schema, schema) {
 			vprint.VV("incomeing schema", schema)
 			vprint.VV("existing schema", sf.schema)
 			return errors.New("dataframe schema's don't match")
 		}
 	}
 	sf.columns = make([]interface{}, len(sf.schema.Fields()))
 	return nil
 }
 func (sf *ShardFile) buildAppenders(maxid int64) {
 	if sf.table != nil {
 		sf.beforeRows = sf.table.NumRows()
 	}
 	if maxid < sf.beforeRows {
 		// no need to add new rows
 		return
 	}
 	newSize := maxid - sf.beforeRows + 1
 	for i := 0; i < len(sf.schema.Fields()); i++ {
 		switch sf.schema.Field(i).Type {
 		case arrow.PrimitiveTypes.Int64:
 			sf.columns[i] = make([]int64, newSize)
 		case arrow.PrimitiveTypes.Float64:
 			sf.columns[i] = make([]float64, newSize)
 		case arrow.BinaryTypes.String:
 			sf.columns[i] = make([]string, newSize)
 		}
 	}
 	sf.added = newSize
 }
 // the row offset must be reset to 0 for the slices being appended
 func (sf *ShardFile) SetIntValue(col int, row int64, val int64) {
 	v := sf.columns[col].([]int64)
 	v[row-sf.beforeRows] = val
 }
 func (sf *ShardFile) SetFloatValue(col int, row int64, val float64) {
 	v := sf.columns[col].([]float64)
 	v[row-sf.beforeRows] = val
 }
 func (sf *ShardFile) SetStringValue(col int, row int64, val string) {
 	v := sf.columns[col].([]string)
 	v[row-sf.beforeRows] = val
 }
 func (sf *ShardFile) Process(cs *ChangesetRequest) error {
 	err := sf.process(cs)
 	if err != nil {
 		return err
 	}
 	rtemp := sf.dest + ".temp"
 	err = sf.Save(rtemp)
 	if err != nil {
 		return err
 	}
 	return os.Rename(rtemp+sf.executor.TableExtension(), sf.dest+sf.executor.TableExtension())
 }
 func (sf *ShardFile) LoadBlobs() error {
 	for col := 0; col < len(sf.schema.Fields()); col++ {
 		column := sf.table.Column(col)
 		switch column.DataType() {
 		case arrow.BinaryTypes.String:
 			for i, chunk := range column.Data().Chunks() {
 				stringData := chunk.(*array.String)
 				k := key{col: col, chunk: i}
 				for j := 0; j < stringData.Len(); j++ {
 					v := stringData.Value(j)
 					sf.strings[k] = append(sf.strings[k], v)
 				}
 			}
 		}
 	}
 	return nil
 }
 func (sf *ShardFile) ReplaceString(col, chunk, l int, s string) {
 	sf.strings[key{col: col, chunk: chunk}][l] = s
 }
 func (sf *ShardFile) process(cs *ChangesetRequest) error {
 	offset := 0
 	if sf.table != nil {
 		// need to load blobs prior
 		sf.LoadBlobs()
 		column := sf.table.Column(0)
 		resolver := dataframe.NewChunkResolver(column)
 		for i, rowid := range cs.ShardIds {
 			offset = i
 			if rowid >= sf.table.NumRows() {
 				break
 			}
 			chunk, l := resolver.Resolve(int(rowid))
 			for col := 0; col < len(sf.schema.Fields()); col++ {
 				column := sf.table.Column(col)
 				switch column.DataType() {
 				case arrow.PrimitiveTypes.Int64:
 					v := column.Data().Chunk(chunk).(*array.Int64).Int64Values()
 					v[l] = cs.Columns[col].([]int64)[i]
 				case arrow.PrimitiveTypes.Float64:
 					v := column.Data().Chunk(chunk).(*array.Float64).Float64Values()
 					v[l] = cs.Columns[col].([]float64)[i]
 				case arrow.BinaryTypes.String:
 					// TODO(twg) 2023/01/09 How to update existing?
 					new := cs.Columns[col].([]string)[i]
 					sf.ReplaceString(col, chunk, l, new)
 				default:
 					panic(fmt.Sprintf("Unknown Type %v", column.DataType()))
 				}
 			}
 		}
 	}
 	max := cs.ShardIds[len(cs.ShardIds)-1]
 	sf.buildAppenders(max)
 	// need to check if only replace and no apend
 	if sf.added > 0 {
 		for i, rowid := range cs.ShardIds[offset:] {
 			i += offset
 			for col := 0; col < len(sf.schema.Fields()); col++ {
 				switch sf.schema.Field(col).Type {
 				case arrow.PrimitiveTypes.Int64:
 					sf.SetIntValue(col, rowid, cs.Columns[col].([]int64)[i])
 				case arrow.PrimitiveTypes.Float64:
 					sf.SetFloatValue(col, rowid, cs.Columns[col].([]float64)[i])
 				case arrow.BinaryTypes.String:
 					sf.SetStringValue(col, rowid, cs.Columns[col].([]string)[i])
 				default:
 					panic(fmt.Sprintf("2 Unknown Type %v", sf.schema.Field(col).Type))
 				}
 			}
 		}
 	}
 	return nil
 }
 type twoSlices struct {
 	id_slice    []int
 	lists_slice [][]string
 }
 type SortByOther twoSlices
 func (sbo SortByOther) Len() int {
 	return len(sbo.id_slice)
 }
 func (sbo SortByOther) Swap(i, j int) {
 	sbo.id_slice[i], sbo.id_slice[j] = sbo.id_slice[j], sbo.id_slice[i]
 	sbo.lists_slice[i], sbo.lists_slice[j] = sbo.lists_slice[j], sbo.lists_slice[i]
 }
 func (sbo SortByOther) Less(i, j int) bool {
 	return sbo.id_slice[i] < sbo.id_slice[j]
 }
 func (sf *ShardFile) buildFromStrings(idx int, mem memory.Allocator) []arrow.Array {
 	ids := make([]int, 0)
 	lists := make([][]string, 0)
 	for k, v := range sf.strings {
 		if k.col == idx { // ugh not ordered :(
 			ids = append(ids, k.chunk)
 			lists = append(lists, v)
 		}
 	}
 	// sort ids/lists
 	parts := twoSlices{id_slice: ids, lists_slice: lists}
 	sort.Sort(SortByOther(parts))
 	builder := array.NewStringBuilder(mem)
 	chunks := make([]arrow.Array, 0)
 	for _, v := range parts.lists_slice {
 		builder.AppendValues(v, nil)
 		newChunk := builder.NewArray()
 		chunks = append(chunks, newChunk)
 	}
 	return chunks
 }
 func (sf *ShardFile) Save(name string) error {
 	parts := make([]arrow.Array, 0)
 	mem := memory.NewGoAllocator()
 	for col := 0; col < len(sf.schema.Fields()); col++ {
 		chunks := make([]arrow.Array, 0)
 		if sf.table != nil {
 			// we append if there was existing file
 			column := sf.table.Column(col)
 			// if primitive type
 			switch column.DataType() {
 			case arrow.BinaryTypes.String:
 				chunks = sf.buildFromStrings(col, mem)
 			default:
 				chunks = append(chunks, column.Data().Chunks()...)
 			}
 			// else binary type
 		}
 		switch sf.schema.Field(col).Type {
 		case arrow.PrimitiveTypes.Int64:
 			// case *arrow.Int64Type:
 			if sf.added > 0 {
 				ibuild := array.NewInt64Builder(mem)
 				ibuild.AppendValues(sf.columns[col].([]int64), nil) // TODO(twg) 2022/09/28 need to handle null
 				newChunk := ibuild.NewArray()
 				chunks = append(chunks, newChunk)
 			}
 			record, err := array.Concatenate(chunks, mem)
 			if err != nil {
 				return err
 			}
 			parts = append(parts, record)
 		case arrow.PrimitiveTypes.Float64:
 			// case *arrow.Float64Type:
 			if sf.added > 0 {
 				fbuild := array.NewFloat64Builder(mem)
 				fbuild.AppendValues(sf.columns[col].([]float64), nil) // TODO(twg) 2022/09/28 need to handle null
 				newChunk := fbuild.NewArray()
 				chunks = append(chunks, newChunk)
 			}
 			record, err := array.Concatenate(chunks, mem)
 			if err != nil {
 				return err
 			}
 			parts = append(parts, record)
 		case arrow.BinaryTypes.String:
 			if sf.added > 0 {
 				fbuild := array.NewStringBuilder(mem)
 				fbuild.AppendValues(sf.columns[col].([]string), nil) // TODO(twg) 2022/09/28 need to handle null
 				newChunk := fbuild.NewArray()
 				chunks = append(chunks, newChunk)
 			}
 			record, err := array.Concatenate(chunks, mem)
 			if err != nil {
 				return err
 			}
 			parts = append(parts, record)
 		default:
 			vprint.VV("UNKNOWN %T", sf.schema.Field(col).Type)
 		}
 	}
 	rec := array.NewRecord(sf.schema, parts, sf.beforeRows+sf.added)
 	table := array.NewTableFromRecords(sf.schema, []arrow.Record{rec})
 	return sf.executor.SaveTable(name, table, mem)
 }
 // TODO(twg) 2022/10/03 Not a huge fan of the global variable will look at adding to executor structure
 // when dataframe is fully integrated
 var (
 	dataframeShardLocks map[uint64]*sync.Mutex
 	muWriteDataframe    sync.Mutex
 )
 func init() {
 	dataframeShardLocks = make(map[uint64]*sync.Mutex)
 }
 func getDataframeWritelock(shard uint64) *sync.Mutex {
 	muWriteDataframe.Lock()
 	defer muWriteDataframe.Unlock()
 	lock, ok := dataframeShardLocks[shard]
 	if ok {
 		return lock
 	}
 	newLock := sync.Mutex{}
 	dataframeShardLocks[shard] = &newLock
 	return &newLock
 }
 func (api *API) ApplyDataframeChangeset(ctx context.Context, index string, cs *ChangesetRequest, shard uint64) error {
 	// TODO(twg) 2022/09/29 need to validate api call
 	idx := api.Holder().Index(index)
 	// check if dataframe exists
 	fname := idx.GetDataFramePath(shard)
 	// only 1 shard writer allowed at at time so wait for it to be available
 	mu := getDataframeWritelock(shard)
 	mu.Lock()
 	defer mu.Unlock()
 	mem := memory.NewGoAllocator()
 	shardFile, err := NewShardFile(ctx, fname, mem, api.server.executor)
 	if err != nil {
 		return err
 	}
 	err = shardFile.EnsureSchema(cs)
 	if err != nil {
 		return err
 	}
 	return shardFile.Process(cs)
 }
 type column struct {
 	Name string
 	Type string
 }
 func (api *API) GetDataframeSchema(ctx context.Context, indexName string) (interface{}, error) {
 	idx, err := api.Index(ctx, indexName)
 	if err != nil {
 		return nil, err
 	}
 	base := idx.DataframesPath()
 	dir, _ := os.Open(base)
 	files, _ := dir.Readdir(0)
 	parts := make([]column, 0)
 	mem := memory.NewGoAllocator()
 	for i := range files {
 		file := files[i]
 		name := file.Name()
 		if api.server.executor.IsDataframeFile(name) {
 			// strip off the parquet extenison
 			name = strings.TrimSuffix(name, filepath.Ext(name))
 			// read the parquet file and extract the schema
 			fname := filepath.Join(base, name)
 			table, err := api.server.executor.getDataTable(ctx, fname, mem)
 			if err != nil {
 				return nil, err
 			}
 			for i := 0; i < int(table.NumCols()); i++ {
 				col := table.Column(i)
 				part := column{Name: col.Name(), Type: col.DataType().String()}
 				parts = append(parts, part)
 			}
 			break // only go on first file
 		}
 	}
 	return parts, nil
 }
--- a/arrow.go
+++ b/arrow.go
@ -1,562 +0,0 @@
 // Copyright 2021 Molecula Corp. All rights reserved.
 package pilosa
 import (
 	"context"
 	"encoding/json"
 	"fmt"
 	"io"
 	"os"
 	"strings"
 	"sync"
 	"github.com/apache/arrow/go/v10/arrow"
 	"github.com/apache/arrow/go/v10/arrow/array"
 	"github.com/apache/arrow/go/v10/arrow/ipc"
 	"github.com/apache/arrow/go/v10/arrow/memory"
 	"github.com/apache/arrow/go/v10/parquet"
 	"github.com/apache/arrow/go/v10/parquet/file"
 	"github.com/apache/arrow/go/v10/parquet/pqarrow"
 	"github.com/featurebasedb/featurebase/v3/pql"
 	"github.com/featurebasedb/featurebase/v3/tracing"
 	"github.com/gomem/gomem/pkg/dataframe"
 	"github.com/pkg/errors"
 )
 /*
 The function Arrow provides filtered access to the raw values stored in the dataframe.
 If Arrow is just provided a bitmap filter, such as ConstRow or any Bitmap Operation,
 all the values associated with each column are returned.  This set can be limited with
 the addition of the header parameter
 Example:
 Arrow(ConstRow(columns=[2,4,6]),header=["fval"])
 */
 // executeApply executes a Arrow() call.
 func (e *executor) executeArrow(ctx context.Context, qcx *Qcx, index string, c *pql.Call, shards []uint64, opt *ExecOptions) (arrow.Table, error) {
 	if !e.dataframeEnabled {
 		return nil, errors.New("Dataframe support not enabled")
 	}
 	span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeArrow")
 	defer span.Finish()
 	if len(c.Children) > 1 {
 		return nil, errors.New("Apply() only accepts a single bitmap input filter")
 	}
 	var columnFilter []string
 	if cols, ok := c.Args["header"].([]interface{}); ok {
 		columnFilter = make([]string, 0, len(cols))
 		for _, v := range cols {
 			columnFilter = append(columnFilter, v.(string))
 		}
 	}
 	mapcounter := 0
 	reducecounter := 0
 	pool := memory.NewGoAllocator() // TODO(twg) 2022/09/01 singledton?
 	// Execute calls in bulk on each remote node and merge.
 	mu := &sync.Mutex{}
 	mapFn := func(ctx context.Context, shard uint64, mopt *mapOptions) (_ interface{}, err error) {
 		mu.Lock()
 		mapcounter++
 		mu.Unlock()
 		return e.executeArrowShard(ctx, qcx, index, c, shard, pool, columnFilter)
 	}
 	tables := make([]*BasicTable, 0)
 	reduceFn := func(ctx context.Context, prev, v interface{}) interface{} {
 		mu.Lock()
 		reducecounter++
 		mu.Unlock()
 		if v == nil {
 			return prev
 		}
 		switch t := v.(type) {
 		case *BasicTable:
 			if t.resolver != nil {
 				mu.Lock()
 				tables = append(tables, t)
 				mu.Unlock()
 			}
 		case arrow.Table:
 			if t.NumRows() > 0 {
 				bt := BasicTableFromArrow(t, pool)
 				mu.Lock()
 				tables = append(tables, bt)
 				mu.Unlock()
 			}
 		}
 		return nil
 	}
 	_, err := e.mapReduce(ctx, index, shards, c, opt, mapFn, reduceFn)
 	if err != nil {
 		return nil, err
 	}
 	if len(tables) == 0 {
 		return &BasicTable{name: "empty"}, nil
 	}
 	tbl := Concat(tables[0].Schema(), tables, pool)
 	r := dataframe.NewChunkResolver(tbl.Column(0))
 	return &BasicTable{resolver: &r, table: tbl}, nil
 }
 type BasicTable struct {
 	resolver dataframe.Resolver
 	table    arrow.Table
 	filtered bool
 	name     string
 }
 func (st *BasicTable) Name() string {
 	return st.name
 }
 func (st *BasicTable) Schema() *arrow.Schema {
 	if st.table != nil {
 		return st.table.Schema()
 	}
 	return &arrow.Schema{}
 }
 func (st *BasicTable) IsFiltered() bool {
 	return st.filtered
 }
 func (st *BasicTable) NumRows() int64 {
 	if st.resolver == nil {
 		return 0
 	}
 	return int64(st.resolver.NumRows())
 }
 func (st *BasicTable) NumCols() int64 {
 	if st.table != nil {
 		return st.table.NumCols()
 	}
 	return 0
 }
 func (st *BasicTable) Column(i int) *arrow.Column {
 	if st.table != nil {
 		return st.table.Column(i)
 	}
 	return nil
 }
 func (st *BasicTable) Retain() {
 	if st.table != nil {
 		st.table.Retain()
 	}
 }
 func (st *BasicTable) Release() {
 	if st.table != nil {
 		st.table.Retain()
 	}
 }
 func (st *BasicTable) Get(column, row int) interface{} {
 	field := st.Schema().Field(column)
 	c, i := st.resolver.Resolve(row)
 	nullable := field.Nullable
 	chunk := st.Column(column).Data().Chunk(c)
 	// TODO(twg) 2023/01/26 potential NULL support?
 	if nullable && chunk.IsNull(i) {
 		return nil
 	}
 	switch field.Type.(type) {
 	case *arrow.BooleanType:
 		return chunk.(*array.Boolean).Value(i)
 	case *arrow.Int8Type:
 		v := chunk.(*array.Int8).Int8Values()
 		return int64(v[i])
 	case *arrow.Int16Type:
 		v := chunk.(*array.Int16).Int16Values()
 		return int64(v[i])
 	case *arrow.Int32Type:
 		v := chunk.(*array.Int32).Int32Values()
 		return int64(v[i])
 	case *arrow.Int64Type:
 		v := chunk.(*array.Int64).Int64Values()
 		return int64(v[i])
 	case *arrow.Uint8Type:
 		v := chunk.(*array.Uint8).Uint8Values()
 		return uint64(v[i])
 	case *arrow.Uint16Type:
 		v := chunk.(*array.Uint16).Uint16Values()
 		return uint64(v[i])
 	case *arrow.Uint32Type:
 		v := chunk.(*array.Uint32).Uint32Values()
 		return uint64(v[i])
 	case *arrow.Uint64Type:
 		v := chunk.(*array.Uint64).Uint64Values()
 		return v[i]
 	case *arrow.Float32Type:
 		v := chunk.(*array.Float32).Float32Values()
 		return float64(v[i])
 	case *arrow.Float64Type:
 		v := chunk.(*array.Float64).Float64Values()
 		return v[i]
 	case *arrow.StringType:
 		return chunk.(*array.String).Value(i)
 	}
 	return 0
 }
 func builderFrom(mem memory.Allocator, dt arrow.DataType, size int64) array.Builder {
 	var bldr array.Builder
 	switch dt := dt.(type) {
 	case *arrow.BooleanType:
 		bldr = array.NewBooleanBuilder(mem)
 	case *arrow.Int8Type:
 		bldr = array.NewInt8Builder(mem)
 	case *arrow.Int16Type:
 		bldr = array.NewInt16Builder(mem)
 	case *arrow.Int32Type:
 		bldr = array.NewInt32Builder(mem)
 	case *arrow.Int64Type:
 		bldr = array.NewInt64Builder(mem)
 	case *arrow.Uint8Type:
 		bldr = array.NewUint8Builder(mem)
 	case *arrow.Uint16Type:
 		bldr = array.NewUint16Builder(mem)
 	case *arrow.Uint32Type:
 		bldr = array.NewUint32Builder(mem)
 	case *arrow.Uint64Type:
 		bldr = array.NewUint64Builder(mem)
 	case *arrow.Float32Type:
 		bldr = array.NewFloat32Builder(mem)
 	case *arrow.Float64Type:
 		bldr = array.NewFloat64Builder(mem)
 	case *arrow.StringType:
 		bldr = array.NewStringBuilder(mem)
 	default:
 		panic(fmt.Errorf("builderFrom: invalid Arrow type %v", dt))
 	}
 	bldr.Reserve(int(size))
 	return bldr
 }
 func appendData(bldr array.Builder, v interface{}) {
 	switch bldr := bldr.(type) {
 	case *array.BooleanBuilder:
 		bldr.Append(v.(bool))
 	case *array.Int8Builder:
 		bldr.Append(v.(int8))
 	case *array.Int16Builder:
 		bldr.Append(v.(int16))
 	case *array.Int32Builder:
 		bldr.Append(v.(int32))
 	case *array.Int64Builder:
 		bldr.Append(v.(int64))
 	case *array.Uint8Builder:
 		bldr.Append(v.(uint8))
 	case *array.Uint16Builder:
 		bldr.Append(v.(uint16))
 	case *array.Uint32Builder:
 		bldr.Append(v.(uint32))
 	case *array.Uint64Builder:
 		bldr.Append(v.(uint64))
 	case *array.Float32Builder:
 		bldr.Append(v.(float32))
 	case *array.Float64Builder:
 		bldr.Append(v.(float64))
 	case *array.StringBuilder:
 		bldr.Append(v.(string))
 	default:
 		panic(fmt.Errorf("appendData: invalid Arrow builder type %T", bldr))
 	}
 }
 func Concat(schema *arrow.Schema, tables []*BasicTable, mem memory.Allocator) arrow.Table {
 	if len(tables) == 1 {
 		if !tables[0].IsFiltered() {
 			return tables[0]
 		}
 	}
 	cols := make([]arrow.Column, len(schema.Fields()))
 	defer func(cols []arrow.Column) {
 		for i := range cols {
 			cols[i].Release()
 		}
 	}(cols)
 	sz := 0
 	for i := range tables {
 		sz += int(tables[i].NumRows())
 	}
 	for i := range cols {
 		field := schema.Field(i)
 		arrs := make([]arrow.Array, 0)
 		builder := builderFrom(mem, field.Type, int64(sz))
 		for t := range tables {
 			table := tables[t]
 			if table.IsFiltered() {
 				for row := 0; row < int(table.NumRows()); row++ {
 					v := table.Get(i, row)
 					appendData(builder, v)
 				}
 				arrs = append(arrs, builder.NewArray())
 			} else {
 				parts := table.Column(i).Data()
 				arrs = append(arrs, parts.Chunks()...)
 			}
 		}
 		chunk := arrow.NewChunked(field.Type, arrs)
 		cols[i] = *arrow.NewColumn(field, chunk)
 		chunk.Release()
 	}
 	return array.NewTable(schema, cols, -1)
 }
 func (st *BasicTable) MarshalJSON() ([]byte, error) {
 	results := make(map[string]interface{})
 	n := 0
 	if st.table != nil {
 		n = int(st.table.NumCols())
 	}
 	for b := 0; b < n; b++ {
 		col := st.table.Column(b)
 		result := make([]interface{}, st.resolver.NumRows())
 		for n := st.resolver.NumRows() - 1; n >= 0; n-- {
 			v := st.Get(b, n)
 			result[n] = v
 		}
 		results[col.Name()] = result
 	}
 	return json.Marshal(results)
 }
 func BasicTableFromArrow(table arrow.Table, mem memory.Allocator) *BasicTable {
 	col := table.Column(0)
 	r := dataframe.NewChunkResolver(col)
 	return &BasicTable{resolver: &r, table: table}
 }
 func filterColumns(filters []string, table arrow.Table) arrow.Table {
 	filters = append(filters, "_ID")
 	schema := table.Schema()
 	// TODO(twg) 2022/11/09 add glob support
 	allFields := schema.Fields()
 	in := func(key string) bool {
 		for _, v := range filters {
 			if v == key {
 				return true
 			}
 		}
 		return false
 	}
 	cols := make([]arrow.Column, 0)
 	fields := make([]arrow.Field, 0)
 	for i := range allFields {
 		field := allFields[i]
 		if in(field.Name) {
 			cols = append(cols, *table.Column(i))
 			fields = append(fields, field)
 		}
 	}
 	filterdSchema := arrow.NewSchema(fields, nil) // TODO(twg) 2022/11/09 handle meta:w
 	return array.NewTable(filterdSchema, cols, table.NumRows())
 }
 func (e *executor) executeArrowShard(ctx context.Context, qcx *Qcx, index string, c *pql.Call, shard uint64, pool memory.Allocator, columnFilter []string) (*BasicTable, error) {
 	name := fmt.Sprintf("a. %v", shard)
 	span, _ := tracing.StartSpanFromContext(ctx, "Executor.executeArrowShard")
 	defer span.Finish()
 	var filter *Row
 	if len(c.Children) == 1 {
 		row, err := e.executeBitmapCallShard(ctx, qcx, index, c.Children[0], shard)
 		if err != nil {
 			return nil, err
 		}
 		filter = row
 		if !filter.Any() {
 			// no need to actuall run the query for its not operating against any values
 			return &BasicTable{name: name}, nil
 		}
 	}
 	//
 	ids := filter.ShardColumns() // needs to be shard columns
 	// Fetch index.
 	idx := e.Holder.Index(index)
 	if idx == nil {
 		return nil, newNotFoundError(ErrIndexNotFound, index)
 	}
 	fname := idx.GetDataFramePath(shard)
 	if !e.dataFrameExists(fname) {
 		return &BasicTable{name: name}, nil
 	}
 	table, err := e.getDataTable(ctx, fname, pool)
 	if err != nil {
 		return nil, errors.Wrap(err, "arrow readTableParquet")
 	}
 	defer table.Release()
 	if len(columnFilter) > 0 {
 		table = filterColumns(columnFilter, table)
 	}
 	df, err := dataframe.NewDataFrameFromTable(pool, table)
 	if err != nil {
 		return nil, errors.Wrap(err, "arrow NewDataFromTable")
 	}
 	p := dataframe.NewChunkResolver(df.ColumnAt(0))
 	var resolver dataframe.Resolver
 	resolver = &p
 	if filter != nil {
 		if len(ids) == 0 {
 			return &BasicTable{name: name}, nil
 		}
 		resolver, err = filterDataframe(resolver, pool, ids)
 		if err != nil {
 			return nil, errors.Wrap(err, "filtering dataframe")
 		}
 	}
 	table.Retain()
 	return &BasicTable{resolver: resolver, table: table, filtered: filter != nil, name: name}, nil
 }
 func (e *executor) dataFrameExists(fname string) bool {
 	if e.typeIsParquet() {
 		if _, err := os.Stat(fname + ".parquet"); os.IsNotExist(err) {
 			return false
 		}
 		return true
 	}
 	if _, err := os.Stat(fname + ".arrow"); os.IsNotExist(err) {
 		return false
 	}
 	return true
 }
 func (e *executor) getDataTable(ctx context.Context, fname string, mem memory.Allocator) (arrow.Table, error) {
 	if e.typeIsParquet() {
 		table, err := readTableParquetCtx(ctx, fname, mem)
 		return table, err
 	}
 	return readTableArrow(fname, mem)
 }
 func (e *executor) typeIsParquet() bool {
 	return e.datafameUseParquet
 }
 func (e *executor) IsDataframeFile(name string) bool {
 	if e.typeIsParquet() {
 		return strings.HasSuffix(name, ".parquet")
 	}
 	return strings.HasSuffix(name, ".arrow")
 }
 func (e *executor) SaveTable(name string, table arrow.Table, mem memory.Allocator) error {
 	if e.typeIsParquet() {
 		return writeTableParquet(table, name)
 	}
 	return writeTableArrow(table, name, mem)
 }
 func (e *executor) TableExtension() string {
 	if e.typeIsParquet() {
 		return ".parquet"
 	}
 	return ".arrow"
 }
 func readTableArrow(filename string, mem memory.Allocator) (arrow.Table, error) {
 	r, err := os.Open(filename + ".arrow")
 	if err != nil {
 		return nil, err
 	}
 	rr, err := ipc.NewFileReader(r, ipc.WithAllocator(mem))
 	if err != nil {
 		return nil, err
 	}
 	defer rr.Close()
 	records := make([]arrow.Record, rr.NumRecords())
 	i := 0
 	for {
 		rec, err := rr.Read()
 		if err == io.EOF {
 			break
 		} else if err != nil {
 			return nil, err
 		}
 		records[i] = rec
 		i++
 	}
 	records = records[:i]
 	table := array.NewTableFromRecords(rr.Schema(), records)
 	return table, nil
 }
 func readTableParquetCtx(ctx context.Context, filename string, mem memory.Allocator) (arrow.Table, error) {
 	r, err := os.Open(filename + ".parquet")
 	if err != nil {
 		return nil, err
 	}
 	defer r.Close()
 	pf, err := file.NewParquetReader(r)
 	if err != nil {
 		return nil, err
 	}
 	reader, err := pqarrow.NewFileReader(pf, pqarrow.ArrowReadProperties{}, mem)
 	if err != nil {
 		return nil, err
 	}
 	return reader.ReadTable(ctx)
 }
 func writeTableParquet(table arrow.Table, filename string) error {
 	f, err := os.Create(filename + ".parquet")
 	if err != nil {
 		return err
 	}
 	defer f.Close()
 	props := parquet.NewWriterProperties(parquet.WithDictionaryDefault(false))
 	arrProps := pqarrow.DefaultWriterProps()
 	chunkSize := 10 * 1024 * 1024
 	err = pqarrow.WriteTable(table, f, int64(chunkSize), props, arrProps)
 	if err != nil {
 		return err
 	}
 	f.Sync()
 	return nil
 }
 func writeTableArrow(table arrow.Table, filename string, mem memory.Allocator) error {
 	f, err := os.Create(filename + ".arrow")
 	if err != nil {
 		return err
 	}
 	defer f.Close()
 	writer, err := ipc.NewFileWriter(f, ipc.WithAllocator(mem), ipc.WithSchema(table.Schema()))
 	if err != nil {
 		panic(err)
 	}
 	chunkSize := int64(0)
 	tr := array.NewTableReader(table, chunkSize)
 	defer tr.Release()
 	n := 0
 	for tr.Next() {
 		arec := tr.Record()
 		err = writer.Write(arec)
 		if err != nil {
 			panic(err)
 		}
 		n++
 	}
 	err = writer.Close()
 	if err != nil {
 		panic(err)
 	}
 	f.Sync()
 	return nil
 }
--- a/arrow_test.go
+++ b/arrow_test.go
@ -1,53 +0,0 @@
 // Copyright 2021 Molecula Corp. All rights reserved.
 package pilosa
 import (
 	"context"
 	"encoding/hex"
 	"math/rand"
 	"os"
 	"path/filepath"
 	"testing"
 	"github.com/apache/arrow/go/v10/arrow"
 	"github.com/apache/arrow/go/v10/arrow/array"
 	"github.com/apache/arrow/go/v10/arrow/memory"
 )
 func TempFileName(prefix string) string {
 	randBytes := make([]byte, 16)
 	rand.Read(randBytes)
 	return filepath.Join(os.TempDir(), prefix+hex.EncodeToString(randBytes))
 }
 func Test_TableParquet(t *testing.T) {
 	// create a arrow table
 	schema := arrow.NewSchema(
 		[]arrow.Field{
 			{Name: "num", Type: arrow.PrimitiveTypes.Float64},
 		},
 		nil, // no metadata
 	)
 	mem := memory.NewGoAllocator()
 	b := array.NewRecordBuilder(mem, schema)
 	defer b.Release()
 	b.Field(0).(*array.Float64Builder).AppendValues([]float64{1.0, 1.5, 2.0}, nil)
 	table := array.NewTableFromRecords(schema, []arrow.Record{b.NewRecord()})
 	defer table.Release()
 	fileName := TempFileName("pq-")
 	// save it as  a parquet file
 	err := writeTableParquet(table, fileName)
 	if err != nil {
 		t.Fatal(err)
 	}
 	defer os.Remove(fileName)
 	// read it back in and compare the result
 	got, err := readTableParquetCtx(context.Background(), fileName, mem)
 	if err != nil {
 		t.Fatalf("readTableParquetCtx() error = %v", err)
 	}
 	if got.NumCols() != table.NumCols() {
 		t.Errorf("got:%v expected:%v", got.NumCols(), table.NumCols())
 	}
 }
--- a/audit_test.go
+++ b/audit_test.go
@ -7,7 +7,7 @@ import (
 	"os"
 	"reflect"
-	"github.com/featurebasedb/featurebase/v3"
+	pilosa "github.com/featurebasedb/featurebase/v3"
 	"github.com/featurebasedb/featurebase/v3/testhook"
 )
--- a/authn/authenticate.go
+++ b/authn/authenticate.go
@ -148,6 +148,7 @@ func (a *Auth) refreshToken(access, refresh string) (string, string, error) {
 			"client_secret": {a.oAuthConfig.ClientSecret},
 		},
 	)
 	if err != nil {
 		return "", "", errors.Wrap(err, "refreshing token")
 	}
@ -366,7 +367,7 @@ func (a *Auth) SetCookie(w http.ResponseWriter, access, refresh string, expiry t
 	return nil
 }
-func (a *Auth) SetGRPCMetadata(ctx context.Context, md metadata.MD, access, refresh string) (context.Context, error) {
+func (a *Auth) SetGRPCMetadata(ctx context.Context, md metadata.MD, access, refresh string) error {
 	mCookies := map[string]string{}
 	if c, ok := md["cookie"]; ok {
 		for _, cookie := range c {
@ -384,7 +385,7 @@ func (a *Auth) SetGRPCMetadata(ctx context.Context, md metadata.MD, access, refr
 	}
 	md["cookie"] = cookies
-	return metadata.NewIncomingContext(ctx, md), grpc.SetHeader(ctx, md)
+	return grpc.SetHeader(ctx, md)
 }
 func decodeHex(hexstr string) ([]byte, error) {
--- a/authn/authenticate_internal_test.go
+++ b/authn/authenticate_internal_test.go
@ -78,7 +78,7 @@ func TestSetGRPCMetadata(t *testing.T) {
 			if !ok {
 				t.Fatalf("expected ok, got: %v", ok)
 			}
-			ctx, err := a.SetGRPCMetadata(ctx, md, "accesstoken!", "refreshtoken!")
+			err := a.SetGRPCMetadata(ctx, md, "accesstoken!", "refreshtoken!")
 			if err != nil {
 				t.Fatalf("expected no errors, got: %v", err)
 			}
@ -377,6 +377,7 @@ func TestAuthenticate_CleanCache(t *testing.T) {
 			t.Errorf("goldy should not have been deleted")
 		}
 	})
 }
 func TestGetGroups(t *testing.T) {
@ -462,6 +463,7 @@ func TestGetGroups(t *testing.T) {
 			}
 		})
 	}
 }
 func TestDecodeHex(t *testing.T) {
@ -613,6 +615,7 @@ func TestHandlers(t *testing.T) {
 			t.Fatalf("expected BadRequest, got %v", resp.StatusCode)
 		}
 	})
 }
 // This type is used for mocking ServerTransportStreams in tests
@ -664,6 +667,7 @@ func assertEqual(exp, got interface{}, t *testing.T) {
 }
 func TestCheckAllowedNetworks(t *testing.T) {
 	tests := []struct {
 		requestIP     string
 		configuredIPs []string
@ -717,6 +721,7 @@ func TestCheckAllowedNetworks(t *testing.T) {
 }
 func TestConvertIP(t *testing.T) {
 	tests := []struct {
 		configuredIPs []string
 		convertedIPs  []net.IPNet
--- a/authn/context.go
+++ b/authn/context.go
@ -1,54 +0,0 @@
 // Copyright 2022 Molecula Corp (DBA FeatureBase). All rights reserved.
 package authn
 import "context"
 // Empty struct to avoid allocations
 type contextKeyAccessToken struct{}
 type contextKeyRefreshToken struct{}
 type contextKeyUserInfo struct{}
 type contextKeyIndexes struct{}
 // GetAccessToken gets the access token from a context.
 func GetAccessToken(ctx context.Context) (token string, ok bool) {
 	token, ok = ctx.Value(contextKeyAccessToken{}).(string)
 	return
 }
 // WithAccessToken makes a new Context with an access token.
 func WithAccessToken(ctx context.Context, token string) context.Context {
 	return context.WithValue(ctx, contextKeyAccessToken{}, token)
 }
 // GetRefreshToken gets the refresh token from a context.
 func GetRefreshToken(ctx context.Context) (token string, ok bool) {
 	token, ok = ctx.Value(contextKeyRefreshToken{}).(string)
 	return
 }
 // WithRefreshToken makes a new Context with a refresh token.
 func WithRefreshToken(ctx context.Context, token string) context.Context {
 	return context.WithValue(ctx, contextKeyRefreshToken{}, token)
 }
 // GetUserInfo gets the UserInfo from a context.
 func GetUserInfo(ctx context.Context) (userInfo *UserInfo, ok bool) {
 	userInfo, ok = ctx.Value(contextKeyUserInfo{}).(*UserInfo)
 	return
 }
 // WithUserInfo makes a new Context with UserInfo.
 func WithUserInfo(ctx context.Context, userInfo *UserInfo) context.Context {
 	return context.WithValue(ctx, contextKeyUserInfo{}, userInfo)
 }
 // GetIndexes get the indices from a context.
 func GetIndexes(ctx context.Context) (indexes []string, ok bool) {
 	indexes, ok = ctx.Value(contextKeyIndexes{}).([]string)
 	return
 }
 // WithIndexes makes a new Context with a []string containing the indicies.
 func WithIndexes(ctx context.Context, indexes []string) context.Context {
 	return context.WithValue(ctx, contextKeyUserInfo{}, indexes)
 }
--- a/batch/Dockerfile-test
+++ b/batch/Dockerfile-test
@ -1,11 +0,0 @@
 ARG GO_VERSION=1.19
 FROM golang:${GO_VERSION}
 WORKDIR /go/src/github.com/featurebasedb/featurebase/
 COPY . .
 WORKDIR /go/src/github.com/featurebasedb/featurebase/batch/
 CMD ["go","test","-v","-mod=vendor","-tags=odbc,dynamic","./..."]
--- a/batch/Dockerfile-wait
+++ b/batch/Dockerfile-wait
@ -1,8 +0,0 @@
 FROM ubuntu:18.04
 RUN ["apt-get", "update", "-y"]
 RUN ["apt-get", "install", "-y", "curl", "netcat"]
 ADD wait.sh /wait
 ENTRYPOINT ["/wait"]
--- a/batch/Makefile
+++ b/batch/Makefile
@ -1,44 +0,0 @@
 GO ?= go
 # We allow setting a custom docker-compose "project". Multiple of the
 # same docker-compose environment can exist simultaneously as long as
 # they use different projects (the project name is prepended to
 # container names and such). This is useful in a CI environment where
 # we might be running multiple instances of the tests concurrently.
 PROJECT ?= batch
 DOCKER_COMPOSE = docker-compose -p $(PROJECT)
 vendor: ../go.mod
 	$(GO) mod vendor
 build-%:
 	$(DOCKER_COMPOSE) build $*
 test-all:
 	$(MAKE) startup
 	$(MAKE) test-run
 	$(MAKE) shutdown
 start-all: build-wait
 	$(DOCKER_COMPOSE) up -d featurebase
 	$(DOCKER_COMPOSE) run -T wait featurebase curl --silent --fail http://featurebase:10101/status
 startup: start-all
 shutdown:
 	$(DOCKER_COMPOSE) down -v --remove-orphans
 save-%-logs:
 	$(DOCKER_COMPOSE) logs $* > ./testdata/$(PROJECT)_$*_logs.txt
 TCMD ?= ./...
 # do "make startup", then e.g. "make test-run-local TCMD='-run=MyFavTest ./kafka'"
 test-run-local: vendor
 	pwd
 	$(DOCKER_COMPOSE) build batch-test
 	$(DOCKER_COMPOSE) run -T batch-test go test -mod=vendor -tags=odbc,dynamic $(TCMD)
 TPKG ?= ../...
 test-run: vendor
 	$(DOCKER_COMPOSE) build batch-test
 	$(DOCKER_COMPOSE) run -T batch-test bash -c "set -o pipefail; go test -v -mod=vendor -tags=odbc,dynamic ./... -covermode=atomic -coverpkg=$(TPKG) -coverprofile=/testdata/$(PROJECT)_base_coverage.out"
--- a/batch/README.md
+++ b/batch/README.md
@ -1,46 +0,0 @@
 # batch
 The `batch` package provides a standard tool set for batching records in a way
 that is most performant for ingesting those records into FeatureBase. The main
 implementation is `Batch` (which can be initated with the `NewBatch()`
 function). The `NewBatch()` function takes an `Importer` which contains all of
 the methods required to interact with FeatureBase; these include methods for
 doing string/id translation as well as for importing shards of data.
 IDK uses the `batch` package internally. Another example where the `batch`
 package is used in the `sql3` package. When an "INSERT INTO" statement is
 executed, the SQL engine uses a `Batch` to do key translation and build import
 batches prior to doing the final import.
 ## Integration tests
 To run the tests, you will need to install the following dependencies:
 1. [Docker](https://docs.docker.com/install/)
 2. [Docker Compose](https://docs.docker.com/compose/install/)
 In addition to these dependancies, you will need to be added to the molecula [Gitlab](https://registry.gitlab.com/molecula) account.
 First start the test environment. This is a docker-compose environment that includes featurebase.
    make startup
 To build and run the integration tests, run:
    make test-run-local
 Then to shut down the test environment, run:
    make shutdown
 The previous command is equivalent to running the following:
    make startup
    sleep 30 # wait for services to come up
    make test-run
    make shutdown
 To run an individual test, you can run the command directly using docker-compose. Note that you must run `docker-compose build batch-test` for docker to run the latest code. Modify the following as needed:
    make startup
    docker-compose build batch-test
    docker-compose run batch-test /usr/local/go/bin/go test -count=1 -mod=vendor -run=TestCmdMainOne .
--- a/batch/batch_test.go
+++ b/batch/batch_test.go
--- a/batch/batcher.go
+++ b/batch/batcher.go
@ -1,20 +0,0 @@
 package batch
 import (
 	"time"
 	"github.com/featurebasedb/featurebase/v3/dax"
 )
 // Batcher is an interface implemented by anything which can allocate new
 // batches.
 type Batcher interface {
 	NewBatch(cfg Config, tbl *dax.Table, fields []*dax.Field) (RecordBatch, error)
 }
 // Config is the configuration options passed to NewBatch for any implementation
 // of the Batcher interface.
 type Config struct {
 	Size         int
 	MaxStaleness time.Duration
 }
--- a/batch/convert.go
+++ b/batch/convert.go
@ -1,145 +0,0 @@
 package batch
 import (
 	"time"
 	featurebase "github.com/featurebasedb/featurebase/v3"
 	"github.com/featurebasedb/featurebase/v3/errors"
 )
 var (
 	MinTimestampNano = time.Unix(-1<<32, 0).UTC()       // 1833-11-24T17:31:44Z
 	MaxTimestampNano = time.Unix(1<<32, 0).UTC()        // 2106-02-07T06:28:16Z
 	MinTimestamp     = time.Unix(-62135596799, 0).UTC() // 0001-01-01T00:00:01Z
 	MaxTimestamp     = time.Unix(253402300799, 0).UTC() // 9999-12-31T23:59:59Z
 	ErrTimestampOutOfRange = errors.New("", "value provided for timestamp field is out of range")
 )
 type TimeUnit string
 const (
 	TimeUnitSeconds      = TimeUnit(featurebase.TimeUnitSeconds)
 	TimeUnitMilliseconds = TimeUnit(featurebase.TimeUnitMilliseconds)
 	TimeUnitMicroseconds = TimeUnit(featurebase.TimeUnitMicroseconds)
 	TimeUnitUSeconds     = TimeUnit(featurebase.TimeUnitUSeconds)
 	TimeUnitNanoseconds  = TimeUnit(featurebase.TimeUnitNanoseconds)
 )
 // TimestampToInt64 converts the provided timestamp to an int64 as the number of
 // units past the epoch.
 func TimestampToInt64(unit TimeUnit, epoch time.Time, ts time.Time) (int64, error) {
 	var err error
 	unit, err = validateTimeUnit(unit)
 	if err != nil {
 		return 0, errors.Wrap(err, "validating time unit")
 	}
 	epoch, err = validateEpoch(epoch)
 	if err != nil {
 		return 0, errors.Wrap(err, "validating epoch")
 	}
 	// Check if the epoch alone is out-of-range. If so, ingest should halt,
 	// regardless of state of the timestamp out-of-range CLI option.
 	if err := validateTimestamp(unit, epoch); err != nil {
 		return 0, errors.Wrap(err, "validating epoch")
 	}
 	epochAsInt64 := timestampToInt(unit, epoch)
 	// Check if the timestamp is out-of-range.
 	if err := validateTimestamp(unit, ts); err != nil {
 		return 0, errors.Wrapf(ErrTimestampOutOfRange, "validating timestamp: %s", ts)
 	}
 	tsAsInt64 := timestampToInt(unit, ts)
 	return tsAsInt64 - epochAsInt64, nil
 }
 // validateTimeUnit checks if the time unit is supported. If the provided unit
 // is blank, validateTimeUnit returns the default TimeUnit.
 func validateTimeUnit(unit TimeUnit) (TimeUnit, error) {
 	switch unit {
 	case "":
 		return TimeUnitSeconds, nil
 	case TimeUnitSeconds,
 		TimeUnitMilliseconds,
 		TimeUnitMicroseconds,
 		TimeUnitUSeconds,
 		TimeUnitNanoseconds:
 		return unit, nil
 	}
 	return "", errors.Errorf("unsupported time unit: %s", unit)
 }
 // validateEpoch checks if the epoch is supported. If the provided epoch
 // is "zero", validateEpoch returns the default epoch value.
 func validateEpoch(epoch time.Time) (time.Time, error) {
 	if epoch.IsZero() {
 		return time.Unix(0, 0), nil
 	}
 	return epoch, nil
 }
 // validateTimestamp checks if the timestamp is within the range of what FB accepts.
 func validateTimestamp(unit TimeUnit, ts time.Time) error {
 	// Min and Max timestamps that Featurebase accepts
 	var minStamp, maxStamp time.Time
 	switch unit {
 	case TimeUnitNanoseconds:
 		minStamp = MinTimestampNano
 		maxStamp = MaxTimestampNano
 	default:
 		minStamp = MinTimestamp
 		maxStamp = MaxTimestamp
 	}
 	if ts.Before(minStamp) || ts.After(maxStamp) {
 		return errors.Errorf("timestamp value (%v) must be within min: %v and max: %v", ts, minStamp, maxStamp)
 	}
 	return nil
 }
 // timestampToInt takes a time unit and a time.Time and converts it to an
 // integer value.
 func timestampToInt(unit TimeUnit, ts time.Time) int64 {
 	switch unit {
 	case TimeUnitSeconds:
 		return ts.Unix()
 	case TimeUnitMilliseconds:
 		return ts.UnixMilli()
 	case TimeUnitMicroseconds, TimeUnitUSeconds:
 		return ts.UnixMicro()
 	case TimeUnitNanoseconds:
 		return ts.UnixNano()
 	}
 	return 0
 }
 // intToTimestamp takes a timeunit and an integer value and converts it to
 // time.Time.
 func intToTimestamp(unit TimeUnit, val int64) (time.Time, error) {
 	switch unit {
 	case TimeUnitSeconds:
 		return time.Unix(val, 0).UTC(), nil
 	case TimeUnitMilliseconds:
 		return time.UnixMilli(val).UTC(), nil
 	case TimeUnitMicroseconds, TimeUnitUSeconds:
 		return time.UnixMicro(val).UTC(), nil
 	case TimeUnitNanoseconds:
 		return time.Unix(0, val).UTC(), nil
 	default:
 		return time.Time{}, errors.Errorf("Unknown time unit: '%v'", unit)
 	}
 }
 // Int64ToTimestamp converts the provided int64 to a timestamp based on the time unit
 // and epoch.
 func Int64ToTimestamp(unit TimeUnit, epoch time.Time, val int64) (time.Time, error) {
 	return intToTimestamp(unit, timestampToInt(unit, epoch)+val)
 }
--- a/batch/docker-compose.yml
+++ b/batch/docker-compose.yml
@ -1,29 +0,0 @@
 version: '3'
 services:
  featurebase:
    build:
      context: ../.
      dockerfile: ./Dockerfile-clustertests
    environment:
      PILOSA_DATA_DIR: /data
      PILOSA_BIND: 0.0.0.0:10101
      PILOSA_BIND_GRPC: 0.0.0.0:20101
      PILOSA_ADVERTISE: featurebase:10101
    command: /featurebase -test.run=TestRunMain -test.coverprofile=/testdata/batch_coverage.out server
    volumes:
      - ./testdata:/testdata
  batch-test:
    build:
      context: ../.
      dockerfile: ./batch/Dockerfile-test
    volumes:
      - ./testdata:/testdata
  wait:
    depends_on:
      - "featurebase"
    build:
      context: .
      dockerfile: Dockerfile-wait
--- a/batch/error.go
+++ b/batch/error.go
@ -1,8 +0,0 @@
 package batch
 import "github.com/pkg/errors"
 // Predefined batch-related errors.
 var (
 	ErrPreconditionFailed = errors.New("Precondition failed")
 )
--- a/batch/testdata/README.md
+++ b/batch/testdata/README.md
@ -1,3 +0,0 @@
 # testdata
 This directory is used in CI tests. I think.
--- a/batch/wait.sh
+++ b/batch/wait.sh
@ -1,26 +0,0 @@
 #!/bin/sh
 name=$1
 shift
 _start_ts=$(date +%s)
 elapsed=0
 timeout=120
 while :
 do
    $@ > /dev/null
    _ret=$?
    _end_ts=$(date +%s)
    if [ $_ret -eq 0 ]; then
        echo "$name is available after $((_end_ts - _start_ts)) seconds."
        break
    else
        echo "Waiting for $name after $((_end_ts - _start_ts)) seconds."
    fi
    sleep 1s
    elapsed=$((elapsed+1))
    if [ $elapsed -ge $timeout ]; then
 	exit 110
    fi
 done
 set -ex
--- a/boltdb/translate.go
+++ b/boltdb/translate.go
@ -1,6 +1,6 @@
 // Copyright 2022 Molecula Corp. (DBA FeatureBase).
 // SPDX-License-Identifier: Apache-2.0
-package pilosa
+package boltdb
 import (
 	"bytes"
@ -10,13 +10,15 @@ import (
 	"io"
 	"os"
 	"path/filepath"
 	"runtime/pprof"
 	"sync"
 	"time"
 	pilosa "github.com/featurebasedb/featurebase/v3"
 	"github.com/featurebasedb/featurebase/v3/roaring"
 	"github.com/pkg/errors"
 	bolt "go.etcd.io/bbolt"
 	"runtime/pprof"
 )
 var _ = pprof.StartCPUProfile
@ -24,7 +26,7 @@ var _ = pprof.StartCPUProfile
 var (
 	// ErrTranslateStoreClosed is returned when reading from an TranslateEntryReader
 	// and the underlying store is closed.
-	ErrBoltTranslateStoreClosed = errors.New("boltdb: translate store closing")
+	ErrTranslateStoreClosed = errors.New("boltdb: translate store closing")
 	// ErrTranslateKeyNotFound is returned when translating key
 	// and the underlying store returns an empty set
@ -44,8 +46,8 @@ const (
 )
 // OpenTranslateStore opens and initializes a boltdb translation store.
-func OpenTranslateStore(path, index, field string, partitionID, partitionN int, fsyncEnabled bool) (TranslateStore, error) {
+func OpenTranslateStore(path, index, field string, partitionID, partitionN int, fsyncEnabled bool) (pilosa.TranslateStore, error) {
-	s := NewBoltTranslateStore(index, field, partitionID, partitionN, fsyncEnabled)
+	s := NewTranslateStore(index, field, partitionID, partitionN, fsyncEnabled)
 	s.Path = path
 	if err := s.Open(); err != nil {
 		return nil, err
@ -54,19 +56,18 @@ func OpenTranslateStore(path, index, field string, partitionID, partitionN int,
 }
 // Ensure type implements interface.
-var _ TranslateStore = &BoltTranslateStore{}
+var _ pilosa.TranslateStore = &TranslateStore{}
-// BoltTranslateStore is an on-disk storage engine for translating string-to-uint64 values.
+// TranslateStore is an on-disk storage engine for translating string-to-uint64 values.
 // An empty string will be converted into the sentinel byte slice:
-//
+// var emptyKey = []byte{
-//	var emptyKey = []byte{
+// 	0x00, 0x00, 0x00,
-//		0x00, 0x00, 0x00,
+// 	0x4d, 0x54, 0x4d, 0x54, // MTMT
-//		0x4d, 0x54, 0x4d, 0x54, // MTMT
+// 	0x00,
-//		0x00,
+// 	0xc2, 0xa0, // NO-BREAK SPACE
-//		0xc2, 0xa0, // NO-BREAK SPACE
+// 	0x00,
-//		0x00,
+// }
-//	}
+type TranslateStore struct {
 type BoltTranslateStore struct {
 	mu sync.RWMutex
 	db *bolt.DB
@ -86,9 +87,9 @@ type BoltTranslateStore struct {
 	Path string
 }
-// NewBoltTranslateStore returns a new instance of TranslateStore.
+// NewTranslateStore returns a new instance of TranslateStore.
-func NewBoltTranslateStore(index, field string, partitionID, partitionN int, fsyncEnabled bool) *BoltTranslateStore {
+func NewTranslateStore(index, field string, partitionID, partitionN int, fsyncEnabled bool) *TranslateStore {
-	return &BoltTranslateStore{
+	return &TranslateStore{
 		index:        index,
 		field:        field,
 		partitionID:  partitionID,
@ -100,7 +101,8 @@ func NewBoltTranslateStore(index, field string, partitionID, partitionN int, fsy
 }
 // Open opens the translate file.
-func (s *BoltTranslateStore) Open() (err error) {
+func (s *TranslateStore) Open() (err error) {
 	// add the path to the problem database if we panic handling it.
 	defer func() {
 		r := recover()
@ -109,9 +111,9 @@ func (s *BoltTranslateStore) Open() (err error) {
 		}
 	}()
-	if err := os.MkdirAll(filepath.Dir(s.Path), 0o750); err != nil {
+	if err := os.MkdirAll(filepath.Dir(s.Path), 0750); err != nil {
 		return errors.Wrapf(err, "mkdir %s", filepath.Dir(s.Path))
-	} else if s.db, err = bolt.Open(s.Path, 0o600, &bolt.Options{Timeout: 1 * time.Second, NoSync: !s.fsyncEnabled, InitialMmapSize: 0}); err != nil {
+	} else if s.db, err = bolt.Open(s.Path, 0600, &bolt.Options{Timeout: 1 * time.Second, NoSync: !s.fsyncEnabled}); err != nil {
 		return errors.Wrapf(err, "open file: %s", err)
 	}
@ -134,7 +136,7 @@ func (s *BoltTranslateStore) Open() (err error) {
 }
 // Close closes the underlying database.
-func (s *BoltTranslateStore) Close() (err error) {
+func (s *TranslateStore) Close() (err error) {
 	s.once.Do(func() { close(s.closing) })
 	if s.db != nil {
@ -146,26 +148,26 @@ func (s *BoltTranslateStore) Close() (err error) {
 }
 // PartitionID returns the partition id the store was initialized with.
-func (s *BoltTranslateStore) PartitionID() int {
+func (s *TranslateStore) PartitionID() int {
 	return s.partitionID
 }
 // ReadOnly returns true if the store is in read-only mode.
-func (s *BoltTranslateStore) ReadOnly() bool {
+func (s *TranslateStore) ReadOnly() bool {
 	s.mu.RLock()
 	defer s.mu.RUnlock()
 	return s.readOnly
 }
 // SetReadOnly toggles whether store is in read-only mode.
-func (s *BoltTranslateStore) SetReadOnly(v bool) {
+func (s *TranslateStore) SetReadOnly(v bool) {
 	s.mu.Lock()
 	defer s.mu.Unlock()
 	s.readOnly = v
 }
 // Size returns the number of bytes in the data file.
-func (s *BoltTranslateStore) Size() int64 {
+func (s *TranslateStore) Size() int64 {
 	if s.db == nil {
 		return 0
 	}
@ -180,7 +182,7 @@ func (s *BoltTranslateStore) Size() int64 {
 // FindKeys looks up the ID for each key.
 // Keys are not created if they do not exist.
 // Missing keys are not considered errors, so the length of the result may be less than that of the input.
-func (s *BoltTranslateStore) FindKeys(keys ...string) (map[string]uint64, error) {
+func (s *TranslateStore) FindKeys(keys ...string) (map[string]uint64, error) {
 	result := make(map[string]uint64, len(keys))
 	err := s.db.View(func(tx *bolt.Tx) error {
 		bkt := tx.Bucket(bucketKeys)
@ -214,9 +216,9 @@ const translateTransactionSize = 16384
 // CreateKeys maps all keys to IDs, creating the IDs if they do not exist.
 // If the translator is read-only, this will return an error.
-func (s *BoltTranslateStore) CreateKeys(keys ...string) (map[string]uint64, error) {
+func (s *TranslateStore) CreateKeys(keys ...string) (map[string]uint64, error) {
 	if s.ReadOnly() {
-		return nil, ErrTranslateStoreReadOnly
+		return nil, pilosa.ErrTranslateStoreReadOnly
 	}
 	written := false
@ -252,7 +254,7 @@ func (s *BoltTranslateStore) CreateKeys(keys ...string) (map[string]uint64, erro
 				}
 				// see if we can re-use any IDs first
 				if id = getter.GetFreeID(); id == 0 {
-					id = GenerateNextPartitionedID(s.index, maxID(tx), s.partitionID, s.partitionN)
+					id = pilosa.GenerateNextPartitionedID(s.index, maxID(tx), s.partitionID, s.partitionN)
 				}
 				idBytes := idScratch[puts*8 : puts*8+8]
 				binary.BigEndian.PutUint64(idBytes, id)
@ -284,7 +286,7 @@ func (s *BoltTranslateStore) CreateKeys(keys ...string) (map[string]uint64, erro
 }
 // Match finds the IDs of all keys matching a filter.
-func (s *BoltTranslateStore) Match(filter func([]byte) bool) ([]uint64, error) {
+func (s *TranslateStore) Match(filter func([]byte) bool) ([]uint64, error) {
 	var matches []uint64
 	err := s.db.View(func(tx *bolt.Tx) error {
 		// This uses the id bucket instead of the key bucket so that matches are produced in sorted order.
@ -314,7 +316,7 @@ func (s *BoltTranslateStore) Match(filter func([]byte) bool) ([]uint64, error) {
 // TranslateID converts an integer ID to a string key.
 // Returns a blank string if ID does not exist.
-func (s *BoltTranslateStore) TranslateID(id uint64) (string, error) {
+func (s *TranslateStore) TranslateID(id uint64) (string, error) {
 	tx, err := s.db.Begin(false)
 	if err != nil {
 		return "", err
@ -324,7 +326,7 @@ func (s *BoltTranslateStore) TranslateID(id uint64) (string, error) {
 }
 // TranslateIDs converts a list of integer IDs to a list of string keys.
-func (s *BoltTranslateStore) TranslateIDs(ids []uint64) ([]string, error) {
+func (s *TranslateStore) TranslateIDs(ids []uint64) ([]string, error) {
 	if len(ids) == 0 {
 		return nil, nil
 	}
@ -345,7 +347,7 @@ func (s *BoltTranslateStore) TranslateIDs(ids []uint64) ([]string, error) {
 }
 // ForceSet writes the id/key pair to the store even if read only. Used by replication.
-func (s *BoltTranslateStore) ForceSet(id uint64, key string) error {
+func (s *TranslateStore) ForceSet(id uint64, key string) error {
 	if err := s.db.Update(func(tx *bolt.Tx) (err error) {
 		if err := tx.Bucket(bucketKeys).Put([]byte(key), u64tob(id)); err != nil {
 			return err
@ -361,13 +363,13 @@ func (s *BoltTranslateStore) ForceSet(id uint64, key string) error {
 }
 // EntryReader returns a reader that streams the underlying data file.
-func (s *BoltTranslateStore) EntryReader(ctx context.Context, offset uint64) (TranslateEntryReader, error) {
+func (s *TranslateStore) EntryReader(ctx context.Context, offset uint64) (pilosa.TranslateEntryReader, error) {
 	ctx, cancel := context.WithCancel(ctx)
-	return &BoltTranslateEntryReader{ctx: ctx, cancel: cancel, store: s, offset: offset}, nil
+	return &TranslateEntryReader{ctx: ctx, cancel: cancel, store: s, offset: offset}, nil
 }
 // WriteNotify returns a channel that is closed when a new entry is written.
-func (s *BoltTranslateStore) WriteNotify() <-chan struct{} {
+func (s *TranslateStore) WriteNotify() <-chan struct{} {
 	s.mu.RLock()
 	ch := s.writeNotify
 	s.mu.RUnlock()
@ -375,7 +377,7 @@ func (s *BoltTranslateStore) WriteNotify() <-chan struct{} {
 }
 // notifyWrite sends a write notification under write lock.
-func (s *BoltTranslateStore) notifyWrite() {
+func (s *TranslateStore) notifyWrite() {
 	s.mu.Lock()
 	defer s.mu.Unlock()
 	close(s.writeNotify)
@ -383,7 +385,7 @@ func (s *BoltTranslateStore) notifyWrite() {
 }
 // MaxID returns the highest id in the store.
-func (s *BoltTranslateStore) MaxID() (max uint64, err error) {
+func (s *TranslateStore) MaxID() (max uint64, err error) {
 	if err := s.db.View(func(tx *bolt.Tx) error {
 		max = maxID(tx)
 		return nil
@ -393,13 +395,18 @@ func (s *BoltTranslateStore) MaxID() (max uint64, err error) {
 	return max, nil
 }
-// Begin starts and returns a transaction on the underlying store.
+// WriteTo writes the contents of the store to the writer.
-func (s *BoltTranslateStore) Begin(write bool) (TranslatorTx, error) {
+func (s *TranslateStore) WriteTo(w io.Writer) (int64, error) {
-	return s.db.Begin(write)
+	tx, err := s.db.Begin(false)
 	if err != nil {
 		return 0, err
 	}
 	defer func() { _ = tx.Rollback() }()
 	return tx.WriteTo(w)
 }
 // ReadFrom reads the content and overwrites the existing store.
-func (s *BoltTranslateStore) ReadFrom(r io.Reader) (n int64, err error) {
+func (s *TranslateStore) ReadFrom(r io.Reader) (n int64, err error) {
 	// Close store.
 	if err := s.Close(); err != nil {
 		return 0, errors.Wrap(err, "closing store")
@ -443,27 +450,27 @@ func maxID(tx *bolt.Tx) uint64 {
 	return 0
 }
-type BoltTranslateEntryReader struct {
+type TranslateEntryReader struct {
 	ctx    context.Context
-	store  *BoltTranslateStore
+	store  *TranslateStore
 	offset uint64
 	cancel func()
 }
 // Close closes the reader.
-func (r *BoltTranslateEntryReader) Close() error {
+func (r *TranslateEntryReader) Close() error {
 	r.cancel()
 	return nil
 }
 // ReadEntry reads the next entry from the underlying translate store.
-func (r *BoltTranslateEntryReader) ReadEntry(entry *TranslateEntry) error {
+func (r *TranslateEntryReader) ReadEntry(entry *pilosa.TranslateEntry) error {
 	// Ensure reader has not been closed before read.
 	select {
 	case <-r.ctx.Done():
 		return r.ctx.Err()
 	case <-r.store.closing:
-		return ErrBoltTranslateStoreClosed
+		return ErrTranslateStoreClosed
 	default:
 	}
@ -503,7 +510,7 @@ func (r *BoltTranslateEntryReader) ReadEntry(entry *TranslateEntry) error {
 		case <-r.ctx.Done():
 			return r.ctx.Err()
 		case <-r.store.closing:
-			return ErrBoltTranslateStoreClosed
+			return ErrTranslateStoreClosed
 		case <-writeNotify:
 		}
 	}
@ -511,6 +518,7 @@ func (r *BoltTranslateEntryReader) ReadEntry(entry *TranslateEntry) error {
 type boltWrapper struct {
 	tx *bolt.Tx
 	db *bolt.DB
 }
 func (w *boltWrapper) Commit() error {
@ -525,8 +533,7 @@ func (w *boltWrapper) Rollback() {
 		w.tx.Rollback()
 	}
 }
-
+func (s *TranslateStore) FreeIDs() (*roaring.Bitmap, error) {
 func (s *BoltTranslateStore) FreeIDs() (*roaring.Bitmap, error) {
 	result := roaring.NewBitmap()
 	err := s.db.View(func(tx *bolt.Tx) error {
 		bkt := tx.Bucket(bucketFree)
@ -542,12 +549,11 @@ func (s *BoltTranslateStore) FreeIDs() (*roaring.Bitmap, error) {
 	})
 	return result, err
 }
-
+func (s *TranslateStore) MergeFree(tx *bolt.Tx, newIDs *roaring.Bitmap) error {
 func (s *BoltTranslateStore) MergeFree(tx *bolt.Tx, newIDs *roaring.Bitmap) error {
 	bkt := tx.Bucket(bucketFree)
 	b := bkt.Get(freeKey)
 	buf := new(bytes.Buffer)
-	if b != nil { // if existing combine with newIDs
+	if b != nil { //if existing combine with newIDs
 		before := roaring.NewBitmap()
 		err := before.UnmarshalBinary(b)
 		if err != nil {
@ -566,7 +572,7 @@ func (s *BoltTranslateStore) MergeFree(tx *bolt.Tx, newIDs *roaring.Bitmap) erro
 // Delete removes the lookeup pairs in order to make avialble for reuse but doesn't commit the
 // transaction for that is tied to the associated rbf transaction being successful
-func (s *BoltTranslateStore) Delete(records *roaring.Bitmap) (Commitor, error) {
+func (s *TranslateStore) Delete(records *roaring.Bitmap) (pilosa.Commitor, error) {
 	tx, err := s.db.Begin(true)
 	if err != nil {
 		return nil, err
--- a/boltdb/translate_internal_test.go
+++ b/boltdb/translate_internal_test.go
@ -1,4 +1,4 @@
-package pilosa
+package boltdb
 import (
 	"path/filepath"
--- a/boltdb/translate_test.go
+++ b/boltdb/translate_test.go
@ -1,6 +1,6 @@
 // Copyright 2022 Molecula Corp. (DBA FeatureBase).
 // SPDX-License-Identifier: Apache-2.0
-package pilosa_test
+package boltdb_test
 import (
 	"bytes"
@ -12,10 +12,10 @@ import (
 	"time"
 	pilosa "github.com/featurebasedb/featurebase/v3"
 	"github.com/featurebasedb/featurebase/v3/boltdb"
 	"github.com/featurebasedb/featurebase/v3/disco"
 	"github.com/featurebasedb/featurebase/v3/roaring"
 	"github.com/featurebasedb/featurebase/v3/testhook"
 	"github.com/stretchr/testify/require"
 )
 //var vv = pilosa.VV
@ -204,7 +204,7 @@ func TestTranslateStore_MaxID(t *testing.T) {
 	}
 }
-func TestBoltTranslateStore_EntryReader(t *testing.T) {
+func TestTranslateStore_EntryReader(t *testing.T) {
 	t.Run("OK", func(t *testing.T) {
 		s := MustOpenNewTranslateStore(t)
 		defer MustCloseTranslateStore(s)
@ -362,7 +362,7 @@ func TestBoltTranslateStore_EntryReader(t *testing.T) {
 		}()
 		var entry pilosa.TranslateEntry
-		if err := r.ReadEntry(&entry); err != pilosa.ErrBoltTranslateStoreClosed {
+		if err := r.ReadEntry(&entry); err != boltdb.ErrTranslateStoreClosed {
 			t.Fatalf("unexpected error: %#v", err)
 		}
@ -375,7 +375,7 @@ func TestBoltTranslateStore_EntryReader(t *testing.T) {
 }
 // MustNewTranslateStore returns a new TranslateStore with a temporary path.
-func MustNewTranslateStore(tb testing.TB) *pilosa.BoltTranslateStore {
+func MustNewTranslateStore(tb testing.TB) *boltdb.TranslateStore {
 	f, err := testhook.TempFile(tb, "translate-store")
 	if err != nil {
 		panic(err)
@ -383,7 +383,7 @@ func MustNewTranslateStore(tb testing.TB) *pilosa.BoltTranslateStore {
 		panic(err)
 	}
-	s := pilosa.NewBoltTranslateStore("I", "F", 0, disco.DefaultPartitionN, false)
+	s := boltdb.NewTranslateStore("I", "F", 0, disco.DefaultPartitionN, false)
 	s.Path = f.Name()
 	return s
 }
@ -458,19 +458,12 @@ func TestTranslateStore_ReadWrite(t *testing.T) {
 		buf := bytes.NewBuffer(nil)
 		expN := s.Size()
-		// wrap in a func so we can defer rollback. Need rollback to
+		// After this, the buffer should contain batch0.
-		// happen before the end of the test. I'm not entirely sure
+		if n, err := s.WriteTo(buf); err != nil {
-		// why, but it hangs if you don't.
+			t.Fatalf("writing to buffer: %s", err)
-		func() {
+		} else if n != expN {
-			tx, err := s.Begin(false)
+			t.Fatalf("expected buffer size: %d, but got: %d", expN, n)
-			require.NoError(t, err)
+		}
 			defer tx.Rollback()
 			// After this, the buffer should contain batch0.
 			n, err := tx.WriteTo(buf)
 			require.NoError(t, err)
 			require.Equal(t, expN, n)
 		}()
 		// Populate the store with the keys in batch1.
 		batch1IDs, err := s.CreateKeys(batch1...)
@ -510,7 +503,7 @@ func TestTranslateStore_ReadWrite(t *testing.T) {
 }
 // MustOpenNewTranslateStore returns a new, opened TranslateStore.
-func MustOpenNewTranslateStore(tb testing.TB) *pilosa.BoltTranslateStore {
+func MustOpenNewTranslateStore(tb testing.TB) *boltdb.TranslateStore {
 	s := MustNewTranslateStore(tb)
 	if err := s.Open(); err != nil {
 		tb.Fatalf("opening s: %v", err)
@ -519,7 +512,7 @@ func MustOpenNewTranslateStore(tb testing.TB) *pilosa.BoltTranslateStore {
 }
 // MustCloseTranslateStore closes s and removes the underlying data file.
-func MustCloseTranslateStore(s *pilosa.BoltTranslateStore) {
+func MustCloseTranslateStore(s *boltdb.TranslateStore) {
 	if err := s.Close(); err != nil {
 		panic(err)
 	}
--- a/broadcast.go
+++ b/broadcast.go
@ -72,7 +72,6 @@ const (
 	messageTypeUNUSED2 // used to be ResizeNodeMessage
 	messageTypeUNUSED3 // used to be ResizeAbortMessage
 	messageTypeUpdateField
 	messageTypeDeleteDataframe
 )
 // MarshalInternalMessage serializes the pilosa message and adds pilosa internal
@ -118,8 +117,6 @@ func getMessage(typ byte) Message {
 		return &TransactionMessage{}
 	case messageTypeUpdateField:
 		return &UpdateFieldMessage{}
 	case messageTypeDeleteDataframe:
 		return &DeleteDataframeMessage{}
 	default:
 		panic(fmt.Sprintf("unknown message type %d", typ))
 	}
@ -157,8 +154,6 @@ func getMessageType(m Message) byte {
 		return messageTypeTransaction
 	case *UpdateFieldMessage:
 		return messageTypeUpdateField
 	case *DeleteDataframeMessage:
 		return messageTypeDeleteDataframe
 	default:
 		panic(fmt.Sprintf("don't have type for message %#v", m))
 	}
--- a/bsi.go
+++ b/bsi.go
@ -8,14 +8,14 @@ import (
 	"github.com/featurebasedb/featurebase/v3/roaring"
 )
-// BSIData contains BSI-structured data.
+// bsiData contains BSI-structured data.
-type BSIData []*Row
+type bsiData []*Row
-// PivotDescending loops over nonzero BSI values in descending order.
+// pivotDescending loops over nonzero BSI values in descending order.
 // For each value, the provided function is called with the value and a slice of the associated columns.
 // If limit or offset are not-nil, they will be applied.
 // Applying a limit or offset may modify the pointed-to value.
-func (bsi BSIData) PivotDescending(filter *Row, branch uint64, limit, offset *uint64, fn func(uint64, ...uint64)) {
+func (bsi bsiData) pivotDescending(filter *Row, branch uint64, limit, offset *uint64, fn func(uint64, ...uint64)) {
 	// This "pivot" algorithm works by treating the BSI data as a tree.
 	// Each branch of this tree corresponds to a power-of-2-sized range of BSI values.
 	// Each range is subdivided into 2 ranges of half size, which form lower branches.
@ -56,8 +56,8 @@ func (bsi BSIData) PivotDescending(filter *Row, branch uint64, limit, offset *ui
 		upperBranch, lowerBranch := branch|(1<<uint(len(bsi)-1)), branch
 		splitBit := bsi[len(bsi)-1]
 		lowerBits := bsi[:len(bsi)-1]
-		lowerBits.PivotDescending(filter.Intersect(splitBit), upperBranch, limit, offset, fn)
+		lowerBits.pivotDescending(filter.Intersect(splitBit), upperBranch, limit, offset, fn)
-		lowerBits.PivotDescending(filter.Difference(splitBit), lowerBranch, limit, offset, fn)
+		lowerBits.pivotDescending(filter.Difference(splitBit), lowerBranch, limit, offset, fn)
 	}
 }
@ -69,7 +69,7 @@ func (bsi BSIData) PivotDescending(filter *Row, branch uint64, limit, offset *ui
 // - TopN on int
 func (bsi bsiData) distribution(filter *Row) bsiData {
 	var dist bsiData
-	bsi.PivotDescending(filter, 0, nil, nil, func(count uint64, values ...uint64) {
+	bsi.pivotDescending(filter, 0, nil, nil, func(count uint64, values ...uint64) {
 		dist.insert(count, uint64(len(values)))
 	})
 	return dist
@ -78,20 +78,20 @@ func (bsi bsiData) distribution(filter *Row) bsiData {
 var placeholderBitmap = roaring.NewBitmap()
-// AddBSI adds two BSI bitmaps together.
+// addBSI adds two BSI bitmaps together.
 // It does not handle sign and has no concept of overflow.
-func AddBSI(x, y BSIData) BSIData {
+func addBSI(x, y bsiData) bsiData {
 	// Accumulate row segments.
-	segments := make([][]RowSegment, len(x)+len(y))
+	segments := make([][]rowSegment, len(x)+len(y))
 	xsegs, ysegs := segments[:len(x)], segments[len(x):]
 	for i, r := range x {
-		xsegs[i] = r.Segments
+		xsegs[i] = r.segments
 	}
 	for i, r := range y {
-		ysegs[i] = r.Segments
+		ysegs[i] = r.segments
 	}
-	var dst BSIData
+	var dst bsiData
 	var xbitmaps, ybitmaps []*roaring.Bitmap
 	for {
 		// Find the next shard.
@ -162,7 +162,7 @@ func AddBSI(x, y BSIData) BSIData {
 			for len(dst) <= i {
 				dst = append(dst, NewRow())
 			}
-			dst[i].Segments = append(dst[i].Segments, RowSegment{
+			dst[i].segments = append(dst[i].segments, rowSegment{
 				shard:    next,
 				writable: true,
 				data:     b,
@ -273,10 +273,10 @@ func (b *bsiBuilder) Insert(col, val uint64) {
 // Build BSI data.
 // This resets the builder.
-func (b *bsiBuilder) Build() BSIData {
+func (b *bsiBuilder) Build() bsiData {
 	builders := *b
 	*b = builders[:0]
-	rows := make(BSIData, len(builders))
+	rows := make(bsiData, len(builders))
 	for i := range builders {
 		rows[i] = builders[i].Build()
 	}
--- a/bsi_test.go
+++ b/bsi_test.go
@ -69,11 +69,11 @@ func TestBSIAdd(t *testing.T) {
 				builderB.Insert(uint64(id), vb)
 			}
 			dataA, dataB := builderA.Build(), builderB.Build()
-			dataC := AddBSI(dataA, dataB)
+			dataC := addBSI(dataA, dataB)
 			// build results from added bsiData; results[i] should hold a[i]+b[i]
 			results := make([]uint64, len(a))
-			dataC.PivotDescending(NewRow().Union(dataC...), 0, nil, nil, func(count uint64, ids ...uint64) {
+			dataC.pivotDescending(NewRow().Union(dataC...), 0, nil, nil, func(count uint64, ids ...uint64) {
 				for _, id := range ids {
 					results[idToIndex[int(id)]] = count
 				}
@ -141,10 +141,10 @@ func TestBSIAddCases(t *testing.T) {
 			}
 			dataA, dataB := builderA.Build(), builderB.Build()
-			dataC := AddBSI(dataA, dataB)
+			dataC := addBSI(dataA, dataB)
 			// maps id to count
 			results := make(map[uint64]uint64)
-			dataC.PivotDescending(NewRow().Union(dataC...), 0, nil, nil, func(count uint64, ids ...uint64) {
+			dataC.pivotDescending(NewRow().Union(dataC...), 0, nil, nil, func(count uint64, ids ...uint64) {
 				for _, id := range ids {
 					results[id] = count
 				}
--- a/buffer/filebuffer.go
+++ b/buffer/filebuffer.go
@ -1,110 +0,0 @@
 package buffer
 import (
 	"bytes"
 	"io"
 	"io/ioutil"
 	"os"
 	"sync"
 )
 // NewFileBuffer returns a file buffer which will use an in-memory buffer, until `max` bytes have been written, at which point it will write the contents of memory to a file, and continue writing future data to the file.
 // The file will be written to `temp` directory. The buffer fulfills the io.Reader and io.Writer interface
 func NewFileBuffer(max int, temp string) *FileBuffer {
 	return &FileBuffer{max: max, tempDir: temp}
 }
 type FileBuffer struct {
 	max     int
 	buf     bytes.Buffer
 	file    *os.File
 	tempDir string
 	reading bool
 	files   []*os.File
 	mu      sync.Mutex
 }
 func (fb *FileBuffer) Write(p []byte) (n int, err error) {
 	if fb.reading {
 		panic("cannot write after read")
 	}
 	if fb.file != nil {
 		return fb.file.Write(p)
 	}
 	n, err = fb.buf.Write(p)
 	if err != nil {
 		return
 	}
 	if fb.buf.Len() > fb.max {
 		fb.file, err = ioutil.TempFile(fb.tempDir, "filebuffer-")
 		if err != nil {
 			return
 		}
 		_, err = io.Copy(fb.file, &fb.buf)
 		fb.buf.Reset()
 	}
 	return
 }
 func (fb *FileBuffer) Len() (int64, error) {
 	if fb.file == nil {
 		return int64(fb.buf.Len()), nil
 	}
 	fi, err := fb.file.Stat()
 	if err != nil {
 		return 0, err
 	}
 	return fi.Size(), nil
 }
 func (fb *FileBuffer) Read(p []byte) (n int, err error) {
 	if fb.file != nil {
 		if !fb.reading {
 			fb.reading = true
 			_, err = fb.file.Seek(0, 0)
 			if err != nil {
 				return
 			}
 		}
 		return fb.file.Read(p)
 	}
 	fb.reading = true
 	return fb.buf.Read(p)
 }
 func (fb *FileBuffer) Close() error {
 	if fb.file != nil {
 		name := fb.file.Name()
 		if err := fb.file.Close(); err != nil {
 			return err
 		}
 		for _, f := range fb.files {
 			f.Close()
 		}
 		fb.files = fb.files[:0]
 		fb.file = nil
 		return os.Remove(name)
 	}
 	return nil
 }
 func (fb *FileBuffer) Reset() error {
 	fb.mu.Lock()
 	defer fb.mu.Unlock()
 	fb.reading = false
 	fb.buf.Reset()
 	return fb.Close()
 }
 func (fb *FileBuffer) NewReader() (io.Reader, error) {
 	fb.mu.Lock()
 	defer fb.mu.Unlock()
 	fb.reading = true
 	if fb.file == nil {
 		return bytes.NewReader(fb.buf.Bytes()), nil
 	}
 	f, err := os.OpenFile(fb.file.Name(), os.O_RDONLY, 0)
 	fb.files = append(fb.files, f)
 	return f, err
 }
--- a/bufferpool/bufferpool.go
+++ b/bufferpool/bufferpool.go
@ -1,262 +0,0 @@
 package bufferpool
 import (
 	"errors"
 	"fmt"
 	"sync"
 )
 // FrameID is the type for frame id
 type FrameID int
 // PageID is the type for page id
 type PageID int
 var pageSyncPool = sync.Pool{
 	New: func() any {
 		pg := new(Page)
 		pg.id = PageID(INVALID_PAGE)
 		pg.isDirty = false
 		pg.pinCount = 0
 		return pg
 	},
 }
 // BufferPool represents a buffer pool of pages
 type BufferPool struct {
 	// the underlying storage
 	diskManager DiskManager
 	// the actual pages in the buffer pool
 	pages []*Page
 	// the replacer that will elect replacements when buffer pool is full
 	replacer *ClockReplacer
 	// the list of free frames
 	freeList []FrameID
 	// the map of frames to page ids to frame ids
 	// frame ids are the offset into pages
 	// if you ask the pool for page 673, this will know at
 	// what offset in pages page 673 will exist
 	pageTable map[PageID]FrameID
 }
 // TODO(pok) implement a lazy writer
 //  * if free list is 'low' then
 //		* increase size of cache if there is physical memory available
 //		* write out old pages and boot them from the cache to increase free list
 // TODO(pok) implement a checkpoint that scans the pool and writes out dirty pages every
 // minute or so
 // NewBufferPool returns a buffer pool
 func NewBufferPool(maxSize int, diskManager DiskManager) *BufferPool {
 	freeList := make([]FrameID, 0)
 	pages := make([]*Page, maxSize)
 	for i := 0; i < maxSize; i++ {
 		frameNumber := FrameID(i)
 		freeList = append(freeList, frameNumber)
 	}
 	clockReplacer := NewClockReplacer(maxSize)
 	return &BufferPool{
 		diskManager: diskManager,
 		pages:       pages,
 		replacer:    clockReplacer,
 		freeList:    freeList,
 		pageTable:   make(map[PageID]FrameID),
 	}
 }
 // Dumps all the pages in the buffer pool
 func (b *BufferPool) Dump() {
 	fmt.Println()
 	fmt.Printf("------------------------------------------------------------------------------------------\n")
 	fmt.Printf("BUFFER POOL\n")
 	for _, p := range b.pages {
 		if p != nil {
 			p.Dump("")
 		}
 	}
 	fmt.Printf("------------------------------------------------------------------------------------------\n")
 	fmt.Println()
 }
 // FetchPage fetches the requested page from the buffer pool.
 func (b *BufferPool) FetchPage(pageID PageID) (*Page, error) {
 	// if it is in buffer pool already then just return it
 	if frameID, ok := b.pageTable[pageID]; ok {
 		page := b.pages[frameID]
 		page.pinCount++
 		b.replacer.Pin(frameID)
 		return page, nil
 	}
 	// not in the buffer pool so try the free list or
 	// the replacer will vote a page off the island
 	frameID, isFromFreeList, err := b.getFrameID()
 	if err != nil {
 		return nil, err
 	}
 	if !isFromFreeList {
 		// if it didn't come from the freelist then
 		// remove page from current frame, writing it out if dirty
 		currentPage := b.pages[frameID]
 		if currentPage != nil {
 			if currentPage.isDirty {
 				b.diskManager.WritePage(currentPage)
 			}
 			delete(b.pageTable, currentPage.id)
 		}
 	}
 	// if we got to here, sorry, have to do an I/O
 	page, err := b.diskManager.ReadPage(pageID)
 	if err != nil {
 		return nil, err
 	}
 	page.pinCount = 1
 	b.pageTable[pageID] = frameID
 	pageSyncPool.Put(b.pages[frameID])
 	b.pages[frameID] = page
 	b.replacer.Pin(frameID)
 	return page, nil
 }
 // UnpinPage unpins the target page from the buffer pool
 func (b *BufferPool) UnpinPage(pageID PageID) error {
 	if frameID, ok := b.pageTable[pageID]; ok {
 		page := b.pages[frameID]
 		page.DecPinCount()
 		if page.pinCount <= 0 {
 			b.replacer.Unpin(frameID)
 		}
 		return nil
 	}
 	return errors.New("could not find page")
 }
 // FlushPage Flushes the target page to disk
 func (b *BufferPool) FlushPage(pageID PageID) bool {
 	if frameID, ok := b.pageTable[pageID]; ok {
 		page := b.pages[frameID]
 		page.DecPinCount()
 		b.diskManager.WritePage(page)
 		page.isDirty = false
 		return true
 	}
 	return false
 }
 // NewPage allocates a new page in the buffer pool with the disk manager help
 func (b *BufferPool) NewPage() (*Page, error) {
 	// get a free frame
 	frameID, isFromFreeList, err := b.getFrameID()
 	if err != nil {
 		return nil, err
 	}
 	if !isFromFreeList {
 		// remove page from current frame
 		currentPage := b.pages[frameID]
 		if currentPage != nil {
 			if currentPage.isDirty {
 				b.diskManager.WritePage(currentPage)
 			}
 			delete(b.pageTable, currentPage.id)
 		}
 	}
 	// allocates new page
 	pageID, err := b.diskManager.AllocatePage()
 	if err != nil {
 		return nil, err
 	}
 	page := &Page{pageID, 1, false, [PAGE_SIZE]byte{}}
 	page.WritePageNumber(int32(pageID))
 	page.WriteFreeSpaceOffset(int16(PAGE_SIZE))
 	page.WriteNextPointer(int32(INVALID_PAGE))
 	page.WritePrevPointer(int32(INVALID_PAGE))
 	// update the frame table
 	b.pageTable[pageID] = frameID
 	pageSyncPool.Put(b.pages[frameID])
 	b.pages[frameID] = page
 	return page, nil
 }
 // ScratchPage returns a page outside the buffer pool - do not use if you intend the page
 // to be in the buffer pool (use NewPage() for that)
 // ScratchPage is intended to be used in cases where you need the Page primitives
 // and will copy the scratch page back over a real page later
 func (b *BufferPool) ScratchPage() *Page {
 	page := &Page{
 		id:       PageID(INVALID_PAGE),
 		pinCount: 0,
 		isDirty:  false,
 		data:     [PAGE_SIZE]byte{},
 	}
 	page.WritePageNumber(int32(INVALID_PAGE))
 	page.WriteFreeSpaceOffset(int16(PAGE_SIZE))
 	page.WriteNextPointer(int32(INVALID_PAGE))
 	page.WritePrevPointer(int32(INVALID_PAGE))
 	return page
 }
 // DeletePage deletes a page from the buffer pool
 func (b *BufferPool) DeletePage(pageID PageID) error {
 	var frameID FrameID
 	var ok bool
 	if frameID, ok = b.pageTable[pageID]; !ok {
 		return nil
 	}
 	page := b.pages[frameID]
 	if page.pinCount > 0 {
 		return errors.New("pin count greater than 0")
 	}
 	delete(b.pageTable, page.id)
 	b.replacer.Pin(frameID)
 	b.diskManager.DeallocatePage(pageID)
 	b.freeList = append(b.freeList, frameID)
 	return nil
 }
 // FlushAllpages flushes all the pages in the buffer pool to disk
 // Yeah, never call this unless you know what you are doing
 func (b *BufferPool) FlushAllpages() {
 	for pageID := range b.pageTable {
 		b.FlushPage(pageID)
 	}
 }
 func (b *BufferPool) getFrameID() (FrameID, bool, error) {
 	if len(b.freeList) > 0 {
 		frameID, newFreeList := b.freeList[0], b.freeList[1:]
 		b.freeList = newFreeList
 		return frameID, true, nil
 	}
 	victim, err := b.replacer.Victim()
 	return victim, false, err
 }
 // OnDiskSize exposes the on disk size of the backing store
 // behind this buffer pool
 func (b *BufferPool) OnDiskSize() int64 {
 	return b.diskManager.FileSize()
 }
 // Close closes the buffer pool
 func (b *BufferPool) Close() {
 	b.diskManager.Close()
 }
--- a/bufferpool/circularlist.go
+++ b/bufferpool/circularlist.go
@ -1,93 +0,0 @@
 package bufferpool
 import (
 	"errors"
 )
 type circularListNode struct {
 	key   interface{}
 	value interface{}
 	next  *circularListNode
 	prev  *circularListNode
 }
 type circularList struct {
 	head     *circularListNode
 	tail     *circularListNode
 	size     int
 	capacity int
 }
 func newCircularList(maxSize int) *circularList {
 	return &circularList{nil, nil, 0, maxSize}
 }
 func (c *circularList) find(key interface{}) *circularListNode {
 	ptr := c.head
 	for i := 0; i < c.size; i++ {
 		if ptr.key == key {
 			return ptr
 		}
 		ptr = ptr.next
 	}
 	return nil
 }
 func (c *circularList) hasKey(key interface{}) bool {
 	return c.find(key) != nil
 }
 func (c *circularList) insert(key interface{}, value interface{}) error {
 	if c.size == c.capacity {
 		return errors.New("list is full")
 	}
 	newNode := &circularListNode{key, value, nil, nil}
 	if c.size == 0 {
 		newNode.next = newNode
 		newNode.prev = newNode
 		c.head = newNode
 		c.tail = newNode
 		c.size++
 		return nil
 	}
 	node := c.find(key)
 	if node != nil {
 		node.value = value
 		return nil
 	}
 	newNode.next = c.head
 	newNode.prev = c.tail
 	c.tail.next = newNode
 	if c.head == c.tail {
 		c.head.next = newNode
 	}
 	c.tail = newNode
 	c.head.prev = c.tail
 	c.size++
 	return nil
 }
 func (c *circularList) remove(key interface{}) {
 	node := c.find(key)
 	if node == nil {
 		return
 	}
 	if c.size == 1 {
 		c.head = nil
 		c.tail = nil
 		c.size--
 		return
 	}
 	if node == c.head {
 		c.head = c.head.next
 	}
 	if node == c.tail {
 		c.tail = c.tail.prev
 	}
 	node.next.prev = node.prev
 	node.prev.next = node.next
 	c.size--
 }
--- a/bufferpool/clockreplacer.go
+++ b/bufferpool/clockreplacer.go
@ -1,64 +0,0 @@
 package bufferpool
 import "errors"
 // ClockReplacer implements a clock replacer algorithm
 type ClockReplacer struct {
 	cList     *circularList
 	clockHand **circularListNode
 }
 // NewClockReplacer instantiates a new clock replacer
 func NewClockReplacer(poolSize int) *ClockReplacer {
 	cList := newCircularList(poolSize)
 	return &ClockReplacer{cList, &cList.head}
 }
 // Victim removes the victim frame as defined by the replacement policy
 func (c *ClockReplacer) Victim() (FrameID, error) {
 	if c.cList.size == 0 {
 		return FrameID(INVALID_PAGE), errors.New("no victims available")
 	}
 	var victimFrameID FrameID
 	currentNode := (*c.clockHand)
 	for {
 		if currentNode.value.(bool) {
 			currentNode.value = false
 			c.clockHand = &currentNode.next
 		} else {
 			frameID := currentNode.key.(FrameID)
 			victimFrameID = frameID
 			c.clockHand = &currentNode.next
 			c.cList.remove(currentNode.key)
 			return victimFrameID, nil
 		}
 	}
 }
 // Unpin unpins a frame, indicating that it can now be victimized
 func (c *ClockReplacer) Unpin(id FrameID) {
 	if !c.cList.hasKey(id) {
 		c.cList.insert(id, true)
 		if c.cList.size == 1 {
 			c.clockHand = &c.cList.head
 		}
 	}
 }
 // Pin pins a frame, indicating that it should not be victimized until it is unpinned
 func (c *ClockReplacer) Pin(id FrameID) {
 	node := c.cList.find(id)
 	if node == nil {
 		return
 	}
 	if (*c.clockHand) == node {
 		c.clockHand = &(*c.clockHand).next
 	}
 	c.cList.remove(id)
 }
 // Size returns the size of the clock
 func (c *ClockReplacer) Size() int {
 	return c.cList.size
 }
--- a/bufferpool/diskmanager.go
+++ b/bufferpool/diskmanager.go
@ -1,21 +0,0 @@
 package bufferpool
 // DiskManager is responsible for interacting with disk
 type DiskManager interface {
 	// reads a page from the disk
 	ReadPage(PageID) (*Page, error)
 	// writes a page to the disk
 	WritePage(*Page) error
 	// allocates a page
 	AllocatePage() (PageID, error)
 	// deallocates a page
 	DeallocatePage(PageID) error
 	// returns on disk file size
 	FileSize() int64
 	// closes and does any clean up
 	Close()
 }
--- a/bufferpool/inmemdiskmanager.go
+++ b/bufferpool/inmemdiskmanager.go
@ -1,162 +0,0 @@
 package bufferpool
 import (
 	"errors"
 	"fmt"
 	"os"
 	uuid "github.com/satori/go.uuid"
 )
 // InMemDiskSpillingDiskManager is a memory implementation for a DiskManager interface
 // that can spill to disk when a threshold is reached
 type InMemDiskSpillingDiskManager struct {
 	// tracks the number of pages
 	numPages int
 	onDiskPages int
 	// tracks the number of pages we can consume before spilling
 	thresholdPages int
 	hasSpilled     *struct{}
 	fd             *os.File
 	// the data buffer
 	data []byte
 }
 // NewInMemDiskSpillingDiskManager returns a in-memory version of disk manager
 func NewInMemDiskSpillingDiskManager(thresholdPages int) *InMemDiskSpillingDiskManager {
 	dm := &InMemDiskSpillingDiskManager{
 		numPages:       0,
 		thresholdPages: thresholdPages,
 		data:           make([]byte, 0),
 	}
 	return dm
 }
 // ReadPage reads a page from pages
 func (d *InMemDiskSpillingDiskManager) ReadPage(pageID PageID) (*Page, error) {
 	// check we're not asking for page out of range
 	if pageID < 0 || int(pageID) >= d.numPages {
 		return nil, errors.New("page not found")
 	}
 	// check that the offset is within range
 	offset := int(pageID) * PAGE_SIZE
 	var page = pageSyncPool.Get().(*Page)
 	// we have to do this stupid check because if -cpuprofile is set for go test, this
 	// the previous line return a weird nil-ish thing...
 	if page == (*Page)(nil) {
 		page = pageSyncPool.New().(*Page)
 	}
 	page.id = pageID
 	// do the read
 	if d.hasSpilled == nil {
 		if offset+PAGE_SIZE > len(d.data) {
 			return nil, errors.New("offset out of range")
 		}
 		b := copy(page.data[:], d.data[offset:offset+PAGE_SIZE])
 		fmt.Printf("bytes read: %d", b)
 	} else {
 		var err error
 		if offset+PAGE_SIZE > d.numPages*PAGE_SIZE {
 			return nil, errors.New("offset out of range")
 		}
 		_, err = d.fd.ReadAt(page.data[:], int64(offset))
 		if err != nil {
 			return nil, err
 		}
 	}
 	return page, nil
 }
 // WritePage writes a page in memory to pages
 func (d *InMemDiskSpillingDiskManager) WritePage(page *Page) error {
 	// make sure the offset is sensible
 	offset := int(page.ID()) * PAGE_SIZE
 	// do the write
 	if d.hasSpilled == nil {
 		if offset+PAGE_SIZE > len(d.data) {
 			return errors.New("offset out of range")
 		}
 		copy(d.data[offset:], page.data[:])
 	} else {
 		var err error
 		if offset+PAGE_SIZE > d.numPages*PAGE_SIZE {
 			return errors.New("offset out of range")
 		}
 		_, err = d.fd.WriteAt(page.data[:], int64(offset))
 		if err != nil {
 			return err
 		}
 		// err = d.fd.Sync()
 		// if err != nil {
 		// 	return err
 		// }
 	}
 	return nil
 }
 // AllocatePage allocates a page and returns the page number
 func (d *InMemDiskSpillingDiskManager) AllocatePage() (PageID, error) {
 	d.numPages = d.numPages + 1
 	pageID := PageID(d.numPages - 1)
 	if d.hasSpilled == nil {
 		// we have not spilled (yet), so make storage bigger
 		newData := make([]byte, PAGE_SIZE)
 		d.data = append(d.data, newData...)
 		// check to see if we need to spill
 		if d.numPages > d.thresholdPages {
 			fileUUID, err := uuid.NewV4()
 			if err != nil {
 				return PageID(INVALID_PAGE), err
 			}
 			// TODO(pok) we should try to tell the OS not to cache this file
 			d.fd, err = os.CreateTemp("", fmt.Sprintf("fb-ehash-%s", fileUUID.String()))
 			if err != nil {
 				return PageID(INVALID_PAGE), err
 			}
 			_, err = d.fd.WriteAt(d.data, 0)
 			if err != nil {
 				return PageID(INVALID_PAGE), err
 			}
 			d.data = []byte{}
 			d.hasSpilled = &struct{}{}
 		}
 	} else {
 		if d.numPages >= d.onDiskPages {
 			// grow the file by a chunk - 512 pages
 			d.onDiskPages += 512
 			var err error
 			size := int64(d.onDiskPages * PAGE_SIZE)
 			_, err = d.fd.WriteAt([]byte{0}, size-1)
 			if err != nil {
 				return PageID(INVALID_PAGE), err
 			}
 		}
 	}
 	return pageID, nil
 }
 // DeallocatePage removes page from disk
 func (d *InMemDiskSpillingDiskManager) DeallocatePage(pageID PageID) error {
 	// nothing to do right now
 	return nil
 }
 func (d *InMemDiskSpillingDiskManager) FileSize() int64 {
 	return int64(len(d.data))
 }
 func (d *InMemDiskSpillingDiskManager) Close() {
 	// close and delete the file if we spilled
 	if d.fd != nil {
 		_ = d.fd.Close()
 		os.Remove(d.fd.Name())
 	}
 }
--- a/bufferpool/page.go
+++ b/bufferpool/page.go
@ -1,371 +0,0 @@
 package bufferpool
 import (
 	"encoding/binary"
 	"errors"
 	"fmt"
 )
 const PAGE_SIZE int = 8192
 const INVALID_PAGE int = -1
 const PAGE_TYPE_BTREE_INTERNAL = 10
 const PAGE_TYPE_BTREE_LEAF = 11
 const PAGE_TYPE_HASH_TABLE = 12
 // PAGE
 // page size 8192 bytes
 // byte aligned, big endian
 // |====================================================|
 // | offset | length        |                           |
 // |----------------------------------------------------|
 // | header                                             |
 // |====================================================|
 // | 0      | 4             |  pageNumber (int32)       |
 // | 4      | 2             |  pageType (int16)         |
 // | 6      | 2             |  slotCount (int16)        |
 // | 8      | 2             |  localDepth (int16)       |
 // | 10     | 2             |  freeSpaceOffset (int16)  |
 // | 12     | 4             |  prevPointer (int32)      |
 // | 16     | 4             |  nextPointer (int32)      |
 // |====================================================|
 // | <start of slot array 1..slotCount>                 |
 // |----------------------------------------------------|
 // | 20     | slotcount     |  slot entry is 2 int16    |
 // |        |  * slotwidth  |   values (payloadOffset,  |
 // |        |  * #slots     |   payloadLength)          |
 // |----------------------------------------------------|
 // | <free space>                                       |
 // |----------------------------------------------------|
 // | <payload starting at freeSpaceOffset>              |
 // |  payload entries are keylength (int16), key bytes, |
 // |  payload length (int32), payload bytes             |
 // |====================================================|
 const PAGE_NUMBER_OFFSET = 0        // offset 0, length 4, end 4
 const PAGE_TYPE_OFFSET = 4          // offset 4, length 2, end 6
 const PAGE_SLOT_COUNT_OFFSET = 6    // offset 6, length 2, end 8
 const PAGE_LOCAL_DEPTH_OFFSET = 8   // offset 8, length 2, end 10
 const PAGE_FREE_SPACE_OFFSET = 10   // offset 10, length 2, end 12
 const PAGE_PREV_POINTER_OFFSET = 12 // offset 12, length 4, end 16
 const PAGE_NEXT_POINTER_OFFSET = 16 // offset 16, length 4, end 20
 const PAGE_SLOTS_START_OFFSET = 20  // offset 20
 // PAGE_SLOT_LENGTH is the size of the page slot key/value.
 //
 // key offset int16    	//offset 0, length 2, end 2
 // value offset int16	    //offset 2, length 2, end 4
 const PAGE_SLOT_LENGTH = 4
 // Page represents a page on disk
 type Page struct {
 	id       PageID
 	pinCount int
 	isDirty  bool
 	data     [PAGE_SIZE]byte
 }
 type PageSlot struct {
 	KeyOffset   int16
 	ValueOffset int16
 }
 func (s *PageSlot) KeyBytes(page *Page) []byte {
 	offset := s.KeyOffset
 	keyLen := int16(binary.BigEndian.Uint16(page.data[offset:]))
 	offset += 2
 	result := make([]byte, keyLen)
 	copy(result, page.data[offset:offset+keyLen])
 	return result
 }
 func (s *PageSlot) KeyAsInt(page *Page) int32 {
 	return int32(binary.BigEndian.Uint32(page.data[s.KeyOffset+2:]))
 }
 func (s *PageSlot) ValueBytes(page *Page) []byte {
 	offset := s.ValueOffset
 	valueLen := int32(binary.BigEndian.Uint32(page.data[offset:]))
 	offset += 4
 	result := make([]byte, valueLen)
 	copy(result, page.data[offset:int32(offset)+valueLen])
 	return result
 }
 func (s *PageSlot) ValueAsPagePointer(page *Page) int32 {
 	return int32(binary.BigEndian.Uint32(page.data[s.ValueOffset+4:]))
 }
 type PageChunk struct {
 	KeyLength int16
 	KeyBytes  []byte
 	// TODO(pok) ValueBytes can be up to int32 long
 	// this requires an overflow page mechanism, that is not implemented
 	// yet, so be aware of this when storing stuff...
 	ValueLength int32
 	ValueBytes  []byte
 }
 func (pc *PageChunk) Length() int {
 	return 2 + len(pc.KeyBytes) + 4 + len(pc.ValueBytes)
 }
 func (pc *PageChunk) ComputeKeyOffset(pageOffset int) int {
 	return pageOffset
 }
 func (pc *PageChunk) ComputeValueOffset(pageOffset int) int {
 	return pageOffset + 2 + len(pc.KeyBytes)
 }
 func (p *Page) WritePageNumber(pageNumber int32) {
 	p.id = PageID(pageNumber)
 	binary.BigEndian.PutUint32(p.data[PAGE_NUMBER_OFFSET:], uint32(pageNumber))
 	p.isDirty = true
 }
 func (p *Page) ReadPageNumber() int {
 	return int(binary.BigEndian.Uint32(p.data[PAGE_NUMBER_OFFSET:]))
 }
 func (p *Page) WritePageType(pageType int16) {
 	binary.BigEndian.PutUint16(p.data[PAGE_TYPE_OFFSET:], uint16(pageType))
 	p.isDirty = true
 }
 func (p *Page) ReadPageType() int16 {
 	return int16(binary.BigEndian.Uint16(p.data[PAGE_TYPE_OFFSET:]))
 }
 func (p *Page) WriteSlotCount(slotCount int16) {
 	binary.BigEndian.PutUint16(p.data[PAGE_SLOT_COUNT_OFFSET:], uint16(slotCount))
 	p.isDirty = true
 }
 func (p *Page) ReadSlotCount() int16 {
 	return int16(binary.BigEndian.Uint16(p.data[PAGE_SLOT_COUNT_OFFSET:]))
 }
 func (p *Page) WriteLocalDepth(localDepth int16) {
 	binary.BigEndian.PutUint16(p.data[PAGE_LOCAL_DEPTH_OFFSET:], uint16(localDepth))
 	p.isDirty = true
 }
 func (p *Page) ReadLocalDepth() int16 {
 	return int16(binary.BigEndian.Uint16(p.data[PAGE_LOCAL_DEPTH_OFFSET:]))
 }
 func (p *Page) ReadSlot(slot int16) PageSlot {
 	offset := PAGE_SLOTS_START_OFFSET + PAGE_SLOT_LENGTH*slot
 	keyOffset := int16(binary.BigEndian.Uint16(p.data[offset:]))
 	offset += 2
 	valueOffset := int16(binary.BigEndian.Uint16(p.data[offset:]))
 	return PageSlot{
 		KeyOffset:   keyOffset,
 		ValueOffset: valueOffset,
 	}
 }
 func (p *Page) WriteSlot(slot int16, value PageSlot) {
 	offset := PAGE_SLOTS_START_OFFSET + PAGE_SLOT_LENGTH*slot
 	binary.BigEndian.PutUint16(p.data[offset:], uint16(value.KeyOffset))
 	offset += 2
 	binary.BigEndian.PutUint16(p.data[offset:], uint16(value.ValueOffset))
 }
 func (p *Page) WriteFreeSpaceOffset(offset int16) {
 	binary.BigEndian.PutUint16(p.data[PAGE_FREE_SPACE_OFFSET:], uint16(offset))
 	p.isDirty = true
 }
 func (p *Page) ReadFreeSpaceOffset() int16 {
 	return int16(binary.BigEndian.Uint16(p.data[PAGE_FREE_SPACE_OFFSET:]))
 }
 func (p *Page) WritePrevPointer(prevPointer int32) {
 	binary.BigEndian.PutUint32(p.data[PAGE_PREV_POINTER_OFFSET:], uint32(prevPointer))
 	p.isDirty = true
 }
 func (p *Page) ReadPrevPointer() int {
 	return int(binary.BigEndian.Uint32(p.data[PAGE_PREV_POINTER_OFFSET:]))
 }
 func (p *Page) WriteNextPointer(nextPointer int32) {
 	binary.BigEndian.PutUint32(p.data[PAGE_NEXT_POINTER_OFFSET:], uint32(nextPointer))
 	p.isDirty = true
 }
 func (p *Page) ReadNextPointer() int {
 	return int(binary.BigEndian.Uint32(p.data[PAGE_NEXT_POINTER_OFFSET:]))
 }
 func (p *Page) WriteChunk(offset int16, chunk PageChunk) {
 	binary.BigEndian.PutUint16(p.data[offset:], uint16(chunk.KeyLength))
 	offset += 2
 	copy(p.data[offset:], chunk.KeyBytes)
 	offset += int16(len(chunk.KeyBytes))
 	binary.BigEndian.PutUint32(p.data[offset:], uint32(chunk.ValueLength))
 	offset += 4
 	copy(p.data[offset:], chunk.ValueBytes)
 	p.isDirty = true
 }
 func (p *Page) ReadChunk(offset int16) PageChunk {
 	keyLen := int16(binary.BigEndian.Uint16(p.data[offset:]))
 	offset += 2
 	keyBytes := make([]byte, keyLen)
 	copy(keyBytes, p.data[offset:offset+keyLen])
 	offset += keyLen
 	valueLen := int32(binary.BigEndian.Uint32(p.data[offset:]))
 	offset += 4
 	valueBytes := make([]byte, valueLen)
 	copy(valueBytes, p.data[offset:int32(offset)+valueLen])
 	return PageChunk{
 		KeyLength:   keyLen,
 		KeyBytes:    keyBytes,
 		ValueLength: valueLen,
 		ValueBytes:  valueBytes,
 	}
 }
 func (p *Page) FreeSpace() int16 {
 	freeSpaceOffset := p.ReadFreeSpaceOffset()
 	freespace := freeSpaceOffset - (p.ReadSlotCount()*PAGE_SLOT_LENGTH + PAGE_SLOT_LENGTH + PAGE_SLOTS_START_OFFSET)
 	return freespace
 }
 func (p *Page) WriteKeyValueInSlot(slotNumber int16, key []byte, value []byte) error {
 	freeSpaceOffset := p.ReadFreeSpaceOffset()
 	// build a chunk
 	chunk := PageChunk{
 		KeyLength:   int16(len(key)),
 		KeyBytes:    key,
 		ValueLength: int32(len(value)),
 		ValueBytes:  value,
 	}
 	// compute the new free space offset
 	freeSpaceOffset -= int16(chunk.Length())
 	// check we won't blow free space on page
 	slotCount := p.ReadSlotCount()
 	slotEndOffset := slotCount*PAGE_SLOT_LENGTH + PAGE_SLOT_LENGTH + PAGE_SLOTS_START_OFFSET
 	// DEBUG!!
 	//fmt.Printf("freeSpaceOffset: %d, slotCount: %d, slotCount*4 + 4 + 20: %d, freeSpace: %d\n", freeSpaceOffset, slotCount, slotEndOffset, freeSpaceOffset-slotEndOffset)
 	if freeSpaceOffset-slotEndOffset <= 0 {
 		return errors.New("page is full")
 	}
 	keyOffset := chunk.ComputeKeyOffset(int(freeSpaceOffset))
 	valueOffset := chunk.ComputeValueOffset(int(freeSpaceOffset))
 	p.WriteChunk(freeSpaceOffset, chunk)
 	// update the free space offset
 	p.WriteFreeSpaceOffset(int16(freeSpaceOffset))
 	// make a slot
 	slot := PageSlot{
 		KeyOffset:   int16(keyOffset),
 		ValueOffset: int16(valueOffset),
 	}
 	// write the slot
 	p.WriteSlot(slotNumber, slot)
 	return nil
 }
 func (p *Page) WritePage(page *Page) {
 	// copy everything but pageNumber & pageType
 	offset := PAGE_SLOT_COUNT_OFFSET
 	copy(page.data[offset:], p.data[offset:offset+PAGE_SIZE-offset])
 }
 func (p *Page) PinCount() int {
 	return p.pinCount
 }
 func (p *Page) ID() PageID {
 	return p.id
 }
 func (p *Page) DecPinCount() {
 	if p.pinCount > 0 {
 		p.pinCount--
 	}
 }
 type PageSlotIterator struct {
 	page      *Page
 	slotCount int16
 	cursor    int16
 }
 func NewPageSlotIterator(page *Page, fromSlot int16) *PageSlotIterator {
 	i := &PageSlotIterator{
 		page:      page,
 		slotCount: page.ReadSlotCount(),
 		cursor:    fromSlot,
 	}
 	return i
 }
 func (i *PageSlotIterator) Next() *PageSlot {
 	if i.cursor < i.slotCount {
 		s := i.page.ReadSlot(i.cursor)
 		i.cursor++
 		return &s
 	}
 	return nil
 }
 func (i *PageSlotIterator) Cursor() int16 {
 	return i.cursor
 }
 func (pg *Page) Dump(label string) {
 	indent := 0
 	if len(label) > 0 {
 		fmt.Printf("%s%s:\n", fmt.Sprintf("%*s", indent, ""), label)
 		indent += 4
 	}
 	pageType := pg.ReadPageType()
 	fmt.Printf("%sPAGE(%d) pageType: %d slotCount: %d, prevPtr: %d, nextPtr: %d\n", fmt.Sprintf("%*s", indent, ""), pg.ID(), pageType, pg.ReadSlotCount(), pg.ReadPrevPointer(), pg.ReadNextPointer())
 	fmt.Printf("%sKEYS: -->\n", fmt.Sprintf("%*s", indent, ""))
 	indent += 4
 	// get the keys off the page
 	keys := make([]int, 0)
 	pointers := make([]int, 0)
 	iter := NewPageSlotIterator(pg, 0)
 	for {
 		ps := iter.Next()
 		if ps == nil {
 			break
 		}
 		keys = append(keys, int(ps.KeyAsInt(pg)))
 		if pageType == /*nodeTypeInternal*/ 10 {
 			pointers = append(pointers, int(ps.ValueAsPagePointer(pg)))
 		}
 	}
 	if pageType == /*nodeTypeLeaf*/ 11 {
 		for _, key := range keys {
 			fmt.Printf("%s(%d)\n", fmt.Sprintf("%*s", indent, ""), key)
 		}
 	} else {
 		for idx, key := range keys {
 			ptr := pointers[idx]
 			fmt.Printf("%s(%d, %d)\n", fmt.Sprintf("%*s", indent, ""), key, ptr)
 		}
 		ptr := pg.ReadNextPointer()
 		fmt.Printf("%s(-->, %d)\n", fmt.Sprintf("%*s", indent, ""), ptr)
 	}
 }
--- a/cache.go
+++ b/cache.go
@ -13,6 +13,7 @@ import (
 	"github.com/featurebasedb/featurebase/v3/lru"
 	pb "github.com/featurebasedb/featurebase/v3/proto"
 	"github.com/featurebasedb/featurebase/v3/stats"
 	"github.com/pkg/errors"
 )
@ -40,6 +41,9 @@ type cache interface {
 	// Returns an ordered list of the top ranked bitmaps.
 	Top() []bitmapPair
 	// SetStats defines the stats client used in the cache.
 	SetStats(s stats.StatsClient)
 	// Clear removes everything from the cache. If possible it should leave allocated structures in place to be reused.
 	Clear()
 }
@ -48,6 +52,7 @@ type cache interface {
 type lruCache struct {
 	cache  *lru.Cache
 	counts map[uint64]uint64
 	stats  stats.StatsClient
 	// maxEntries is saved to support Clear which recreates the cache.
 	maxEntries uint32
 }
@ -57,6 +62,7 @@ func newLRUCache(maxEntries uint32) *lruCache {
 	c := &lruCache{
 		cache:      lru.New(int(maxEntries)),
 		counts:     make(map[uint64]uint64),
 		stats:      stats.NopStatsClient,
 		maxEntries: maxEntries,
 	}
 	c.cache.OnEvicted = c.onEvicted
@ -114,6 +120,11 @@ func (c *lruCache) Top() []bitmapPair {
 	return a
 }
 // SetStats defines the stats client used in the cache.
 func (c *lruCache) SetStats(s stats.StatsClient) {
 	c.stats = s
 }
 func (c *lruCache) Clear() {
 	for k := range c.counts {
 		delete(c.counts, k)
@ -147,6 +158,8 @@ type rankCache struct {
 	// thresholdValue is the value of the last item in the cache
 	thresholdValue uint64
 	stats stats.StatsClient
 }
 // NewRankCache returns a new instance of RankCache.
@ -155,6 +168,7 @@ func NewRankCache(maxEntries uint32) *rankCache {
 		maxEntries:      maxEntries,
 		thresholdBuffer: int(thresholdFactor * float64(maxEntries)),
 		entries:         make(map[uint64]uint64),
 		stats:           stats.NopStatsClient,
 	}
 }
@ -215,7 +229,7 @@ func (c *rankCache) BulkAdd(id uint64, n uint64) {
 	// as this can take up an upbounded amount of memory. This is especially
 	// true when restoring shards as all rows will be added.
 	if len(c.entries) > int(2*c.maxEntries) {
-		CounterRecalculateCache.Inc()
+		c.stats.Count(MetricRecalculateCache, 1, 1.0)
 		c.recalculate()
 	}
 }
@ -260,7 +274,7 @@ func (c *rankCache) Invalidate() {
 func (c *rankCache) Recalculate() {
 	c.mu.Lock()
 	defer c.mu.Unlock()
-	CounterRecalculateCache.Inc()
+	c.stats.Count(MetricRecalculateCache, 1, 1.0)
 	c.recalculate()
 }
@ -272,12 +286,12 @@ func (c *rankCache) invalidate() {
 		// This is somewhat necessary for now since recalculation is not cheap.
 		// The cache will remain flagged as dirty and will be recalculated if Top is called.
 		// This may cause unexpected memory growth, so record it in metrics for debugging purposes.
-		CounterInvalidateCacheSkipped.Inc()
+		c.stats.Count(MetricInvalidateCacheSkipped, 1, 1.0)
 		// Ensure that we're marked as dirty even if we weren't otherwise.
 		c.dirty = true
 		return
 	}
-	CounterInvalidateCache.Inc()
+	c.stats.Count(MetricInvalidateCache, 1, 1.0)
 	c.recalculate()
 }
@ -303,7 +317,7 @@ func (c *rankCache) recalculate() {
 	// Store the count of the item at the threshold index.
 	length := len(c.rankings)
-	GaugeRankCacheLength.Set(float64(length))
+	c.stats.Gauge(MetricRankCacheLength, float64(length), 1.0)
 	var removeItems []bitmapPair // cached, ordered list
 	if length > int(c.maxEntries) {
@ -319,7 +333,7 @@ func (c *rankCache) recalculate() {
 	// If size is larger than the threshold then trim it.
 	if len(c.entries) > c.thresholdBuffer {
-		CounterCacheThresholdReached.Inc()
+		c.stats.Count(MetricCacheThresholdReached, 1, 1.0)
 		for _, pair := range removeItems {
 			delete(c.entries, pair.ID)
 		}
@ -329,6 +343,11 @@ func (c *rankCache) recalculate() {
 	c.dirty = false
 }
 // SetStats defines the stats client used in the cache.
 func (c *rankCache) SetStats(s stats.StatsClient) {
 	c.stats = s
 }
 // Top returns an ordered list of pairs.
 func (c *rankCache) Top() []bitmapPair {
 	c.mu.Lock()
@ -336,7 +355,7 @@ func (c *rankCache) Top() []bitmapPair {
 	if c.dirty {
 		// The cache is dirty, so we need to recalculate it to get a consistent view.
-		CounterReadDirtyCache.Inc()
+		c.stats.Count(MetricReadDirtyCache, 1, 1.0)
 		c.recalculate()
 	}
@ -587,21 +606,25 @@ func (p uint64Slice) Len() int           { return len(p) }
 func (p uint64Slice) Less(i, j int) bool { return p[i] < p[j] }
 // nopCache represents a no-op Cache implementation.
-type nopCache struct{}
+type nopCache struct {
 	stats stats.StatsClient
 }
 // Ensure NopCache implements Cache.
-var globalNopCache cache = nopCache{}
+var globalNopCache cache = nopCache{
 	stats: stats.NopStatsClient,
 }
 func (c nopCache) Add(uint64, uint64)     {}
 func (c nopCache) BulkAdd(uint64, uint64) {}
 func (c nopCache) Get(uint64) uint64      { return 0 }
 func (c nopCache) IDs() []uint64          { return []uint64{} }
-func (c nopCache) Invalidate()  {}
+func (c nopCache) Invalidate()                {}
-func (c nopCache) Len() int     { return 0 }
+func (c nopCache) Len() int                   { return 0 }
-func (c nopCache) Recalculate() {}
+func (c nopCache) Recalculate()               {}
-
+func (c nopCache) SetStats(stats.StatsClient) {}
-func (c nopCache) Clear() {}
+func (c nopCache) Clear()                     {}
 func (c nopCache) Top() []bitmapPair {
 	return []bitmapPair{}
--- a/cache_test.go
+++ b/cache_test.go
@ -62,7 +62,7 @@ func TestCache_Rank_Dirty(t *testing.T) {
 		cache.Add(v.ID, v.Count)
 	}
-	var got []pair //nolint:prealloc
+	var got []pair
 	for _, p := range cache.Top() {
 		got = append(got, pair(p))
 	}
--- a/catcher.go
+++ b/catcher.go
@ -27,6 +27,10 @@ func init() {
 var _ Tx = (*catcherTx)(nil)
 func (c *catcherTx) NewTxIterator(index, field, view string, shard uint64) *roaring.Iterator {
 	return c.b.NewTxIterator(index, field, view, shard)
 }
 func (c *catcherTx) ImportRoaringBits(index, field, view string, shard uint64, rit roaring.RoaringIterator, clear bool, log bool, rowSize uint64) (changed int, rowSet map[uint64]int, err error) {
 	defer func() {
 		if r := recover(); r != nil {
@ -124,17 +128,6 @@ func (c *catcherTx) Remove(index, field, view string, shard uint64, a ...uint64)
 	return c.b.Remove(index, field, view, shard, a...)
 }
 func (c *catcherTx) Removed(index, field, view string, shard uint64, a ...uint64) (changed []uint64, err error) {
 	defer func() {
 		if r := recover(); r != nil {
 			vprint.AlwaysPrintf("see Removed() PanicOn '%v' at '%v'", r, vprint.Stack())
 			vprint.PanicOn(r)
 		}
 	}()
 	return c.b.Removed(index, field, view, shard, a...)
 }
 func (c *catcherTx) Contains(index, field, view string, shard uint64, key uint64) (exists bool, err error) {
 	defer func() {
@ -157,6 +150,28 @@ func (c *catcherTx) ContainerIterator(index, field, view string, shard uint64, f
 	return c.b.ContainerIterator(index, field, view, shard, firstRoaringContainerKey)
 }
 func (c *catcherTx) ForEach(index, field, view string, shard uint64, fn func(i uint64) error) error {
 	defer func() {
 		if r := recover(); r != nil {
 			vprint.AlwaysPrintf("see ForEach() PanicOn '%v' at '%v'", r, vprint.Stack())
 			vprint.PanicOn(r)
 		}
 	}()
 	return c.b.ForEach(index, field, view, shard, fn)
 }
 func (c *catcherTx) ForEachRange(index, field, view string, shard uint64, start, end uint64, fn func(uint64) error) error {
 	defer func() {
 		if r := recover(); r != nil {
 			vprint.AlwaysPrintf("see ForEachRange() PanicOn '%v' at '%v'", r, vprint.Stack())
 			vprint.PanicOn(r)
 		}
 	}()
 	return c.b.ForEachRange(index, field, view, shard, start, end, fn)
 }
 func (c *catcherTx) Count(index, field, view string, shard uint64) (uint64, error) {
 	defer func() {
@ -228,6 +243,6 @@ func (c *catcherTx) GetSortedFieldViewList(idx *Index, shard uint64) (fvs []txke
 	return c.b.GetSortedFieldViewList(idx, shard)
 }
-func (c *catcherTx) GetFieldSizeBytes(index, field string) (uint64, error) {
+func (tx *catcherTx) GetFieldSizeBytes(index, field string) (uint64, error) {
 	return 0, nil
 }
--- a/cli/Makefile
+++ b/cli/Makefile
@ -1,15 +0,0 @@
 .PHONY: test testv test-integration testv-integration
 GO=go
 test:
 	$(GO) test ./... -short
 testv:
 	$(GO) test -v ./... -short
 test-integration:
 	$(GO) test . -count 1 -timeout 20m -run TestCLIIntegration/$(RUN)
 testv-integration:
 	$(GO) test -v . -count 1 -timeout 20m -run TestCLIIntegration/$(RUN)
--- a/cli/batch/inserter.go
+++ b/cli/batch/inserter.go
@ -1,8 +0,0 @@
 package batch
 // Inserter can be implemented by anything which can handle a SQL statement
 // representing a write operation. An example is `BULK INSERT`. The Insert()
 // method on this interface does not return any results other than an error.
 type Inserter interface {
 	Insert(sql string) error
 }
--- a/cli/batch/sql.go
+++ b/cli/batch/sql.go
@ -1,215 +0,0 @@
 package batch
 import (
 	"encoding/json"
 	"fmt"
 	"strings"
 	"time"
 	fbbatch "github.com/featurebasedb/featurebase/v3/batch"
 	"github.com/featurebasedb/featurebase/v3/dax"
 	"github.com/featurebasedb/featurebase/v3/errors"
 	"github.com/featurebasedb/featurebase/v3/pql"
 )
 // Ensure type implements interface.
 var _ fbbatch.Batcher = (*sqlBatcher)(nil)
 type sqlBatcher struct {
 	inserter Inserter
 	fields   []*dax.Field
 }
 func NewSQLBatcher(i Inserter, flds []*dax.Field) *sqlBatcher {
 	return &sqlBatcher{
 		inserter: i,
 		fields:   flds,
 	}
 }
 func (b *sqlBatcher) NewBatch(cfg fbbatch.Config, tbl *dax.Table, flds []*dax.Field) (fbbatch.RecordBatch, error) {
 	fields := flds
 	if b.fields != nil {
 		fields = b.fields
 	}
 	return &sqlBatch{
 		table:        tbl,
 		fields:       fields,
 		size:         cfg.Size,
 		maxStaleness: cfg.MaxStaleness,
 		ids:          make([]interface{}, 0, cfg.Size),
 		rows:         make([][]interface{}, 0, cfg.Size),
 		inserter:     b.inserter,
 	}, nil
 }
 // Ensure type implements interface.
 var _ fbbatch.RecordBatch = (*sqlBatch)(nil)
 type sqlBatch struct {
 	table  *dax.Table
 	fields []*dax.Field
 	size   int
 	ids  []interface{}
 	rows [][]interface{}
 	// staleTime tracks the time the first record of the batch was inserted
 	// plus the maxStaleness, in order to raise ErrBatchNowStale if the
 	// maxStaleness has elapsed
 	staleTime    time.Time
 	maxStaleness time.Duration
 	// inserter handles SQL INSERT statements generated for each batch.
 	inserter Inserter
 }
 func (b *sqlBatch) Add(rec fbbatch.Row) error {
 	// Clear rec.Values and rec.Clears upon return.
 	defer func() {
 		for i := range rec.Values {
 			rec.Values[i] = nil
 		}
 		for k := range rec.Clears {
 			delete(rec.Clears, k)
 		}
 	}()
 	if len(b.ids) == cap(b.ids) {
 		return fbbatch.ErrBatchAlreadyFull
 	}
 	if len(rec.Values) != len(b.fields) {
 		return errors.Errorf("record needs to match up with batch fields, got %d fields and %d record", len(b.fields), len(rec.Values))
 	}
 	// Append the ID to b.ids.
 	b.ids = append(b.ids, rec.ID)
 	// Convert decimal fields (which come in as int64, along with the scale in
 	// field) to pql.Decimal.
 	for i, fld := range b.fields {
 		switch b.fields[i].Type {
 		case dax.BaseTypeDecimal:
 			if val, ok := rec.Values[i].(int64); ok {
 				rec.Values[i] = pql.NewDecimal(val, fld.Options.Scale)
 			}
 		case dax.BaseTypeTimestamp:
 			if val, ok := rec.Values[i].(int64); ok {
 				ts := time.Unix(val, 0)
 				rec.Values[i] = ts.Format(time.RFC3339)
 			}
 		}
 	}
 	// Append the record to b.rows.
 	vals := make([]interface{}, 0, len(rec.Values))
 	vals = append(vals, rec.Values...)
 	b.rows = append(b.rows, vals)
 	// Check for batch full or stale.
 	if len(b.ids) == cap(b.ids) {
 		return fbbatch.ErrBatchNowFull
 	}
 	if b.maxStaleness != time.Duration(0) { // set maxStaleness to 0 to disable staleness checking
 		if len(b.ids) == 1 {
 			b.staleTime = time.Now().Add(b.maxStaleness)
 		} else if time.Now().After(b.staleTime) {
 			return fbbatch.ErrBatchNowStale
 		}
 	}
 	return nil
 }
 func (b *sqlBatch) Import() error {
 	if len(b.rows) == 0 {
 		return nil
 	}
 	// Construct the BULK INSERT statement based on the table and fields.
 	sql, err := buildBulkInsert(b.table, b.fields, b.ids, b.rows)
 	if err != nil {
 		return errors.Wrap(err, "building bulk insert statement")
 	}
 	// Reset batch data.
 	b.reset()
 	// Submit the SQL statement.
 	return b.inserter.Insert(sql)
 }
 func (b *sqlBatch) reset() {
 	b.ids = b.ids[:0]
 	b.rows = b.rows[:0]
 }
 func (b *sqlBatch) Len() int {
 	return len(b.rows)
 }
 func (b *sqlBatch) Flush() error {
 	return nil
 }
 func buildBulkInsert(tbl *dax.Table, fields []*dax.Field, ids []interface{}, rows [][]interface{}) (string, error) {
 	// Validation.
 	if tbl.Name == "" {
 		return "", errors.New(errors.ErrUncoded, "table name is required")
 	} else if len(fields) == 0 {
 		return "", errors.New(errors.ErrUncoded, "at least one field is required")
 	}
 	var sb strings.Builder
 	sb.WriteString(`BULK INSERT INTO `)
 	sb.WriteString(string(tbl.Name))
 	sb.WriteString(` (_id,`)
 	flds := make([]string, 0, len(fields))
 	maps := make([]string, 0, len(fields))
 	for i := range fields {
 		flds = append(flds, string(fields[i].Name))
 		maps = append(maps, fmt.Sprintf("'$.col_%d' %s", i, fields[i].FullType()))
 	}
 	// Fields
 	sb.WriteString(strings.Join(flds, ","))
 	// MAP
 	keyType := dax.BaseTypeID
 	if tbl.StringKeys() {
 		keyType = dax.BaseTypeString
 	}
 	sb.WriteString(`) MAP ('$._id' `)
 	sb.WriteString(keyType)
 	sb.WriteString(`,`)
 	sb.WriteString(strings.Join(maps, ","))
 	sb.WriteString(`) FROM x'`)
 	// Row values.
 	// m is a map representing a single row to be marshalled and added to the
 	// bulk insert as one line in the NDJSON payload. We re-use the map for each
 	// row.
 	m := make(map[string]interface{})
 	for i := range rows {
 		// Write the ID value.
 		m[string(dax.PrimaryKeyFieldName)] = ids[i]
 		// Write the rest of the data values.
 		for col := range rows[i] {
 			m[fmt.Sprintf("col_%d", col)] = rows[i][col]
 		}
 		// Marshal the map to json and add to the sql statement.
 		if j, err := json.Marshal(m); err != nil {
 			return "", errors.Wrap(err, "marshalling row to json")
 		} else {
 			sb.Write(j)
 			sb.WriteString("\n")
 		}
 	}
 	// WITH
 	sb.WriteString(fmt.Sprintf(`' WITH BATCHSIZE %d FORMAT 'NDJSON' INPUT 'STREAM'`, len(rows)))
 	return sb.String(), nil
 }
--- a/cli/batch/sql_test.go
+++ b/cli/batch/sql_test.go
@ -1,47 +0,0 @@
 package batch
 import (
 	"testing"
 	"github.com/featurebasedb/featurebase/v3/dax"
 	"github.com/stretchr/testify/assert"
 )
 func TestBatchSQL(t *testing.T) {
 	tbl := &dax.Table{
 		Name: "foo",
 	}
 	fields := []*dax.Field{
 		{
 			Name: "name",
 			Type: dax.BaseTypeString,
 		},
 		{
 			Name: "age",
 			Type: dax.BaseTypeInt,
 		},
 	}
 	ids := []interface{}{
 		0, 1, 2,
 	}
 	rows := [][]interface{}{
 		{
 			[]interface{}{"Alice", int64(11)},
 		},
 		{
 			[]interface{}{"Bob", int64(22)},
 		},
 		{
 			[]interface{}{"Carl,Comma", int64(33)},
 		},
 	}
 	s, err := buildBulkInsert(tbl, fields, ids, rows)
 	assert.NoError(t, err)
 	exp := `BULK INSERT INTO foo (_id,name,age) MAP ('$._id' id,'$.col_0' string,'$.col_1' int) FROM x'{"_id":0,"col_0":["Alice",11]}
 {"_id":1,"col_0":["Bob",22]}
 {"_id":2,"col_0":["Carl,Comma",33]}
 ' WITH BATCHSIZE 3 FORMAT 'NDJSON' INPUT 'STREAM'`
 	assert.Equal(t, exp, s)
 }
--- a/cli/buffer.go
+++ b/cli/buffer.go
@ -1,91 +0,0 @@
 package cli
 import (
 	"io"
 	"strings"
 	"github.com/featurebasedb/featurebase/v3/errors"
 )
 // buffer is a query buffer for SQL statements. Note that this is not a query
 // buffer as you would find on a database server (buffering query results).
 // Rather, this buffers the working SQL statement. The buffer has two
 // components: the buffer of query parts making up the working, incomplete SQL
 // statement, and the last completed SQL statement submitted to the Queryer.
 type buffer struct {
 	parts     []queryPart
 	lastQuery query
 	hasBatchFile bool
 }
 func newBuffer() *buffer {
 	return &buffer{}
 }
 // addPart adds the given queryPart to the buffer. If the part is of type
 // `partTerminator` (which is generally singified in the CLI by a ";"), the
 // buffer will finalize the query and return it. In all other cases, the
 // returned query is nil.
 func (b *buffer) addPart(part queryPart) (query, error) {
 	// Check for part type compatibility. For example, multiple batchFile parts
 	// are not allowed in the same query.
 	switch part.(type) {
 	case *partBatchFile:
 		if b.hasBatchFile {
 			return nil, errors.Errorf("multiple batch files in one query is not supported")
 		}
 		b.hasBatchFile = true
 	case *partTerminator:
 		return b.finalize(), nil
 	}
 	b.parts = append(b.parts, part)
 	return nil, nil
 }
 // finalize copies the contents (queryParts) of buffer to lastQuery and then
 // resets the buffer. It returns the query that was finalized.
 func (b *buffer) finalize() query {
 	q := make(query, len(b.parts))
 	copy(q, b.parts)
 	b.lastQuery = q
 	b.reset()
 	return q
 }
 // print returns the contents of the buffer as a string. This is generally used
 // to visually inspect the state of the buffer (for example, when a user issues
 // a `\p` meta-command in the CLI).
 func (b *buffer) print() string {
 	if len(b.parts) > 0 {
 		return query(b.parts).String()
 	} else if b.lastQuery != nil {
 		return b.lastQuery.String() + ";"
 	}
 	return "Query buffer is empty."
 }
 // reset clears the buffer. It returns a message which may optionally be used to
 // display to a user.
 func (b *buffer) reset() string {
 	b.parts = b.parts[:0]
 	b.hasBatchFile = false
 	return "Query buffer reset (cleared)."
 }
 func (b *buffer) Reader() io.Reader {
 	if len(b.parts) > 0 {
 		return query(b.parts).Reader()
 	} else if b.lastQuery != nil {
 		r := b.lastQuery.Reader()
 		// TODO(tlt): terminating the query here results in a line feed just
 		// before the semi-colon (for example, when you print out the query
 		// buffer using `\w [FILE]`). The removal and re-introduction of line
 		// feeds is kind of a mess.
 		term := strings.NewReader(";")
 		return io.MultiReader(r, term)
 	}
 	return strings.NewReader("")
 }
--- a/cli/cli.go
+++ b/cli/cli.go
@ -1,816 +0,0 @@
 // Package cli contains a FeatureBase command line interface.
 package cli
 import (
 	"context"
 	"fmt"
 	"io"
 	"net/http"
 	"os"
 	"path/filepath"
 	"strings"
 	"time"
 	"github.com/chzyer/readline"
 	featurebase "github.com/featurebasedb/featurebase/v3"
 	"github.com/featurebasedb/featurebase/v3/cli/batch"
 	"github.com/featurebasedb/featurebase/v3/cli/fbcloud"
 	"github.com/featurebasedb/featurebase/v3/errors"
 	"github.com/featurebasedb/featurebase/v3/logger"
 )
 const (
 	defaultHost     string = "localhost"
 	defaultClientID string = "6i2gs7mu215ab23cnvmshdoq6t" // production Cognito client ID
 	defaultRegion   string = "us-east-2"
 	terminationChar string = ";"
 	nullValue       string = "NULL"
 )
 var (
 	Stdin  io.ReadCloser = os.Stdin
 	Stdout io.Writer     = os.Stdout
 	Stderr io.Writer     = os.Stderr
 )
 var splash string = fmt.Sprintf(`FeatureBase CLI (%s)
 Type "\q" to quit.
 `, featurebase.Version)
 // Ensure type implments interfaces.
 var _ printer = (*Command)(nil)
 var _ batch.Inserter = (*Command)(nil)
 type Command struct {
 	host string
 	port string
 	splitter   *splitter
 	buffer     *buffer
 	workingDir *workingDir
 	organizationID string
 	database       string
 	databaseID     string
 	databaseName   string
 	Queryer Queryer `json:"-"`
 	stdin  io.ReadCloser `json:"-"`
 	stdout io.Writer     `json:"-"`
 	stderr io.Writer     `json:"-"`
 	// output is where actual results are written. This might point to stdout,
 	// or to a file, based on the current configuration.
 	output       io.Writer `json:"-"`
 	writeOptions *writeOptions
 	Config *Config `json:"config"`
 	historyPath string
 	// Commands contains optional commands provided via one or more `-c` (or
 	// `--command`) flags. If this is non-empty, the cli will run in
 	// non-interactive mode; i.e. it will quit after the command is complete.
 	Commands []string `json:"commands"`
 	// Files contains optional files provided via one or more `-f` (or `--file`)
 	// flags. If this is non-empty, the cli will run in non-interactive mode;
 	// i.e. it will quit after the command is complete.
 	Files []string `json:"files"`
 	// variables holds the variables created with the \set meta-command.
 	variables map[string]string
 	// nonInteractiveMode is set to true when fbsql is running in
 	// non-ineracative mode. And example of this is when the user has provided a
 	// `-c` flag in the command line.
 	nonInteractiveMode bool
 	// quit gets closed when Run should stop listening for input.
 	quit chan struct{}
 }
 func NewCommand(logdest logger.Logger) *Command {
 	variables := make(map[string]string)
 	return &Command{
 		Config: &Config{
 			Host: defaultHost,
 			Port: "",
 			OrganizationID: "",
 			Database:       "",
 			CloudAuth: CloudAuthConfig{
 				ClientID: defaultClientID,
 				Region:   defaultRegion,
 				Email:    "",
 				Password: "",
 			},
 			HistoryPath: "",
 			CSV: false,
 		},
 		buffer:     newBuffer(),
 		splitter:   newSplitter(newReplacer(variables)),
 		workingDir: newWorkingDir(),
 		stdin:  Stdin,
 		stdout: Stdout,
 		stderr: Stderr,
 		output:       Stdout,
 		writeOptions: defaultWriteOptions(),
 		variables: variables,
 		quit: make(chan struct{}),
 	}
 }
 // SetStdin sets stdin. This is useful for initial configuration in tests.
 func (cmd *Command) SetStdin(rc io.ReadCloser) {
 	cmd.stdin = rc
 }
 // SetStdout sets both stdout and output to the value provided. This is useful
 // for initial configuration in tests.
 func (cmd *Command) SetStdout(w io.Writer) {
 	cmd.stdout = w
 	cmd.output = w
 }
 // SetStderr sets stderr. This is useful for initial configuration in tests.
 func (cmd *Command) SetStderr(w io.Writer) {
 	cmd.stderr = w
 }
 // Run is the main entry-point to the CLI.
 func (cmd *Command) Run(ctx context.Context) error {
 	if err := cmd.run(ctx); err != nil {
 		cmd.Errorf(err.Error() + "\n")
 		return err
 	}
 	return nil
 }
 // run is effectively wrapped by the Run() method, but it's split out this way
 // so that run() can simply return errors, rather than worrying about how errors
 // should be printed; printing errors returned by run() is left up to the Run()
 // method.
 func (cmd *Command) run(ctx context.Context) error {
 	if err := cmd.setupConfig(); err != nil {
 		return errors.Wrap(err, "setting up config")
 	}
 	// Check to see if Command needs to run in non-interactive mode.
 	if len(cmd.Commands) > 0 ||
 		len(cmd.Files) > 0 ||
 		cmd.Config.KafkaConfig != "" ||
 		cmd.Config.CSV {
 		cmd.nonInteractiveMode = true
 	}
 	// Print the splash message.
 	if !cmd.nonInteractiveMode {
 		cmd.Printf(splash)
 	}
 	if err := cmd.setupClient(); err != nil {
 		return errors.Wrap(err, "setting up client")
 	}
 	// Print the connection info.
 	if !cmd.nonInteractiveMode {
 		cmd.printConnInfo()
 	}
 	if err := cmd.connectToDatabase(cmd.database); err != nil {
 		cmd.Errorf(errors.Wrap(err, "connecting to database").Error() + "\n")
 		// We intentionally do not return err here.
 	}
 	// Run in non-interactive mode based on flags and configuration.
 	// This includes either handling `-c` and/or `-f` flags, or handling a
 	// `--kafka-config` flag.
 	if len(cmd.Commands) > 0 || len(cmd.Files) > 0 {
 		// Run Commands.
 		for _, line := range cmd.Commands {
 			if err := cmd.handleLine(line); err != nil {
 				return errors.Wrapf(err, "handling line: %s", line)
 			}
 		}
 		// Run Files.
 		for _, fname := range cmd.Files {
 			if _, err := executeFile(cmd, fname); err != nil {
 				return errors.Wrapf(err, "executing file: %s", fname)
 			}
 		}
 		return nil
 	} else if cmd.Config.KafkaConfig != "" {
 		runner, err := cmd.newKafkaRunner(cmd.Config.KafkaConfig)
 		if err != nil {
 			return errors.Wrap(err, "getting new kafka runner")
 		}
 		if err := runner.Main.Run(); err != nil {
 			return errors.Wrap(err, "running kafka")
 		}
 		return nil
 	}
 	// From this point on, we should be in interactive mode.
 	// Set up history for saving user input.
 	cmd.setupHistory()
 	rl, err := readline.NewEx(&readline.Config{
 		Prompt:                 cmd.prompt(false),
 		HistoryFile:            cmd.historyPath,
 		HistoryLimit:           100000,
 		DisableAutoSaveHistory: true,
 		Stdin:  cmd.stdin,
 		Stdout: cmd.stdout,
 		Stderr: cmd.stderr,
 	})
 	if err != nil {
 		return errors.Wrap(err, "getting readline")
 	}
 	defer rl.Close()
 	// inMidCommand indicates whether a partial command has been received and
 	// we're still waiting for a termination character.
 	var inMidCommand bool
 	for {
 		rl.SetPrompt(cmd.prompt(inMidCommand))
 		// Read user provided input.
 		line, err := rl.Readline()
 		if err == readline.ErrInterrupt {
 			inMidCommand = false
 			cmd.buffer.reset()
 			continue
 		} else if err != nil {
 			return errors.Wrap(err, "reading line")
 		}
 		// We append a line feed at the end of each line because at this point
 		// we have effectively stripped any intentional line feeds (since we are
 		// reading a line at a time), and we don't want to do that. An example
 		// of an intentional line feed is in a BULK INSERT CSV STREAM like this
 		// example:
 		//
 		// bulk replace
 		// 	 into foo (_id, age)
 		// 	 map (0 id, 1 int)
 		// from
 		// 	 x'3,33
 		// 	 4,44
 		// 	 5,55'
 		// with
 		// 	 format 'CSV'
 		// 	 input 'STREAM';
 		//
 		// We want to preserve the line feeds that are contained in the x''
 		// block; those are intentional as they demarc records within the csv.
 		qps, mcs, err := cmd.splitter.split(line + "\n")
 		if err != nil {
 			cmd.Errorf("error splitting line: %s\n", err)
 			continue
 		}
 		// Save line in the history.
 		if err := rl.SaveHistory(line); err != nil {
 			cmd.Errorf("Couldn't save history: %v\n", err)
 		}
 		// This is wrapped in an anonymous function so we can capture any
 		// errors, ignore the rest of the line, and return back to a prompt.
 		if err := func() error {
 			for i := range qps {
 				if qry, err := cmd.buffer.addPart(qps[i]); err != nil {
 					return errors.Wrap(err, "adding part to buffer")
 				} else if qry != nil {
 					if err := cmd.executeAndWriteQuery(qry); err != nil {
 						return errors.Wrap(err, "executing query")
 					}
 					// In addition to saving each line in the history, we also
 					// save each successful query.
 					if err := rl.SaveHistory(qry.String() + ";"); err != nil {
 						cmd.Errorf("Couldn't save query in history: %v\n", err)
 					}
 					inMidCommand = false
 				} else {
 					inMidCommand = true
 				}
 			}
 			return nil
 		}(); err != nil {
 			cmd.Errorf(err.Error() + "\n")
 			inMidCommand = false
 			continue
 		}
 		// This is wrapped in an anonymous function so we can capture any
 		// errors, ignore the rest of the line, and return back to a prompt.
 		if err := func() error {
 			for i := range mcs {
 				action, err := mcs[i].execute(cmd)
 				if err != nil {
 					return errors.Wrap(err, "executing meta command")
 				}
 				switch action {
 				case actionQuit:
 					close(cmd.quit)
 					return nil
 				case actionReset:
 					inMidCommand = false
 				}
 			}
 			return nil
 		}(); err != nil {
 			cmd.Errorf(err.Error() + "\n")
 			inMidCommand = false
 			continue
 		}
 		select {
 		case <-cmd.quit:
 			if err := cmd.close(); err != nil {
 				cmd.Errorf("closing: %s\n", err)
 			}
 			return nil
 		default:
 			// pass
 		}
 	}
 }
 // prompt constructs the prompt that the user sees based on the currently
 // connected database and whether the user is in the middle of a sql statement.
 func (cmd *Command) prompt(mid bool) string {
 	db := "fbsql" // default prompt when a database is not set.
 	if cmd.databaseName != "" {
 		db = cmd.databaseName
 	}
 	if mid {
 		return strings.Repeat(" ", len(db)) + "-# "
 	}
 	return db + "=# "
 }
 // close is called upon quitting. It should close any remaining open file
 // handles used by the CLICommand.
 func (cmd *Command) close() error {
 	return cmd.closeOutput()
 }
 // setupConfig sets up private struct members based on values provided via the
 // configuration flags.
 func (cmd *Command) setupConfig() error {
 	if cmd.Config == nil {
 		return nil
 	}
 	cmd.host = cmd.Config.Host
 	cmd.port = cmd.Config.Port
 	cmd.organizationID = cmd.Config.OrganizationID
 	cmd.database = cmd.Config.Database
 	cmd.historyPath = cmd.Config.HistoryPath
 	// Apply any pset flag arguments.
 	for _, pset := range cmd.Config.PSets {
 		if err := cmd.applyPSet(pset); err != nil {
 			return errors.Wrapf(err, "applying pset: %s", pset)
 		}
 	}
 	// If running with the `--csv` flag, configure things to ensure the output
 	// is correct (i.e. that it's just the csv).
 	if cmd.Config.CSV {
 		cmd.writeOptions.format = formatCSV
 	}
 	return nil
 }
 // applyPSet takes a pset string of the form `arg` or `arg=val` and applies it
 // as if the user had run `\pset arg val`. The only difference is that applying
 // pset here suppresses any output to stdout.
 func (cmd *Command) applyPSet(pset string) error {
 	// We expect arg to be one of the folowing formats:
 	// arg
 	// arg=val
 	args := strings.SplitN(pset, "=", 2)
 	// This is kind of hacky, but until we re-think the metaCommand interface to
 	// take a printer interface somewhere (so we can pass in the nopPrinter
 	// here), we're just going to discard stdout for the duration of this apply,
 	// and then set stdout back to its previous writer after the apply.
 	hold := cmd.stdout
 	cmd.stdout = io.Discard
 	defer func() {
 		cmd.stdout = hold
 	}()
 	_, err := newMetaPSet(args).execute(cmd)
 	return err
 }
 func (cmd *Command) executeAndWriteQuery(qry query) error {
 	queryResponse, err := cmd.executeQuery(qry)
 	if err != nil {
 		if errors.Is(err, ErrOrganizationRequired) {
 			// Print an error message and return nil, effectively aborting any
 			// further writes for this query.
 			cmd.Errorf("Organization required. Use \\org to set an organization.\n")
 			return nil
 		}
 		return errors.Wrap(err, "making query")
 	}
 	if err := writeOutput(queryResponse, cmd.writeOptions, cmd.output, cmd.stdout, cmd.stderr); err != nil {
 		return errors.Wrap(err, "writing out response")
 	}
 	return nil
 }
 func (cmd *Command) executeQuery(qry query) (*featurebase.WireQueryResponse, error) {
 	wqr, err := cmd.Queryer.Query(cmd.organizationID, cmd.databaseID, qry.Reader())
 	if err != nil {
 		return nil, errors.Wrap(err, "executing query")
 	}
 	// If we're running in non-interactive mode, we need to check the error that
 	// comes back in the WireQueryResponse. If there's an error, we want to
 	// return it now (rather than just printing it later) so that we immediately
 	// stop any further execution of commands.
 	if cmd.nonInteractiveMode && wqr.Error != "" {
 		return nil, errors.Errorf(wqr.Error)
 	}
 	return wqr, nil
 }
 // printer is an interface which encapsulates the methods used to print output
 // to the various io.Writers.
 type printer interface {
 	Printf(format string, a ...any)
 	Outputf(format string, a ...any)
 	Errorf(format string, a ...any)
 }
 type nopPrinter struct{}
 func newNopPrinter() *nopPrinter {
 	return &nopPrinter{}
 }
 func (n *nopPrinter) Printf(format string, a ...any)  {}
 func (n *nopPrinter) Outputf(format string, a ...any) {}
 func (n *nopPrinter) Errorf(format string, a ...any)  {}
 // Printf is a helper method which sends the given payload to stdout.
 func (cmd *Command) Printf(format string, a ...any) {
 	out := fmt.Sprintf(format, a...)
 	cmd.stdout.Write([]byte(out))
 }
 // Outputf is a helper method which sends the given payload to output.
 func (cmd *Command) Outputf(format string, a ...any) {
 	out := fmt.Sprintf(format, a...)
 	cmd.output.Write([]byte(out))
 }
 // Errorf is a helper method which sends the given payload to stderr.
 func (cmd *Command) Errorf(format string, a ...any) {
 	out := fmt.Sprintf(format, a...)
 	cmd.stderr.Write([]byte(out))
 }
 func (cmd *Command) setupHistory() {
 	// If HistoryPath has already been configured (i.e. with a command flag),
 	// don't bother setting up the default in the home directory.
 	if cmd.historyPath != "" {
 		return
 	}
 	historyPath := ""
 	if home, err := os.UserHomeDir(); err != nil {
 		cmd.Errorf("Error getting home directory, command history persistence will be disabled: %v\n", err)
 	} else {
 		historyDir := filepath.Join(home, ".featurebase")
 		err := os.MkdirAll(historyDir, 0o750)
 		if err != nil {
 			cmd.Errorf("Creating directory for history: %v\n", err)
 		} else {
 			historyPath = filepath.Join(historyDir, "fbsql_history")
 		}
 	}
 	cmd.historyPath = historyPath
 }
 // printConnInfo displays the currently set host.
 // TODO(tlt): extend this to be the output of the /conninfo meta-command.
 func (cmd *Command) printConnInfo() {
 	cmd.Printf("Host: %s\n", hostPort(cmd.host, cmd.port))
 }
 func (cmd *Command) connectToDatabase(dbName string) error {
 	var p printer = cmd
 	if cmd.nonInteractiveMode {
 		p = newNopPrinter()
 	}
 	// Providing a blank ("") or hyphen ("-") dbName is the equivalent of
 	// disconnecting from the current database. We support the hyphen option
 	// because calling the `\c` meta-command without an argument is how you
 	// print the current connection.
 	switch dbName {
 	case "-", "":
 		cmd.databaseID = ""
 		cmd.databaseName = ""
 		p.Printf(cmd.connectionMessage())
 		return nil
 	}
 	// Look up dbID based on dbName.
 	wqr, err := cmd.executeQuery(newRawQuery("SHOW DATABASES"))
 	if err != nil {
 		return errors.Wrap(err, "executing query")
 	}
 	for _, db := range wqr.Data {
 		// 0: _id
 		// 1: name
 		if db[1] == dbName {
 			cmd.databaseName = dbName
 			cmd.databaseID = db[0].(string)
 			p.Printf(cmd.connectionMessage())
 			return nil
 		}
 	}
 	return errors.Errorf("invalid database: %s", dbName)
 }
 func (cmd *Command) orgMessage() string {
 	if cmd.organizationID == "" {
 		return "You have not set an organization.\n"
 	}
 	return fmt.Sprintf("You have set organization \"%s\".\n", cmd.organizationID)
 }
 func (cmd *Command) connectionMessage() string {
 	if cmd.databaseName == "" {
 		return "You are not connected to a database.\n"
 	}
 	return fmt.Sprintf("You are now connected to database \"%s\" (%s).\n", cmd.databaseName, cmd.databaseID)
 }
 func (cmd *Command) setupClient() error {
 	// If the Queryer has already been set (in tests for example), don't bother
 	// trying to detect it.
 	if cmd.Queryer != nil {
 		return nil
 	}
 	var p printer = cmd
 	if cmd.nonInteractiveMode {
 		p = newNopPrinter()
 	}
 	if strings.TrimSpace(cmd.host) == "" {
 		return errors.Errorf("no host provided\n")
 	}
 	if !strings.HasPrefix(cmd.host, "http") {
 		cmd.host = "http://" + cmd.host
 	}
 	typ, err := cmd.detectFBType()
 	if err != nil {
 		return errors.Wrap(err, "detecting FeatureBase deployment type")
 	}
 	switch typ {
 	case featurebaseTypeOnPremClassic:
 		p.Printf("Detected on-prem, classic deployment.\n")
 		cmd.Queryer = &standardQueryer{
 			Host: cmd.host,
 			Port: cmd.port,
 		}
 	case featurebaseTypeOnPremServerless:
 		p.Printf("Detected on-prem, serverless deployment.\n")
 		cmd.Queryer = &serverlessQueryer{
 			Host: cmd.host,
 			Port: cmd.port,
 		}
 	case featurebaseTypeCloud:
 		p.Printf("Detected cloud deployment.\n")
 		cmd.Queryer = &fbcloud.Queryer{
 			Host: hostPort(cmd.host, cmd.port),
 			ClientID: cmd.Config.CloudAuth.ClientID,
 			Region:   cmd.Config.CloudAuth.Region,
 			Email:    cmd.Config.CloudAuth.Email,
 			Password: cmd.Config.CloudAuth.Password,
 		}
 	case featurebaseTypeUnknown:
 		p.Printf("Could not detect deployment\n")
 		// cmd.Queryer = &nopQueryer{}
 		// Instead of using a no-op queryer when the type can't be detected, we
 		// default to using a cloud queryer.
 		cmd.Queryer = &fbcloud.Queryer{
 			Host: hostPort(cmd.host, cmd.port),
 			ClientID: cmd.Config.CloudAuth.ClientID,
 			Region:   cmd.Config.CloudAuth.Region,
 			Email:    cmd.Config.CloudAuth.Email,
 			Password: cmd.Config.CloudAuth.Password,
 		}
 	default:
 		return errors.Errorf("unknown type: %s", typ)
 	}
 	return nil
 }
 type featurebaseType string
 const (
 	featurebaseTypeUnknown          featurebaseType = "unknown"            // unknown
 	featurebaseTypeOnPremClassic    featurebaseType = "on-prem-standard"   // on-prem, classic
 	featurebaseTypeOnPremServerless featurebaseType = "on-prem-serverless" // on-prem, serverless
 	featurebaseTypeCloud            featurebaseType = "cloud"              // cloud, (both classic and serverless)?
 )
 func hostPort(host, port string) string {
 	if port == "" {
 		return host
 	}
 	return host + ":" + port
 }
 // detectFBType determines if we're talking to standalone FeatureBase
 // or FeatureBase Cloud
 func (cmd *Command) detectFBType() (featurebaseType, error) {
 	type trial struct {
 		port   string
 		health string
 		typ    featurebaseType
 	}
 	// trials is populated with the url/endpoints to try in order to detect if a
 	// process is running there which can support the cli requests.
 	trials := []trial{}
 	var clientTimeout time.Duration
 	if cmd.port != "" {
 		clientTimeout = 100 * time.Millisecond
 		trials = append(trials,
 			// on-prem, serverless
 			trial{
 				port:   cmd.port,
 				health: "/queryer/health",
 				typ:    featurebaseTypeOnPremServerless,
 			},
 			// on-prem, classic
 			trial{
 				port:   cmd.port,
 				health: "/status",
 				typ:    featurebaseTypeOnPremClassic,
 			},
 		)
 	} else if strings.HasPrefix(cmd.host, "https") {
 		// https suggesting we might be connecting to a cloud host
 		clientTimeout = 1 * time.Second
 		trials = append(trials,
 			// cloud
 			trial{
 				port:   "",
 				health: "/health",
 				typ:    featurebaseTypeCloud,
 			},
 		)
 	} else {
 		// Try default ports just in case.
 		clientTimeout = 100 * time.Millisecond
 		trials = append(trials,
 			// on-prem, serverless
 			trial{
 				port:   "8080",
 				health: "/queryer/health",
 				typ:    featurebaseTypeOnPremServerless,
 			},
 			// on-prem, classic
 			trial{
 				port:   "10101",
 				health: "/status",
 				typ:    featurebaseTypeOnPremClassic,
 			},
 		)
 	}
 	client := http.Client{
 		Timeout: clientTimeout,
 	}
 	for _, trial := range trials {
 		url := hostPort(cmd.host, trial.port) + trial.health
 		if resp, err := client.Get(url); err != nil {
 			continue
 		} else if resp.StatusCode/100 == 2 {
 			cmd.port = trial.port
 			return trial.typ, nil
 		}
 	}
 	return featurebaseTypeUnknown, nil
 }
 func (cmd *Command) closeOutput() error {
 	if cmd.output == nil {
 		return nil
 	}
 	if closer, ok := cmd.output.(io.Closer); ok {
 		return closer.Close()
 	}
 	return nil
 }
 func (cmd *Command) handleLine(line string) error {
 	// For single-line command handling, we handle either a meta-command, or
 	// query parts, but not both. The logic is that any line which begins with
 	// "\" will be handled as a meta-command, otherwise it will be handled as a
 	// query.
 	if len(line) == 0 {
 		return nil
 	} else if line[0] == byte('\\') {
 		return cmd.handleLineAsMetaCommand(line)
 	} else {
 		return cmd.handleLineAsQueryParts(line)
 	}
 }
 func (cmd *Command) handleLineAsMetaCommand(line string) error {
 	_, mcs, err := cmd.splitter.split(line)
 	if err != nil {
 		return errors.Wrapf(err, "splitting line")
 	}
 	for i := range mcs {
 		_, err := mcs[i].execute(cmd)
 		if err != nil {
 			return errors.Wrap(err, "executing meta command")
 		}
 	}
 	return nil
 }
 func (cmd *Command) handleLineAsQueryParts(line string) error {
 	qps, mcs, err := cmd.splitter.split(line)
 	if err != nil {
 		return errors.Wrapf(err, "splitting line")
 	} else if len(mcs) > 0 {
 		return errors.Errorf("--command does not support meta-commands")
 	}
 	// Add a termintor part to the end of []queryPart. We do this because the
 	// command is coming in from the --command flag, it may not end with a
 	// semi-colon, but we still want to execute it.
 	if len(qps) > 0 {
 		if _, ok := qps[len(qps)-1].(*partTerminator); !ok {
 			qps = append(qps, newPartTerminator())
 		}
 	}
 	for i := range qps {
 		if qry, err := cmd.buffer.addPart(qps[i]); err != nil {
 			return errors.Wrap(err, "adding part to buffer")
 		} else if qry != nil {
 			if err := cmd.executeAndWriteQuery(qry); err != nil {
 				return errors.Wrap(err, "executing query")
 			}
 		}
 	}
 	return nil
 }
 func (cmd *Command) Insert(sql string) error {
 	wqr, err := cmd.executeQuery(newRawQuery(sql))
 	if wqr.Error != "" {
 		return errors.Errorf(wqr.Error)
 	}
 	return err
 }
--- a/cli/cli_integration_test.go
+++ b/cli/cli_integration_test.go
@ -1,257 +0,0 @@
 package cli_test
 import (
 	"bufio"
 	"context"
 	"fmt"
 	"os"
 	"strings"
 	"testing"
 	"time"
 	"github.com/featurebasedb/featurebase/v3/cli"
 	"github.com/featurebasedb/featurebase/v3/dax/server/test"
 	"github.com/featurebasedb/featurebase/v3/logger"
 	"github.com/stretchr/testify/require"
 )
 func TestCLIIntegration(t *testing.T) {
 	if testing.Short() {
 		t.Skip("skipping integration test")
 	}
 	ctx := context.Background()
 	t.Run("Stubbed Framework", func(t *testing.T) {
 		mc := test.MustRunManagedCommand(t)
 		defer mc.Close()
 		addr := mc.Address()
 		capture := newCapture(t)
 		comparer := newComparer(t)
 		comparer.run()
 		fbsql := cli.NewCommand(logger.StderrLogger)
 		fbsql.SetStdin(capture)
 		fbsql.SetStdout(comparer)
 		fbsql.SetStderr(comparer)
 		fbsql.Config = &cli.Config{
 			Host: addr.Host(),
 			Port: fmt.Sprintf("%d", addr.Port()),
 		}
 		// Run fbsql in a goroutine so we can continue to send it commands
 		// below.
 		didQuit := make(chan struct{})
 		go func() {
 			require.NoError(t, fbsql.Run(ctx))
 			close(didQuit)
 		}()
 		// testFiles reference files located in the cli/testdata directory. All
 		// tests should be placed there; other than adding another test file to
 		// this list, you probably shouldn't be editing this file unless you are
 		// trying to modify the way the test framework itself works.
 		testFiles := []string{
 			"setup",
 			"database",
 			"table",
 			// the tests below may be dependent on the previous tests, which do
 			// setup and some shared database and table creation.
 			"query_buffer",
 			// meta commands
 			"meta_bang",
 			"meta_cd",
 			"meta_echo",
 			"meta_describe",
 			"meta_file",
 			"meta_pset_border",
 			"meta_pset_expanded",
 			"meta_pset_format_csv",
 			"meta_pset_tuples_only",
 			"meta_include",
 			"meta_output",
 			"meta_set",
 			"meta_timing",
 			"meta_write",
 		}
 		for _, testFile := range testFiles {
 			t.Run(testFile, func(t *testing.T) {
 				f, err := os.Open("testdata/" + testFile)
 				require.NoError(t, err)
 				scanner := bufio.NewScanner(f)
 				var lineNo int
 				for scanner.Scan() {
 					line := scanner.Text()
 					lineNo++
 					// Empty lines and comments (//) are ignored.
 					if line == "" {
 						continue
 					} else if strings.HasPrefix(line, "//") {
 						continue
 					}
 					parts := strings.SplitN(line, ":", 2)
 					switch parts[0] {
 					case "SEND":
 						v := ""
 						if len(parts) == 2 {
 							v = parts[1]
 						}
 						capture.sendLine(v)
 					case "EXPECT":
 						v := ""
 						if len(parts) == 2 {
 							v = parts[1]
 						}
 						comparer.expectLine(v, testFile, lineNo)
 					case "EXPECTCOMP":
 						if len(parts) == 2 {
 							comps := strings.SplitN(parts[1], ":", 2)
 							v := ""
 							if len(comps) == 2 {
 								v = comps[1]
 							}
 							comparer.expectLineComp(comparator(comps[0]), v, testFile, lineNo)
 						} else {
 							t.Errorf("unexpected line: %s[%d]:%s", testFile, lineNo, line)
 						}
 					default:
 						t.Errorf("unexpected line: %s[%d]:%s", testFile, lineNo, line)
 					}
 				}
 				require.NoError(t, scanner.Err())
 			})
 		}
 		// End with quit to ensure that fbsql closes without error.
 		capture.sendLine(`\q`)
 		// Ensure fbsql quits cleanly.
 		select {
 		case <-didQuit:
 		case <-time.After(time.Second):
 			t.Fatalf("expected fbsql to quit")
 		}
 	})
 }
 // compare is used to compare fbsql output written to its Stdout with expected
 // lines.
 type comparer struct {
 	t       *testing.T
 	out     chan byte
 	outline chan []byte
 	exp     chan []byte
 }
 func newComparer(t *testing.T) *comparer {
 	return &comparer{
 		t:       t,
 		out:     make(chan byte, 1024),
 		outline: make(chan []byte, 128),
 		exp:     make(chan []byte, 1024),
 	}
 }
 func (c *comparer) run() {
 	// Read bytes off output, and for every line (designated by a line feed "\n"),
 	// push the line onto the outline channel.
 	go func() {
 		var line []byte
 		for {
 			b := <-c.out
 			if b == byte('\n') {
 				c.outline <- line
 				line = []byte{}
 				continue
 			}
 			line = append(line, b)
 		}
 	}()
 }
 type comparator string
 const (
 	compEquals     = "Equals"
 	compHasPrefix  = "HasPrefix"
 	compWithFormat = "WithFormat"
 )
 // expectLine is a convenience method which calls expectLineComp with the compEq
 // comparator and the given line.
 func (c *comparer) expectLine(line string, fileName string, lineNo int) {
 	c.expectLineComp(compEquals, line, fileName, lineNo)
 }
 // expectLineComp reads the next line from the outline channel and compares it
 // with the given `line`. A comparator can be provided to inform how the lines
 // should be compared (for example, the compHasPrefix comparator will just
 // compare the beginning part of the outline).
 func (c *comparer) expectLineComp(comp comparator, line string, fileName string, lineNo int) {
 	var outline []byte
 	select {
 	case outline = <-c.outline:
 	case <-time.After(10 * time.Second):
 		// TODO(tlt): this is 10 seconds to account for the fb_views creation on
 		// a local mac. This should really be something like 2 seconds. Put this
 		// back to 2 once fb_views issue is addressed.
 		c.t.Fatalf("expected output line %s[%d]: >%s<", fileName, lineNo, line)
 	}
 	// msg is included in any require which fails.
 	msg := []interface{}{"exp: %s[%d], got: >%s<", fileName, lineNo, outline}
 	switch comp {
 	case compEquals:
 		require.Equal(c.t, []byte(line), outline, msg...)
 	case compHasPrefix:
 		require.True(c.t, strings.HasPrefix(string(outline), line), msg...)
 	case compWithFormat:
 		require.True(c.t, compareByteSlices(outline, []byte(line)), msg...)
 	default:
 		c.t.Fatalf("invalid comparator: %s", comp)
 	}
 }
 func (c *comparer) Write(b []byte) (n int, err error) {
 	for i := range b {
 		c.out <- b[i]
 	}
 	return len(b), err
 }
 // compareByteSlices compares a byte slice s with another byte slice format and
 // returns true if they are the same. It will accept underscore as a
 // single-character wildcard anywhere in slice format.
 func compareByteSlices(s, format []byte) bool {
 	// Replace some helpers in format before comparing.
 	f := string(format)
 	f = strings.ReplaceAll(f, `{uuid}`, `________-____-____-____-____________`)
 	f = strings.ReplaceAll(f, `{timestamp}`, `____-__-__T__:__:__Z`)
 	format = []byte(f)
 	if len(s) != len(format) {
 		return false
 	}
 	for i := range s {
 		if format[i] == '_' {
 			continue
 		}
 		if s[i] != format[i] {
 			// log.Printf("DEBUG: characters differ: (%d): '%v' != '%v'", i, s[i], format[i])
 			return false
 		}
 	}
 	return true
 }
--- a/cli/cli_test.go
+++ b/cli/cli_test.go
@ -1,199 +0,0 @@
 package cli_test
 import (
 	"context"
 	"io"
 	"strings"
 	"sync"
 	"testing"
 	"time"
 	featurebase "github.com/featurebasedb/featurebase/v3"
 	"github.com/featurebasedb/featurebase/v3/cli"
 	"github.com/featurebasedb/featurebase/v3/logger"
 	"github.com/pkg/errors"
 	"github.com/stretchr/testify/assert"
 )
 func TestCLI(t *testing.T) {
 	t.Run("Input", func(t *testing.T) {
 		ctx := context.Background()
 		capture := newCapture(t)
 		cli := cli.NewCommand(logger.StderrLogger)
 		cli.SetStdin(capture)
 		cli.SetStdout(capture)
 		cli.Queryer = capture
 		go func() {
 			assert.NoError(t, cli.Run(ctx))
 		}()
 		none := []string{}
 		// One statement, one line.
 		capture.Assert("one;", []string{"one\n"})
 		// One statement, multiple lines.
 		capture.Assert("one", none)
 		capture.Assert(" two ", none)
 		capture.Assert("three;", []string{"one\ntwo\nthree\n"})
 		// Multiple statements, one line.
 		capture.Assert("foo; bar;", []string{"foo\n", "bar\n"})
 		// Multiple statements, multiple lines.
 		capture.Assert("a1", none)
 		capture.Assert("a2; b1", []string{"a1\na2\n"})
 		capture.Assert("b2;", []string{"b1\nb2\n"})
 		// Blank lines.
 		capture.Assert("one", none)
 		capture.Assert("", none)
 		capture.Assert("three;", []string{"one\nthree\n"})
 		// Just a semi-colon.
 		capture.Assert(";", []string{""})
 		// Multi-line with just a semi-colon.
 		capture.Assert("one", none)
 		capture.Assert(";", []string{"one\n"})
 		// Ensure a clean exit with no errors.
 		assert.NoError(t, capture.Exit())
 	})
 }
 ////////////////////////////////////////////////////////
 // Ensure type implementes interface.
 var _ io.ReadCloser = (*capture)(nil)
 var _ io.Writer = (*capture)(nil)
 var _ cli.Queryer = (*capture)(nil)
 // capture implements the various CLI interfaces in order to capture test input
 // and submit it as though that input were being read from the command line. It
 // also captures calls made to the Queryer.Query method and ensures the sql they
 // contain is expected.
 type capture struct {
 	t *testing.T
 	// ch is a channel of strings (one line at a time) of CLI input.
 	ch chan string
 	mu   sync.RWMutex
 	sqls []string
 	// queryDone will receive an event any time the Query method is called and
 	// has completed. This is to tell the Assert method that it's safe to
 	// compare the sqls slice.
 	queryDone chan struct{}
 	asserting chan struct{}
 	err error
 }
 func newCapture(t *testing.T) *capture {
 	return &capture{
 		t:         t,
 		ch:        make(chan string),
 		sqls:      make([]string, 0),
 		queryDone: make(chan struct{}),
 	}
 }
 func (c *capture) Exit() error {
 	c.sendLine(`\q`)
 	c.mu.RLock()
 	defer c.mu.RUnlock()
 	return c.err
 }
 func (c *capture) Assert(in string, out []string) {
 	c.asserting = make(chan struct{})
 	c.sendLine(in)
 	// Wait for the CLI command to complete processing the input and send the
 	// sql to Query() by blocking on the queryDone channel. Because Query gets
 	// called for every sql statement in the input, an input resulting in
 	// multiple sql statements needs to wait for all expected queries to
 	// complete. A timeout is included to this so it doesn't deadlock in the
 	// case where Query is expected to be called, but isn't; after the timeout,
 	// the test should fail completely. In summary: we wait on queryDone the
 	// number of sql statements we expect. If we receive fewer than expected,
 	// the timeout will occur. If we receive more than expected, the Query()
 	// method will effectively deadlock, reach its own timout, then write to
 	// capture.err, which will be reported upon Exit().
 	for range out {
 		select {
 		case <-c.queryDone:
 		case <-time.After(2 * time.Second):
 			c.t.Fatalf("expected Query() to be called")
 		}
 	}
 	close(c.asserting)
 	c.mu.Lock()
 	defer c.mu.Unlock()
 	assert.Equal(c.t, out, c.sqls)
 	// Reset the slice.
 	c.sqls = c.sqls[:0]
 }
 // sendLine sends the given string as a line input to the CLI command. It
 // appends a line feed to the end of string in order to mimic the user hitting
 // the return key.
 func (c *capture) sendLine(s string) {
 	// Add a line feed before putting s on the channel in order to mimic the
 	// user hitting the return key.
 	c.ch <- s + "\n"
 }
 // Read is read by the CLI in place of user input. It effectively sends lines of
 // input to the CLI, getting each line to be sent off the channel.
 func (c *capture) Read(b []byte) (n int, err error) {
 	s := <-c.ch
 	return strings.NewReader(s).Read(b)
 }
 func (c *capture) Close() error {
 	close(c.ch)
 	return nil
 }
 // Write is called with anything written to output. This would included results
 // from calling Query() under normal, non-testing conditions, as well as other
 // informational text sent to output, such as the splash message.
 func (c *capture) Write(b []byte) (n int, err error) {
 	return 0, nil
 }
 // Query is called by the CLI command once a full SQL statement is received
 // (signified by the terminator: `;`).
 func (c *capture) Query(org string, db string, sql io.Reader) (*featurebase.WireQueryResponse, error) {
 	tmpBuf := new(strings.Builder)
 	_, err := io.Copy(tmpBuf, sql)
 	if err != nil {
 		return nil, err
 	}
 	c.mu.Lock()
 	c.sqls = append(c.sqls, tmpBuf.String())
 	c.mu.Unlock()
 	select {
 	case c.queryDone <- struct{}{}:
 	case <-c.asserting:
 		c.mu.Lock()
 		c.err = errors.Errorf("unexpected query: %s", sql)
 		c.mu.Unlock()
 	}
 	return &featurebase.WireQueryResponse{}, nil
 }
--- a/cli/config.go
+++ b/cli/config.go
@ -1,31 +0,0 @@
 package cli
 // Config represents the configuration for the command.
 type Config struct {
 	Host string `json:"host"`
 	Port string `json:"port"`
 	OrganizationID string `json:"org-id"`
 	Database       string `json:"db"`
 	// CloudAuth
 	CloudAuth CloudAuthConfig `json:"cloud-auth"`
 	// Kafka
 	KafkaConfig string `json:"kafka-config"`
 	HistoryPath string `json:"history-path"`
 	// CSV (Comma-Separated Values) table output mode.
 	CSV bool `json:"csv"`
 	// PSet takes one or more pset arguments of the form: `--pset=VAR[=ARG]`.
 	PSets []string `json:"pset"`
 }
 type CloudAuthConfig struct {
 	ClientID string `json:"client-id"`
 	Region   string `json:"region"`
 	Email    string `json:"email"`
 	Password string `json:"password"`
 }
--- a/cli/errors.go
+++ b/cli/errors.go
@ -1,16 +0,0 @@
 package cli
 import (
 	"github.com/featurebasedb/featurebase/v3/errors"
 )
 const (
 	ErrOrganizationRequired errors.Code = "OrganizationRequired"
 )
 func NewErrOrganizationRequired() error {
 	return errors.New(
 		ErrOrganizationRequired,
 		"organization required",
 	)
 }
--- a/cli/fbcloud/auth.go
+++ b/cli/fbcloud/auth.go
@ -1,79 +0,0 @@
 package fbcloud
 import (
 	"bytes"
 	"encoding/json"
 	"fmt"
 	"io"
 	"net/http"
 	"github.com/pkg/errors"
 )
 const (
 	authFlow           = "USER_PASSWORD_AUTH"
 	cognitoURLTemplate = "https://cognito-idp.%s.amazonaws.com"
 )
 type cognitoParameters struct {
 	Email    string `json:"USERNAME"`
 	Password string `json:"PASSWORD"`
 }
 type cognitoAuthRequest struct {
 	AuthParameters cognitoParameters `json:"AuthParameters"`
 	AuthFlow       string            `json:"AuthFlow"`
 	AppClientID    string            `json:"ClientId"`
 }
 type cognitoAuthResult struct {
 	IDToken string `json:"IdToken"`
 }
 type cognitoAuthResponse struct {
 	Result cognitoAuthResult `json:"AuthenticationResult"`
 }
 func authenticate(clientID, region, email, password string) (string, error) {
 	authPayload := cognitoAuthRequest{
 		AuthParameters: cognitoParameters{
 			Email:    email,
 			Password: password,
 		},
 		AuthFlow:    authFlow,
 		AppClientID: clientID,
 	}
 	data, err := json.Marshal(authPayload)
 	if err != nil {
 		return "", errors.Wrap(err, "marshaling json")
 	}
 	url := fmt.Sprintf(cognitoURLTemplate, region)
 	req, err := http.NewRequest(http.MethodPost, url, bytes.NewBuffer(data))
 	if err != nil {
 		return "", errors.Wrap(err, "creating authentication request object")
 	}
 	req.Header.Add("Content-Type", "application/x-amz-json-1.1")
 	req.Header.Add("X-Amz-Target", "AWSCognitoIdentityProviderService.InitiateAuth")
 	resp, err := http.DefaultClient.Do(req)
 	if err != nil {
 		return "", errors.Wrap(err, "making request")
 	}
 	defer resp.Body.Close()
 	fullbod, err := io.ReadAll(resp.Body)
 	if resp.StatusCode != http.StatusOK || err != nil {
 		return "", errors.Errorf("HTTP status code=%d from Cognito authentication response. reading body: %v, body: '%s'", resp.StatusCode, err, fullbod)
 	}
 	var auth cognitoAuthResponse
 	err = json.Unmarshal(fullbod, &auth)
 	if err != nil {
 		return "", errors.Wrap(err, "decoding cognito auth response")
 	}
 	return auth.Result.IDToken, nil
 }
--- a/cli/fbcloud/client.go
+++ b/cli/fbcloud/client.go
@ -1,131 +0,0 @@
 package fbcloud
 import (
 	"encoding/json"
 	"fmt"
 	"io"
 	"net/http"
 	"strings"
 	"time"
 	featurebase "github.com/featurebasedb/featurebase/v3"
 	"github.com/pkg/errors"
 )
 // TokenRefreshTimeout is currently hardcoded to be just under the
 // Cognito token timeout for cloud which is 15 minutes (I think I
 // heard that somewhere anyway). It seems to work.
 const TokenRefreshTimeout = time.Minute * 13
 type Queryer struct {
 	Host string
 	ClientID string
 	Region   string
 	Email    string
 	Password string
 	token       string
 	lastRefresh time.Time
 }
 func (cq *Queryer) tokenRefresh() error {
 	token, err := authenticate(cq.ClientID, cq.Region, cq.Email, cq.Password)
 	if err != nil {
 		return errors.Wrap(err, "getting token")
 	}
 	cq.token = token
 	cq.lastRefresh = time.Now()
 	return nil
 }
 // Query issues a SQL query formatted for the FeatureBase cloud query endpoint.
 func (cq *Queryer) Query(org string, db string, sql io.Reader) (*featurebase.WireQueryResponse, error) {
 	if time.Since(cq.lastRefresh) > TokenRefreshTimeout {
 		if err := cq.tokenRefresh(); err != nil {
 			return nil, errors.Wrap(err, "refreshing token")
 		}
 	}
 	url := fmt.Sprintf("%s/databases/%s/sql", cq.Host, db)
 	if db == "" {
 		url = fmt.Sprintf("%s/sql", cq.Host)
 	}
 	client := &http.Client{
 		Timeout: time.Second * 30,
 	}
 	req, err := http.NewRequest(http.MethodPost, url, sql)
 	if err != nil {
 		return nil, errors.Wrap(err, "creating new post request")
 	}
 	req.Header.Add("Content-Type", "text/plain")
 	req.Header.Add("Authorization", cq.token)
 	var resp *http.Response
 	if resp, err = client.Do(req); err != nil {
 		return nil, errors.Wrap(err, "executing post request")
 	}
 	fullbod, err := io.ReadAll(resp.Body)
 	if err != nil {
 		return nil, errors.Wrap(err, "reading cloud response")
 	}
 	if resp.StatusCode/100 != 2 {
 		return nil, errors.Errorf("unexpected status: %s, full body: '%s'", resp.Status, fullbod)
 	}
 	var sqlResponse featurebase.WireQueryResponse
 	if err := json.Unmarshal(fullbod, &sqlResponse); err != nil {
 		return nil, errors.Wrapf(err, "decoding cloud response, body:\n%s", fullbod)
 	}
 	return &sqlResponse, nil
 }
 // HTTPRequest can make an arbitrary http request to the host and
 // tries to json unmarshal the response body into v if v is
 // non-nil. This is handy for hitting cloud endpoints other than the
 // query endpoint which is handled by Query. I don't think this is
 // currently used, but I'd like to keep it around for debugging.
 func (cq *Queryer) HTTPRequest(method, path, body string, v interface{}) ([]byte, error) {
 	if time.Since(cq.lastRefresh) > TokenRefreshTimeout {
 		if err := cq.tokenRefresh(); err != nil {
 			return nil, errors.Wrap(err, "refreshing token")
 		}
 	}
 	var bod io.Reader
 	if body == "" {
 		bod = nil
 	} else {
 		bod = strings.NewReader(body)
 	}
 	req, err := http.NewRequest(method, fmt.Sprintf("%s%s", cq.Host, path), bod)
 	if err != nil {
 		return nil, errors.Errorf("creating request: %v", err)
 	}
 	// fmt.Printf("%+v\n", req)
 	req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", cq.token))
 	if bod != nil {
 		req.Header.Add("Content-Type", "application/json")
 	}
 	resp, err := http.DefaultClient.Do(req)
 	if err != nil {
 		return nil, errors.Errorf("doing request: %v", err)
 	}
 	bodbytes, err := io.ReadAll(resp.Body)
 	if err != nil {
 		return nil, errors.Errorf("reading response body: %v", err)
 	}
 	if resp.StatusCode/100 != 2 {
 		return nil, errors.Errorf("bad status: %s. body: '%s'", resp.Status, bodbytes)
 	}
 	if v != nil {
 		err = json.Unmarshal(bodbytes, v)
 		if err != nil {
 			return nil, errors.Errorf("unmarshaling: %v", err)
 		}
 	}
 	return bodbytes, nil
 }
--- a/cli/kafka.go
+++ b/cli/kafka.go
@ -1,70 +0,0 @@
 package cli
 import (
 	"fmt"
 	"github.com/featurebasedb/featurebase/v3/cli/batch"
 	"github.com/featurebasedb/featurebase/v3/cli/kafka"
 	"github.com/featurebasedb/featurebase/v3/errors"
 	"github.com/spf13/viper"
 )
 func (cmd *Command) newKafkaRunner(cfgFile string) (*kafka.Runner, error) {
 	// Read the kafka config file.
 	v := viper.New()
 	v.SetConfigFile(cfgFile)
 	v.SetConfigType("toml")
 	err := v.ReadInConfig()
 	if err != nil {
 		return nil, fmt.Errorf("error reading configuration file '%s': %v", cfgFile, err)
 	}
 	cfg := kafka.Config{}
 	if err := v.Unmarshal(&cfg); err != nil {
 		return nil, errors.Wrap(err, "unmarshalling config")
 	}
 	if err := kafka.ValidateConfig(cfg); err != nil {
 		return nil, errors.Wrap(err, "validating config")
 	}
 	// Create a new config with defaults.
 	// Look up fields based on table provided in the config.
 	wqr, err := cmd.executeQuery(newRawQuery("SHOW COLUMNS FROM " + cfg.Table))
 	if err != nil {
 		return nil, errors.Wrap(err, "executing query")
 	}
 	scr, err := wqr.ShowColumnsResponse()
 	if err != nil {
 		return nil, errors.Wrap(err, "getting show columns from wire query response")
 	}
 	// If no fields were provided in the config, use the fields defined on the
 	// table and assume a 1-to-1 mapping of source to destination.
 	if len(cfg.Fields) == 0 {
 		cfg.Fields = kafka.FieldsToConfig(scr.Fields)
 	} else {
 		cfg.Fields, err = kafka.CheckFieldCompatibility(cfg.Fields, scr)
 		if err != nil {
 			return nil, errors.Wrap(err, "validating config fields")
 		}
 	}
 	idkCfg, err := kafka.ConvertConfig(cfg)
 	if err != nil {
 		return nil, errors.Wrap(err, "cleaning config")
 	}
 	flds, err := kafka.ConfigToFields(cfg)
 	if err != nil {
 		return nil, errors.Wrap(err, "getting fields from config")
 	}
 	return kafka.NewRunner(
 		idkCfg,
 		batch.NewSQLBatcher(cmd, flds),
 		cmd.stderr,
 	), nil
 }
--- a/cli/kafka/config.go
+++ b/cli/kafka/config.go
@ -1,251 +0,0 @@
 package kafka
 import (
 	"fmt"
 	"time"
 	featurebase "github.com/featurebasedb/featurebase/v3"
 	"github.com/featurebasedb/featurebase/v3/dax"
 	"github.com/featurebasedb/featurebase/v3/idk"
 	"github.com/pkg/errors"
 )
 // Config is the user-facing configuration for kafka support in the CLI. This is
 // unmarshalled from the the toml config file supplied by the user.
 type Config struct {
 	Hosts  []string `mapstructure:"hosts" help:"Kafka hosts."`
 	Group  string   `mapstructure:"group" help:"Kafka group."`
 	Topics []string `mapstructure:"topics" help:"Kafka topics to read from."`
 	BatchSize         int           `mapstructure:"batch-size" help:"Batch size."`
 	BatchMaxStaleness time.Duration `mapstructure:"batch-max-staleness" help:"Maximum length of time that the oldest record in a batch can exist before flushing the batch. Note that this can potentially stack with timeouts waiting for the source."`
 	Timeout           time.Duration `mapstructure:"timeout" help:"Time to wait for more records from Kafka before flushing a batch. 0 to disable."`
 	Table  string  `mapstructure:"table" help:"Destination table name."`
 	Fields []Field `mapstructure:"fields"`
 }
 // Field is a user-facing configuration field.
 type Field struct {
 	Name       string   `mapstructure:"name"`
 	SourceType string   `mapstructure:"source-type"`
 	SourcePath []string `mapstructure:"source-path"`
 	PrimaryKey bool     `mapstructure:"primary-key"`
 }
 // ConfigForIDK represents Config converted to values suitable for IDK. In
 // particular, the idk.RawField is used in parsing the schema in IDK.
 type ConfigForIDK struct {
 	Hosts  []string
 	Group  string
 	Topics []string
 	BatchSize         int
 	BatchMaxStaleness time.Duration
 	Timeout           time.Duration
 	Table   string
 	IDField string
 	Fields  []idk.RawField
 }
 // ValidateConfig validates the config is usable.
 func ValidateConfig(c Config) error {
 	if c.Table == "" {
 		return errors.Errorf("table is required")
 	} else if len(c.Topics) == 0 {
 		return errors.Errorf("at least one topic is required")
 	} else if len(c.Fields) > 0 {
 		// We only need to do these checks if any fields are specified at all.
 		// If no fields are specified, that's ok because then we default to
 		// using fields based off the existing table.
 		if len(c.Fields) < 2 {
 			return errors.Errorf("at least two fields are required (one should be a primary key)")
 		} else {
 			var found int
 			for i := range c.Fields {
 				if c.Fields[i].PrimaryKey {
 					found++
 				}
 				if c.Fields[i].Name == "" {
 					return errors.Errorf("a name attribute (which isn't equal to \"\") should exist for all fields")
 				}
 			}
 			if found != 1 {
 				return errors.Errorf("exactly one primary key field is required")
 			}
 		}
 	}
 	return nil
 }
 // ConvertConfig converts a Config to one that suitable for IDK.
 func ConvertConfig(c Config) (ConfigForIDK, error) {
 	// Set a default kafka host in case one isn't provided.
 	hosts := []string{"localhost:9092"}
 	if len(c.Hosts) > 0 {
 		hosts = c.Hosts
 	}
 	// Copy all the shared members from Config to ConfigForIDK.
 	out := ConfigForIDK{
 		Hosts:             hosts,
 		Group:             c.Group,
 		Topics:            c.Topics,
 		BatchSize:         c.BatchSize,
 		BatchMaxStaleness: c.BatchMaxStaleness,
 		Timeout:           c.Timeout,
 		Table:             c.Table,
 	}
 	if len(c.Fields) == 0 {
 		return out, errors.New("fields cannot be empty")
 	}
 	// rawFields wil be the same as c.Fields, but possibly enhanced.
 	rawFields := make([]idk.RawField, 0, len(c.Fields))
 	var foundPK bool
 	for _, fld := range c.Fields {
 		if fld.PrimaryKey {
 			out.IDField = fld.Name
 			foundPK = true
 		}
 		typ, quals, err := dax.SplitFieldType(fld.SourceType)
 		if err != nil {
 			return out, errors.Wrap(err, "getting base type")
 		}
 		rawFld := idk.RawField{
 			Name: fld.Name,
 			Type: string(typ),
 			Path: fld.SourcePath,
 		}
 		// If a SourcePath wasn't provided, default to using the field name.
 		if len(rawFld.Path) == 0 {
 			rawFld.Path = []string{fld.Name}
 		}
 		switch typ {
 		case dax.BaseTypeInt:
 			// We don't have to handle min/max because we don't create the table.
 		case dax.BaseTypeDecimal:
 			if len(quals) != 1 {
 				return out, errors.Errorf("expected decimal scale")
 			}
 			rawFld.Config = []byte(fmt.Sprintf(`{"scale":%d}`, quals[0]))
 		case dax.BaseTypeID:
 			rawFld.Config = []byte("{\"mutex\":true}")
 		case dax.BaseTypeIDSet:
 			rawFld.Type = "ids"
 		case dax.BaseTypeString:
 			rawFld.Config = []byte("{\"mutex\":true}")
 		case dax.BaseTypeStringSet:
 			rawFld.Type = "strings"
 		case dax.BaseTypeTimestamp:
 			// No timestamp options are handled for now.
 		}
 		rawFields = append(rawFields, rawFld)
 	}
 	if !foundPK {
 		return out, errors.New("primary-key not found in fields")
 	}
 	out.Fields = rawFields
 	return out, nil
 }
 // ConfigToFields returns a list of *dax.Field based on the IDField and Fields
 // in the Config.
 func ConfigToFields(c Config) ([]*dax.Field, error) {
 	// We don't know if a primary key will be found, so we can't set the
 	// capacity to `len(c.Fields)-1`.
 	out := make([]*dax.Field, 0, len(c.Fields))
 	for _, fld := range c.Fields {
 		if fld.PrimaryKey {
 			continue
 		}
 		typ, quals, err := dax.SplitFieldType(fld.SourceType)
 		if err != nil {
 			return nil, errors.Wrap(err, "splitting field type")
 		}
 		dfld := &dax.Field{
 			Name: dax.FieldName(fld.Name),
 			Type: typ,
 		}
 		switch typ {
 		case dax.BaseTypeDecimal:
 			if len(quals) != 1 {
 				return nil, errors.Errorf("expected decimal scale")
 			}
 			scale, ok := quals[0].(int64)
 			if !ok {
 				return nil, errors.Errorf("invalid decimal scale: %v", quals[0])
 			}
 			dfld.Options.Scale = scale
 		}
 		out = append(out, dfld)
 	}
 	return out, nil
 }
 // FieldsToConfig returns a Config.Fields based on a list of *dax.Field.
 func FieldsToConfig(flds []*dax.Field) []Field {
 	out := make([]Field, 0, len(flds))
 	for _, fld := range flds {
 		out = append(out, Field{
 			Name:       string(fld.Name),
 			SourceType: fld.FullType(),
 			PrimaryKey: fld.IsPrimaryKey(),
 		})
 	}
 	return out
 }
 // CheckFieldCompatibility ensures that the fields provided in the kafka config
 // are compatible with the fields in the existing table. It returns a copy of
 // the kafka config fields with empty values defaulted to the table field
 // configuration.
 func CheckFieldCompatibility(cflds []Field, scr *featurebase.ShowColumnsResponse) ([]Field, error) {
 	out := make([]Field, len(cflds))
 	for i, cfld := range cflds {
 		out[i] = cfld
 		cfldName := dax.FieldName(cfld.Name)
 		// Primary key field.
 		if cfld.PrimaryKey {
 			f := scr.Field(dax.PrimaryKeyFieldName)
 			if f == nil {
 				return nil, dax.NewErrFieldDoesNotExist(dax.PrimaryKeyFieldName) // It should be impossible to hit this.
 			}
 			if out[i].SourceType == "" {
 				if f.StringKeys() {
 					out[i].SourceType = dax.BaseTypeString
 				} else {
 					out[i].SourceType = dax.BaseTypeID
 				}
 			}
 			continue
 		}
 		// Non primary key fields.
 		if cfldName == dax.PrimaryKeyFieldName {
 			return nil, errors.Errorf("field named '%s' must be a primary key", dax.PrimaryKeyFieldName)
 		}
 		f := scr.Field(cfldName)
 		if f == nil {
 			return nil, dax.NewErrFieldDoesNotExist(cfldName)
 		}
 		if out[i].SourceType == "" {
 			out[i].SourceType = f.FullType()
 		}
 	}
 	return out, nil
 }
--- a/cli/kafka/runner.go
+++ b/cli/kafka/runner.go
@ -1,67 +0,0 @@
 package kafka
 import (
 	"io"
 	"time"
 	fbbatch "github.com/featurebasedb/featurebase/v3/batch"
 	"github.com/featurebasedb/featurebase/v3/errors"
 	"github.com/featurebasedb/featurebase/v3/idk"
 	"github.com/featurebasedb/featurebase/v3/idk/kafka_static"
 	"github.com/featurebasedb/featurebase/v3/logger"
 )
 // Runner is a CLI-specific kafka consumer. It's similar to
 // idk.kafka_static.Main in that it embeds idk.Main and contains additional
 // functionality specific to its use case.
 type Runner struct {
 	idk.Main   `flag:"!embed"`
 	KafkaHosts []string       `help:"Comma separated list of host:port pairs for Kafka."`
 	Group      string         `help:"Kafka group."`
 	Topics     []string       `help:"Kafka topics to read from."`
 	Timeout    time.Duration  `help:"Time to wait for more records from Kafka before flushing a batch. 0 to disable."`
 	Header     []idk.RawField `help:"Header configuration."`
 }
 func NewRunner(cfg ConfigForIDK, batcher fbbatch.Batcher, logWriter io.Writer) *Runner {
 	idkMain := idk.NewMain()
 	idkMain.IDField = cfg.IDField
 	idkMain.Index = cfg.Table
 	idkMain.Batcher = batcher
 	idkMain.BatchSize = cfg.BatchSize
 	idkMain.BatchMaxStaleness = cfg.BatchMaxStaleness
 	idkMain.SetBasic()
 	idkMain.SetLog(logger.NewStandardLogger(logWriter))
 	kr := &Runner{
 		Main:       *idkMain,
 		KafkaHosts: cfg.Hosts,
 		Group:      cfg.Group,
 		Topics:     cfg.Topics,
 		Header:     cfg.Fields,
 		Timeout:    cfg.Timeout,
 	}
 	kr.OffsetMode = true
 	kr.Main.Namespace = "cli_kafka_runner"
 	kr.Main.Pprof = "" // don't initialize pprof until we actually use it in tests
 	kr.NewSource = func() (idk.Source, error) {
 		source := kafka_static.NewSource()
 		source.Hosts = kr.KafkaHosts
 		source.Group = kr.Group
 		source.Topics = kr.Topics
 		source.Log = kr.Main.Log()
 		// source.TLS = m.KafkaTLS
 		source.Timeout = kr.Timeout
 		// source.SkipOld = m.SkipOld
 		source.HeaderFields = kr.Header
 		// source.S3Region = m.S3Region
 		// source.AllowMissingFields = m.AllowMissingFields
 		err := source.Open()
 		if err != nil {
 			return nil, errors.Wrap(err, "opening source")
 		}
 		return source, nil
 	}
 	return kr
 }
--- a/cli/meta.go
+++ b/cli/meta.go
--- a/cli/parts.go
+++ b/cli/parts.go
@ -1,136 +0,0 @@
 package cli
 import (
 	"fmt"
 	"io"
 	"os"
 	"strings"
 )
 // query is a collection of queryParts which, when applied together, make up an
 // executable SQL query.
 type query []queryPart
 func (q query) String() string {
 	var sb strings.Builder
 	for i := range q {
 		sb.WriteString(q[i].String())
 		if i < len(q)-1 {
 			sb.WriteRune('\n')
 		}
 	}
 	return sb.String()
 }
 // Reader returns the query as an io.Reader so that it can be passed to, for
 // example, http.Post().
 func (q query) Reader() io.Reader {
 	readers := make([]io.Reader, 0, len(q))
 	for i := range q {
 		readers = append(readers, q[i].Reader())
 	}
 	return io.MultiReader(readers...)
 }
 // queryPart is an interface representing anything which can use to build up a
 // query.
 type queryPart interface {
 	fmt.Stringer
 	Reader() io.Reader
 }
 func newRawQuery(s string) query {
 	return []queryPart{
 		newPartRaw(s),
 	}
 }
 // ////////////////////////////////////////////////////////////////////////////
 // raw
 // ////////////////////////////////////////////////////////////////////////////
 // Ensure type implements interface.
 var _ queryPart = (*partRaw)(nil)
 type partRaw struct {
 	raw string
 }
 func newPartRaw(s string) *partRaw {
 	return &partRaw{
 		raw: s,
 	}
 }
 func (p *partRaw) Reader() io.Reader {
 	return strings.NewReader(p.raw + "\n")
 }
 func (p *partRaw) String() string {
 	return p.raw
 }
 // ////////////////////////////////////////////////////////////////////////////
 // file
 // ////////////////////////////////////////////////////////////////////////////
 // Ensure type implements interface.
 var _ queryPart = (*partFile)(nil)
 type partFile struct {
 	file *os.File
 }
 func newPartFile(f *os.File) *partFile {
 	return &partFile{
 		file: f,
 	}
 }
 func (p *partFile) Reader() io.Reader {
 	return p.file
 }
 func (p *partFile) String() string {
 	return fmt.Sprintf("[file: %s]", p.file.Name())
 }
 // ////////////////////////////////////////////////////////////////////////////
 // batch file
 // ////////////////////////////////////////////////////////////////////////////
 // Ensure type implements interface.
 var _ queryPart = (*partBatchFile)(nil)
 type partBatchFile struct {
 	file *os.File
 }
 func (p *partBatchFile) Reader() io.Reader {
 	return p.file
 }
 func (p *partBatchFile) String() string {
 	return p.file.Name()
 }
 // ////////////////////////////////////////////////////////////////////////////
 // terminator (i.e. ";")
 // ////////////////////////////////////////////////////////////////////////////
 // Ensure type implements interface.
 var _ queryPart = (*partTerminator)(nil)
 type partTerminator struct{}
 func newPartTerminator() *partTerminator {
 	return &partTerminator{}
 }
 func (p *partTerminator) Reader() io.Reader {
 	return nil
 }
 func (p *partTerminator) String() string {
 	return terminationChar
 }
--- a/cli/queryer.go
+++ b/cli/queryer.go
@ -1,108 +0,0 @@
 package cli
 import (
 	"fmt"
 	"io"
 	"net/http"
 	"time"
 	featurebase "github.com/featurebasedb/featurebase/v3"
 	"github.com/pkg/errors"
 )
 type Queryer interface {
 	Query(org string, db string, sql io.Reader) (*featurebase.WireQueryResponse, error)
 }
 // Ensure type implements interface.
 var _ Queryer = (*nopQueryer)(nil)
 type nopQueryer struct{}
 func (qryr *nopQueryer) Query(org string, db string, sql io.Reader) (*featurebase.WireQueryResponse, error) {
 	return nil, errors.Errorf("no-op queryer")
 }
 // Ensure type implements interface.
 var _ Queryer = (*standardQueryer)(nil)
 // standardQueryer supports a standard featurebase deployment hitting the /sql
 // endpoint with a payload containing only the sql statement.
 type standardQueryer struct {
 	Host string
 	Port string
 }
 func (qryr *standardQueryer) Query(org string, db string, sql io.Reader) (*featurebase.WireQueryResponse, error) {
 	url := fmt.Sprintf("%s/sql", hostPort(qryr.Host, qryr.Port))
 	resp, err := http.Post(url, "application/json", sql)
 	if err != nil {
 		return nil, errors.Wrapf(err, "posting query")
 	}
 	fullbod, err := io.ReadAll(resp.Body)
 	if err != nil {
 		return nil, errors.Wrap(err, "reading response")
 	}
 	sqlResponse := &featurebase.WireQueryResponse{}
 	// TODO(tlt): switch this back once all responses are typed
 	// TODO(twg) 2023/03/01 using json.Number to decode large ints so care must be made
 	// if err := json.Unmarshal(fullbod, sqlResponse); err != nil {
 	if err := sqlResponse.UnmarshalJSONTyped(fullbod, true); err != nil {
 		return nil, errors.Wrapf(err, "unmarshaling query response, body:\n'%s'\n", fullbod)
 	}
 	return sqlResponse, nil
 }
 // Ensure type implements interface.
 var _ Queryer = (*serverlessQueryer)(nil)
 // serverlessQueryer is similar to the standardQueryer except that it hits a
 // different endpoint, and its payload is database-aware.
 type serverlessQueryer struct {
 	Host string
 	Port string
 }
 func (qryr *serverlessQueryer) Query(org string, db string, sql io.Reader) (*featurebase.WireQueryResponse, error) {
 	if org == "" {
 		return nil, NewErrOrganizationRequired()
 	}
 	url := fmt.Sprintf("%s/queryer/databases/%s/sql", hostPort(qryr.Host, qryr.Port), db)
 	if db == "" {
 		url = fmt.Sprintf("%s/queryer/sql", hostPort(qryr.Host, qryr.Port))
 	}
 	client := &http.Client{
 		Timeout: time.Second * 30,
 	}
 	req, err := http.NewRequest(http.MethodPost, url, sql)
 	if err != nil {
 		return nil, errors.Wrap(err, "creating new post request")
 	}
 	req.Header.Add("Content-Type", "text/plain")
 	req.Header.Add("OrganizationID", org)
 	var resp *http.Response
 	if resp, err = client.Do(req); err != nil {
 		return nil, errors.Wrap(err, "executing post request")
 	}
 	fullbod, err := io.ReadAll(resp.Body)
 	if err != nil {
 		return nil, errors.Wrap(err, "reading response")
 	}
 	sqlResponse := &featurebase.WireQueryResponse{}
 	// TODO(tlt): switch this back once all responses are typed
 	// if err := json.Unmarshal(fullbod, sqlResponse); err != nil {
 	if err := sqlResponse.UnmarshalJSONTyped(fullbod, true); err != nil {
 		return nil, errors.Wrapf(err, "unmarshaling query response, body:\n'%s'\n", fullbod)
 	}
 	return sqlResponse, nil
 }
--- a/cli/replacer.go
+++ b/cli/replacer.go
@ -1,109 +0,0 @@
 package cli
 import (
 	"strings"
 	"github.com/benhoyt/goawk/lexer"
 )
 // replacer can replace parts of a string based on some rules and the provided
 // map[string]string. For example, the Command can replace strings with values
 // in its `variables` map.
 type replacer struct {
 	m map[string]string
 }
 func newReplacer(m map[string]string) *replacer {
 	return &replacer{
 		m: m,
 	}
 }
 // replace replaces all instances of the string pattern `:key` with the value at
 // m[key]. For example we want something like this:
 //
 // GIVEN: `start :one,:'two', :"three" ::four ::`
 //
 // with map
 //
 //	map[string]string{
 //		"one":   "repl1",
 //		"three": "repl3",
 //	}
 //
 // WANT: `start repl1,:'two', "repl3" ::four ::`
 func (r *replacer) replace(s string) string {
 	// If no variables have been added to the map, there's no need to parse the
 	// string for variable replacement.
 	if len(r.m) == 0 {
 		return s
 	}
 	line := []byte(s)
 	lex := lexer.NewLexer(line)
 	// finger contains the index into line at the start of non-variable text
 	// that we want to include, as-is in the output.
 	var finger int
 	// sb builds the string which will be the final output.
 	var sb strings.Builder
 	for {
 		pos, tok, _ := lex.Scan()
 		switch tok {
 		case lexer.COLON:
 			// last is the last normal character position before the colon.
 			last := pos.Column - 1
 			// Get the next byte to see if the colon value is quoted, and if so,
 			// whether its has single or double quotes.
 			b := lex.PeekByte()
 			// padding is the amount of padding we have to consider around the
 			// variable name. If the variable is not quoted, it doesn't require
 			// any padding. But if it has quotes, it needs 2 characters of
 			// paddings to accomodate the quotes.
 			padding := 0
 			// quote holds the character to use to quote the final, replaced
 			// output value. Because the lexer doesn't tell us how a certain
 			// `string` token was quoted, we need to keep track of that here so
 			// we can put them back.
 			quote := ""
 			switch b {
 			case byte('\''): // single quote
 				quote = `'`
 				padding = 2
 			case byte('"'): // double quote
 				quote = `"`
 				padding = 2
 			}
 			pos, tok, key := lex.Scan()
 			switch tok {
 			case lexer.NAME, lexer.STRING:
 				// Write the normal text up to the variable replacement
 				// position.
 				sb.Write(line[finger:last])
 				if v, ok := r.m[key]; ok {
 					// Write replaced variable with the quotes it had.
 					sb.WriteString(quote + v + quote)
 				} else {
 					// Since the variable was not found in the map, just write
 					// back what was already there.
 					sb.WriteString(":" + quote + key + quote)
 				}
 				// Reset finger to point to the next position after the
 				// variable.
 				finger = pos.Column + len(key) + padding - 1
 			}
 		case lexer.EOF:
 			// Write the remainder of the string and return.
 			sb.Write(line[finger:])
 			return sb.String()
 		}
 	}
 }
--- a/cli/replacer_test.go
+++ b/cli/replacer_test.go
@ -1,109 +0,0 @@
 package cli
 import (
 	"fmt"
 	"testing"
 	"github.com/stretchr/testify/assert"
 )
 func TestReplacer(t *testing.T) {
 	t.Run("general replace function", func(t *testing.T) {
 		m := map[string]string{
 			"v1": "newVone",
 			"v2": "newVtwo",
 		}
 		tests := []struct {
 			s   string
 			m   map[string]string
 			exp string
 		}{
 			{
 				// no variables present
 				s:   "foo",
 				m:   m,
 				exp: "foo",
 			},
 			{
 				// variable prefix, but not in map
 				s:   ":foo",
 				m:   m,
 				exp: ":foo",
 			},
 			{
 				// variable name match, but missing prefix
 				s:   "v1",
 				m:   m,
 				exp: "v1",
 			},
 			{
 				// variable name match
 				s:   ":v1",
 				m:   m,
 				exp: "newVone",
 			},
 			{
 				// two variables, the same, no space
 				s:   ":v1:v1",
 				m:   m,
 				exp: "newVonenewVone",
 			},
 			{
 				// two variables, different, no space
 				s:   ":v1:v2",
 				m:   m,
 				exp: "newVonenewVtwo",
 			},
 			{
 				// two variables, different, spaces
 				s:   ":v1  :v2",
 				m:   m,
 				exp: "newVone  newVtwo",
 			},
 			{
 				// one variable, one non-variable, no space
 				s:   ":v1:foo",
 				m:   m,
 				exp: "newVone:foo",
 			},
 			{
 				// one non-variable, one variable, no space
 				s:   "foo:v1",
 				m:   m,
 				exp: "foonewVone",
 			},
 			{
 				// two variables, different, comma
 				s:   ":v1, :v2",
 				m:   m,
 				exp: "newVone, newVtwo",
 			},
 			{
 				// single quotes
 				s:   ":'v1'",
 				m:   m,
 				exp: "'newVone'",
 			},
 			{
 				// double quotes
 				s:   `:"v2"`,
 				m:   m,
 				exp: `"newVtwo"`,
 			},
 			{
 				// more quotes
 				s:   `start :v1,:'two', :"v2" ::four :: `,
 				m:   m,
 				exp: `start newVone,:'two', "newVtwo" ::four :: `,
 			},
 		}
 		for i, test := range tests {
 			t.Run(fmt.Sprintf("test-%d", i), func(t *testing.T) {
 				replacer := newReplacer(test.m)
 				assert.Equal(t, test.exp, replacer.replace(test.s))
 			})
 		}
 	})
 }
--- a/cli/splitter.go
+++ b/cli/splitter.go
@ -1,133 +0,0 @@
 package cli
 import (
 	"strings"
 	"github.com/pkg/errors"
 )
 // splitter is a line splitter which splits a line into queryParts and
 // metaCommands. It may not be necessary to have this be a separate struct since
 // it contains no members and just has the one `split()` method, but here we
 // are.
 type splitter struct {
 	replacer *replacer
 }
 func newSplitter(r *replacer) *splitter {
 	return &splitter{
 		replacer: r,
 	}
 }
 // split splits the given line into queryParts and metaCommands.
 // If a metaCommand is found, everything after that is considered either arguments to that
 // metaCommand, or additional metaCommands. In other words, queryParts can not follow
 // metaCommands in the same line.
 //
 // A line can contain any of the following patterns:
 // 1- [queryParts...]: "select * from tbl; select"
 // 2- [metaCommands...]: "\! pwd \q"
 // 3- [queryParts...][metaCommands...]: "select * from \i file.sql"
 func (s *splitter) split(line string) ([]queryPart, []metaCommand, error) {
 	// Look for a comment line.
 	if strings.HasPrefix(line, "--") {
 		return nil, nil, nil
 	}
 	// Look for a meta command.
 	parts := strings.SplitN(line, `\`, 2)
 	switch len(parts) {
 	case 1:
 		// slice of queryParts (pattern 1)
 		if qps, err := s.splitQueryParts(strings.TrimSpace(parts[0])); err != nil {
 			return nil, nil, errors.Wrap(err, "splitting query parts")
 		} else {
 			return qps, nil, nil
 		}
 	case 2:
 		// slice of parts + slice of meta commands (pattern 3)
 		// or
 		// slice of meta commands (pattern 2)
 		qps, err := s.splitQueryParts(strings.TrimSpace(parts[0]))
 		if err != nil {
 			return nil, nil, errors.Wrap(err, "splitting query parts")
 		}
 		mcs, err := s.splitMetaCommands(strings.TrimSpace(parts[1]))
 		if err != nil {
 			return nil, nil, errors.Wrap(err, "splitting meta commands")
 		}
 		return qps, mcs, nil
 	}
 	return nil, nil, nil
 }
 func (s *splitter) splitQueryParts(line string) ([]queryPart, error) {
 	if line == "" {
 		return nil, nil
 	}
 	// Look for a termination character;
 	parts := strings.Split(line, terminationChar)
 	// Do variable replacement.
 	for i := range parts {
 		parts[i] = s.replacer.replace(parts[i])
 	}
 	if len(parts) == 1 {
 		part0 := strings.TrimSpace(parts[0])
 		return []queryPart{
 			newPartRaw(part0),
 		}, nil
 	}
 	qps := make([]queryPart, 0)
 	for i := range parts {
 		part := strings.TrimSpace(parts[i])
 		if part == "" {
 			// If the line starts with a ";", treat it as a terminator for a
 			// previous line.
 			if i == 0 {
 				qps = append(qps, &partTerminator{})
 			}
 			continue
 		}
 		qps = append(qps, newPartRaw(part))
 		if i < len(parts)-1 {
 			qps = append(qps, &partTerminator{})
 		}
 	}
 	return qps, nil
 }
 func (s *splitter) splitMetaCommands(in string) ([]metaCommand, error) {
 	parts := strings.Split(in, `\`)
 	if len(parts) == 1 {
 		mc, err := splitMetaCommand(parts[0], s.replacer)
 		if err != nil {
 			return nil, errors.Wrapf(err, "splitting meta command: %s", parts[0])
 		}
 		return []metaCommand{mc}, nil
 	}
 	mcs := make([]metaCommand, 0)
 	for i := range parts {
 		part := strings.TrimSpace(parts[i])
 		if part == "" {
 			continue
 		}
 		mc, err := splitMetaCommand(part, s.replacer)
 		if err != nil {
 			return nil, errors.Wrapf(err, "splitting meta command: %s", part)
 		}
 		mcs = append(mcs, mc)
 	}
 	return mcs, nil
 }
--- a/cli/splitter_test.go
+++ b/cli/splitter_test.go
@ -1,146 +0,0 @@
 package cli
 import (
 	"fmt"
 	"testing"
 	"github.com/stretchr/testify/assert"
 )
 func TestSplitter(t *testing.T) {
 	s := newSplitter(newReplacer(nil))
 	t.Run("Split", func(t *testing.T) {
 		tests := []struct {
 			line            string
 			expQueryParts   []queryPart
 			expMetaCommands []metaCommand
 			expError        string
 		}{
 			{
 				line: `foo`,
 				expQueryParts: []queryPart{
 					newPartRaw("foo"),
 				},
 			},
 			{
 				line: `foo;`,
 				expQueryParts: []queryPart{
 					newPartRaw("foo"),
 					newPartTerminator(),
 				},
 			},
 			{
 				line: `foo; `,
 				expQueryParts: []queryPart{
 					newPartRaw("foo"),
 					newPartTerminator(),
 				},
 			},
 			{
 				line: `foo; ; `,
 				expQueryParts: []queryPart{
 					newPartRaw("foo"),
 					newPartTerminator(),
 				},
 			},
 			{
 				line: `foo; bar`,
 				expQueryParts: []queryPart{
 					newPartRaw("foo"),
 					newPartTerminator(),
 					newPartRaw("bar"),
 				},
 			},
 			{
 				line: `foo; bar;`,
 				expQueryParts: []queryPart{
 					newPartRaw("foo"),
 					newPartTerminator(),
 					newPartRaw("bar"),
 					newPartTerminator(),
 				},
 			},
 			{
 				line: `\q`,
 				expMetaCommands: []metaCommand{
 					&metaQuit{},
 				},
 			},
 			{
 				line: ` \p`,
 				expMetaCommands: []metaCommand{
 					&metaPrint{},
 				},
 			},
 			{
 				line: `\q \p`,
 				expMetaCommands: []metaCommand{
 					&metaQuit{},
 					&metaPrint{},
 				},
 			},
 			{
 				line: `\q \p arg1 arg2`,
 				expMetaCommands: []metaCommand{
 					&metaQuit{},
 					&metaPrint{},
 				},
 			},
 			{
 				line: `\set`,
 				expMetaCommands: []metaCommand{
 					&metaSet{
 						args: []string{},
 					},
 				},
 			},
 			{
 				line: `\set arg1 arg2`,
 				expMetaCommands: []metaCommand{
 					&metaSet{
 						args: []string{"arg1", "arg2"},
 					},
 				},
 			},
 			{
 				line: `\set 'arg1' 'arg2'`,
 				expMetaCommands: []metaCommand{
 					&metaSet{
 						args: []string{"arg1", "arg2"},
 					},
 				},
 			},
 			{
 				line: `\set 'arg1' '"arg2"'`,
 				expMetaCommands: []metaCommand{
 					&metaSet{
 						args: []string{"arg1", "\"arg2\""},
 					},
 				},
 			},
 			{
 				line:     `\`,
 				expError: "unsupported meta-command:",
 			},
 			{
 				line:     `\xyzxyz`,
 				expError: "unsupported meta-command:",
 			},
 		}
 		for i, tt := range tests {
 			t.Run(fmt.Sprintf("test-%d-%s", i, tt.line), func(t *testing.T) {
 				qps, mcs, err := s.split(tt.line)
 				if tt.expError != "" {
 					if assert.Error(t, err) {
 						assert.Contains(t, err.Error(), tt.expError)
 					}
 					return
 				}
 				assert.NoError(t, err)
 				assert.ElementsMatch(t, tt.expQueryParts, qps)
 				assert.ElementsMatch(t, tt.expMetaCommands, mcs)
 			})
 		}
 	})
 }
--- a/cli/testdata/database
+++ b/cli/testdata/database
@ -1,53 +0,0 @@
 // Show databases now that we have set org.
 SEND:SHOW DATABASES;
 EXPECT:+-----+------+-------+------------+------------+------------+-------+-------------+
 EXPECT:| _id | name | owner | updated_by | created_at | updated_at | units | description |
 EXPECT:+-----+------+-------+------------+------------+------------+-------+-------------+
 EXPECT:+-----+------+-------+------------+------------+------------+-------+-------------+
 EXPECT:
 // Create db1.
 SEND:CREATE DATABASE db1 WITH UNITS 1;
 EXPECT:
 // List databases via SHOW DATABASES.
 SEND:SHOW DATABASES;
 EXPECT:+--------------------------------------+------+-------+------------+----------------------+----------------------+-------+-------------+
 EXPECT:| _id                                  | name | owner | updated_by | created_at           | updated_at           | units | description |
 EXPECT:+--------------------------------------+------+-------+------------+----------------------+----------------------+-------+-------------+
 EXPECTCOMP:WithFormat:| {uuid} | db1  |       |            | {timestamp} | {timestamp} |     1 |             |
 EXPECT:+--------------------------------------+------+-------+------------+----------------------+----------------------+-------+-------------+
 EXPECT:
 // List databases via SHOW DATABASES.
 SEND:\l
 EXPECT:+--------------------------------------+------+-------+------------+----------------------+----------------------+-------+-------------+
 EXPECT:| _id                                  | name | owner | updated_by | created_at           | updated_at           | units | description |
 EXPECT:+--------------------------------------+------+-------+------------+----------------------+----------------------+-------+-------------+
 EXPECTCOMP:WithFormat:| {uuid} | db1  |       |            | {timestamp} | {timestamp} |     1 |             |
 EXPECT:+--------------------------------------+------+-------+------------+----------------------+----------------------+-------+-------------+
 EXPECT:
 // Check database connection.
 SEND:\c
 EXPECT:You are not connected to a database.
 // Try connecting to an invalid database.
 SEND:\c invalid
 EXPECT:executing meta command: invalid database: invalid
 // Try connecting with too many arguments.
 SEND:\c db1 extra
 EXPECT:executing meta command: meta command 'connect' takes zero or one argument
 // Connect to a database.
 SEND:\c db1
 EXPECTCOMP:WithFormat:You are now connected to database "db1" ({uuid}).
 // Disconnect from the current database.
 SEND:\c -
 EXPECT:You are not connected to a database.
 // Connect to a database again.
 SEND:\c db1
 EXPECTCOMP:WithFormat:You are now connected to database "db1" ({uuid}).
--- a/cli/testdata/famous.csv
+++ b/cli/testdata/famous.csv
@ -1,10 +0,0 @@
 "Id", "Name", "Short description", "Gender", "Country", "Occupation", "Birth year", "Death year", "Manner of death", "Age of death"
 1, "George Washington", "1st president of the United States (1732–1799)", "Male", "United States of America; Kingdom of Great Britain", "Politician", "1732", "1799", "natural causes", "67"
 2, "Douglas Adams", "English writer and humorist", "Male", "United Kingdom", "Artist", "1952", "2001", "natural causes", "49"
 3, "Abraham Lincoln", "16th president of the United States (1809-1865)", "Male", "United States of America", "Politician", "1809", "1865", "homicide", "56"
 4, "Wolfgang Amadeus Mozart", "Austrian composer of the Classical period", "Male", "Archduchy of Austria; Archbishopric of Salzburg", "Artist", "1756", "1791", "0", "35"
 5, "Ludwig van Beethoven", "German classical and romantic composer", "Male", "Holy Roman Empire; Austrian Empire", "Artist", "1770", "1827", "0", "57"
 6, "Jean-François Champollion", "French classical scholar", "Male", "Kingdom of France; First French Empire", "Egyptologist", "1790", "1832", "natural causes", "42"
 7, "Paul Morand", "French writer", "Male", "France", "Artist", "1888", "1976", "0", "88"
 8, "Claude Monet", "French impressionist painter (1840-1926)", "Male", "France", "Artist", "1840", "1926", "natural causes", "86"
--- a/cli/testdata/meta_bang
+++ b/cli/testdata/meta_bang
@ -1,8 +0,0 @@
 SEND:\! echo 'foo'
 EXPECT:foo
 SEND:\! echo "foo"
 EXPECT:"foo"
 SEND:\!
 EXPECT:executing meta command: meta command '!' requires at least one argument
--- a/cli/testdata/meta_cd
+++ b/cli/testdata/meta_cd
@ -1,17 +0,0 @@
 // Make a directory so we can test \cd'ing into it.
 SEND:\! mkdir cli-test-dir
 SEND:\cd cli-test-dir
 SEND:\cd ..
 SEND:\! rmdir cli-test-dir
 // TODO(tlt): before we do this, we should implement the ability to execute
 // commands in a \set like:
 // \set homedir `pwd`
 // then we can store what directory we're in so we can move back to it 
 // at the end of the test
 // Switch to home directory.
 // SEND:\cd
 // Expect error on extra argument to \cd.
 SEND:\cd dir extra
 EXPECT:executing meta command: meta command 'cd' takes zero or one argument
--- a/cli/testdata/meta_describe
+++ b/cli/testdata/meta_describe
@ -1,33 +0,0 @@
 // TODO(tlt): we can't run this test until we get the system tables under control (i.e. sorted). Currently, fb_views is in a map with users, so the following can fail 50% of the time.
 // Show tables for database by calling describe with no args.
 // SEND:\d
 // EXPECT:+-------------------------+-------------------------+-------+------------+----------------------+----------------------+-------+------------+------------------------+
 // EXPECT:| _id                     | name                    | owner | updated_by | created_at           | updated_at           | keys  | space_used | description            |
 // EXPECT:+-------------------------+-------------------------+-------+------------+----------------------+----------------------+-------+------------+------------------------+
 // EXPECTCOMP:WithFormat:| fb_veiws                | fb_views                |       |            | {timestamp} | {timestamp} | true  |          0 | system table for views |
 // EXPECTCOMP:WithFormat:| users                   | users                   |       |            | {timestamp} | {timestamp} | false |          0 |                        |
 // EXPECTCOMP:WithFormat:| fb_____________________ | fb_____________________ |       |            | {timestamp} | {timestamp} | false |          0 |                        |
 // EXPECTCOMP:WithFormat:| fb_____________________ | fb_____________________ |       |            | {timestamp} | {timestamp} | false |          0 |                        |
 // EXPECTCOMP:WithFormat:| fb_____________________ | fb_____________________ |       |            | {timestamp} | {timestamp} | false |          0 |                        |
 // EXPECTCOMP:WithFormat:| fb_____________________ | fb_____________________ |       |            | {timestamp} | {timestamp} | false |          0 |                        |
 // EXPECTCOMP:WithFormat:| fb_____________________ | fb_____________________ |       |            | {timestamp} | {timestamp} | false |          0 |                        |
 // EXPECT:+-------------------------+-------------------------+-------+------------+----------------------+----------------------+-------+------------+------------------------+
 // EXPECT:
 // Show columns for table.
 SEND:\d users
 EXPECT:+------+------+--------+----------------------+-------+------------+------------+-------+----------------------+---------------------+----------+-------+-------------+-----+
 EXPECT:| _id  | name | type   | created_at           | keys  | cache_type | cache_size | scale |                  min |                 max | timeunit | epoch | timequantum | ttl |
 EXPECT:+------+------+--------+----------------------+-------+------------+------------+-------+----------------------+---------------------+----------+-------+-------------+-----+
 EXPECTCOMP:WithFormat:| _id  | _id  | id     | {timestamp} | false |            |          0 |     0 |                    0 |                   0 |          |     0 |             | 0s  |
 EXPECTCOMP:WithFormat:| name | name | string | {timestamp} | true  | ranked     |      50000 |     0 |                    0 |                   0 |          |     0 |             | 0s  |
 EXPECTCOMP:WithFormat:| age  | age  | int    | {timestamp} | false |            |          0 |     0 | -9223372036854775808 | 9223372036854775807 |          |     0 |             | 0s  |
 EXPECT:+------+------+--------+----------------------+-------+------------+------------+-------+----------------------+---------------------+----------+-------+-------------+-----+
 EXPECT:
 // Show columns for an invalid table.
 SEND:\d invalid
 EXPECT:Error: compiling plan: [1:19] table 'invalid' not found
 SEND:\d users extra
 EXPECT:executing meta command: meta command 'describe' takes zero or one argument
--- a/cli/testdata/meta_echo
+++ b/cli/testdata/meta_echo
@ -1,6 +0,0 @@
 SEND:\echo
 EXPECT:
 // Simple \echo.
 SEND:\echo foo bar
 EXPECT:foo bar
--- a/cli/testdata/meta_file
+++ b/cli/testdata/meta_file
@ -1,70 +0,0 @@
 // Create a table.
 SEND:CREATE TABLE famous (
 SEND:    _id ID,
 SEND:    name STRING,
 SEND:    description STRING,
 SEND:    gender STRING,
 SEND:    country STRING,
 SEND:    occupation STRING,
 SEND:    birth_year INT min -32767 max 32767,
 SEND:    death_year INT min -32767 max 32767,
 SEND:    death_manner STRING,
 SEND:    birth_age INT min -32767 max 32767
 SEND:);
 EXPECT:
 // Open bulk insert.
 SEND:BULK INSERT
 SEND:INTO famous (_id, name, description, gender, country, occupation,
 SEND:    birth_year, death_year, death_manner, birth_age )
 SEND:MAP(0 INT,
 SEND:1 STRING,
 SEND:2 STRING,
 SEND:3 STRING,
 SEND:4 STRING,
 SEND:5 STRING,
 SEND:6 INT,
 SEND:7 INT,
 SEND:8 STRING,
 SEND:9 INT )
 SEND:FROM
 SEND:    x'
 // Call \file
 SEND:\file testdata/famous.csv
 // Close bulk insert.
 SEND:'
 SEND:WITH
 SEND:    BATCHSIZE 100000
 SEND:    FORMAT 'CSV'
 SEND:    INPUT 'STREAM'
 SEND:    HEADER_ROW;
 EXPECT:
 // Query table to ensure we have data.
 SEND:SELECT * FROM famous;
 EXPECT:+-----+---------------------------+-------------------------------------------------+--------+----------------------------------------------------+--------------+------------+------------+----------------+-----------+
 EXPECT:| _id | name                      | description                                     | gender | country                                            | occupation   | birth_year | death_year | death_manner   | birth_age |
 EXPECT:+-----+---------------------------+-------------------------------------------------+--------+----------------------------------------------------+--------------+------------+------------+----------------+-----------+
 EXPECT:|   1 | George Washington         | 1st president of the United States (1732–1799)  | Male   | United States of America; Kingdom of Great Britain | Politician   |       1732 |       1799 | natural causes |        67 |
 EXPECT:|   2 | Douglas Adams             | English writer and humorist                     | Male   | United Kingdom                                     | Artist       |       1952 |       2001 | natural causes |        49 |
 EXPECT:|   3 | Abraham Lincoln           | 16th president of the United States (1809-1865) | Male   | United States of America                           | Politician   |       1809 |       1865 | homicide       |        56 |
 EXPECT:|   4 | Wolfgang Amadeus Mozart   | Austrian composer of the Classical period       | Male   | Archduchy of Austria; Archbishopric of Salzburg    | Artist       |       1756 |       1791 | 0              |        35 |
 EXPECT:|   5 | Ludwig van Beethoven      | German classical and romantic composer          | Male   | Holy Roman Empire; Austrian Empire                 | Artist       |       1770 |       1827 | 0              |        57 |
 EXPECT:|   6 | Jean-François Champollion | French classical scholar                        | Male   | Kingdom of France; First French Empire             | Egyptologist |       1790 |       1832 | natural causes |        42 |
 EXPECT:|   7 | Paul Morand               | French writer                                   | Male   | France                                             | Artist       |       1888 |       1976 | 0              |        88 |
 EXPECT:|   8 | Claude Monet              | French impressionist painter (1840-1926)        | Male   | France                                             | Artist       |       1840 |       1926 | natural causes |        86 |
 EXPECT:+-----+---------------------------+-------------------------------------------------+--------+----------------------------------------------------+--------------+------------+------------+----------------+-----------+
 EXPECT:
 // TODO(tlt): dropping the table seems to cause problems.
 // Drop the table.
 //SEND:DROP TABLE famous;
 // Ensure that invalid aruments (none or too many) return an error.
 SEND:\file
 EXPECT:executing meta command: meta command 'file' requires exactly one argument
 SEND:\file filename extra
 EXPECT:executing meta command: meta command 'file' requires exactly one argument
--- a/cli/testdata/meta_include
+++ b/cli/testdata/meta_include
@ -1,24 +0,0 @@
 // Include with no argument should error.
 SEND:\i
 EXPECT:executing meta command: meta command 'include' requires exactly one argument
 // Include with too many arguments should error.
 SEND:\include testdata/people.sql extra
 EXPECT:executing meta command: meta command 'include' requires exactly one argument
 // Invalid file should error.
 SEND:\include invalid.file
 EXPECT:executing meta command: opening file: invalid.file: open invalid.file: no such file or directory
 SEND:\include testdata/people.sql
 EXPECT:
 EXPECT:
 EXPECT:+-----+------+-----+
 EXPECT:| _id | name | age |
 EXPECT:+-----+------+-----+
 EXPECT:|   1 | Amy  |  42 |
 EXPECT:|   2 | Bob  |  27 |
 EXPECT:|   3 | Carl |  33 |
 EXPECT:+-----+------+-----+
 EXPECT:
 EXPECT:mix in a meta command
--- a/cli/testdata/meta_output
+++ b/cli/testdata/meta_output
@ -1,67 +0,0 @@
 SEND:SELECT * FROM users;
 EXPECT:+-----+-------+-----+
 EXPECT:| _id | name  | age |
 EXPECT:+-----+-------+-----+
 EXPECT:|   1 | Anne  |  38 |
 EXPECT:|   2 | Bill  |  23 |
 EXPECT:|   3 | Cindy |  64 |
 EXPECT:+-----+-------+-----+
 EXPECT:
 // Redirect output to a file.
 SEND:\o test-output-file
 SEND:SELECT * FROM users;
 // Ensure the output went to the file.
 SEND:\! cat test-output-file
 EXPECT:+-----+-------+-----+
 EXPECT:| _id | name  | age |
 EXPECT:+-----+-------+-----+
 EXPECT:|   1 | Anne  |  38 |
 EXPECT:|   2 | Bill  |  23 |
 EXPECT:|   3 | Cindy |  64 |
 EXPECT:+-----+-------+-----+
 EXPECT:
 // Let's test some qecho stuff here while we're at it.
 SEND:\qecho string with "double quotes"
 SEND:\qecho -n one
 SEND:\qecho -n two
 SEND:\qecho three
 SEND:\qecho four
 SEND:\! cat test-output-file
 EXPECT:+-----+-------+-----+
 EXPECT:| _id | name  | age |
 EXPECT:+-----+-------+-----+
 EXPECT:|   1 | Anne  |  38 |
 EXPECT:|   2 | Bill  |  23 |
 EXPECT:|   3 | Cindy |  64 |
 EXPECT:+-----+-------+-----+
 EXPECT:
 EXPECT:string with "double quotes"
 EXPECT:onetwothree
 EXPECT:four
 // And \warn messages should still go to stderr, not the file.
 SEND:\warn a warning string
 EXPECT:a warning string
 // Remove the file.
 SEND:\! rm test-output-file
 // Set the output back to stdout.
 SEND:\o
 SEND:SELECT * FROM users;
 EXPECT:+-----+-------+-----+
 EXPECT:| _id | name  | age |
 EXPECT:+-----+-------+-----+
 EXPECT:|   1 | Anne  |  38 |
 EXPECT:|   2 | Bill  |  23 |
 EXPECT:|   3 | Cindy |  64 |
 EXPECT:+-----+-------+-----+
 EXPECT:
 // Ensure extra arguments to \output causes an error.
 SEND:\o filename extra
 EXPECT:executing meta command: meta command 'output' takes zero or one argument
--- a/cli/testdata/meta_pset_border
+++ b/cli/testdata/meta_pset_border
@ -1,52 +0,0 @@
 SEND:\pset border 2
 EXPECT:Border style is 2.
 SEND:SELECT 1 as foo, 'baz' as bar;
 EXPECT:+-----+-----+
 EXPECT:| foo | bar |
 EXPECT:+-----+-----+
 EXPECT:|   1 | baz |
 EXPECT:+-----+-----+
 EXPECT:
 SEND:\pset border
 EXPECT:Border style is 2.
 SEND:\pset border 999
 EXPECT:Border style is 0.
 SEND:\pset border 1
 EXPECT:Border style is 1.
 SEND:SELECT 1 as foo, 'baz' as bar;
 EXPECT: foo | bar 
 EXPECT:-----+-----
 EXPECT:   1 | baz 
 EXPECT:
 SEND:\pset border 2
 EXPECT:Border style is 2.
 SEND:SELECT 1 as foo, 'baz' as bar;
 EXPECT:+-----+-----+
 EXPECT:| foo | bar |
 EXPECT:+-----+-----+
 EXPECT:|   1 | baz |
 EXPECT:+-----+-----+
 EXPECT:
 SEND:\pset border 0
 EXPECT:Border style is 0.
 SEND:SELECT 1 as foo, 'baz' as bar;
 EXPECT:foo bar
 EXPECT:--- ---
 EXPECT:  1 baz
 EXPECT:
 SEND:\pset border 1 extra
 EXPECT:executing meta command: meta command 'pset' takes zero, one, or two arguments
 // Set border back to the testing default.
 SEND:\pset border 2
 EXPECT:Border style is 2.
--- a/cli/testdata/meta_pset_expanded
+++ b/cli/testdata/meta_pset_expanded
@ -1,35 +0,0 @@
 // Set to off.
 SEND:\pset expanded off
 EXPECT:Expanded display is off.
 // Set to on.
 SEND:\pset expanded on
 EXPECT:Expanded display is on.
 // Toggle to off.
 SEND:\pset expanded
 EXPECT:Expanded display is off.
 // Toggle to on.
 SEND:\pset expanded
 EXPECT:Expanded display is on.
 // Set to something invalid.
 SEND:\pset expanded invalid
 EXPECT:executing meta command: unrecognized value "invalid" for "expanded": Boolean expected
 // Ensure expanded shows results vertically.
 SEND:SELECT 1 as foo, 'baz' as bar;
 EXPECT:+-----+-----+
 EXPECT:| foo | 1   |
 EXPECT:| bar | baz |
 EXPECT:+-----+-----+
 EXPECT:
 // Set back to off as we started.
 SEND:\pset expanded off
 EXPECT:Expanded display is off.
 // make sure the \x meta-command returns expected errors
 SEND:\x on extra
 EXPECT:executing meta command: meta command 'expanded' takes zero or one argument
--- a/cli/testdata/meta_pset_format_csv
+++ b/cli/testdata/meta_pset_format_csv
@ -1,47 +0,0 @@
 SEND:\pset format csv
 EXPECT:Output format is csv.
 SEND:SELECT * FROM users;
 EXPECT:_id,name,age
 EXPECT:1,Anne,38
 EXPECT:2,Bill,23
 EXPECT:3,Cindy,64
 // Exclude headers.
 SEND:\t on
 EXPECT:Tuples only is on.
 SEND:SELECT * FROM users;
 EXPECT:1,Anne,38
 EXPECT:2,Bill,23
 EXPECT:3,Cindy,64
 // Reset headers.
 SEND:\t off
 EXPECT:Tuples only is off.
 // Set expanded to on.
 SEND:\x on
 EXPECT:Expanded display is on.
 SEND:SELECT * FROM users;
 EXPECT:_id,1
 EXPECT:name,Anne
 EXPECT:age,38
 EXPECT:_id,2
 EXPECT:name,Bill
 EXPECT:age,23
 EXPECT:_id,3
 EXPECT:name,Cindy
 EXPECT:age,64
 // Set expanded back to off.
 SEND:\x off
 EXPECT:Expanded display is off.
 // Set format back to aligned as we started.
 SEND:\pset format aligned
 EXPECT:Output format is aligned.
 SEND:\pset format invalid
 EXPECT:executing meta command: \pset: allowed formats are aligned, csv
--- a/cli/testdata/meta_pset_tuples_only
+++ b/cli/testdata/meta_pset_tuples_only
@ -1,34 +0,0 @@
 // Set to off.
 SEND:\pset tuples_only off
 EXPECT:Tuples only is off.
 // Set to on.
 SEND:\pset tuples_only on
 EXPECT:Tuples only is on.
 // Toggle to off.
 SEND:\pset tuples_only
 EXPECT:Tuples only is off.
 // Toggle to on.
 SEND:\pset tuples_only
 EXPECT:Tuples only is on.
 // Set to something invalid.
 SEND:\pset tuples_only invalid
 EXPECT:executing meta command: unrecognized value "invalid" for "tuples_only": Boolean expected
 // Ensure tuples_only shows only tuples.
 SEND:SELECT 1 as foo, 'baz' as bar;
 EXPECT:+---+-----+
 EXPECT:| 1 | baz |
 EXPECT:+---+-----+
 EXPECT:
 // Set back to off as we started.
 SEND:\pset tuples_only off
 EXPECT:Tuples only is off.
 // make sure the \t meta-command returns expected errors
 SEND:\t off extra
 EXPECT:executing meta command: meta command 'tuples_only' takes zero or one argument
--- a/cli/testdata/meta_set
+++ b/cli/testdata/meta_set
@ -1,31 +0,0 @@
 SEND:\set
 SEND:\set var1 foo
 SEND:\set
 EXPECT:var1 = 'foo'
 SEND:\set var2 bar
 SEND:\set
 EXPECT:var1 = 'foo'
 EXPECT:var2 = 'bar'
 SEND:\set var3 zoo
 SEND:\set
 EXPECT:var1 = 'foo'
 EXPECT:var2 = 'bar'
 EXPECT:var3 = 'zoo'
 SEND:\unset
 EXPECT:\unset: missing required argument
 SEND:\unset non-existent-key
 SEND:\unset var1
 SEND:\set
 EXPECT:var2 = 'bar'
 EXPECT:var3 = 'zoo'
 SEND:\unset var2 extra
 EXPECT:\unset: extra argument "extra" ignored
 SEND:\set
 EXPECT:var3 = 'zoo'
--- a/cli/testdata/meta_timing
+++ b/cli/testdata/meta_timing
@ -1,50 +0,0 @@
 // Start by ensuring timing is off.
 SEND:\timing off
 EXPECT:Timing is off.
 // Set timing on.
 SEND:\timing on
 EXPECT:Timing is on.
 // Toggle timing.
 SEND:\timing
 EXPECT:Timing is off.
 // Toggle timing again.
 SEND:\timing
 EXPECT:Timing is on.
 // Send extra argument to \timing.
 SEND:\timing on extra
 EXPECT:executing meta command: meta command 'timing' takes zero or one argument
 SEND:SELECT * FROM users;
 EXPECT:+-----+-------+-----+
 EXPECT:| _id | name  | age |
 EXPECT:+-----+-------+-----+
 EXPECT:|   1 | Anne  |  38 |
 EXPECT:|   2 | Bill  |  23 |
 EXPECT:|   3 | Cindy |  64 |
 EXPECT:+-----+-------+-----+
 EXPECT:
 EXPECTCOMP:HasPrefix:Execution time:
 // Turn timing back off.
 SEND:\timing off
 EXPECT:Timing is off.
 // Ensure we don't get timing.
 SEND:SELECT * FROM users;
 EXPECT:+-----+-------+-----+
 EXPECT:| _id | name  | age |
 EXPECT:+-----+-------+-----+
 EXPECT:|   1 | Anne  |  38 |
 EXPECT:|   2 | Bill  |  23 |
 EXPECT:|   3 | Cindy |  64 |
 EXPECT:+-----+-------+-----+
 EXPECT:
 // Ensure an invalid timing value returns an error.
 SEND:\timing invalid
 EXPECT:executing meta command: unrecognized value "invalid" for "\timing": Boolean expected
--- a/cli/testdata/meta_write
+++ b/cli/testdata/meta_write
@ -1,24 +0,0 @@
 // Make sure there's something in the query buffer.
 // This is left unterminated because we don't need to execute the query;
 // we just need there to be something in the buffer.
 SEND:SELECT * FROM invalid-table
 SEND:\write query-buffer-contents
 // Reset the buffer.
 SEND:\r
 EXPECT:Query buffer reset (cleared).
 // Read from the file.
 SEND:\! cat query-buffer-contents
 EXPECT:SELECT * FROM invalid-table
 // Remove the file.
 SEND:\! rm query-buffer-contents
 // Send \write with no arguments.
 SEND:\write
 EXPECT:\w: missing required argument
 // Send \write with extra arguments.
 SEND:\write filename extra
 EXPECT:executing meta command: meta command 'w' exactly one argument
--- a/cli/testdata/people.sql
+++ b/cli/testdata/people.sql
@ -1,12 +0,0 @@
 -- Create a table.
 create table people (_id id, name string, age int);
 -- Insert some values.
 insert into people values (1, 'Amy', 42), (2, 'Bob', 27), (3, 'Carl', 33);
 -- Get all rows from the table.
 select * from people;
 -- Mix in a meta-command to show that both are supported
 -- in the include file.
 \echo mix in a meta command
--- a/cli/testdata/query_buffer
+++ b/cli/testdata/query_buffer
@ -1,40 +0,0 @@
 SEND:select 1 as foo;
 EXPECT:+-----+
 EXPECT:| foo |
 EXPECT:+-----+
 EXPECT:|   1 |
 EXPECT:+-----+
 EXPECT:
 SEND:\p
 EXPECT:select 1 as foo;
 SEND:select 2
 SEND:\p
 EXPECT:select 2
 SEND:\r
 EXPECT:Query buffer reset (cleared).
 SEND:\p
 EXPECT:select 1 as foo;
 SEND:select 3
 SEND:\p
 EXPECT:select 3
 SEND:as foo
 SEND:\p
 EXPECT:select 3
 EXPECT:as foo
 SEND:;
 EXPECT:+-----+
 EXPECT:| foo |
 EXPECT:+-----+
 EXPECT:|   3 |
 EXPECT:+-----+
 EXPECT:
 SEND:\p
 EXPECT:select 3
 EXPECT:as foo;
--- a/cli/testdata/setup
+++ b/cli/testdata/setup
@ -1,51 +0,0 @@
 // Startup splash.
 EXPECT:FeatureBase CLI ()
 EXPECT:Type "\q" to quit.
 EXPECT:Detected on-prem, serverless deployment.
 EXPECTCOMP:HasPrefix:Host: http://localhost:
 EXPECT:You are not connected to a database.
 // Show databases.
 SEND:SHOW DATABASES;
 EXPECT:Organization required. Use \org to set an organization.
 // Get current org.
 SEND:\org
 EXPECT:You have not set an organization.
 // Set org.
 SEND:\org acme
 EXPECT:You have set organization "acme".
 // Try to set org with too many arguments.
 SEND:\org acme extra
 EXPECT:executing meta command: meta command 'org' takes zero or one argument
 // Set location to UTC so that expected timestamp size is consistent.
 // Without this, a test running locally in may have a timestamp that
 // ends in a timezone offset such as `-06:00`, while one running as UTC
 // will have `Z`. Since these string lengths differ, our generic
 // {timestamp} comparison will fail.
 SEND:\pset location UTC
 EXPECT:Location is UTC.
 // Set an invalid location.
 SEND:\pset location invalid
 EXPECT:executing meta command: loading location: invalid: unknown time zone invalid
 // Try to set location with too many arguments.
 SEND:\pset location UTC extra
 EXPECT:executing meta command: meta command 'pset' takes zero, one, or two arguments
 // Set border to 2 for testing because it makes it easier to visually see
 // what the tests are expecting (because lines don't end in spaces).
 SEND:\pset border 2
 EXPECT:Border style is 2.
 // Check the state of pset.
 SEND:\pset
 EXPECT:border      2
 EXPECT:expanded    off
 EXPECT:format      aligned
 EXPECT:location    UTC
 EXPECT:tuples_only off
--- a/cli/testdata/table
+++ b/cli/testdata/table
@ -1,72 +0,0 @@
 // Show tables for database using SHOW TABLES WITH SYSTEM.
 SEND:SHOW TABLES WITH SYSTEM;
 EXPECT:+-------------------------+-------------------------+-------+------------+----------------------+----------------------+-------+------------+-------------+
 EXPECT:| _id                     | name                    | owner | updated_by | created_at           | updated_at           | keys  | space_used | description |
 EXPECT:+-------------------------+-------------------------+-------+------------+----------------------+----------------------+-------+------------+-------------+
 EXPECTCOMP:WithFormat:| fb_____________________ | fb_____________________ |       |            | {timestamp} | {timestamp} | false |          0 |             |
 EXPECTCOMP:WithFormat:| fb_____________________ | fb_____________________ |       |            | {timestamp} | {timestamp} | false |          0 |             |
 EXPECTCOMP:WithFormat:| fb_____________________ | fb_____________________ |       |            | {timestamp} | {timestamp} | false |          0 |             |
 EXPECTCOMP:WithFormat:| fb_____________________ | fb_____________________ |       |            | {timestamp} | {timestamp} | false |          0 |             |
 EXPECTCOMP:WithFormat:| fb_____________________ | fb_____________________ |       |            | {timestamp} | {timestamp} | false |          0 |             |
 EXPECT:+-------------------------+-------------------------+-------+------------+----------------------+----------------------+-------+------------+-------------+
 EXPECT:
 // Show tables for database using \d.
 SEND:\d
 EXPECT:+-------------------------+-------------------------+-------+------------+----------------------+----------------------+-------+------------+-------------+
 EXPECT:| _id                     | name                    | owner | updated_by | created_at           | updated_at           | keys  | space_used | description |
 EXPECT:+-------------------------+-------------------------+-------+------------+----------------------+----------------------+-------+------------+-------------+
 EXPECTCOMP:WithFormat:| fb_____________________ | fb_____________________ |       |            | {timestamp} | {timestamp} | false |          0 |             |
 EXPECTCOMP:WithFormat:| fb_____________________ | fb_____________________ |       |            | {timestamp} | {timestamp} | false |          0 |             |
 EXPECTCOMP:WithFormat:| fb_____________________ | fb_____________________ |       |            | {timestamp} | {timestamp} | false |          0 |             |
 EXPECTCOMP:WithFormat:| fb_____________________ | fb_____________________ |       |            | {timestamp} | {timestamp} | false |          0 |             |
 EXPECTCOMP:WithFormat:| fb_____________________ | fb_____________________ |       |            | {timestamp} | {timestamp} | false |          0 |             |
 EXPECT:+-------------------------+-------------------------+-------+------------+----------------------+----------------------+-------+------------+-------------+
 EXPECT:
 // Show tables for database using SHOW TABLES.
 SEND:SHOW TABLES;
 EXPECT:+-----+------+-------+------------+------------+------------+------+------------+-------------+
 EXPECT:| _id | name | owner | updated_by | created_at | updated_at | keys | space_used | description |
 EXPECT:+-----+------+-------+------------+------------+------------+------+------------+-------------+
 EXPECT:+-----+------+-------+------------+------------+------------+------+------------+-------------+
 EXPECT:
 // Show tables for database using \dt.
 SEND:\dt
 EXPECT:+-----+------+-------+------------+------------+------------+------+------------+-------------+
 EXPECT:| _id | name | owner | updated_by | created_at | updated_at | keys | space_used | description |
 EXPECT:+-----+------+-------+------------+------------+------------+------+------------+-------------+
 EXPECT:+-----+------+-------+------------+------------+------------+------+------------+-------------+
 EXPECT:
 // Create a table. That can be used for general testing.
 SEND:CREATE TABLE users (_id id, name string, age int);
 EXPECT:
 SEND:INSERT INTO users VALUES (1, 'Anne', 38), (2, 'Bill', 23), (3, 'Cindy', 64);
 EXPECT:
 // Show tables for database to get the newly created table.
 SEND:\dt
 EXPECT:+-------+-------+-------+------------+----------------------+----------------------+-------+------------+-------------+
 EXPECT:| _id   | name  | owner | updated_by | created_at           | updated_at           | keys  | space_used | description |
 EXPECT:+-------+-------+-------+------------+----------------------+----------------------+-------+------------+-------------+
 EXPECTCOMP:WithFormat:| users | users |       |            | {timestamp} | {timestamp} | false |          0 |             |
 EXPECT:+-------+-------+-------+------------+----------------------+----------------------+-------+------------+-------------+
 EXPECT:
 // We don't select from users until AFTER we check SHOW TABLES above because
 // running this creates the fb_views sytem table which has a description.
 // And it's annoying to mask out all of the description fields because we
 // don't know which row fb_views will fall into.
 SEND:SELECT * FROM users;
 EXPECT:+-----+-------+-----+
 EXPECT:| _id | name  | age |
 EXPECT:+-----+-------+-----+
 EXPECT:|   1 | Anne  |  38 |
 EXPECT:|   2 | Bill  |  23 |
 EXPECT:|   3 | Cindy |  64 |
 EXPECT:+-----+-------+-----+
 EXPECT:
--- a/Show more
+++ b/Show more
		`@ -1,3 +0,0 @@`
			`# testdata`

			`This directory is used in CI tests. I think.`