Update README.md

changed links to the community help repo

.github/workflows/ci.yml

@@ -22,6 +22,22 @@ jobs:
           github-token: ${{ secrets.GITHUB_TOKEN }}
           dry: true
 
+  # golangci-lint must be run separately from "validate" as there are go mod issues if you run it after the vet
+  golangci:
+    name: lint
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/setup-go@v3
+        with:
+          go-version: 1.19
+      - uses: actions/checkout@v3
+      - name: golangci-lint
+        uses: golangci/golangci-lint-action@v3
+        with:
+          # Optional: version of golangci-lint to use in form of v1.2 or v1.2.3 or `latest` to use the latest version
+          # version: v1.29
+          args: --timeout=8m
+
   validate:
     name: Code Checks
     runs-on: ubuntu-latest
@@ -39,11 +55,3 @@ jobs:
 
       - name: go vet
         run: go vet ./...
-
-      - name: golangci-lint
-        uses: golangci/golangci-lint-action@v3
-        with:
-          args: --timeout=5m
-
-      - name: test
-        run: go test ./...

.gitignore

@@ -83,3 +83,5 @@ staticcheck.conf
 dax/dax-data
 
 coverage-from-docker
+*.client_id.txt
+

.gitlab/.gitlab-ci.yml

@@ -0,0 +1,747 @@
+# You will see a couple of instances of:
+# PKG_LIST=$(go list ./... | grep -Ev 'internal/clustertests|simulacraData|batch|idk|v3/dax/test/dax' | paste -s -d, -)
+# This gets us a package list, comma-separated, which excludes the batch
+# and IDK tests, and a couple of subdirs with specialized stuff. We can
+# then use this with -coverpkg, or we can use ${PKG_LIST//,/ } to get a
+# space-separated list for use with `go test` to run the tests for those
+# directories only.
+
+include:
+  - local: /.gitlab/batch-ci.yml
+  - template: Security/SAST.gitlab-ci.yml
+  - template: Security/License-Scanning.gitlab-ci.yml
+  - template: Security/Dependency-Scanning.gitlab-ci.yml
+
+default:
+  retry:
+    max: 2 # This is confusing but this means "3 runs at max".
+    when:
+      - unknown_failure
+      - api_failure
+      - runner_system_failure
+      - job_execution_timeout
+      - stuck_or_timeout_failure
+
+variables:
+  GOVERSION: "1.19.3"
+  GOFUTURE: "latest"
+  CI_IMAGE: "${CI_REGISTRY_IMAGE}/ci-builder:0.0.1"
+  CI_PRE_CLONE_SCRIPT: |
+    set -x
+
+stages:
+  - ci_image_build
+  - lint
+  - test
+  - build
+  - post build
+  - integration
+  - gauntlet
+  - performance
+  - nonblocking
+  - cleanup_build
+
+gosec-sast:
+  allow_failure: false
+  before_script:
+    - export GOPRIVATE=github.com/molecula/*
+    - apk add openssh-client
+    - eval $(ssh-agent -s)
+    - echo "$FB_SSH_PRIVATE_KEY" | tr -d '\r' | ssh-add -
+    - mkdir -p ~/.ssh
+    - chmod 700 ~/.ssh
+    - git config --global --add url."ssh://git@github.com/".insteadOf "https://github.com/"
+    - ssh-keygen -F github.com || echo "$SSH_KNOWN_HOSTS_HASHED" >> ~/.ssh/known_hosts
+    - chmod 644 ~/.ssh/known_hosts
+  script:
+    ## securego/gosec works for scanning, but not converting to the gitlab report format.
+    - go install github.com/securego/gosec/v2/cmd/gosec@v2.12.0
+    - gosec -fmt=json -out=gosec.json -tests ./... || true
+    ## gitlab's wrapper for gosec works for converting, but not for scanning.
+    - go install 'gitlab.com/gitlab-org/security-products/analyzers/gosec@v1.4.0'
+    - gosec convert gosec.json > gl-sast-report.json
+
+.go-cache:
+  variables:
+    GOPATH: $CI_PROJECT_DIR/.go
+  before_script:
+    - mkdir -p .go
+  cache:
+    # this caching strategy makes it so each branch uses the same cache
+    key: "$CI_COMMIT_REF_SLUG"
+    paths:
+      - .go/pkg/mod/
+
+smoke build:
+  image: golang:$GOVERSION
+  extends: .go-cache
+  stage: lint
+  allow_failure: false
+  rules:
+    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
+  script:
+    - echo "Let's just see if it compiles... (sometimes the linter gives unclear errors if it doesn't)"
+    - go build ./...
+
+golangci-lint:
+  image: golangci/golangci-lint:v1.46.2
+  extends: .go-cache
+  stage: lint
+  allow_failure: true
+  rules:
+    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
+  script:
+    - echo "Checking for issues in new code"
+    - golangci-lint run -v --timeout=8m
+
+go mod tidy:
+  stage: lint
+  image: golang:$GOVERSION
+  extends: .go-cache
+  rules:
+    - if: '$CI_COMMIT_TAG == null && ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web")'
+  script:
+    - go mod tidy
+    - git diff --exit-code -- go.mod go.sum
+
+build lattice:
+  stage: test
+  image: node:14
+  variables:
+    AWS_PROFILE: "service-fb-ci"
+    AWS_ACCESS_KEY_ID: $AWS_FBCI_ACCESS_KEY_ID
+    AWS_SECRET_ACCESS_KEY: $AWS_FBCI_SECRET_ACCESS_KEY
+    CI: "false"
+  rules:
+    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
+  before_script:
+    - curl -sS "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
+    - unzip -qq awscliv2.zip
+    - ./aws/install
+    - aws --version
+    - aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID
+    - aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY
+    - aws configure set region "us-east-2"
+    - aws configure set aws_profile $AWS_PROFILE
+    - aws sts get-caller-identity # ensure we have a valid AWS login
+  script:
+    - cd lattice
+    - cache=$(find . -type f -print0 | sort -z | xargs -0 sha1sum | sha1sum | cut -d ' ' -f 1)
+    - echo "'$cache'"
+    - echo "looking for s3://molecula-artifact-storage/lattice/$cache/build.tar.gz"
+    # if is for if we had a cache object in S3
+    # else is for if we didn't have a cache object (and have to build).
+    - |
+      if aws s3api head-object --bucket molecula-artifact-storage --key "lattice/$cache/build.tar.gz"; then
+        # download object, extract, name the folder `build`
+        aws s3 cp "s3://molecula-artifact-storage/lattice/$cache/build.tar.gz" build.tar.gz
+        tar -xf build.tar.gz
+      else # cache file not found
+        yarn install --frozen-lockfile # CI needs to enforce that the lockfile doesn't need to be updated
+        yarn build
+        tar -czvf "$cache.tar.gz" build/
+        aws s3 mv "$cache.tar.gz" "s3://molecula-artifact-storage/lattice/$cache/build.tar.gz"
+        touch "$CI_COMMIT_SHA"
+        aws s3 mv "$CI_COMMIT_SHA" "s3://molecula-artifact-storage/lattice/$cache/$CI_COMMIT_SHA"
+      fi
+    - | # Ensure that we have build directory after the caching step
+      if [ ! -d build ]; then
+        echo "no build directory, erroring out" || exit 1
+      fi
+    - mv build ../
+    - cd ../
+    - rm -r lattice
+    - mv build lattice
+    - tar -czvf lattice.tar.gz lattice
+  artifacts:
+    paths:
+      - lattice.tar.gz
+
+build featurebase:
+  stage: test
+  image: golang:$GOVERSION
+  extends: .go-cache
+  rules:
+    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
+  script:
+    - rm -r lattice
+    - tar -xvf lattice.tar.gz
+    - go install github.com/rakyll/statik@v0.1.7
+    - $GOPATH/bin/statik -src=lattice
+    - export SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct)
+    - |
+      for goos in "darwin" "linux"; do
+        for goarch in "amd64" "arm64"; do
+          GOOS="${goos}" GOARCH="${goarch}" make build FLAGS="-o featurebase_${goos}_${goarch}"
+        done
+      done
+  artifacts:
+    paths:
+      - featurebase_*
+  needs:
+    - job: build lattice
+
+build fbsql amd64:
+  stage: test
+  variables:
+    BUILD_NAME: build_${CI_COMMIT_SHA}_${CI_CONCURRENT_ID}
+  tags:
+    - shell
+  rules:
+    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
+  script:
+    - export SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct)
+    - date
+    - GOOS="linux" GOARCH="amd64" make docker-build-fbsql BUILD_CGO=1
+    - GOOS="darwin" GOARCH="amd64" make docker-build-fbsql
+  artifacts:
+    paths:
+      - ./build/fbsql_*
+
+build fbsql arm64:
+  stage: test
+  variables:
+    BUILD_NAME: build_${CI_COMMIT_SHA}_${CI_CONCURRENT_ID}
+  tags:
+    - shell-arm64
+  rules:
+    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
+  script:
+    - export SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct)
+    - date
+    - GOOS="linux" GOARCH="arm64" make docker-build-fbsql BUILD_CGO=1
+    - GOOS="darwin" GOARCH="arm64" make docker-build-fbsql
+  artifacts:
+    paths:
+      - ./build/fbsql_*
+
+build amd container fb:
+  stage: test
+  tags:
+    - shell
+  rules:
+    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
+  before_script:
+    - echo "${DOCKER_DEPLOY_TOKEN}" | docker login -u ${DOCKER_DEPLOY_USER} --password-stdin ${CI_REGISTRY}
+  script:
+    - tag=${CI_REGISTRY_IMAGE}/featurebase:linux-amd64-${CI_COMMIT_REF_SLUG}
+    - export SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct)
+    - docker build --build-arg GO_VERSION=$GOVERSION --build-arg ARCH=amd64 --build-arg SOURCE_DATE_EPOCH=$SOURCE_DATE_EPOCH -t $tag -f .gitlab/Dockerfile .
+    - docker push $tag
+    - echo Created docker featurebase image with tag "$tag"
+  needs:
+    - job: build featurebase
+
+run jest tests:
+  stage: test
+  image: node:14
+  variables:
+    CI: "true"
+  rules:
+    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
+  script:
+    - echo "Testing lattice..."
+    - cd lattice
+    - npm install --force
+    - npm test -- --coverage --testResultsProcessor=jest-sonar-reporter
+  artifacts:
+    paths:
+      - lattice/coverage/lcov.info
+
+# We run go test -race on all the standard packages, skipping the ones that have their
+# own separate tests. we spin this off as nonblocking because it used to take a really
+# long time and even now it's pretty slow.
+run go tests race:
+  stage: nonblocking # don't let this job block any other jobs because it takes much longer than the other tests.
+  image: golang:$GOVERSION
+  extends: .go-cache
+  variables:
+    FEATUREBASE_CONTROLLER_CONFIG_SQLDB_DATABASE: run_go_tests_race
+    POSTGRES_DB: run_go_tests_race
+    FEATUREBASE_CONTROLLER_CONFIG_SQLDB_USER: postgres
+    POSTGRES_USER: postgres
+    FEATUREBASE_CONTROLLER_CONFIG_SQLDB_PASSWORD: $POSTGRES_PASSWORD
+    POSTGRES_PASSWORD: $POSTGRES_PASSWORD
+    FEATUREBASE_CONTROLLER_CONFIG_SQLDB_HOST: postgres
+  services:
+    - postgres:14.7
+  rules:
+    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
+  needs: ["smoke build"] # we do block on smoke build though bc it's pretty dumb to test stuff if it doesn't build
+  script:
+    - echo "Running featurebase race tests..."
+    - PKG_LIST=$(go list ./... | grep -Ev 'internal/clustertests|simulacraData|batch|idk|v3/dax/test/dax' | paste -s -d, -)
+    - export TMPDIR=/mnt/ramdisk/test-$CI_JOB_ID
+    - mkdir -p $TMPDIR
+    - go test -race -v -timeout=10m ${PKG_LIST//,/ }
+  after_script:
+    - rm -rf /mnt/ramdisk/test-$CI_JOB_ID
+  tags:
+    - docker
+
+# We run our base tests against $GOVERSION (a reasonably current version that we trust)
+# and use shardwidth22 for them. This gives us a canary for things breaking for
+# unusual shard widths.
+run go tests:
+  stage: test
+  image: golang:$GOVERSION
+  extends: .go-cache
+  variables:
+    FEATUREBASE_CONTROLLER_CONFIG_SQLDB_DATABASE: run_go_tests
+    POSTGRES_DB: run_go_tests
+    FEATUREBASE_CONTROLLER_CONFIG_SQLDB_USER: postgres
+    POSTGRES_USER: postgres
+    FEATUREBASE_CONTROLLER_CONFIG_SQLDB_PASSWORD: $POSTGRES_PASSWORD
+    POSTGRES_PASSWORD: $POSTGRES_PASSWORD
+    FEATUREBASE_CONTROLLER_CONFIG_SQLDB_HOST: postgres
+  services:
+    - postgres:14.7
+  rules:
+    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
+  script:
+    - echo "Running featurebase unit tests..."
+    - PKG_LIST=$(go list ./... | grep -Ev 'internal/clustertests|simulacraData|batch|idk|v3/dax/test/dax' | paste -s -d, -)
+    - export TMPDIR=/mnt/ramdisk/test-$CI_JOB_ID
+    - mkdir -p $TMPDIR
+    - go test -tags=shardwidth22 -timeout=10m -coverprofile=coverage.out -covermode=atomic -coverpkg=${PKG_LIST} ${PKG_LIST//,/ }
+  after_script:
+    - rm -rf /mnt/ramdisk/test-$CI_JOB_ID
+  artifacts:
+    paths:
+      - coverage.out
+  tags:
+    - docker
+
+run go tests dax/test/dax:
+  stage: test
+  image: golang:$GOVERSION
+  extends: .go-cache
+  tags:
+    - docker
+  variables:
+    FEATUREBASE_CONTROLLER_CONFIG_SQLDB_DATABASE: run_go_tests_dax
+    POSTGRES_DB: run_go_tests_dax
+    FEATUREBASE_CONTROLLER_CONFIG_SQLDB_USER: postgres
+    POSTGRES_USER: postgres
+    FEATUREBASE_CONTROLLER_CONFIG_SQLDB_PASSWORD: $POSTGRES_PASSWORD
+    POSTGRES_PASSWORD: $POSTGRES_PASSWORD
+    FEATUREBASE_CONTROLLER_CONFIG_SQLDB_HOST: postgres
+  services:
+    - postgres:14.7
+  rules:
+    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
+  script:
+    - echo "Building FB and Datagen docker images for DAX tests"
+    - PKG_LIST=$(go list ./... | grep -Ev 'internal/clustertests|simulacraData' | paste -s -d, -)
+    - export TMPDIR=/mnt/ramdisk/test-$CI_JOB_ID
+    - mkdir -p $TMPDIR
+    - go test -coverprofile=coverage-dax-integration.out -covermode=atomic -coverpkg=${PKG_LIST} -timeout=20m ./dax/test/dax
+  after_script:
+    - rm -rf /mnt/ramdisk/test-$CI_JOB_ID
+  artifacts:
+    paths:
+      - coverage-dax-integration.out
+
+# idk tests
+run go tests idk race:
+  variables:
+    PROJECT: race_${CI_CONCURRENT_ID}
+  stage: nonblocking
+  retry: 1
+  script:
+    - echo "Running test-all-race"
+    - cd ./idk/
+    - echo $PROJECT
+    - echo $CI_COMMIT_REF_SLUG
+    - BRANCH_NAME=${CI_COMMIT_REF_SLUG} IDK_FEATUREBASE_TAG=${CI_COMMIT_TAG} IDK_FEATUREBASE_HASH=${CI_COMMIT_SHA} make test-all-race
+  after_script:
+    - cd ./idk/
+    - make save-pilosa-logs
+    - make shutdown
+  artifacts:
+    paths:
+      - ./idk/testdata/*_coverage.out
+  tags:
+    - shell
+    - aws
+  needs:
+    - job: build amd container fb
+
+run go tests idk shard transactional:
+  variables:
+    IDK_DEFAULT_SHARD_TRANSACTIONAL: 1
+    PROJECT: shardttrans_${CI_CONCURRENT_ID}
+  stage: nonblocking
+  retry: 1
+  script:
+    - echo "Running shard transactional tests"
+    - cd ./idk/
+    - echo $PROJECT
+    - echo $CI_COMMIT_REF_SLUG
+    - BRANCH_NAME=${CI_COMMIT_REF_SLUG} IDK_FEATUREBASE_TAG=${CI_COMMIT_TAG} IDK_FEATUREBASE_HASH=${CI_COMMIT_SHA} make test-all
+  after_script:
+    - cd ./idk/
+    - make save-pilosa-logs
+    - make shutdown
+  artifacts:
+    paths:
+      - ./idk/testdata/*_coverage.out
+      - ./idk/testdata/*_logs.txt
+  tags:
+    - shell
+    - aws
+  needs:
+    - job: build amd container fb
+
+run go tests idk 533:
+  variables:
+    USERNAME: fb-idk-access
+    PROJECT: test533_${CI_CONCURRENT_ID}
+  stage: nonblocking
+  retry: 1
+  script:
+    - echo "Running confluent 5.3.3 test-all"
+    - cd ./idk/
+    - echo $PROJECT
+    - echo $CI_COMMIT_REF_SLUG
+    - CONFLUENT_VERSION=5.3.3 BRANCH_NAME=${CI_COMMIT_REF_SLUG} IDK_FEATUREBASE_TAG=${CI_COMMIT_TAG} IDK_FEATUREBASE_HASH=${CI_COMMIT_SHA} make test-all
+  after_script:
+    - cd ./idk/
+    - make save-pilosa-logs
+    - make shutdown
+  tags:
+    - shell
+    - aws
+  artifacts:
+    paths:
+      - ./idk/testdata/*_coverage.out
+  needs:
+    - job: build amd container fb
+
+run go tests idk sasl:
+  variables:
+    PROJECT: sasl_${CI_CONCURRENT_ID}
+  stage: nonblocking
+  retry: 1
+  script:
+    - echo "Running test-all-kafka-sasl"
+    - cd ./idk/
+    - echo $PROJECT
+    - echo $CI_COMMIT_REF_SLUG
+    - BRANCH_NAME=${CI_COMMIT_REF_SLUG} IDK_FEATUREBASE_TAG=${CI_COMMIT_TAG} IDK_FEATUREBASE_HASH=${CI_COMMIT_SHA} make test-all-kafka-sasl
+  after_script:
+    - cd ./idk/
+    - make save-pilosa-logs
+    - make shutdown
+  tags:
+    - shell
+    - aws
+  artifacts:
+    paths:
+      - ./idk/testdata/*_coverage.out
+  needs:
+    - job: build amd container fb
+
+upload to sonarcloud:
+  stage: nonblocking
+  image: sonarsource/sonar-scanner-cli:4.7
+  variables:
+    SONAR_TOKEN: $SONAR_TOKEN
+  rules:
+    - if: '$CI_COMMIT_TAG == null && ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web")'
+  script:
+    - sonar-scanner -Dsonar.projectKey=molecula_featurebase -Dsonar.organization=molecula -Dsonar.sources=. -Dsonar.host.url=https://sonarcloud.io -Dsonar.go.coverage.reportPaths=coverage*.out,results/coverage*out,idk/testdata/*coverage.out,batch/testdata/*coverage.out,coverage-from-docker/*.out -Dsonar.javascript.lcov.reportPaths=lattice/coverage/lcov.info
+  needs:
+    - job: run go tests
+    - job: run jest tests
+    - job: external lookup tests
+    - job: run go tests idk race
+      optional: true
+    - job: run go tests idk shard transactional
+      optional: true
+    - job: run go tests idk sasl
+      optional: true
+    - job: run go tests idk 533
+      optional: true
+    - job: run go tests batch
+      optional: true
+    - job: run go tests dax/test/dax
+      optional: true
+
+package for linux amd64:
+  stage: build
+  image: golang:$GOVERSION
+  extends: .go-cache
+  rules:
+    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
+  variables:
+    GOOS: "linux"
+    GOARCH: "amd64"
+  script:
+    - echo 'deb [trusted=yes] https://repo.goreleaser.com/apt/ /' | tee /etc/apt/sources.list.d/goreleaser.list
+    - apt update && apt install nfpm=2.11.3
+    - make package
+  artifacts:
+    paths:
+      - "*.deb"
+      - "*.rpm"
+
+upload_artifacts_to_nexus:
+  stage: post build
+  image: golang:$GOVERSION
+  rules:
+    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
+  script:
+    - curl -v --user "$NEXUS_YUM_CREDS" --upload-file *.arm64.rpm https://nexus.molecula.com/repository/molecula-yum/release/
+    - curl -v --user "$NEXUS_YUM_CREDS" --upload-file *.amd64.rpm https://nexus.molecula.com/repository/molecula-yum/release/
+  dependencies:
+    - package for linux amd64
+    - package for linux arm64
+
+trigger_m-cloud-images:
+  stage: post build
+  rules:
+    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
+  needs:
+    - upload_artifacts_to_nexus
+  trigger: molecula/m-cloud-images
+
+package for linux arm64:
+  stage: build
+  image: golang:$GOVERSION
+  extends: .go-cache
+  rules:
+    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
+  variables:
+    GOOS: "linux"
+    GOARCH: "arm64"
+  script:
+    - echo 'deb [trusted=yes] https://repo.goreleaser.com/apt/ /' | tee /etc/apt/sources.list.d/goreleaser.list
+    - apt update && apt install nfpm=2.11.3
+    - make package
+  artifacts:
+    paths:
+      - "*.deb"
+      - "*.rpm"
+
+build arm container fb:
+  stage: build
+  needs:
+    - "build featurebase"
+  tags:
+    - shell
+  rules:
+    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
+  before_script:
+    - echo "${DOCKER_DEPLOY_TOKEN}" | docker login -u ${DOCKER_DEPLOY_USER} --password-stdin ${CI_REGISTRY}
+  script:
+    - echo $CI_COMMIT_REF_SLUG
+    - tag=${CI_REGISTRY_IMAGE}/featurebase:linux-arm64-${CI_COMMIT_REF_SLUG}
+    - export SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct)
+    - docker build --build-arg GO_VERSION=$GOVERSION --build-arg ARCH=arm64 --build-arg SOURCE_DATE_EPOCH=$SOURCE_DATE_EPOCH -t $tag -f .gitlab/Dockerfile .
+    - docker push $tag
+    - echo Created docker featurebase image with tag "$tag"
+
+### start idk builds ###
+# building them all serially because otherwise you get container name conflicts.
+idk build_amd64:
+  stage: build
+  variables:
+    BUILD_NAME: build_${CI_COMMIT_SHA}_${CI_CONCURRENT_ID}
+  tags:
+    - shell
+  rules:
+    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
+  script:
+    - export SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct)
+    - cd ./idk/
+    - date
+    - make docker-build GOOS="linux" GOARCH="amd64" BUILD_CGO=1
+    - date
+    - make docker-build GOOS="darwin" GOARCH="amd64"
+    - date
+  artifacts:
+    paths:
+      - ./idk/build/*
+  needs:
+    # doesn't actually need this... just want it to start executing before *all* the tests finish
+    - job: run go tests
+
+idk build_arm64:
+  stage: build
+  tags:
+    - shell-arm64
+  rules:
+    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
+  script:
+    - export SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct)
+    - cd ./idk/
+    - date
+    - make docker-build GOOS="linux" GOARCH="arm64" BUILD_CGO=1 BUILD_NAME="linux-arm64"
+    - date
+    - make docker-build GOOS="darwin" GOARCH="arm64"
+    - date
+  artifacts:
+    paths:
+      - ./idk/build/*
+  needs:
+    # doesn't actually need this... just want it to start executing before *all* the tests finish
+    - job: run go tests
+
+# building them all serially because otherwise you get container name conflicts.
+# only do containers on default branch
+idk package_docker_all:
+  stage: build
+  tags:
+    - shell
+  rules:
+    - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web")'
+  script:
+    - export SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct)
+    - make docker-idk GOOS="linux" GOARCH="amd64"
+    - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
+    - make docker-idk-tag-push GOOS="linux" GOARCH="amd64"
+  needs:
+    - job: idk build_amd64
+    - job: idk build_arm64
+
+idk s3 dump:
+  stage: post build
+  allow_failure: false
+  variables:
+    PROFILE: "service-fb-ci"
+    AWS_SSH_PRIVATE_KEY: $AWS_FBCI_SSH_KEY
+    AWS_ACCESS_KEY_ID: $AWS_FBCI_ACCESS_KEY_ID
+    AWS_SECRET_ACCESS_KEY: $AWS_FBCI_SECRET_ACCESS_KEY
+  tags:
+    - shell
+  rules:
+    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
+  script:
+    - aws configure set aws_access_key_id $AWS_FBCI_ACCESS_KEY_ID
+    - aws configure set aws_secret_access_key $AWS_FBCI_SECRET_ACCESS_KEY
+    - aws configure set region "us-east-2"
+    - aws configure set aws_profile $PROFILE
+    - aws s3 cp ./idk/build/ s3://molecula-artifact-storage/idk/${CI_COMMIT_BRANCH}/${CI_COMMIT_SHORT_SHA}/ --recursive
+    - aws s3 cp ./idk/build/ s3://molecula-artifact-storage/idk/${CI_COMMIT_BRANCH}/_latest/ --recursive
+  needs:
+    - job: idk build_amd64
+    - job: idk build_arm64
+
+idk s3 dump tag:
+  stage: post build
+  variables:
+    PROFILE: "service-fb-ci"
+    AWS_SSH_PRIVATE_KEY: $AWS_FBCI_SSH_KEY
+    AWS_ACCESS_KEY_ID: $AWS_FBCI_ACCESS_KEY_ID
+    AWS_SECRET_ACCESS_KEY: $AWS_FBCI_SECRET_ACCESS_KEY
+    LOCATION: molecula-artifact-storage/idk/_tags
+  tags:
+    - shell
+  rules:
+    - if: '$CI_COMMIT_TAG != null && ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web")'
+  script:
+    - aws configure set aws_access_key_id $AWS_FBCI_ACCESS_KEY_ID
+    - aws configure set aws_secret_access_key $AWS_FBCI_SECRET_ACCESS_KEY
+    - aws configure set region "us-east-2"
+    - aws configure set aws_profile $PROFILE
+    - |
+      for goos in "darwin" "linux"; do
+        for goarch in "amd64" "arm64"; do
+          dir=idk-${CI_COMMIT_TAG}-${goos}-${goarch}
+          echo "Directory ${dir}"
+          mkdir ${dir}
+          mv ./idk/build/idk-${goos}-${goarch}/molecula-consumer-* ${dir}/
+          tar cvzf ${dir}.tar.gz ${dir}
+          aws s3 cp ${dir} s3://${LOCATION}/${CI_COMMIT_TAG}/${dir}/ --recursive
+          aws s3 cp ${dir}.tar.gz s3://${LOCATION}/${CI_COMMIT_TAG}/
+        done
+      done
+  needs:
+    - job: idk build_amd64
+    - job: idk build_arm64
+### end idk builds ###
+
+external lookup tests:
+  stage: integration
+  image: golang:$GOVERSION
+  extends: .go-cache
+  # TODO: no rules here, do we need to add the rules line?
+  variables:
+    POSTGRES_DB: $POSTGRES_DB
+    POSTGRES_USER: $POSTGRES_USER
+    POSTGRES_PASSWORD: $POSTGRES_PASSWORD
+    POSTGRES_HOST_AUTH_METHOD: trust
+  services:
+    - postgres:13.5
+  script:
+    - apt-get update --allow-releaseinfo-change -y
+    - apt-get install -y postgresql-client
+    - go test . -run "^TestExternalLookup" -externalLookupDSN postgresql://$POSTGRES_USER:$POSTGRES_PASSWORD@postgres/$POSTGRES_DB?sslmode=disable
+
+s3 dump:
+  stage: post build
+  variables:
+    PROFILE: "service-fb-ci"
+    AWS_SSH_PRIVATE_KEY: $AWS_FBCI_SSH_KEY
+    AWS_ACCESS_KEY_ID: $AWS_FBCI_ACCESS_KEY_ID
+    AWS_SECRET_ACCESS_KEY: $AWS_FBCI_SECRET_ACCESS_KEY
+  tags:
+    - shell
+  rules:
+    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "web"'
+  script:
+    - aws configure set aws_access_key_id $AWS_FBCI_ACCESS_KEY_ID
+    - aws configure set aws_secret_access_key $AWS_FBCI_SECRET_ACCESS_KEY
+    - aws configure set region "us-east-2"
+    - aws configure set aws_profile $PROFILE
+    - |
+      for goos in "darwin" "linux"; do
+        for goarch in "amd64" "arm64"; do
+          aws s3 cp featurebase_${goos}_${goarch} s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/${CI_COMMIT_SHORT_SHA}/featurebase_${goos}_${goarch}
+          aws s3 cp featurebase_${goos}_${goarch} s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/_latest/featurebase_${goos}_${goarch}
+          aws s3 cp ./build/fbsql_${goos}_${goarch} s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/${CI_COMMIT_SHORT_SHA}/fbsql_${goos}_${goarch}
+          aws s3 cp ./build/fbsql_${goos}_${goarch} s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/_latest/fbsql_${goos}_${goarch}
+        done
+      done
+  needs:
+    - job: build featurebase
+    - job: build fbsql amd64
+    - job: build fbsql arm64
+
+s3 dump tag:
+  stage: post build
+  variables:
+    PROFILE: "service-fb-ci"
+    AWS_SSH_PRIVATE_KEY: $AWS_FBCI_SSH_KEY
+    AWS_ACCESS_KEY_ID: $AWS_FBCI_ACCESS_KEY_ID
+    AWS_SECRET_ACCESS_KEY: $AWS_FBCI_SECRET_ACCESS_KEY
+    LOCATION: molecula-artifact-storage/featurebase/_tags
+  tags:
+    - shell
+  rules:
+    - if: '$CI_COMMIT_TAG != null && ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "web")'
+  script:
+    - aws configure set aws_access_key_id $AWS_FBCI_ACCESS_KEY_ID
+    - aws configure set aws_secret_access_key $AWS_FBCI_SECRET_ACCESS_KEY
+    - aws configure set region "us-east-2"
+    - aws configure set aws_profile $PROFILE
+    - |
+      for goos in "darwin" "linux"; do
+        for goarch in "amd64" "arm64"; do
+          dir=featurebase-${CI_COMMIT_TAG}-${goos}-${goarch}
+          echo "Directory ${dir}"
+          mkdir $dir
+          mv featurebase_${goos}_${goarch} ${dir}/featurebase
+          mv ./build/fbsql_${goos}_${goarch} ${dir}/fbsql
+          cp NOTICE install/featurebase.conf install/featurebase.*.service ${dir}/
+          tar cvzf ${dir}.tar.gz ${dir}
+          aws s3 cp ${dir} s3://${LOCATION}/${CI_COMMIT_TAG}/${dir}/ --recursive
+          aws s3 cp ${dir}.tar.gz s3://${LOCATION}/${CI_COMMIT_TAG}/
+        done
+      done
+
+  needs:
+    - job: build featurebase
+    - job: build fbsql amd64
+    - job: build fbsql arm64

.gitlab/Dockerfile

@@ -0,0 +1,24 @@
+FROM alpine:3.14.2
+
+LABEL maintainer "dev@molecula.com"
+LABEL org.opencontainers.image.authors="dev@molecula.com"
+
+ARG ARCH
+
+WORKDIR /
+
+RUN apk add --no-cache curl jq
+
+COPY NOTICE .
+COPY featurebase_linux_$ARCH featurebase
+
+RUN chmod ugo+x .
+EXPOSE 10101
+VOLUME /data
+
+ENV PILOSA_DATA_DIR /data
+ENV PILOSA_BIND 0.0.0.0:10101
+ENV PILOSA_BIND_GRPC 0.0.0.0:20101
+
+ENTRYPOINT ["/featurebase"]
+CMD ["server"]

.gitlab/batch-ci.yml

@@ -0,0 +1,28 @@
+run go tests batch:
+  variables:
+    PROJECT: batch_${CI_CONCURRENT_ID}
+  # this test relies on stuff that happens after build-lattice, which
+  # makes it pause the entire CI run waiting for this. we accept the
+  # small risk of wasting a build against the near certainty of spending
+  # five minutes running only one job.
+  stage: nonblocking
+  retry: 1
+  script:
+    - echo "Running test-all"
+    - cd ./batch/
+    - echo $PROJECT
+    - make build-featurebase
+    - make test-all
+  after_script:
+    - cd ./batch/
+    - make save-featurebase-logs
+    - make shutdown
+  artifacts:
+    paths:
+      - ./batch/testdata/*_coverage.out
+      - ./batch/testdata/*_logs.txt
+  tags:
+    - shell
+    - aws
+  needs:
+    - job: build amd container fb

Dockerfile-clustertests

@@ -20,12 +20,12 @@ RUN apt install -y docker.io
 ADD https://github.com/docker/compose/releases/latest/download/docker-compose-Linux-x86_64 /usr/local/bin/docker-compose
 RUN chmod +x /usr/local/bin/docker-compose
 
-WORKDIR /go/src/github.com/molecula/featurebase/cmd/featurebase
+WORKDIR /go/src/github.com/featurebasedb/featurebase/cmd/featurebase
 
 # generate an instrumented binary to allow for calculating code coverage for clustertests
 # the entrypoint for the binary is TestRunMain, which is wrapper for main
 RUN go test -covermode=atomic -coverpkg=../../... -c -tags testrunmain -o featurebase
-RUN cp /go/src/github.com/molecula/featurebase/cmd/featurebase/featurebase /featurebase
+RUN cp /go/src/github.com/featurebasedb/featurebase/cmd/featurebase/featurebase /featurebase
 
 COPY NOTICE /NOTICE
 

Dockerfile-clustertests-client

@@ -19,10 +19,10 @@ RUN apt install -y docker.io
 ADD https://github.com/docker/compose/releases/latest/download/docker-compose-Linux-x86_64 /usr/local/bin/docker-compose
 RUN chmod +x /usr/local/bin/docker-compose
 
-WORKDIR /go/src/github.com/molecula/featurebase/cmd/featurebase
+WORKDIR /go/src/github.com/featurebasedb/featurebase/cmd/featurebase
 
 RUN go test -covermode=atomic -coverpkg=../../... -c -tags testrunmain -o featurebase
-RUN cp /go/src/github.com/molecula/featurebase/cmd/featurebase/featurebase /featurebase
+RUN cp /go/src/github.com/featurebasedb/featurebase/cmd/featurebase/featurebase /featurebase
 
 
 COPY NOTICE /NOTICE
@@ -32,6 +32,6 @@ COPY ./internal/clustertests /go/src/github.com/featurebasedb/featurebase/intern
 EXPOSE 10101
 VOLUME /data
 
-WORKDIR /go/src/github.com/molecula/featurebase
+WORKDIR /go/src/github.com/featurebasedb/featurebase
 
 CMD ["/featurebase", "-test.run=TestRunMain", "-test.coverprofile=/results/coverage.out", "server", "--data-dir", "/data", "--bind", "http://0.0.0.0:10101"]

Dockerfile-dax

@@ -16,7 +16,7 @@ RUN make build FLAGS="-o build/featurebase" ${MAKE_FLAGS}
 ### FeatureBase runner ###
 ##########################
 
-FROM alpine:3.13.2 as runner
+FROM golang:alpine as runner
 
 LABEL maintainer "dev@featurebase.com"
 

Dockerfile-fbsql

@@ -0,0 +1,48 @@
+ARG GO_VERSION=1.19
+
+FROM golang:1.19-buster as builder
+
+WORKDIR /
+RUN apt-get update -y -qq && apt-get install -y -qq \
+  build-essential \
+  git \
+  musl-tools \
+  netcat \
+  unixodbc \
+  unixodbc-dev \
+  && rm -rf /var/lib/apt/lists/*
+RUN ["git", "clone", "https://github.com/edenhill/librdkafka.git"]
+WORKDIR /librdkafka
+RUN ./configure --prefix /usr && \
+  make && \
+  make install
+
+WORKDIR /featurebase
+
+COPY . .
+
+ARG MAKE_FLAGS
+ARG GO_BUILD_FLAGS
+ARG SOURCE_DATE_EPOCH
+
+WORKDIR /featurebase/
+
+ENV SOURCE_DATE_EPOCH=${SOURCE_DATE_EPOCH}
+RUN make build-fbsql GO_BUILD_FLAGS="-mod=vendor ${GO_BUILD_FLAGS}" ${MAKE_FLAGS}
+
+FROM ubuntu:20.04 as runner
+
+RUN apt-get update -y -qq && apt-get install -y -qq \
+  ca-certificates \
+  musl-tools \
+  netcat \
+  unixodbc-dev \
+  && rm -rf /var/lib/apt/lists/*
+
+COPY --from=builder /featurebase/fbsql /usr/local/bin/
+
+# Verify that the linker can find everything.
+FROM runner as linkcheck
+RUN if [ -e /usr/local/bin/fbsql ] ; then ldd /usr/local/bin/fbsql; fi
+
+FROM runner

LICENSE

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright 2023 Molecula Corp. All rights reserved.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

Makefile

@@ -1,5 +1,6 @@
-.PHONY: build clean build-lattice cover cover-viz default docker docker-build docker-tag-push generate generate-protoc generate-pql generate-statik generate-stringer install install-protoc-gen-gofast install-protoc install-statik install-peg test docker-login
+.PHONY: build clean build-lattice cover cover-viz default docker docker-build docker-build-fbsql docker-tag-push generate generate-protoc generate-pql generate-statik generate-stringer install install-protoc-gen-gofast install-protoc install-statik install-peg test docker-login
 
+SHELL := /bin/bash
 VERSION := $(shell git describe --tags 2> /dev/null || echo unknown)
 VARIANT = Molecula
 GO=go
@@ -18,11 +19,15 @@ SHARD_WIDTH = 20
 COMMIT := $(shell git describe --exact-match >/dev/null 2>&1 || git rev-parse --short HEAD)
 LDFLAGS="-X github.com/featurebasedb/featurebase/v3.Version=$(VERSION) -X github.com/featurebasedb/featurebase/v3.BuildTime=$(BUILD_TIME) -X github.com/featurebasedb/featurebase/v3.Variant=$(VARIANT) -X github.com/featurebasedb/featurebase/v3.Commit=$(COMMIT) -X github.com/featurebasedb/featurebase/v3.TrialDeadline=$(TRIAL_DEADLINE)"
 GO_VERSION=1.19
+GO_BUILD_FLAGS=
 DOCKER_BUILD= # set to 1 to use `docker-build` instead of `build` when creating a release
 BUILD_TAGS += 
 TEST_TAGS = roaringparanoia
 TEST_TIMEOUT=10m
 RACE_TEST_TIMEOUT=10m
+# size in GB to use for ramdisk, ?= so you can override it with env
+# 4GB is not enough for `make test`, 8GB usually is.
+RAMDISK_SIZE ?= 8
 
 export GO111MODULE=on
 export GOPRIVATE=github.com/molecula
@@ -46,11 +51,11 @@ version:
 
 # We build a list of packages that omits the IDK and batch packages because
 # those packages require fancy environment setup.
-GOPACKAGES := $(shell $(GO) list ./... | grep -v "/idk" | grep -v "/batch")
+GOPACKAGES := $(shell $(GO) list ./... | grep -v "/v3/idk" | grep -v "/v3/batch")
 
 # Run test suite
 test:
-	$(GO) test $(GOPACKAGES) -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) -v -timeout $(TEST_TIMEOUT)
+	$(GO) test $(GOPACKAGES) -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) -v -timeout $(TEST_TIMEOUT) -count=1
 
 # Run test suite with race flag
 test-race:
@@ -76,6 +81,21 @@ testvsub:
 			echo; echo "999 done testing subpkg $$pkg"; \
 		done
 
+# make a $(RAMDISK_SIZE)GB RAMDisk. Speed up tests by running
+# them with TMPDIR=/mnt/ramdisk.
+ramdisk-linux:
+	mount -o size=$(RAMDISK__SIZE)G -t tmpfs none /mnt/ramdisk
+
+# make a $(RAMDISK_SIZE)GB RAMDisk. Speed up tests by running
+# them with TMPDIR=/Volumes/RAMDisk. This is more important on
+# OS X than it is on Linux, because there's performance issues
+# with fsync on OS X that can make the SSD slow down to moving-platters
+# drive speeds. Oops.
+ramdisk-osx:
+	diskutil erasevolume HFS+ 'RAMDisk' $$(hdiutil attach -nobrowse -nomount ram://$$(expr 2097152 \* $(RAMDISK_SIZE)))
+
+detach-ramdisk-osx:
+	hdiutil detach /Volumes/RAMDisk
 
 testvsub-race:
 	@set -e; for pkg in $(GOPACKAGES); do \
@@ -97,12 +117,13 @@ cover:
 cover-viz: cover
 	$(GO) tool cover -html=build/coverage.out
 
-# Compile Pilosa
+# Build featurebase
 build:
 	$(GO) build -tags='$(BUILD_TAGS)' -ldflags $(LDFLAGS) $(FLAGS) ./cmd/featurebase
 
 package:
-	GOOS=$(GOOS) GOARCH=$(GOARCH) FLAGS="-o featurebase" $(MAKE) build
+	GOOS=$(GOOS) GOARCH=$(GOARCH) $(MAKE) build
+	GOOS=$(GOOS) GOARCH=$(GOARCH) $(MAKE) build-fbsql
 	GOARCH=$(GOARCH) VERSION=$(VERSION) nfpm package --packager deb --target featurebase.$(VERSION).$(GOARCH).deb
 	GOARCH=$(GOARCH) VERSION=$(VERSION) nfpm package --packager rpm --target featurebase.$(VERSION).$(GOARCH).rpm
 
@@ -133,7 +154,7 @@ authclustertests: vendor
 	CLUSTERTESTS_FB_ARGS=$(AUTH_ARGS) $(DOCKER_COMPOSE) -f internal/clustertests/docker-compose.yml down
 
 # Install FeatureBase and IDK
-install: install-featurebase install-idk
+install: install-featurebase install-idk install-fbsql
 
 install-featurebase:
 	$(GO) install -tags='$(BUILD_TAGS)' -ldflags $(LDFLAGS) $(FLAGS) ./cmd/featurebase
@@ -141,6 +162,9 @@ install-featurebase:
 install-idk:
 	$(MAKE) -C ./idk install
 
+install-fbsql:
+	CGO_ENABLED=1 $(GO) install ./cmd/fbsql
+
 # Build the lattice assets
 build-lattice:
 	docker build -t lattice:build ./lattice
@@ -209,6 +233,13 @@ docker-image-featurebase: vendor
 		--file Dockerfile-dax \
 		--tag dax/featurebase .
 
+docker-image-featurebase-linux-amd64: vendor
+	docker build \
+	    --build-arg GO_VERSION=$(GO_VERSION) \
+	    --platform linux/amd64 \
+	    --file Dockerfile-dax \
+	    --tag dax/featurebase .
+
 docker-image-featurebase-test: vendor
 	docker build \
 	    --build-arg GO_VERSION=$(GO_VERSION) \
@@ -228,21 +259,27 @@ build-for-quick:
 docker-image-featurebase-quick: build-for-quick
 	docker build \
 	    --build-arg GO_VERSION=$(GO_VERSION) \
-	    --file Dockerfile-dax-quick ./.quick/
+	    --file Dockerfile-dax-quick \
+	    --tag dax/featurebase ./.quick/
 
 
 docker-image-datagen: vendor
 	docker build --tag dax/datagen --file Dockerfile-datagen .
 
+get-account-id:
+	$(eval AWS_ACCOUNTID := $(shell aws sts get-caller-identity --output=json | jq -r .Account))
+
+
 ecr-push-featurebase: docker-login
-	docker tag dax/featurebase:latest $(AWS_ACCOUNTID).dkr.ecr.us-east-2.amazonaws.com/dax:latest
-	docker push $(AWS_ACCOUNTID).dkr.ecr.us-east-2.amazonaws.com/dax:latest
+	echo "Pushing to account $(AWS_ACCOUNTID), profile $(AWS_PROFILE)"
+	docker tag dax/featurebase:latest $(AWS_ACCOUNTID).dkr.ecr.us-east-2.amazonaws.com/dax/featurebase:latest
+	docker push $(AWS_ACCOUNTID).dkr.ecr.us-east-2.amazonaws.com/dax/featurebase:latest
 
 ecr-push-datagen: docker-login
 	docker tag dax/datagen:latest $(AWS_ACCOUNTID).dkr.ecr.us-east-2.amazonaws.com/dax/datagen:latest
 	docker push $(AWS_ACCOUNTID).dkr.ecr.us-east-2.amazonaws.com/dax/datagen:latest
 
-docker-login:
+docker-login: get-account-id
 	aws ecr get-login-password --region us-east-2 | docker login --username AWS --password-stdin $(AWS_ACCOUNTID).dkr.ecr.us-east-2.amazonaws.com
 
 # Create docker image (alias)
@@ -316,3 +353,53 @@ test-external-lookup:
 
 bnf:
 	ebnf2railroad --no-overview-diagram  --no-optimizations ./sql3/sql3.ebnf
+
+#################################
+# fbsql builds in docker
+#################################
+
+# This allows multiple concurrent builds to happen in CI without
+# creating container name conflicts and such. (different BUILD_NAMEs
+# are passed in from gitlab-ci.yml)
+BUILD_NAME ?= fbsql-build
+
+LDFLAGS_STATIC="-linkmode external -extldflags \"-static\" -X 'github.com/featurebasedb/featurebase/v3/fbsql.Version=$(VERSION)' -X 'github.com/featurebasedb/featurebase/v3/fbsql.BuildTime=$(BUILD_TIME)' "
+
+UNAME_P := $(shell uname -p)
+BUILD_CGO ?= 0
+
+# Build fbsql
+build-fbsql:
+	@echo GOOS=$(GOOS) GOARCH=$(GOARCH) uname -p=$(UNAME_P) build_cgo=$(BUILD_CGO)
+ifeq ($(BUILD_CGO), 0)
+	make build-fbsql-non-cgo
+endif
+ifeq ($(BUILD_CGO), 1)
+	make build-fbsql-cgo
+endif
+
+build-fbsql-non-cgo:
+	CGO_ENABLED=0 $(GO) build -ldflags $(LDFLAGS) $(GO_BUILD_FLAGS) -o fbsql ./cmd/fbsql
+
+build-fbsql-cgo:
+ifeq ($(GOARCH), arm64)
+	CGO_ENABLED=1 $(GO) build -tags dynamic $(GO_BUILD_FLAGS) -o fbsql ./cmd/fbsql
+endif
+ifeq ($(GOARCH), amd64)
+	CC=/usr/bin/musl-gcc CGO_ENABLED=1 $(GO) build -tags "musl static" -ldflags $(LDFLAGS_STATIC) $(GO_BUILD_FLAGS) -o fbsql ./cmd/fbsql
+endif
+
+docker-build-fbsql: vendor
+	DOCKER_BUILDKIT=0 docker build \
+		--file Dockerfile-fbsql \
+	    --build-arg GO_VERSION=$(GO_VERSION) \
+	    --build-arg MAKE_FLAGS="GOOS=$(GOOS) GOARCH=$(GOARCH) BUILD_CGO=$(BUILD_CGO)" \
+	    --build-arg GO_BUILD_FLAGS=$(GO_BUILD_FLAGS) \
+		--build-arg SOURCE_DATE_EPOCH=$(SOURCE_DATE_EPOCH) \
+	    --target builder \
+	    --tag fbsql:$(BUILD_NAME) .
+	mkdir -p build
+	docker create --name $(BUILD_NAME) fbsql:$(BUILD_NAME)
+	docker cp $(BUILD_NAME):/featurebase/fbsql ./build/fbsql_$(GOOS)_$(GOARCH)
+	docker rm $(BUILD_NAME)
+

OPENSOURCE.md

@@ -0,0 +1,45 @@
+## User Contribution Guidelines for FeatureBase
+
+Thank you for your interest in contributing to FeatureBase! We appreciate your support in making this open-source project even better. Here are some guidelines to help you get started with contributing to FeatureBase:
+
+1. Familiarize Yourself with the Project:
+   - Visit the FeatureBase website at www.featurebase.com to understand the project's goals, capabilities, and features.
+   - Read the documentation available on the website, including the installation guide, configuration options, and data modeling concepts.
+   - Explore the codebase by cloning the repository and reviewing the source code.
+
+2. Join the Community:
+   - Visit the FeatureBase community page at https://www.featurebase.com/community to learn more about the project's community and how to get involved.
+   - Join the Discord server at https://discord.gg/FBn2vEp7Na to chat with other contributors and users, ask questions, and share your ideas.
+
+3. Set Up Your Development Environment:
+   - Ensure you have Go installed on your machine. Make sure your shell's search path includes the go/bin directory.
+   - Clone the FeatureBase repository or download it as a zip file from the repository's page.
+   - Follow the "Build FeatureBase Server from source" instructions in the README file to compile the server binary and the ingester binaries.
+
+4. Choose a Contribution Area:
+   - Identify the area you'd like to contribute to, such as bug fixes, new features, performance improvements, documentation updates, or community support.
+   - Check the issue tracker on the repository or the FeatureBase community for open issues or feature requests that align with your interests and skills. Alternatively, propose your own idea by creating a new issue.
+
+5. Create a New Branch:
+   - Before making any changes, create a new branch in the repository's Git repository. This branch will contain your contributions.
+   - Give your branch a descriptive name that reflects the nature of your contribution.
+
+6. Make Your Changes:
+   - Follow the coding style and conventions used in the existing codebase.
+   - Write clear and concise commit messages for each logical change.
+   - If you're introducing new features or modifying existing behavior, make sure to update the documentation to reflect the changes.
+
+7. Test Your Changes:
+   - Run the existing test suite to ensure that your modifications do not introduce any regressions.
+   - If applicable, write additional tests to cover the changes you made.
+   - Document any new testing procedures required for your contribution.
+
+8. Submitting Your Contribution:
+   - Push your branch to the main repository or create a fork and submit a pull request to the main repository.
+   - Provide a detailed description of your changes, including the problem you solved and the approach you took.
+   - Be responsive to any feedback or suggestions provided by the project maintainers or other contributors.
+   - Once your contribution is approved, it will be reviewed and merged into the main codebase.
+
+Please note that by contributing to FeatureBase, you agree that your contributions will be licensed under the Apache 2.0 license, which governs the project.
+
+Thank you for considering contributing to FeatureBase! Your contributions are valuable and help improve the project for everyone.

README.md

@@ -1,4 +1,10 @@
-# FeatureBase
+# FeatureBase Community
+
+FeatureBase Community is now archived and no longer maintained.
+
+* [FeatureBase Community Help](https://github.com/FeatureBaseDB/FB-community-help)
+
+
 
 ## Pilosa is now FeatureBase
 
@@ -10,6 +16,8 @@ For more information about FeatureBase, please visit [www.featurebase.com][HomeP
 
 ## Getting Started
 
+* [Learn how to install FeatureBase Community](https://github.com/FeatureBaseDB/FB-community-help/blob/main/docs/community/com-getstart/com-getstart-home.md)
+
 ### Build FeatureBase Server from source
 
 0. Install go. Ensure that your shell's search path includes the go/bin directory.
@@ -19,42 +27,20 @@ For more information about FeatureBase, please visit [www.featurebase.com][HomeP
 4. Run `featurebase server --handler.allowed-origins=http://localhost:3000` to run FeatureBase server with default settings (learn more about configuring FeatureBase at the link below). The `--handler.allowed-origins` parameter allows the standalone web UI to talk to the server; this can be omitted if the web UI is not needed.
 5. Run `curl localhost:10101/status` to verify the server is running and accessible.
 
-### Ingest Data and Query
-
-1. Run 
-```
-molecula-consumer-csv \
-    --index repository \
-    --header "language__ID_F,project_id__ID_F" \
-    --id-field project_id \
-    --batch-size 1000 \
-    --files example.csv
-```
-
-This will ingest the `example.csv` file into a FeatureBase table called `repository`. If the table does not exist, it will be automatically created. Learn more about ingesting into FeatureBase: [https://docs.featurebase.com/data-ingestion/enterprise/ingesters][Ingest]
-
-2. Query your data. 
-```
-curl localhost:10101/index/repository/query \
-     -X POST \
-     -d 'Row(example=5)'
-```
-Learn about supported [SQL][SQL], native [Pilosa Query Language (PQL)][PQL].
-
 ### Data Model
 
 Because FeatureBase is built on bitmaps, there is bit of a learning curve to grasp how your data is represented. 
-Data Model Guide: [https://docs.featurebase.com/data-modeling-guide/data-modeling][DataModel]
 
-### More Information
+* [Learn about Data Modeling](https://github.com/FeatureBaseDB/FB-community-help/blob/main/docs/concepts/concepts-home.md)
 
-Installation:[https://docs.featurebase.com/setting-up-featurebase/enterprise/installing-featurebase][Install]
 
-Configuration: [https://docs.featurebase.com/setting-up-featurebase/enterprise/featurebase-configuration][Config]
+### Ingest Data and Query
+
+* [Learn how to ingest data from multiple data sources](https://github.com/FeatureBaseDB/FB-community-help/blob/main/docs/community/com-ingest/com-ingest-manage.md)
 
 ## Community
 
-You can email us at community@featurebase.com or learn more about contributing at [https://www.featurebase.com/community][Community].
+You can email us at community@featurebase.com and [learn more about contributing](https://github.com/FeatureBaseDB/featurebase/blob/master/OPENSOURCE.md).
 
 Chat with us: [https://discord.gg/FBn2vEp7Na][Discord]
 
@@ -73,13 +59,14 @@ A lot has changed since the days of Pilosa. This list highlights some new capabi
 
 FeatureBase is licensed under the [Apache License, Version 2.0][License]
 
-[Community]: http://www.featurebase.com/community?utm_campaign=Open%20Source&utm_source=GitHub
-[Config]: https://docs.featurebase.com/setting-up-featurebase/enterprise/featurebase-configuration?utm_campaign=Open%20Source&utm_source=GitHub
-[DataModel]: https://docs.featurebase.com/data-modeling-guide/data-modeling?utm_campaign=Open%20Source&utm_source=GitHub
+[Community]: https://github.com/FeatureBaseDB/FB-community-help/tree/main
+[Install]:https://github.com/FeatureBaseDB/FB-community-help/blob/main/docs/community/com-getstart/com-getstart-home.md
+[Config]: https://github.com/FeatureBaseDB/FB-community-help/tree/main/docs/community/com-config
+[DataModel]: https://github.com/FeatureBaseDB/FB-community-help/blob/main/docs/concepts/concepts-home.md
 [Discord]: https://discord.gg/FBn2vEp7Na
 [HomePage]: http://featurebase.com?utm_campaign=Open%20Source&utm_source=GitHub
-[Ingest]: https://docs.featurebase.com/data-ingestion/enterprise/ingesters?utm_campaign=Open%20Source&utm_source=GitHub
-[Install]: http://docs.featurebase.com/setting-up-featurebase/enterprise/installing-featurebase?utm_campaign=Open%20Source&utm_source=GitHub
+[Ingest]: https://github.com/FeatureBaseDB/FB-community-help/blob/main/docs/community/com-ingest/com-ingest-manage.md
+ 
 [License]: http://www.apache.org/licenses/LICENSE-2.0
-[PQL]: http://docs.featurebase.com/data-querying/pql/introduction?utm_campaign=Open%20Source&utm_source=GitHub
-[SQL]: http://docs.featurebase.com/data-querying/sql?utm_campaign=Open%20Source&utm_source=GitHub
+[PQL]: https://docs.featurebase.com/docs/pql-guide/pql-home/?utm_campaign=Open%20Source&utm_source=GitHub
+[SQL]: https://docs.featurebase.com/docs/sql-guide/sql-guide-home/?utm_campaign=Open%20Source&utm_source=GitHub

api.go

@@ -25,14 +25,16 @@ import (
 	fbcontext "github.com/featurebasedb/featurebase/v3/context"
 	"github.com/featurebasedb/featurebase/v3/dax"
 	"github.com/featurebasedb/featurebase/v3/dax/computer"
+	"github.com/featurebasedb/featurebase/v3/dax/storage"
 	"github.com/featurebasedb/featurebase/v3/disco"
+	"github.com/featurebasedb/featurebase/v3/logger"
 	"github.com/featurebasedb/featurebase/v3/rbf"
+	"github.com/prometheus/client_golang/prometheus"
 
 	//"github.com/featurebasedb/featurebase/v3/pg"
 	"github.com/featurebasedb/featurebase/v3/pql"
 	"github.com/featurebasedb/featurebase/v3/roaring"
 	planner_types "github.com/featurebasedb/featurebase/v3/sql3/planner/types"
-	"github.com/featurebasedb/featurebase/v3/stats"
 	"github.com/featurebasedb/featurebase/v3/tracing"
 	"github.com/pkg/errors"
 	"golang.org/x/sync/errgroup"
@@ -55,9 +57,7 @@ type API struct {
 
 	Serializer Serializer
 
-	writeLogReader     computer.WriteLogReader
-	writeLogWriter     computer.WriteLogWriter
-	snapshotReadWriter computer.SnapshotReadWriter
+	serverlessStorage *storage.ResourceManager
 
 	directiveWorkerPoolSize int
 
@@ -70,6 +70,10 @@ func (api *API) Holder() *Holder {
 	return api.holder
 }
 
+func (api *API) logger() logger.Logger {
+	return api.server.logger
+}
+
 // apiOption is a functional option type for pilosa.API
 type apiOption func(*API) error
 
@@ -83,6 +87,13 @@ func OptAPIServer(s *Server) apiOption {
 	}
 }
 
+func OptAPIServerlessStorage(mm *storage.ResourceManager) apiOption {
+	return func(a *API) error {
+		a.serverlessStorage = mm
+		return nil
+	}
+}
+
 func OptAPIImportWorkerPoolSize(size int) apiOption {
 	return func(a *API) error {
 		a.importWorkerPoolSize = size
@@ -90,27 +101,6 @@ func OptAPIImportWorkerPoolSize(size int) apiOption {
 	}
 }
 
-func OptAPIWriteLogReader(wlr computer.WriteLogReader) apiOption {
-	return func(a *API) error {
-		a.writeLogReader = wlr
-		return nil
-	}
-}
-
-func OptAPIWriteLogWriter(wlw computer.WriteLogWriter) apiOption {
-	return func(a *API) error {
-		a.writeLogWriter = wlw
-		return nil
-	}
-}
-
-func OptAPISnapshotter(snap computer.SnapshotReadWriter) apiOption {
-	return func(a *API) error {
-		a.snapshotReadWriter = snap
-		return nil
-	}
-}
-
 func OptAPIDirectiveWorkerPoolSize(size int) apiOption {
 	return func(a *API) error {
 		a.directiveWorkerPoolSize = size
@@ -129,9 +119,6 @@ func OptAPIIsComputeNode(is bool) apiOption {
 func NewAPI(opts ...apiOption) (*API, error) {
 	api := &API{
 		importWorkerPoolSize: 2,
-		writeLogReader:       computer.NewNopWriteLogReader(),
-		writeLogWriter:       computer.NewNopWriteLogWriter(),
-		snapshotReadWriter:   computer.NewNopSnapshotReadWriter(),
 
 		directiveWorkerPoolSize: 2,
 	}
@@ -250,7 +237,7 @@ func (api *API) query(ctx context.Context, req *QueryRequest) (QueryResponse, er
 		EmbeddedData:  req.EmbeddedData, // precomputed values that needed to be passed with the request
 		MaxMemory:     req.MaxMemory,
 	}
-	resp, err := api.server.executor.Execute(ctx, req.Index, q, req.Shards, execOpts)
+	resp, err := api.server.executor.Execute(ctx, dax.StringTableKeyer(req.Index), q, req.Shards, execOpts)
 	if err != nil {
 		return QueryResponse{}, errors.Wrap(err, "executing")
 	}
@@ -291,7 +278,7 @@ func (api *API) CreateIndex(ctx context.Context, indexName string, options Index
 		return nil, errors.Wrap(err, "creating index")
 	}
 
-	api.holder.Stats.Count(MetricCreateIndex, 1, 1.0)
+	CounterCreateIndex.Inc()
 	return index, nil
 }
 
@@ -332,7 +319,7 @@ func (api *API) DeleteDataframe(ctx context.Context, indexName string) error {
 		api.server.logger.Errorf("problem sending DeleteIndex message: %s", err)
 		return errors.Wrap(err, "sending DeleteIndex message")
 	}
-	api.holder.Stats.Count(MetricDeleteDataframe, 1, 1.0)
+	CounterDeleteDataframe.Inc()
 	return nil
 }
 
@@ -351,6 +338,14 @@ func (api *API) DeleteIndex(ctx context.Context, indexName string) error {
 	if err != nil {
 		return errors.Wrap(err, "deleting index")
 	}
+
+	// Remove from writelogger/snapshotter if serverless.
+	if api.isComputeNode {
+		if err := api.serverlessStorage.RemoveTable(dax.TableKey(indexName).QualifiedTableID()); err != nil {
+			return errors.Wrapf(err, "removing table from serverless storage: %s", indexName)
+		}
+	}
+
 	// Send the delete index message to all nodes.
 	err = api.server.SendSync(
 		&DeleteIndexMessage{
@@ -367,13 +362,13 @@ func (api *API) DeleteIndex(ctx context.Context, indexName string) error {
 			return errors.Wrap(err, "deleting id allocation for index")
 		}
 	}
-	api.holder.Stats.Count(MetricDeleteIndex, 1, 1.0)
+	CounterDeleteIndex.Inc()
 	return nil
 }
 
 // CreateField makes the named field in the named index with the given options.
-// This method currently only takes a single functional option, but that may be
-// changed in the future to support multiple options.
+//
+// The resulting field will always have TrackExistence set.
 func (api *API) CreateField(ctx context.Context, indexName string, fieldName string, opts ...FieldOption) (*Field, error) {
 	span, _ := tracing.StartSpanFromContext(ctx, "API.CreateField")
 	defer span.Finish()
@@ -386,6 +381,11 @@ func (api *API) CreateField(ctx context.Context, indexName string, fieldName str
 	// authN/Z info
 	requestUserID, _ := fbcontext.UserID(ctx) // requestUserID is "" if not in ctx
 
+	// newFieldOptions is also used in the path through the index creating
+	// a field from an update from DAX, so it can't assume it can always
+	// override this. But we're the call path for creating new fields, and
+	// new fields should always have TrackExistence on.
+	opts = append(opts, OptFieldTrackExistence())
 	// Apply and validate functional options.
 	fo, err := newFieldOptions(opts...)
 	if err != nil {
@@ -419,7 +419,7 @@ func (api *API) CreateField(ctx context.Context, indexName string, fieldName str
 		return nil, errors.Wrap(err, "sending CreateField message")
 	}
 
-	api.holder.Stats.CountWithCustomTags(MetricCreateField, 1, 1.0, []string{fmt.Sprintf("index:%s", indexName)})
+	CounterCreateField.With(prometheus.Labels{"index": indexName})
 	return field, nil
 }
 
@@ -499,16 +499,9 @@ func importWorker(importWork chan importJob) {
 	for j := range importWork {
 		err := func() (err0 error) {
 			for viewName, viewData := range j.req.Views {
-				// The logic here corresponds to the logic in fragment.cleanViewName().
-				// Unfortunately, the logic in that method is not completely exclusive
-				// (i.e. an "other" view named with format YYYYMMDD would be handled
-				// incorrectly). One way to address this would be to change the logic
-				// overall so there weren't conflicts. For now, we just
-				// rely on the field type to inform the intended view name.
-				if viewName == "" {
-					viewName = viewStandard
-				} else if j.field.Type() == FieldTypeTime {
-					viewName = fmt.Sprintf("%s_%s", viewStandard, viewName)
+				viewName, err0 = j.field.cleanupViewName(viewName)
+				if err0 != nil {
+					return err0
 				}
 				if len(viewData) == 0 {
 					return fmt.Errorf("no data to import for view: %s", viewName)
@@ -709,20 +702,20 @@ func (api *API) ImportRoaring(ctx context.Context, indexName, fieldName string,
 					Views:           req.Views,
 				}
 
-				// Get the current version for shard.
-				version, err := api.getOrCreateShardVersion(ctx, indexName, shard)
-				if err != nil {
-					return errors.Wrap(err, "get or creating shard version")
-				}
-
 				tkey := dax.TableKey(indexName)
 				qtid := tkey.QualifiedTableID()
 				partitionNum := dax.PartitionNum(partition)
 				shardNum := dax.ShardNum(shard)
 
-				api.server.logger.Debugf("importroaring writing to writelogger: %+v, %[1]T len(msg.Views): %d, table: %s", api.writeLogWriter, len(msg.Views), msg.Table)
-				if err := api.writeLogWriter.WriteShard(ctx, qtid, partitionNum, shardNum, version, msg); err != nil {
-					return err
+				b, err := computer.MarshalLogMessage(msg, computer.EncodeTypeJSON)
+				if err != nil {
+					return errors.Wrap(err, "marshalling log message")
+				}
+
+				resource := api.serverlessStorage.GetShardResource(qtid, partitionNum, shardNum)
+				err = resource.Append(b)
+				if err != nil {
+					return errors.Wrap(err, "appending shard data") // TODO do we need to set err0 or something?
 				}
 			}
 
@@ -731,29 +724,6 @@ func (api *API) ImportRoaring(ctx context.Context, indexName, fieldName string,
 	}
 }
 
-func (api *API) getOrCreateShardVersion(ctx context.Context, indexName string, shard uint64) (int, error) {
-	tableName := dax.TableName(indexName)
-	shardNum := dax.ShardNum(shard)
-
-	// Here we assume that indexName is the string encoding of QualifiedTableID.
-	qtid, err := dax.QualifiedTableIDFromKey(indexName)
-	if err != nil {
-		return -1, errors.Wrap(err, "decoding qtid from key (indexName)")
-	}
-
-	version, found, err := api.holder.versionStore.ShardVersion(ctx, qtid, shardNum)
-	if err != nil {
-		return -1, errors.Wrap(err, "getting shard version")
-	} else if !found {
-		version = 0
-		api.server.logger.Printf("could not find version for shard: %s, %d, so creating 0", tableName, shardNum)
-		if err := api.holder.versionStore.AddShards(ctx, qtid, dax.NewVersionedShard(shardNum, version)); err != nil {
-			return -1, errors.Wrap(err, "adding shard 0")
-		}
-	}
-	return version, nil
-}
-
 // DeleteField removes the named field from the named index. If the index is not
 // found, an error is returned. If the field is not found, it is ignored and no
 // action is taken.
@@ -786,7 +756,7 @@ func (api *API) DeleteField(ctx context.Context, indexName string, fieldName str
 		api.server.logger.Errorf("problem sending DeleteField message: %s", err)
 		return errors.Wrap(err, "sending DeleteField message")
 	}
-	api.holder.Stats.CountWithCustomTags(MetricDeleteField, 1, 1.0, []string{fmt.Sprintf("index:%s", indexName)})
+	CounterDeleteField.With(prometheus.Labels{"index": indexName})
 	return nil
 }
 
@@ -818,7 +788,7 @@ func (api *API) DeleteAvailableShard(_ context.Context, indexName, fieldName str
 		api.server.logger.Errorf("problem sending DeleteAvailableShard message: %s", err)
 		return errors.Wrap(err, "sending DeleteAvailableShard message")
 	}
-	api.holder.Stats.CountWithCustomTags(MetricDeleteAvailableShard, 1, 1.0, []string{fmt.Sprintf("index:%s", indexName)})
+	CounterDeleteAvailableShard.With(prometheus.Labels{"index": indexName}).Inc()
 	return nil
 }
 
@@ -982,7 +952,7 @@ func (r RedirectError) Error() string {
 }
 
 // TranslateData returns all translation data in the specified partition.
-func (api *API) TranslateData(ctx context.Context, indexName string, partition int) (io.WriterTo, error) {
+func (api *API) TranslateData(ctx context.Context, indexName string, partition int) (TranslateStore, error) {
 	span, _ := tracing.StartSpanFromContext(ctx, "API.TranslateData")
 	defer span.Finish()
 
@@ -1037,7 +1007,7 @@ func (api *API) TranslateData(ctx context.Context, indexName string, partition i
 }
 
 // FieldTranslateData returns all translation data in the specified field.
-func (api *API) FieldTranslateData(ctx context.Context, indexName, fieldName string) (io.WriterTo, error) {
+func (api *API) FieldTranslateData(ctx context.Context, indexName, fieldName string) (TranslateStore, error) {
 	span, _ := tracing.StartSpanFromContext(ctx, "API.FieldTranslateData")
 	defer span.Finish()
 	if err := api.validate(apiFieldTranslateData); err != nil {
@@ -1286,8 +1256,13 @@ func (api *API) DeleteView(ctx context.Context, indexName string, fieldName stri
 	return errors.Wrap(err, "sending DeleteView message")
 }
 
-// IndexShardSnapshot returns a reader that contains the contents of an RBF snapshot for an index/shard.
-func (api *API) IndexShardSnapshot(ctx context.Context, indexName string, shard uint64) (io.ReadCloser, error) {
+// IndexShardSnapshot returns a reader that contains the contents of
+// an RBF snapshot for an index/shard. When snapshotting for
+// serverless, we need to be able to transactionally move the write
+// log to the new version, so we expose writeTx to allow the caller to
+// request a write transaction for the snapshot even though we'll just
+// be reading inside RBF.
+func (api *API) IndexShardSnapshot(ctx context.Context, indexName string, shard uint64, writeTx bool) (io.ReadCloser, error) {
 	span, _ := tracing.StartSpanFromContext(ctx, "API.IndexShardSnapshot")
 	defer span.Finish()
 
@@ -1298,7 +1273,7 @@ func (api *API) IndexShardSnapshot(ctx context.Context, indexName string, shard
 	}
 
 	// Start transaction.
-	tx := index.holder.txf.NewTx(Txo{Index: index, Shard: shard})
+	tx := index.holder.txf.NewTx(Txo{Index: index, Shard: shard, Write: writeTx})
 
 	// Ensure transaction is an RBF transaction.
 	rtx, ok := tx.(*RBFTx)
@@ -1339,7 +1314,6 @@ type ImportOptions struct {
 	Clear          bool
 	IgnoreKeyCheck bool
 	Presorted      bool
-	fullySorted    bool // format-aware sorting, internal use only please.
 	suppressLog    bool
 
 	// test Tx atomicity if > 0
@@ -1517,20 +1491,20 @@ func (api *API) Import(ctx context.Context, qcx *Qcx, req *ImportRequest, opts .
 	}
 
 	if api.isComputeNode && !options.suppressLog {
-		// Get the current version for shard.
-		version, err := api.getOrCreateShardVersion(ctx, req.Index, req.Shard)
-		if err != nil {
-			return errors.Wrap(err, "get or creating shard version")
-		}
-
 		tkey := dax.TableKey(req.Index)
 		qtid := tkey.QualifiedTableID()
 		partitionNum := dax.PartitionNum(partition)
 		shardNum := dax.ShardNum(req.Shard)
 
-		// Write the request to the write logger.
-		if err := api.writeLogWriter.WriteShard(ctx, qtid, partitionNum, shardNum, version, msg); err != nil {
-			return err
+		b, err := computer.MarshalLogMessage(msg, computer.EncodeTypeJSON)
+		if err != nil {
+			return errors.Wrap(err, "marshalling log message")
+		}
+
+		resource := api.serverlessStorage.GetShardResource(qtid, partitionNum, shardNum)
+		err = resource.Append(b)
+		if err != nil {
+			return errors.Wrap(err, "appending shard data") // TODO do we need to set err0 or something?
 		}
 	}
 
@@ -1546,7 +1520,6 @@ func (api *API) ImportWithTx(ctx context.Context, qcx *Qcx, req *ImportRequest,
 		return errors.Wrap(err, "validating api method")
 	}
 
-	api.server.logger.Debugf("ImportWithTx: %v %v %v", req.Index, req.Field, req.Shard)
 	idx, field, err := api.indexField(req.Index, req.Field, req.Shard)
 	if err != nil {
 		return errors.Wrap(err, "getting index and field")
@@ -1665,6 +1638,12 @@ func (api *API) ImportWithTx(ctx context.Context, qcx *Qcx, req *ImportRequest,
 // across many fields in a single shard. It can both set and clear
 // bits and updates caches/bitDepth as appropriate, although only the
 // bitmap parts happen truly transactionally.
+//
+// This function does not attempt to do existence tracking, because
+// it can't; there's no way to distinguish empty sets from not setting
+// bits. As a result, users of this endpoint are responsible for
+// providing corrected existence views for fields with existence
+// tracking. Our batch API does that.
 func (api *API) ImportRoaringShard(ctx context.Context, indexName string, shard uint64, req *ImportRoaringShardRequest) error {
 	index, err := api.Index(ctx, indexName)
 	if err != nil {
@@ -1695,7 +1674,7 @@ func (api *API) ImportRoaringShard(ctx context.Context, indexName string, shard
 		}
 
 		fieldType := field.Options().Type
-		if err1 = cleanupView(fieldType, &viewUpdate); err1 != nil {
+		if viewUpdate.View, err1 = field.cleanupViewName(viewUpdate.View); err1 != nil {
 			return err1
 		}
 
@@ -1765,21 +1744,21 @@ func (api *API) ImportRoaringShard(ctx context.Context, indexName string, shard
 				ClearRecords: view.ClearRecords,
 			}
 		}
-		// Get the current version for shard.
-		version, err := api.getOrCreateShardVersion(ctx, indexName, shard)
-		if err != nil {
-			err1 = errors.Wrap(err, "get or creating shard version")
-			return err1
-		}
+
 		tkey := dax.TableKey(indexName)
 		qtid := tkey.QualifiedTableID()
 		partitionNum := dax.PartitionNum(partition)
 		shardNum := dax.ShardNum(shard)
 
-		api.server.logger.Debugf("importroaringshard writing shard to writelogger: %+v, len(msg.Views): %d, table: %s", api.writeLogWriter, len(msg.Views), msg.Table)
+		b, err := computer.MarshalLogMessage(msg, computer.EncodeTypeJSON)
+		if err != nil {
+			err1 = errors.Wrap(err, "marshalling log message")
+			return err1
+		}
 
-		if err := api.writeLogWriter.WriteShard(ctx, qtid, partitionNum, shardNum, version, msg); err != nil {
-			err1 = errors.Wrap(err, "writing import-roaring-shard to writelogger")
+		resource := api.serverlessStorage.GetShardResource(qtid, partitionNum, shardNum)
+		err1 = errors.Wrap(resource.Append(b), "appending shard data")
+		if err1 != nil {
 			return err1
 		}
 	}
@@ -1787,27 +1766,6 @@ func (api *API) ImportRoaringShard(ctx context.Context, indexName string, shard
 	return nil
 }
 
-func cleanupView(fieldType string, viewUpdate *RoaringUpdate) error {
-	// TODO wouldn't hurt to have consolidated logic somewhere for validating view names.
-	switch fieldType {
-	case FieldTypeSet, FieldTypeTime:
-		if viewUpdate.View == "" {
-			viewUpdate.View = "standard"
-		}
-		// add 'standard_' if we just have a time... this is how IDK works by default
-		if fieldType == FieldTypeTime && !strings.HasPrefix(viewUpdate.View, viewStandard) {
-			viewUpdate.View = fmt.Sprintf("%s_%s", viewStandard, viewUpdate.View)
-		}
-	case FieldTypeInt, FieldTypeDecimal, FieldTypeTimestamp:
-		if viewUpdate.View == "" {
-			viewUpdate.View = "bsig_" + viewUpdate.Field
-		} else if viewUpdate.View != "bsig_"+viewUpdate.Field {
-			return NewBadRequestError(errors.Errorf("invalid view name (%s) for field %s of type %s", viewUpdate.View, viewUpdate.Field, fieldType))
-		}
-	}
-	return nil
-}
-
 // ImportValue is a wrapper around the common code in ImportValueWithTx, which
 // currently just translates req.Clear into a clear ImportOption.
 func (api *API) ImportValue(ctx context.Context, qcx *Qcx, req *ImportValueRequest, opts ...ImportOption) error {
@@ -1860,20 +1818,21 @@ func (api *API) ImportValue(ctx context.Context, qcx *Qcx, req *ImportValueReque
 
 	if api.isComputeNode && !options.suppressLog {
 		// Get the current version for shard.
-		version, err := api.getOrCreateShardVersion(ctx, req.Index, req.Shard)
-		if err != nil {
-			return errors.Wrap(err, "get or creating shard version")
-		}
-
 		tkey := dax.TableKey(req.Index)
 		qtid := tkey.QualifiedTableID()
 		partitionNum := dax.PartitionNum(partition)
 		shardNum := dax.ShardNum(req.Shard)
-
-		// Write the request to the write logger.
-		if err := api.writeLogWriter.WriteShard(ctx, qtid, partitionNum, shardNum, version, msg); err != nil {
-			return errors.Wrap(err, "writing shard to write logger")
+		b, err := computer.MarshalLogMessage(msg, computer.EncodeTypeJSON)
+		if err != nil {
+			return errors.Wrap(err, "marshalling log message")
 		}
+
+		resource := api.serverlessStorage.GetShardResource(qtid, partitionNum, shardNum)
+		err = resource.Append(b)
+		if err != nil {
+			return errors.Wrap(err, "appending shard data") // TODO do we need to set err0 or something?
+		}
+
 	}
 
 	return nil
@@ -2060,21 +2019,20 @@ func (api *API) ImportValueWithTx(ctx context.Context, qcx *Qcx, req *ImportValu
 	return nil
 }
 
-func importExistenceColumns(qcx *Qcx, index *Index, columnIDs []uint64, shard uint64) error {
+func importExistenceColumns(qcx *Qcx, index *Index, columnIDs []uint64, shard uint64) (err0 error) {
 	ef := index.existenceField()
 	if ef == nil {
 		return nil
 	}
-
-	existenceRowIDs := make([]uint64, len(columnIDs))
-	// If we don't gratuitously hand-duplicate things in field.Import,
-	// the fact that fragment.bulkImport rewrites its row and column
-	// lists can burn us if we don't make a copy before doing the
-	// existence field write.
-	columnCopy := make([]uint64, len(columnIDs))
-	copy(columnCopy, columnIDs)
-	options := ImportOptions{}
-	return ef.Import(qcx, existenceRowIDs, columnCopy, nil, shard, &options)
+	tx, finisher, err := qcx.GetTx(Txo{Write: true, Index: index, Shard: shard})
+	if err != nil {
+		return err
+	}
+	defer finisher(&err0)
+	// markExistingInView is simpler/faster than Import, but unusually, we use the
+	// standard view of the existence field, instead of the existence view of
+	// a specific field, when doing the index-wide update.
+	return ef.markExistingInView(tx, columnIDs, viewStandard, shard)
 }
 
 // ShardDistribution returns an object representing the distribution of shards
@@ -2126,15 +2084,6 @@ func (api *API) AvailableShards(ctx context.Context, indexName string) (*roaring
 	return index.AvailableShards(false), nil
 }
 
-// StatsWithTags returns an instance of whatever implementation of StatsClient
-// pilosa is using with the given tags.
-func (api *API) StatsWithTags(tags []string) stats.StatsClient {
-	if api.holder == nil || api.cluster == nil {
-		return nil
-	}
-	return api.holder.Stats.WithTags(tags...)
-}
-
 // LongQueryTime returns the configured threshold for logging/statting
 // long running queries.
 func (api *API) LongQueryTime() time.Duration {
@@ -2421,19 +2370,19 @@ func (api *API) StartTransaction(ctx context.Context, id string, timeout time.Du
 	switch err {
 	case nil:
 		if exclusive {
-			api.holder.Stats.Count(MetricExclusiveTransactionRequest, 1, 1.0)
+			CounterExclusiveTransactionRequest.Inc()
 		} else {
-			api.holder.Stats.Count(MetricTransactionStart, 1, 1.0)
+			CounterTransactionStart.Inc()
 		}
 	case ErrTransactionExclusive:
 		if exclusive {
-			api.holder.Stats.Count(MetricExclusiveTransactionBlocked, 1, 1.0)
+			CounterExclusiveTransactionBlocked.Inc()
 		} else {
-			api.holder.Stats.Count(MetricTransactionBlocked, 1, 1.0)
+			CounterTransactionBlocked.Inc()
 		}
 	}
 	if exclusive && t != nil && t.Active {
-		api.holder.Stats.Count(MetricExclusiveTransactionActive, 1, 1.0)
+		CounterExclusiveTransactionActive.Inc()
 	}
 	return t, err
 }
@@ -2445,9 +2394,9 @@ func (api *API) FinishTransaction(ctx context.Context, id string, remote bool) (
 	t, err := api.server.FinishTransaction(ctx, id, remote)
 	if err == nil {
 		if t.Exclusive {
-			api.holder.Stats.Count(MetricExclusiveTransactionEnd, 1, 1.0)
+			CounterExclusiveTransactionEnd.Inc()
 		} else {
-			api.holder.Stats.Count(MetricTransactionEnd, 1, 1.0)
+			CounterTransactionEnd.Inc()
 		}
 	}
 	return t, err
@@ -2467,7 +2416,7 @@ func (api *API) GetTransaction(ctx context.Context, id string, remote bool) (*Tr
 	t, err := api.server.GetTransaction(ctx, id, remote)
 	if err == nil {
 		if t.Exclusive && t.Active {
-			api.holder.Stats.Count(MetricExclusiveTransactionActive, 1, 1.0)
+			CounterExclusiveTransactionActive.Inc()
 		}
 	}
 	return t, err
@@ -3076,6 +3025,10 @@ func (api *API) CompilePlan(ctx context.Context, q string) (planner_types.PlanOp
 	return api.server.CompileExecutionPlan(ctx, q)
 }
 
+func (api *API) RehydratePlanOperator(ctx context.Context, reader io.Reader) (planner_types.PlanOperator, error) {
+	return api.server.RehydratePlanOperator(ctx, reader)
+}
+
 func (api *API) RBFDebugInfo() map[string]*rbf.DebugInfo {
 	infos := make(map[string]*rbf.DebugInfo)
 
@@ -3097,9 +3050,9 @@ func (api *API) Directive(ctx context.Context, d *dax.Directive) error {
 }
 
 // DirectiveApplied returns true if the computer's current Directive has been
-// applied and is ready to be queried. This it temporary (primarily for tests)
-// and needs to be refactored as we improve the logic around mds-to-computer
-// communication.
+// applied and is ready to be queried. This is temporary (primarily for tests)
+// and needs to be refactored as we improve the logic around
+// controller-to-computer communication.
 func (api *API) DirectiveApplied(ctx context.Context) (bool, error) {
 	return api.holder.DirectiveApplied(), nil
 }
@@ -3107,52 +3060,43 @@ func (api *API) DirectiveApplied(ctx context.Context) (bool, error) {
 // SnapshotShardData triggers the node to perform a shard snapshot based on the
 // provided SnapshotShardDataRequest.
 func (api *API) SnapshotShardData(ctx context.Context, req *dax.SnapshotShardDataRequest) error {
-	qtid := req.TableKey.QualifiedTableID()
-
-	// Confirm that this node is currently responsible for table/shard/fromVersion.
-	var version int
-	if v, ok, err := api.holder.versionStore.ShardVersion(ctx, qtid, req.ShardNum); err != nil {
-		return err
-	} else if !ok {
-		return errors.Errorf("shard not managed by this node: %s, %d", req.TableKey, req.ShardNum)
-	} else if v != req.FromVersion {
-		return errors.Errorf("shard managed by this node is at version: %d, not: %d", v, req.FromVersion)
-	} else {
-		version = v
+	if !api.holder.DirectiveApplied() {
+		return errors.New("don't have directive yet, can't snapshot shard")
 	}
+	// TODO(jaffee) confirm this node is actually responsible for the given
+	// shard? Not sure we need to given that this request comes from
+	// the Controller, but might be a belt&suspenders situation.
+
+	qtid := req.TableKey.QualifiedTableID()
 
 	partition := disco.ShardToShardPartition(string(req.TableKey), uint64(req.ShardNum), disco.DefaultPartitionN)
 	partitionNum := dax.PartitionNum(partition)
 
-	// Create the snapshot for the current version.
-	rc, err := api.IndexShardSnapshot(ctx, string(req.TableKey), uint64(req.ShardNum))
+	// Open a write Tx snapshotting current version.
+	rc, err := api.IndexShardSnapshot(ctx, string(req.TableKey), uint64(req.ShardNum), true)
 	if err != nil {
 		return errors.Wrap(err, "getting index/shard readcloser")
 	}
+	defer rc.Close()
 
-	// The following closes rc, the ReadCloser.
-	if err := api.snapshotReadWriter.WriteShardData(ctx, qtid, partitionNum, req.ShardNum, version, rc); err != nil {
-		return errors.Wrap(err, "snapshotting shard data")
+	resource := api.serverlessStorage.GetShardResource(qtid, partitionNum, req.ShardNum)
+	// Bump writelog version while write Tx is held.
+	if ok, err := resource.IncrementWLVersion(); err != nil {
+		return errors.Wrap(err, "incrementing write log version")
+	} else if !ok {
+		return nil
 	}
-
-	// Increment the version of the shard managed by this node.
-	if err := api.holder.versionStore.AddShards(ctx, qtid,
-		dax.NewVersionedShard(req.ShardNum, req.ToVersion),
-	); err != nil {
-		return errors.Wrap(err, "incrementing shard version locally")
-	}
-
-	// Update the cached directive on the holder.
-	api.holder.SetDirective(&req.Directive)
-	api.holder.SetDirectiveApplied(true)
-
-	// Finally, delete the log file for the previous version.
-	return api.writeLogWriter.DeleteShard(ctx, qtid, partitionNum, req.ShardNum, req.FromVersion)
+	// TODO(jaffee) look into downgrading Tx on RBF to read lock here now that WL version is incremented.
+	err = resource.Snapshot(rc)
+	return errors.Wrap(err, "snapshotting shard data")
 }
 
 // SnapshotTableKeys triggers the node to perform a table keys snapshot based on
 // the provided SnapshotTableKeysRequest.
 func (api *API) SnapshotTableKeys(ctx context.Context, req *dax.SnapshotTableKeysRequest) error {
+	if !api.holder.DirectiveApplied() {
+		return errors.New("don't have directive yet, can't snapshot table keys")
+	}
 	// If the index is not keyed, no-op on snapshotting its keys.
 	if idx, err := api.Index(ctx, string(req.TableKey)); err != nil {
 		return newNotFoundError(ErrIndexNotFound, string(req.TableKey))
@@ -3162,83 +3106,60 @@ func (api *API) SnapshotTableKeys(ctx context.Context, req *dax.SnapshotTableKey
 
 	qtid := req.TableKey.QualifiedTableID()
 
-	// Confirm that this node is currently responsible for table/partition/fromVersion.
-	var version int
-	if v, ok, err := api.holder.versionStore.PartitionVersion(ctx, qtid, req.PartitionNum); err != nil {
-		return err
-	} else if !ok {
-		return errors.Errorf("partition not managed by this node: %s, %d", req.TableKey, req.PartitionNum)
-	} else if v != req.FromVersion {
-		return errors.Errorf("partition managed by this node is at version: %d, not: %d", v, req.FromVersion)
-	} else {
-		version = v
-	}
-
 	// Create the snapshot for the current version.
-	wrTo, err := api.TranslateData(ctx, string(req.TableKey), int(req.PartitionNum))
+	trans, err := api.TranslateData(ctx, string(req.TableKey), int(req.PartitionNum))
 	if err != nil {
-		return errors.Wrapf(err, "getting index/partition writeto: %s/%d", req.TableKey, req.PartitionNum)
+		return errors.Wrapf(err, "getting index/partition translate store: %s/%d", req.TableKey, req.PartitionNum)
 	}
-
-	if err := api.snapshotReadWriter.WriteTableKeys(ctx, qtid, req.PartitionNum, version, wrTo); err != nil {
-		return errors.Wrap(err, "snapshotting table keys")
+	// get a write tx to ensure no other writes while incrementing WL version.
+	wrTo, err := trans.Begin(true)
+	if err != nil {
+		return errors.Wrap(err, "beginning table translate write tx")
 	}
+	defer wrTo.Rollback()
 
-	// Increment the version of the partition managed by this node.
-	if err := api.holder.versionStore.AddPartitions(ctx, qtid,
-		dax.NewVersionedPartition(req.PartitionNum, req.ToVersion),
-	); err != nil {
-		return errors.Wrap(err, "incrementing partition version locally")
+	resource := api.serverlessStorage.GetTableKeyResource(qtid, req.PartitionNum)
+	if ok, err := resource.IncrementWLVersion(); err != nil {
+		return errors.Wrap(err, "incrementing write log version")
+	} else if !ok {
+		// no need to snapshot, no writes
+		return nil
 	}
-
-	// Update the cached directive on the holder.
-	api.holder.SetDirective(&req.Directive)
-	api.holder.SetDirectiveApplied(true)
-
-	// Finally, delete the log file for the previous version.
-	return api.writeLogWriter.DeleteTableKeys(ctx, qtid, req.PartitionNum, req.FromVersion)
+	// TODO(jaffee) downgrade write tx to read-only
+	err = resource.SnapshotTo(wrTo)
+	return errors.Wrap(err, "snapshotting table keys")
 }
 
 // SnapshotFieldKeys triggers the node to perform a field keys snapshot based on
 // the provided SnapshotFieldKeysRequest.
 func (api *API) SnapshotFieldKeys(ctx context.Context, req *dax.SnapshotFieldKeysRequest) error {
+	if !api.holder.DirectiveApplied() {
+		return errors.New("don't have directive yet, can't snapshot field keys")
+	}
 	qtid := req.TableKey.QualifiedTableID()
 
-	// Confirm that this node is currently responsible for table/field/fromVersion.
-	var version int
-	if v, ok, err := api.holder.versionStore.FieldVersion(ctx, qtid, req.Field); err != nil {
-		return err
-	} else if !ok {
-		return errors.Errorf("field not managed by this node: %s, %s", req.TableKey, req.Field)
-	} else if v != req.FromVersion {
-		return errors.Errorf("field managed by this node is at version: %d, not: %d", v, req.FromVersion)
-	} else {
-		version = v
-	}
-
 	// Create the snapshot for the current version.
-	wrTo, err := api.FieldTranslateData(ctx, string(req.TableKey), string(req.Field))
+	trans, err := api.FieldTranslateData(ctx, string(req.TableKey), string(req.Field))
 	if err != nil {
-		return errors.Wrap(err, "getting index/field writeto")
+		return errors.Wrap(err, "getting index/field translator")
 	}
-
-	if err := api.snapshotReadWriter.WriteFieldKeys(ctx, qtid, req.Field, version, wrTo); err != nil {
-		return errors.Wrap(err, "snapshotting field keys")
+	// get a write tx to ensure no other writes while incrementing WL version.
+	wrTo, err := trans.Begin(true)
+	if err != nil {
+		return errors.Wrap(err, "beginning field translate write tx")
 	}
+	defer wrTo.Rollback()
 
-	// Increment the version of the field managed by this node.
-	if err := api.holder.versionStore.AddFields(ctx, qtid,
-		dax.NewVersionedField(req.Field, req.ToVersion),
-	); err != nil {
-		return errors.Wrap(err, "incrementing field version locally")
+	resource := api.serverlessStorage.GetFieldKeyResource(qtid, req.Field)
+	if ok, err := resource.IncrementWLVersion(); err != nil {
+		return errors.Wrap(err, "incrementing writelog version")
+	} else if !ok {
+		// no need to snapshot, no writes
+		return nil
 	}
-
-	// Update the cached directive on the holder.
-	api.holder.SetDirective(&req.Directive)
-	api.holder.SetDirectiveApplied(true)
-
-	// Finally, delete the log file for the previous version.
-	return api.writeLogWriter.DeleteFieldKeys(ctx, qtid, req.Field, req.FromVersion)
+	// TODO(jaffee) downgrade to read tx
+	err = resource.SnapshotTo(wrTo)
+	return errors.Wrap(err, "snapshotTo in FieldKeys")
 }
 
 type serverInfo struct {
@@ -3365,9 +3286,9 @@ var methodsNormal = map[apiMethod]struct{}{
 	apiDeleteDataframe:      {},
 }
 
-func shardInShards(i dax.ShardNum, s dax.VersionedShards) bool {
+func shardInShards(i dax.ShardNum, s dax.ShardNums) bool {
 	for _, o := range s {
-		if i == o.Num {
+		if i == o {
 			return true
 		}
 	}
@@ -3375,6 +3296,14 @@ func shardInShards(i dax.ShardNum, s dax.VersionedShards) bool {
 }
 
 type SchemaAPI interface {
+	CreateDatabase(context.Context, *dax.Database) error
+	DropDatabase(context.Context, dax.DatabaseID) error
+
+	DatabaseByName(ctx context.Context, dbname dax.DatabaseName) (*dax.Database, error)
+	DatabaseByID(ctx context.Context, dbid dax.DatabaseID) (*dax.Database, error)
+	SetDatabaseOption(ctx context.Context, dbid dax.DatabaseID, option string, value string) error
+	Databases(context.Context, ...dax.DatabaseID) ([]*dax.Database, error)
+
 	TableByName(ctx context.Context, tname dax.TableName) (*dax.Table, error)
 	TableByID(ctx context.Context, tid dax.TableID) (*dax.Table, error)
 	Tables(ctx context.Context) ([]*dax.Table, error)
@@ -3386,13 +3315,49 @@ type SchemaAPI interface {
 	DeleteField(ctx context.Context, tname dax.TableName, fname dax.FieldName) error
 }
 
-type SchemaInfoAPI interface {
-	IndexInfo(ctx context.Context, indexName string) (*IndexInfo, error)
-	FieldInfo(ctx context.Context, indexName, fieldName string) (*FieldInfo, error)
+// Ensure type implements interface.
+var _ SchemaAPI = (*NopSchemaAPI)(nil)
+
+// NopSchemaAPI is a no-op implementation of the SchemaAPI.
+type NopSchemaAPI struct{}
+
+func (n *NopSchemaAPI) ClusterName() string {
+	return ""
+}
+func (n *NopSchemaAPI) CreateDatabase(context.Context, *dax.Database) error { return nil }
+func (n *NopSchemaAPI) DropDatabase(context.Context, dax.DatabaseID) error  { return nil }
+func (n *NopSchemaAPI) DatabaseByName(ctx context.Context, dbname dax.DatabaseName) (*dax.Database, error) {
+	return nil, nil
+}
+func (n *NopSchemaAPI) DatabaseByID(ctx context.Context, dbid dax.DatabaseID) (*dax.Database, error) {
+	return nil, nil
+}
+func (n *NopSchemaAPI) SetDatabaseOption(ctx context.Context, dbid dax.DatabaseID, option string, value string) error {
+	return nil
+}
+func (n *NopSchemaAPI) Databases(context.Context, ...dax.DatabaseID) ([]*dax.Database, error) {
+	return nil, nil
+}
+func (n *NopSchemaAPI) TableByName(ctx context.Context, tname dax.TableName) (*dax.Table, error) {
+	return nil, nil
+}
+func (n *NopSchemaAPI) TableByID(ctx context.Context, tid dax.TableID) (*dax.Table, error) {
+	return nil, nil
+}
+func (n *NopSchemaAPI) Tables(ctx context.Context) ([]*dax.Table, error) { return nil, nil }
+
+func (n *NopSchemaAPI) CreateTable(ctx context.Context, tbl *dax.Table) error { return nil }
+func (n *NopSchemaAPI) CreateField(ctx context.Context, tname dax.TableName, fld *dax.Field) error {
+	return nil
+}
+func (n *NopSchemaAPI) DeleteTable(ctx context.Context, tname dax.TableName) error { return nil }
+func (n *NopSchemaAPI) DeleteField(ctx context.Context, tname dax.TableName, fname dax.FieldName) error {
+	return nil
 }
 
 type ClusterNode struct {
 	ID        string
+	Type      string
 	State     string
 	URI       string
 	GRPCURI   string
@@ -3408,7 +3373,9 @@ type SystemAPI interface {
 	ClusterReplicaCount() int
 	ShardWidth() int
 	ClusterState() string
+	DataDir() string
 
+	NodeID() string
 	ClusterNodes() []ClusterNode
 }
 
@@ -3424,6 +3391,9 @@ type QueryAPI interface {
 	Query(ctx context.Context, req *QueryRequest) (QueryResponse, error)
 }
 
+// Ensure type implements interface.
+var _ SystemAPI = (*FeatureBaseSystemAPI)(nil)
+
 // FeatureBaseSystemAPI is a wrapper around pilosa.API. It implements the
 // SystemAPI interface
 type FeatureBaseSystemAPI struct {
@@ -3476,6 +3446,14 @@ func (fsapi *FeatureBaseSystemAPI) ClusterState() string {
 	return string(state)
 }
 
+func (fsapi *FeatureBaseSystemAPI) DataDir() string {
+	return fsapi.server.dataDir
+}
+
+func (fsapi *FeatureBaseSystemAPI) NodeID() string {
+	return fsapi.cluster.Node.ID
+}
+
 func (fsapi *FeatureBaseSystemAPI) ClusterNodes() []ClusterNode {
 	result := make([]ClusterNode, 0)
 
@@ -3494,3 +3472,54 @@ func (fsapi *FeatureBaseSystemAPI) ClusterNodes() []ClusterNode {
 
 	return result
 }
+
+// Ensure type implements interface.
+var _ SystemAPI = (*NopSystemAPI)(nil)
+
+// NopSystemAPI is a no-op implementation of the SystemAPI.
+type NopSystemAPI struct{}
+
+func (napi *NopSystemAPI) ClusterName() string {
+	return ""
+}
+
+func (napi *NopSystemAPI) Version() string {
+	return ""
+}
+
+func (napi *NopSystemAPI) PlatformDescription() string {
+	return ""
+}
+
+func (napi *NopSystemAPI) PlatformVersion() string {
+	return ""
+}
+
+func (napi *NopSystemAPI) ClusterNodeCount() int {
+	return 0
+}
+
+func (napi *NopSystemAPI) ClusterReplicaCount() int {
+	return 0
+}
+
+func (napi *NopSystemAPI) ShardWidth() int {
+	return 0
+}
+
+func (napi *NopSystemAPI) ClusterState() string {
+	return ""
+}
+
+func (napi *NopSystemAPI) DataDir() string {
+	return ""
+}
+
+func (napi *NopSystemAPI) NodeID() string {
+	return ""
+}
+
+func (napi *NopSystemAPI) ClusterNodes() []ClusterNode {
+	result := make([]ClusterNode, 0)
+	return result
+}

api_directive.go

@@ -9,6 +9,7 @@ import (
 
 	"github.com/featurebasedb/featurebase/v3/dax"
 	"github.com/featurebasedb/featurebase/v3/dax/computer"
+	"github.com/featurebasedb/featurebase/v3/dax/storage"
 	"github.com/featurebasedb/featurebase/v3/disco"
 	"github.com/pkg/errors"
 )
@@ -34,6 +35,22 @@ func (api *API) ApplyDirective(ctx context.Context, d *dax.Directive) error {
 	// Handle the operations based on the directive method.
 	switch d.Method {
 	case dax.DirectiveMethodDiff:
+		// In order to prevent adding too much code specific to handling a diff
+		// directive (e.g. adding something like an `enactDirectiveDiff()`
+		// method), we are instead going to build a full Directive based on the
+		// diff, and then proceed normally as if we had received a full
+		// Directive. We do that by copying the previous Directive and then
+		// applying the diffs to the copy.
+		newD := previousDirective.Copy()
+
+		// Apply the diffs from the incoming Directive to the new, copied
+		// Directive.
+		newD.ApplyDiff(d)
+
+		// Now proceed with the new diff as if we had received it as a full diff.
+		d = newD
+
+	case dax.DirectiveMethodFull:
 		// pass: normal operation
 
 	case dax.DirectiveMethodReset:
@@ -41,7 +58,6 @@ func (api *API) ApplyDirective(ctx context.Context, d *dax.Directive) error {
 		if err := api.deleteAllIndexes(ctx); err != nil {
 			return errors.Wrap(err, "deleting all indexes")
 		}
-
 		// Set previousDirective to empty so the diff handles everything as new.
 		previousDirective = dax.Directive{}
 
@@ -60,7 +76,7 @@ func (api *API) ApplyDirective(ctx context.Context, d *dax.Directive) error {
 	// the "enactDirective" stage of ApplyDirective which validates against this
 	// cached Directive, so it's important that it be set before calling
 	// enactDirective(). An example: when loading partition data from the
-	// WriteLogger, there are validations to ensure that the partition being
+	// Writelogger, there are validations to ensure that the partition being
 	// loaded is meant to be handled by this node; that validation is done
 	// against the cached Directive.
 	// TODO(tlt): despite what this comment says, this logic is not sound; we
@@ -102,19 +118,19 @@ type directiveJobTableKeys struct {
 	directiveJobType
 	idx       *Index
 	tkey      dax.TableKey
-	partition dax.VersionedPartition
+	partition dax.PartitionNum
 }
 
 type directiveJobFieldKeys struct {
 	directiveJobType
 	tkey  dax.TableKey
-	field dax.VersionedField
+	field dax.FieldName
 }
 
 type directiveJobShards struct {
 	directiveJobType
 	tkey  dax.TableKey
-	shard dax.VersionedShard
+	shard dax.ShardNum
 }
 
 // directiveWorker is a worker in a worker pool which handles portions of a
@@ -246,7 +262,7 @@ func (api *API) enactTables(ctx context.Context, fromD, toD *dax.Directive) erro
 	// Remove all indexes that are no longer part of the directive.
 	for _, tkey := range sc.removed() {
 		idx := string(tkey)
-		if err := api.holder.deleteIndex(idx); err != nil {
+		if err := api.DeleteIndex(ctx, idx); err != nil {
 			return errors.Wrapf(err, "deleting index: %s", tkey)
 		}
 	}
@@ -324,7 +340,18 @@ func (api *API) pushJobsTableKeys(ctx context.Context, jobs chan<- directiveJobT
 	// Get the diff between from/to directive.partitions.
 	partComp := newPartitionsComparer(fromD.TranslatePartitionsMap(), toPartitionsMap)
 
-	// Loop over the partition map and load from WriteLogger.
+	// Remove any partitions which are no longer assigned to this worker.
+	// TODO(tlt): currently, this is just removing the file lock on the
+	// resource; it's not actually removing the resource from the local
+	// computer. We should do that.
+	for tkey, partitions := range partComp.removed() {
+		qtid := tkey.QualifiedTableID()
+		for _, partition := range partitions {
+			api.serverlessStorage.RemoveTableKeyResource(qtid, partition)
+		}
+	}
+
+	// Loop over the partition map and load from Writelogger.
 	for tkey, partitions := range partComp.added() {
 		// Get index in order to find the translate stores (by partition) for
 		// the table.
@@ -348,42 +375,38 @@ func (api *API) pushJobsTableKeys(ctx context.Context, jobs chan<- directiveJobT
 	}
 }
 
-func (api *API) loadTableKeys(ctx context.Context, idx *Index, tkey dax.TableKey, partition dax.VersionedPartition) error {
+func (api *API) loadTableKeys(ctx context.Context, idx *Index, tkey dax.TableKey, partition dax.PartitionNum) error {
 	qtid := tkey.QualifiedTableID()
 
-	// Load the previous snapshot. Version 0 doesn't have a snapshot
-	// file; it only has log entries.
-	if partition.Version > 0 {
-		// Load partition snapshot: version - 1
-		previousVersion := partition.Version - 1
-		rc, err := api.snapshotReadWriter.ReadTableKeys(ctx, qtid, partition.Num, previousVersion)
-		if err != nil {
-			return errors.Wrap(err, "reading table keys snapshot")
-		}
+	resource := api.serverlessStorage.GetTableKeyResource(qtid, partition)
+	if resource.IsLocked() {
+		api.logger().Warnf("skipping loadTableKeys (already held) %s %d", tkey, partition)
+		return nil
+	}
+
+	// load latest snapshot
+	if rc, err := resource.LoadLatestSnapshot(); err != nil {
+		return errors.Wrap(err, "loading table key snapshot")
+	} else if rc != nil {
 		defer rc.Close()
-
-		if err := api.TranslateIndexDB(ctx, string(tkey), int(partition.Num), rc); err != nil {
+		if err := api.TranslateIndexDB(ctx, string(tkey), int(partition), rc); err != nil {
 			return errors.Wrap(err, "restoring table keys")
-
 		}
 	}
 
-	if err := func() error {
-		store := idx.TranslateStore(int(partition.Num))
-
-		reader := api.writeLogReader.TableKeyReader(ctx, qtid, partition.Num, partition.Version)
-		if err := reader.Open(); err != nil {
-			// TODO: this log can be confusing because on a create
-			// table, there is no log file yet, so an error is expected.
-			// Instead of swallowing this error, we need to check the
-			// error code and handle it differently. This means the
-			// writelogger will need to return an error indicating that
-			// the log file does not exist, but that that is expected.
-			// log.Printf("could not open log file for table: %s, partition: %d: version: %d, err: %s", table, partition.Num, partition.Version, err)
+	// define write log loading in a function since we have to do it
+	// before and after locking
+	loadWriteLog := func() error {
+		writelog, err := resource.LoadWriteLog()
+		if err != nil {
+			return errors.Wrap(err, "getting write log reader for table keys")
+		}
+		if writelog == nil {
 			return nil
 		}
+		reader := storage.NewTableKeyReader(qtid, partition, writelog)
 		defer reader.Close()
-
+		store := idx.TranslateStore(int(partition))
 		for msg, err := reader.Read(); err != io.EOF; msg, err = reader.Read() {
 			if err != nil {
 				return errors.Wrap(err, "reading from log reader")
@@ -394,25 +417,39 @@ func (api *API) loadTableKeys(ctx context.Context, idx *Index, tkey dax.TableKey
 				}
 			}
 		}
-
 		return nil
-	}(); err != nil {
+	}
+	// 1st write log load
+	if err := loadWriteLog(); err != nil {
 		return err
 	}
 
-	// Set the table/partition/version in the holder.
-	if err := api.holder.versionStore.AddPartitions(ctx, qtid, partition); err != nil {
-		return errors.Wrap(err, "adding partition to sharder")
+	// acquire lock on this partition's keys
+	if err := resource.Lock(); err != nil {
+		return errors.Wrap(err, "locking table key partition")
 	}
 
-	return nil
+	// reload writelog in case of changes between last load and
+	// lock. The resource object takes care of only loading new data.
+	return loadWriteLog()
 }
 
 func (api *API) pushJobsFieldKeys(ctx context.Context, jobs chan<- directiveJobType, fromD, toD *dax.Directive) {
 	// Get the diff between from/to directive.fields.
 	fieldComp := newFieldsComparer(fromD.TranslateFieldsMap(), toD.TranslateFieldsMap())
 
-	// Loop over the field map and load from WriteLogger.
+	// Remove any field keys which are no longer assigned to this worker.
+	// TODO(tlt): currently, this is just removing the file lock on the
+	// resource; it's not actually removing the resource from the local
+	// computer. We should do that.
+	for tkey, fields := range fieldComp.removed() {
+		qtid := tkey.QualifiedTableID()
+		for _, field := range fields {
+			api.serverlessStorage.RemoveFieldKeyResource(qtid, field)
+		}
+	}
+
+	// Loop over the field map and load from Writelogger.
 	for tkey, fields := range fieldComp.added() {
 		for _, field := range fields {
 			jobs <- directiveJobFieldKeys{
@@ -423,47 +460,45 @@ func (api *API) pushJobsFieldKeys(ctx context.Context, jobs chan<- directiveJobT
 	}
 }
 
-func (api *API) loadFieldKeys(ctx context.Context, tkey dax.TableKey, field dax.VersionedField) error {
+func (api *API) loadFieldKeys(ctx context.Context, tkey dax.TableKey, field dax.FieldName) error {
 	qtid := tkey.QualifiedTableID()
 
-	// Load the previous snapshot. Version 0 doesn't have a snapshot
-	// file; it only has log entries.
-	if field.Version > 0 {
-		// Load field snapshot: version - 1
-		previousVersion := field.Version - 1
-		rc, err := api.snapshotReadWriter.ReadFieldKeys(ctx, qtid, field.Name, previousVersion)
-		if err != nil {
-			return errors.Wrap(err, "reading field keys snapshot")
-		}
-		defer rc.Close()
+	resource := api.serverlessStorage.GetFieldKeyResource(qtid, field)
+	if resource.IsLocked() {
+		api.logger().Warnf("skipping loadFieldKeys (already held) %s %s", tkey, field)
+		return nil
+	}
 
-		if err := api.TranslateFieldDB(ctx, string(tkey), string(field.Name), rc); err != nil {
+	// load latest snapshot
+	if rc, err := resource.LoadLatestSnapshot(); err != nil {
+		return errors.Wrap(err, "loading field key snapshot")
+	} else if rc != nil {
+		defer rc.Close()
+		if err := api.TranslateFieldDB(ctx, string(tkey), string(field), rc); err != nil {
 			return errors.Wrap(err, "restoring field keys")
 		}
 	}
 
-	if err := func() error {
+	// define write log loading in a function since we have to do it
+	// before and after locking
+	loadWriteLog := func() error {
+		writelog, err := resource.LoadWriteLog()
+		if err != nil {
+			return errors.Wrap(err, "getting write log reader for field keys")
+		}
+		if writelog == nil {
+			return nil
+		}
+		reader := storage.NewFieldKeyReader(qtid, field, writelog)
+		defer reader.Close()
 		// Get field in order to find the translate store.
-		fld := api.holder.Field(string(tkey), string(field.Name))
+		fld := api.holder.Field(string(tkey), string(field))
 		if fld == nil {
-			log.Printf("field not found in holder: %s", field.Name)
+			log.Printf("field not found in holder: %s", field)
 			return nil
 		}
 		store := fld.TranslateStore()
 
-		reader := api.writeLogReader.FieldKeyReader(ctx, qtid, field.Name, field.Version)
-		if err := reader.Open(); err != nil {
-			// TODO: this log can be confusing because on a create
-			// table, there is no log file yet, so an error is expected.
-			// Instead of swallowing this error, we need to check the
-			// error code and handle it differently. This means the
-			// writelogger will need to return an error indicating that
-			// the log file does not exist, but that that is expected.
-			// log.Printf("could not open log file for table: %s, field: %s: version: %d, err: %s", table, field.Name, field.Version, err)
-			return nil
-		}
-		defer reader.Close()
-
 		for msg, err := reader.Read(); err != io.EOF; msg, err = reader.Read() {
 			if err != nil {
 				return errors.Wrap(err, "reading from log reader")
@@ -474,18 +509,21 @@ func (api *API) loadFieldKeys(ctx context.Context, tkey dax.TableKey, field dax.
 				}
 			}
 		}
-
 		return nil
-	}(); err != nil {
+	}
+	// 1st write log load
+	if err := loadWriteLog(); err != nil {
 		return err
 	}
 
-	// Set the table/field/version in the holder.
-	if err := api.holder.versionStore.AddFields(ctx, qtid, field); err != nil {
-		return errors.Wrap(err, "adding field to sharder")
+	// acquire lock on this partition's keys
+	if err := resource.Lock(); err != nil {
+		return errors.Wrap(err, "locking field key partition")
 	}
 
-	return nil
+	// reload writelog in case of changes between last load and
+	// lock. The resource object takes care of only loading new data.
+	return loadWriteLog()
 }
 
 func (api *API) pushJobsShards(ctx context.Context, jobs chan<- directiveJobType, fromD, toD *dax.Directive) {
@@ -495,7 +533,19 @@ func (api *API) pushJobsShards(ctx context.Context, jobs chan<- directiveJobType
 	// Get the diff between from/to directive shards.
 	shardComp := newShardsComparer(fromD.ComputeShardsMap(), shardMap)
 
-	// Loop over the shard map and load from WriteLogger.
+	// Remove any shards which are no longer assigned to this worker.
+	// TODO(tlt): currently, this is just removing the file lock on the
+	// resource; it's not actually removing the resource from the local
+	// computer. We should do that.
+	for tkey, shards := range shardComp.removed() {
+		qtid := tkey.QualifiedTableID()
+		for _, shard := range shards {
+			partition := dax.PartitionNum(disco.ShardToShardPartition(string(tkey), uint64(shard), disco.DefaultPartitionN))
+			api.serverlessStorage.RemoveShardResource(qtid, partition, shard)
+		}
+	}
+
+	// Loop over the shard map and load from Writelogger.
 	for tkey, shards := range shardComp.added() {
 		for _, shard := range shards {
 			jobs <- directiveJobShards{
@@ -506,46 +556,43 @@ func (api *API) pushJobsShards(ctx context.Context, jobs chan<- directiveJobType
 	}
 }
 
-func (api *API) loadShard(ctx context.Context, tkey dax.TableKey, shard dax.VersionedShard) error {
+func (api *API) loadShard(ctx context.Context, tkey dax.TableKey, shard dax.ShardNum) error {
 	qtid := tkey.QualifiedTableID()
 
-	partition := disco.ShardToShardPartition(string(tkey), uint64(shard.Num), disco.DefaultPartitionN)
-	partitionNum := dax.PartitionNum(partition)
+	partition := dax.PartitionNum(disco.ShardToShardPartition(string(tkey), uint64(shard), disco.DefaultPartitionN))
 
-	// Load the previous snapshot. Version 0 doesn't have a snapshot
-	// file; it only has log entries.
-	if shard.Version > 0 {
-		// Load shard snapshot: version - 1
-		previousVersion := shard.Version - 1
-		rc, err := api.snapshotReadWriter.ReadShardData(ctx, qtid, partitionNum, shard.Num, previousVersion)
-		if err != nil {
-			return errors.Wrap(err, "reading shard data snapshot")
-		}
+	resource := api.serverlessStorage.GetShardResource(qtid, partition, shard)
+	if resource.IsLocked() {
+		api.logger().Warnf("skipping loadShard (already held) %s %d", tkey, shard)
+		return nil
+	}
 
-		if err := api.RestoreShard(ctx, string(tkey), uint64(shard.Num), rc); err != nil {
+	if rc, err := resource.LoadLatestSnapshot(); err != nil {
+		return errors.Wrap(err, "reading latest snapshot for shard")
+	} else if rc != nil {
+		defer rc.Close()
+		if err := api.RestoreShard(ctx, string(tkey), uint64(shard), rc); err != nil {
 			return errors.Wrap(err, "restoring shard data")
 		}
 	}
 
-	// WriteLog reader.
-	if err := func() error {
-		reader := api.writeLogReader.ShardReader(ctx, qtid, partitionNum, shard.Num, shard.Version)
-		if err := reader.Open(); err != nil {
-			// TODO: this log can be confusing because on a create
-			// table, there is no log file yet, so an error is expected.
-			// Instead of swallowing this error, we need to check the
-			// error code and handle it differently. This means the
-			// writelogger will need to return an error indicating that
-			// the log file does not exist, but that that is expected.
-			// log.Printf("could not open log file for table: %s, partition: %d: version: %d, shard: %d, err: %s", table, partition, shard.Version, shard.Num, err)
+	// define write log loading in a func because we do it twice.
+	loadWriteLog := func() error {
+		writelog, err := resource.LoadWriteLog()
+		if err != nil {
+			return errors.Wrap(err, "")
+		}
+		if writelog == nil {
 			return nil
 		}
-		defer reader.Close()
 
+		reader := storage.NewShardReader(qtid, partition, shard, writelog)
+		defer reader.Close()
 		for logMsg, err := reader.Read(); err != io.EOF; logMsg, err = reader.Read() {
 			if err != nil {
 				return errors.Wrap(err, "reading from log reader")
 			}
+
 			switch msg := logMsg.(type) {
 			case *computer.ImportRoaringMessage:
 				req := &ImportRoaringRequest{
@@ -632,18 +679,21 @@ func (api *API) loadShard(ctx context.Context, tkey dax.TableKey, shard dax.Vers
 				}
 			}
 		}
-
 		return nil
-	}(); err != nil {
+	}
+	// 1st write log load
+	if err := loadWriteLog(); err != nil {
 		return err
 	}
 
-	// Set the table/shard/version in the holder.
-	if err := api.holder.versionStore.AddShards(ctx, qtid, shard); err != nil {
-		return errors.Wrap(err, "adding shard to sharder")
+	// acquire lock on this partition's keys
+	if err := resource.Lock(); err != nil {
+		return errors.Wrap(err, "locking field key partition")
 	}
 
-	return nil
+	// reload writelog in case of changes between last load and
+	// lock. The resource object takes care of only loading new data.
+	return loadWriteLog()
 }
 
 //////////////////////////////////////////////////////////////
@@ -707,11 +757,11 @@ func thingsAdded[K comparable](from []K, to []K) []K {
 // partitionsComparer is used to compare the differences between two maps of
 // table:[]partition.
 type partitionsComparer struct {
-	from map[dax.TableKey]dax.VersionedPartitions
-	to   map[dax.TableKey]dax.VersionedPartitions
+	from map[dax.TableKey]dax.PartitionNums
+	to   map[dax.TableKey]dax.PartitionNums
 }
 
-func newPartitionsComparer(from map[dax.TableKey]dax.VersionedPartitions, to map[dax.TableKey]dax.VersionedPartitions) *partitionsComparer {
+func newPartitionsComparer(from map[dax.TableKey]dax.PartitionNums, to map[dax.TableKey]dax.PartitionNums) *partitionsComparer {
 	return &partitionsComparer{
 		from: from,
 		to:   to,
@@ -720,23 +770,23 @@ func newPartitionsComparer(from map[dax.TableKey]dax.VersionedPartitions, to map
 
 // added returns the partitions which are present in `to` but not in `from`. The
 // results remain in the format of a map of table:[]partition.
-func (p *partitionsComparer) added() map[dax.TableKey]dax.VersionedPartitions {
+func (p *partitionsComparer) added() map[dax.TableKey]dax.PartitionNums {
 	return partitionsAdded(p.from, p.to)
 }
 
 // removed returns the partitions which are present in `from` but not in `to`.
 // The results remain in the format of a map of table:[]partition.
-func (p *partitionsComparer) removed() map[dax.TableKey]dax.VersionedPartitions {
+func (p *partitionsComparer) removed() map[dax.TableKey]dax.PartitionNums {
 	return partitionsAdded(p.to, p.from)
 }
 
 // partitionsAdded returns the partitions which are present in `to` but not in `from`.
-func partitionsAdded(from map[dax.TableKey]dax.VersionedPartitions, to map[dax.TableKey]dax.VersionedPartitions) map[dax.TableKey]dax.VersionedPartitions {
+func partitionsAdded(from map[dax.TableKey]dax.PartitionNums, to map[dax.TableKey]dax.PartitionNums) map[dax.TableKey]dax.PartitionNums {
 	if from == nil {
 		return to
 	}
 
-	added := make(map[dax.TableKey]dax.VersionedPartitions)
+	added := make(map[dax.TableKey]dax.PartitionNums)
 	for tt, tps := range to {
 		fps, found := from[tt]
 		if !found {
@@ -744,7 +794,7 @@ func partitionsAdded(from map[dax.TableKey]dax.VersionedPartitions, to map[dax.T
 			continue
 		}
 
-		addedPartitions := dax.VersionedPartitions{}
+		addedPartitions := dax.PartitionNums{}
 		for i := range tps {
 			var found bool
 			for j := range fps {
@@ -768,11 +818,11 @@ func partitionsAdded(from map[dax.TableKey]dax.VersionedPartitions, to map[dax.T
 // fieldsComparer is used to compare the differences between two maps of
 // table:[]fieldVersion.
 type fieldsComparer struct {
-	from map[dax.TableKey]dax.VersionedFields
-	to   map[dax.TableKey]dax.VersionedFields
+	from map[dax.TableKey][]dax.FieldName
+	to   map[dax.TableKey][]dax.FieldName
 }
 
-func newFieldsComparer(from map[dax.TableKey]dax.VersionedFields, to map[dax.TableKey]dax.VersionedFields) *fieldsComparer {
+func newFieldsComparer(from map[dax.TableKey][]dax.FieldName, to map[dax.TableKey][]dax.FieldName) *fieldsComparer {
 	return &fieldsComparer{
 		from: from,
 		to:   to,
@@ -781,23 +831,23 @@ func newFieldsComparer(from map[dax.TableKey]dax.VersionedFields, to map[dax.Tab
 
 // added returns the fields which are present in `to` but not in `from`. The
 // results remain in the format of a map of table:[]field.
-func (f *fieldsComparer) added() map[dax.TableKey]dax.VersionedFields {
+func (f *fieldsComparer) added() map[dax.TableKey][]dax.FieldName {
 	return fieldsAdded(f.from, f.to)
 }
 
 // removed returns the fields which are present in `from` but not in `to`.
 // The results remain in the format of a map of table:[]field.
-func (f *fieldsComparer) removed() map[dax.TableKey]dax.VersionedFields {
+func (f *fieldsComparer) removed() map[dax.TableKey][]dax.FieldName {
 	return fieldsAdded(f.to, f.from)
 }
 
 // fieldsAdded returns the fields which are present in `to` but not in `from`.
-func fieldsAdded(from map[dax.TableKey]dax.VersionedFields, to map[dax.TableKey]dax.VersionedFields) map[dax.TableKey]dax.VersionedFields {
+func fieldsAdded(from map[dax.TableKey][]dax.FieldName, to map[dax.TableKey][]dax.FieldName) map[dax.TableKey][]dax.FieldName {
 	if from == nil {
 		return to
 	}
 
-	added := make(map[dax.TableKey]dax.VersionedFields)
+	added := make(map[dax.TableKey][]dax.FieldName)
 	for tt, tps := range to {
 		fps, found := from[tt]
 		if !found {
@@ -805,7 +855,7 @@ func fieldsAdded(from map[dax.TableKey]dax.VersionedFields, to map[dax.TableKey]
 			continue
 		}
 
-		addedFieldVersions := dax.VersionedFields{}
+		addedFieldVersions := []dax.FieldName{}
 		for i := range tps {
 			var found bool
 			for j := range fps {
@@ -829,11 +879,11 @@ func fieldsAdded(from map[dax.TableKey]dax.VersionedFields, to map[dax.TableKey]
 // shardsComparer is used to compare the differences between two maps of
 // table:[]shardV.
 type shardsComparer struct {
-	from map[dax.TableKey]dax.VersionedShards
-	to   map[dax.TableKey]dax.VersionedShards
+	from map[dax.TableKey]dax.ShardNums
+	to   map[dax.TableKey]dax.ShardNums
 }
 
-func newShardsComparer(from map[dax.TableKey]dax.VersionedShards, to map[dax.TableKey]dax.VersionedShards) *shardsComparer {
+func newShardsComparer(from map[dax.TableKey]dax.ShardNums, to map[dax.TableKey]dax.ShardNums) *shardsComparer {
 	return &shardsComparer{
 		from: from,
 		to:   to,
@@ -842,23 +892,23 @@ func newShardsComparer(from map[dax.TableKey]dax.VersionedShards, to map[dax.Tab
 
 // added returns the shards which are present in `to` but not in `from`. The
 // results remain in the format of a map of table:[]shard.
-func (s *shardsComparer) added() map[dax.TableKey]dax.VersionedShards {
+func (s *shardsComparer) added() map[dax.TableKey]dax.ShardNums {
 	return shardsAdded(s.from, s.to)
 }
 
 // removed returns the shards which are present in `from` but not in `to`. The
 // results remain in the format of a map of table:[]shard.
-func (s *shardsComparer) removed() map[dax.TableKey]dax.VersionedShards {
+func (s *shardsComparer) removed() map[dax.TableKey]dax.ShardNums {
 	return shardsAdded(s.to, s.from)
 }
 
 // shardsAdded returns the shards which are present in `to` but not in `from`.
-func shardsAdded(from map[dax.TableKey]dax.VersionedShards, to map[dax.TableKey]dax.VersionedShards) map[dax.TableKey]dax.VersionedShards {
+func shardsAdded(from map[dax.TableKey]dax.ShardNums, to map[dax.TableKey]dax.ShardNums) map[dax.TableKey]dax.ShardNums {
 	if from == nil {
 		return to
 	}
 
-	added := make(map[dax.TableKey]dax.VersionedShards)
+	added := make(map[dax.TableKey]dax.ShardNums)
 	for tt, tss := range to {
 		fss, found := from[tt]
 		if !found {
@@ -866,7 +916,7 @@ func shardsAdded(from map[dax.TableKey]dax.VersionedShards, to map[dax.TableKey]
 			continue
 		}
 
-		addedShards := dax.VersionedShards{}
+		addedShards := dax.ShardNums{}
 		for i := range tss {
 			var found bool
 			for j := range fss {
@@ -889,7 +939,7 @@ func shardsAdded(from map[dax.TableKey]dax.VersionedShards, to map[dax.TableKey]
 
 // createTableAndFields creates the FeatureBase Tables and Fields provided in
 // the dax.Directive format.
-func (api *API) createTableAndFields(tbl *dax.QualifiedTable, partitions dax.VersionedPartitions) error {
+func (api *API) createTableAndFields(tbl *dax.QualifiedTable, partitions dax.PartitionNums) error {
 	cim := &CreateIndexMessage{
 		Index:     string(tbl.Key()),
 		CreatedAt: 0,
@@ -930,7 +980,7 @@ func createField(idx *Index, fld *dax.Field) error {
 		return errors.Wrapf(err, "creating field options from field: %s", fld.Name)
 	}
 
-	if _, err := idx.CreateField(string(fld.Name), "", opts...); err != nil {
+	if _, err := idx.createNullableField(string(fld.Name), "", opts...); err != nil {
 		return errors.Wrapf(err, "creating field on index: %s", fld.Name)
 	}
 	return nil

api_directive_test.go

@@ -20,17 +20,17 @@ func TestAPI_Directive(t *testing.T) {
 	api := c.GetPrimary().API
 	ctx := context.Background()
 
-	qual := dax.NewTableQualifier("acme", "db1")
-	tbl1 := daxtest.TestQualifiedTableWithID(t, qual, "1", "tbl1", 12, false)
-	tbl2 := daxtest.TestQualifiedTableWithID(t, qual, "2", "tbl2", 12, false)
-	tbl3 := daxtest.TestQualifiedTableWithID(t, qual, "3", "tbl3", 12, false)
+	qdbid := dax.NewQualifiedDatabaseID("acme", "db1")
+	tbl1 := daxtest.TestQualifiedTableWithID(t, qdbid, "1", "tbl1", 12, false)
+	tbl2 := daxtest.TestQualifiedTableWithID(t, qdbid, "2", "tbl2", 12, false)
+	tbl3 := daxtest.TestQualifiedTableWithID(t, qdbid, "3", "tbl3", 12, false)
 
 	t.Run("Schema", func(t *testing.T) {
 
 		// Empty directive (and empty holder).
 		{
 			d := &dax.Directive{
-				Method:  dax.DirectiveMethodDiff,
+				Method:  dax.DirectiveMethodFull,
 				Version: 1,
 			}
 			err := api.ApplyDirective(ctx, d)
@@ -41,7 +41,7 @@ func TestAPI_Directive(t *testing.T) {
 		// Add a new table.
 		{
 			d := &dax.Directive{
-				Method: dax.DirectiveMethodDiff,
+				Method: dax.DirectiveMethodFull,
 				Tables: []*dax.QualifiedTable{
 					tbl1,
 				},
@@ -55,7 +55,7 @@ func TestAPI_Directive(t *testing.T) {
 		// Add a new table, and keep the existing table.
 		{
 			d := &dax.Directive{
-				Method: dax.DirectiveMethodDiff,
+				Method: dax.DirectiveMethodFull,
 				Tables: []*dax.QualifiedTable{
 					tbl1,
 					tbl2,
@@ -70,7 +70,7 @@ func TestAPI_Directive(t *testing.T) {
 		// Add a new table and remove one of the existing tables.
 		{
 			d := &dax.Directive{
-				Method: dax.DirectiveMethodDiff,
+				Method: dax.DirectiveMethodFull,
 				Tables: []*dax.QualifiedTable{
 					tbl2,
 					tbl3,

api_test.go

@@ -837,7 +837,7 @@ func TestAPI_IDAlloc(t *testing.T) {
 			t.Fatalf("obtaining random bytes: %v", err)
 		}
 		ids3, err := primary.ReserveIDs(key, session, 0, 2)
-		var esync pilosa.ErrIDOffsetDesync
+		var esync pilosa.IDOffsetDesyncError
 		if errors.As(err, &esync) {
 			if esync.Requested != 0 {
 				t.Errorf("incorrect requested offset in error: provided %d but got %d", 0, esync.Requested)

apply.go

@@ -7,18 +7,17 @@ import (
 	"io"
 	"os"
 	"path/filepath"
+	"sort"
 	"strings"
 	"sync"
 
 	"github.com/apache/arrow/go/v10/arrow"
 	"github.com/apache/arrow/go/v10/arrow/array"
 	"github.com/apache/arrow/go/v10/arrow/memory"
-	"github.com/apache/arrow/go/v10/parquet/file"
-	"github.com/apache/arrow/go/v10/parquet/pqarrow"
-	"github.com/gomem/gomem/pkg/dataframe"
 	"github.com/featurebasedb/featurebase/v3/pql"
 	"github.com/featurebasedb/featurebase/v3/tracing"
 	"github.com/featurebasedb/featurebase/v3/vprint"
+	"github.com/gomem/gomem/pkg/dataframe"
 	"github.com/pkg/errors"
 
 	ivy "robpike.io/ivy/arrow"
@@ -96,7 +95,7 @@ func IvyReduce(reduceCode string, opCode string, opt *ExecOptions) (func(ctx con
 			col := value.ToArrowColumn(accumulator, pool)
 			return dataframe.NewDataFrameFromColumns(pool, []arrow.Column{*col})
 		}
-		// only acutally reduce on the initiating node i hate the network
+		// only actually reduce on the initiating node i hate the network
 		// over head but oh well
 		ctxIvy.AssignGlobal("_", accumulator)
 		ok, err := runIvyString(ctxIvy, reduceCode)
@@ -220,12 +219,14 @@ func (e *executor) executeApplyShard(ctx context.Context, qcx *Qcx, index string
 	if idx == nil {
 		return nil, newNotFoundError(ErrIndexNotFound, index)
 	}
+
 	fname := idx.GetDataFramePath(shard)
-	if _, err := os.Stat(fname + ".parquet"); os.IsNotExist(err) {
+
+	if !e.dataFrameExists(fname) {
 		return value.NewVector([]value.Value{}), nil
 	}
 
-	table, err := readTableParquet(fname)
+	table, err := e.getDataTable(ctx, fname, pool)
 	if err != nil {
 		return nil, err
 	}
@@ -254,57 +255,20 @@ func (e *executor) executeApplyShard(ctx context.Context, qcx *Qcx, index string
 	return context.Global("_"), nil
 }
 
-func readTableParquet(filename string) (arrow.Table, error) {
-	r, err := os.Open(filename + ".parquet")
-	if err != nil {
-		return nil, err
-	}
-
-	pf, err := file.NewParquetReader(r)
-	if err != nil {
-		return nil, err
-	}
-
-	reader, err := pqarrow.NewFileReader(pf, pqarrow.ArrowReadProperties{}, memory.DefaultAllocator)
-	if err != nil {
-		return nil, err
-	}
-	return reader.ReadTable(context.Background())
-}
-
-func readTableParquetCtx(ctx context.Context, filename string, mem memory.Allocator) (arrow.Table, error) {
-	r, err := os.Open(filename + ".parquet")
-	if err != nil {
-		return nil, err
-	}
-
-	pf, err := file.NewParquetReader(r)
-	if err != nil {
-		return nil, err
-	}
-
-	reader, err := pqarrow.NewFileReader(pf, pqarrow.ArrowReadProperties{}, mem)
-	if err != nil {
-		return nil, err
-	}
-	return reader.ReadTable(ctx)
-}
-
 // ///////////////////////////////////////////////////////
 // all the ingest supporting functions
 // ///////////////////////////////////////////////////////
 
-func NewShardFile(name string) (*ShardFile, error) {
-	if _, err := os.Stat(name + ".parquet"); os.IsNotExist(err) {
-		return &ShardFile{dest: name}, nil
+func NewShardFile(ctx context.Context, name string, mem memory.Allocator, e *executor) (*ShardFile, error) {
+	if !e.dataFrameExists(name) {
+		return &ShardFile{dest: name, executor: e, strings: make(map[key][]string)}, nil
 	}
-
 	// else read in existing
-	table, err := readTableParquet(name)
+	table, err := e.getDataTable(ctx, name, mem)
 	if err != nil {
 		return nil, err
 	}
-	return &ShardFile{table: table, schema: table.Schema(), dest: name}, nil
+	return &ShardFile{table: table, schema: table.Schema(), dest: name, executor: e, strings: make(map[key][]string)}, nil
 }
 
 type NameType struct {
@@ -328,6 +292,8 @@ func cast(v interface{}) arrow.DataType {
 		return arrow.PrimitiveTypes.Float64
 	case float64:
 		return arrow.PrimitiveTypes.Float64
+	case *arrow.StringType:
+		return arrow.BinaryTypes.String
 	default:
 		vprint.VV("%T .... %v", v, v)
 	}
@@ -342,6 +308,11 @@ func (cr *ChangesetRequest) ArrowSchema() *arrow.Schema {
 	return arrow.NewSchema(fields, nil)
 }
 
+type key struct {
+	col   int
+	chunk int
+}
+
 type ShardFile struct {
 	table      arrow.Table
 	schema     *arrow.Schema
@@ -349,6 +320,8 @@ type ShardFile struct {
 	added      int64
 	columns    []interface{}
 	dest       string
+	executor   *executor
+	strings    map[key][]string
 }
 
 func compareSchema(s1, s2 *arrow.Schema) bool {
@@ -401,6 +374,8 @@ func (sf *ShardFile) buildAppenders(maxid int64) {
 			sf.columns[i] = make([]int64, newSize)
 		case arrow.PrimitiveTypes.Float64:
 			sf.columns[i] = make([]float64, newSize)
+		case arrow.BinaryTypes.String:
+			sf.columns[i] = make([]string, newSize)
 		}
 	}
 	sf.added = newSize
@@ -417,6 +392,11 @@ func (sf *ShardFile) SetFloatValue(col int, row int64, val float64) {
 	v[row-sf.beforeRows] = val
 }
 
+func (sf *ShardFile) SetStringValue(col int, row int64, val string) {
+	v := sf.columns[col].([]string)
+	v[row-sf.beforeRows] = val
+}
+
 func (sf *ShardFile) Process(cs *ChangesetRequest) error {
 	err := sf.process(cs)
 	if err != nil {
@@ -427,12 +407,37 @@ func (sf *ShardFile) Process(cs *ChangesetRequest) error {
 	if err != nil {
 		return err
 	}
-	return os.Rename(rtemp+".parquet", sf.dest+".parquet")
+	return os.Rename(rtemp+sf.executor.TableExtension(), sf.dest+sf.executor.TableExtension())
+}
+
+func (sf *ShardFile) LoadBlobs() error {
+	for col := 0; col < len(sf.schema.Fields()); col++ {
+		column := sf.table.Column(col)
+		switch column.DataType() {
+		case arrow.BinaryTypes.String:
+			for i, chunk := range column.Data().Chunks() {
+				stringData := chunk.(*array.String)
+				k := key{col: col, chunk: i}
+				for j := 0; j < stringData.Len(); j++ {
+					v := stringData.Value(j)
+					sf.strings[k] = append(sf.strings[k], v)
+
+				}
+			}
+		}
+	}
+	return nil
+}
+
+func (sf *ShardFile) ReplaceString(col, chunk, l int, s string) {
+	sf.strings[key{col: col, chunk: chunk}][l] = s
 }
 
 func (sf *ShardFile) process(cs *ChangesetRequest) error {
 	offset := 0
 	if sf.table != nil {
+		// need to load blobs prior
+		sf.LoadBlobs()
 		column := sf.table.Column(0)
 		resolver := dataframe.NewChunkResolver(column)
 		for i, rowid := range cs.ShardIds {
@@ -450,6 +455,10 @@ func (sf *ShardFile) process(cs *ChangesetRequest) error {
 				case arrow.PrimitiveTypes.Float64:
 					v := column.Data().Chunk(chunk).(*array.Float64).Float64Values()
 					v[l] = cs.Columns[col].([]float64)[i]
+				case arrow.BinaryTypes.String:
+					// TODO(twg) 2023/01/09 How to update existing?
+					new := cs.Columns[col].([]string)[i]
+					sf.ReplaceString(col, chunk, l, new)
 				default:
 					panic(fmt.Sprintf("Unknown Type %v", column.DataType()))
 				}
@@ -469,6 +478,8 @@ func (sf *ShardFile) process(cs *ChangesetRequest) error {
 					sf.SetIntValue(col, rowid, cs.Columns[col].([]int64)[i])
 				case arrow.PrimitiveTypes.Float64:
 					sf.SetFloatValue(col, rowid, cs.Columns[col].([]float64)[i])
+				case arrow.BinaryTypes.String:
+					sf.SetStringValue(col, rowid, cs.Columns[col].([]string)[i])
 				default:
 					panic(fmt.Sprintf("2 Unknown Type %v", sf.schema.Field(col).Type))
 				}
@@ -479,15 +490,65 @@ func (sf *ShardFile) process(cs *ChangesetRequest) error {
 	return nil
 }
 
+type twoSlices struct {
+	id_slice    []int
+	lists_slice [][]string
+}
+
+type SortByOther twoSlices
+
+func (sbo SortByOther) Len() int {
+	return len(sbo.id_slice)
+}
+
+func (sbo SortByOther) Swap(i, j int) {
+	sbo.id_slice[i], sbo.id_slice[j] = sbo.id_slice[j], sbo.id_slice[i]
+	sbo.lists_slice[i], sbo.lists_slice[j] = sbo.lists_slice[j], sbo.lists_slice[i]
+}
+
+func (sbo SortByOther) Less(i, j int) bool {
+	return sbo.id_slice[i] < sbo.id_slice[j]
+}
+
+func (sf *ShardFile) buildFromStrings(idx int, mem memory.Allocator) []arrow.Array {
+	ids := make([]int, 0)
+	lists := make([][]string, 0)
+	for k, v := range sf.strings {
+		if k.col == idx { // ugh not ordered :(
+			ids = append(ids, k.chunk)
+			lists = append(lists, v)
+		}
+	}
+	// sort ids/lists
+	parts := twoSlices{id_slice: ids, lists_slice: lists}
+	sort.Sort(SortByOther(parts))
+
+	builder := array.NewStringBuilder(mem)
+	chunks := make([]arrow.Array, 0)
+	for _, v := range parts.lists_slice {
+		builder.AppendValues(v, nil)
+		newChunk := builder.NewArray()
+		chunks = append(chunks, newChunk)
+	}
+	return chunks
+}
+
 func (sf *ShardFile) Save(name string) error {
 	parts := make([]arrow.Array, 0)
 	mem := memory.NewGoAllocator()
 	for col := 0; col < len(sf.schema.Fields()); col++ {
 		chunks := make([]arrow.Array, 0)
 		if sf.table != nil {
-			// we append if there was existing parquet file
+			// we append if there was existing file
 			column := sf.table.Column(col)
-			chunks = append(chunks, column.Data().Chunks()...)
+			// if primitive type
+			switch column.DataType() {
+			case arrow.BinaryTypes.String:
+				chunks = sf.buildFromStrings(col, mem)
+			default:
+				chunks = append(chunks, column.Data().Chunks()...)
+			}
+			// else binary type
 		}
 		switch sf.schema.Field(col).Type {
 		case arrow.PrimitiveTypes.Int64:
@@ -516,27 +577,26 @@ func (sf *ShardFile) Save(name string) error {
 				return err
 			}
 			parts = append(parts, record)
+		case arrow.BinaryTypes.String:
+			if sf.added > 0 {
+				fbuild := array.NewStringBuilder(mem)
+				fbuild.AppendValues(sf.columns[col].([]string), nil) // TODO(twg) 2022/09/28 need to handle null
+				newChunk := fbuild.NewArray()
+				chunks = append(chunks, newChunk)
+			}
+			record, err := array.Concatenate(chunks, mem)
+			if err != nil {
+				return err
+			}
+			parts = append(parts, record)
 		default:
 			vprint.VV("UNKNOWN %T", sf.schema.Field(col).Type)
 		}
 	}
 	rec := array.NewRecord(sf.schema, parts, sf.beforeRows+sf.added)
-	df, err := dataframe.NewDataFrameFromRecord(mem, rec)
-	if err != nil {
-		return err
-	}
-	// confirm change
-	w, err := os.Create(name + ".parquet")
-	if err != nil {
-		return err
-	}
+	table := array.NewTableFromRecords(sf.schema, []arrow.Record{rec})
 
-	err = df.ToParquet(w, 1024)
-	if err != nil {
-		return err
-	}
-	w.Close()
-	return nil
+	return sf.executor.SaveTable(name, table, mem)
 }
 
 // TODO(twg) 2022/10/03 Not a huge fan of the global variable will look at adding to executor structure
@@ -573,7 +633,8 @@ func (api *API) ApplyDataframeChangeset(ctx context.Context, index string, cs *C
 	mu := getDataframeWritelock(shard)
 	mu.Lock()
 	defer mu.Unlock()
-	shardFile, err := NewShardFile(fname)
+	mem := memory.NewGoAllocator()
+	shardFile, err := NewShardFile(ctx, fname, mem, api.server.executor)
 	if err != nil {
 		return err
 	}
@@ -600,14 +661,16 @@ func (api *API) GetDataframeSchema(ctx context.Context, indexName string) (inter
 	dir, _ := os.Open(base)
 	files, _ := dir.Readdir(0)
 	parts := make([]column, 0)
+	mem := memory.NewGoAllocator()
 	for i := range files {
 		file := files[i]
 		name := file.Name()
-		if strings.HasSuffix(name, ".parquet") {
+		if api.server.executor.IsDataframeFile(name) {
 			// strip off the parquet extenison
 			name = strings.TrimSuffix(name, filepath.Ext(name))
 			// read the parquet file and extract the schema
-			table, err := readTableParquet(filepath.Join(base, name))
+			fname := filepath.Join(base, name)
+			table, err := api.server.executor.getDataTable(ctx, fname, mem)
 			if err != nil {
 				return nil, err
 			}

arrow.go

@@ -5,15 +5,21 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"io"
 	"os"
+	"strings"
 	"sync"
 
 	"github.com/apache/arrow/go/v10/arrow"
 	"github.com/apache/arrow/go/v10/arrow/array"
+	"github.com/apache/arrow/go/v10/arrow/ipc"
 	"github.com/apache/arrow/go/v10/arrow/memory"
-	"github.com/gomem/gomem/pkg/dataframe"
+	"github.com/apache/arrow/go/v10/parquet"
+	"github.com/apache/arrow/go/v10/parquet/file"
+	"github.com/apache/arrow/go/v10/parquet/pqarrow"
 	"github.com/featurebasedb/featurebase/v3/pql"
 	"github.com/featurebasedb/featurebase/v3/tracing"
+	"github.com/gomem/gomem/pkg/dataframe"
 	"github.com/pkg/errors"
 )
 
@@ -54,7 +60,7 @@ func (e *executor) executeArrow(ctx context.Context, qcx *Qcx, index string, c *
 		mu.Unlock()
 		return e.executeArrowShard(ctx, qcx, index, c, shard, pool, columnFilter)
 	}
-	tables := make([]*basicTable, 0)
+	tables := make([]*BasicTable, 0)
 
 	reduceFn := func(ctx context.Context, prev, v interface{}) interface{} {
 		mu.Lock()
@@ -64,7 +70,7 @@ func (e *executor) executeArrow(ctx context.Context, qcx *Qcx, index string, c *
 			return prev
 		}
 		switch t := v.(type) {
-		case *basicTable:
+		case *BasicTable:
 
 			if t.resolver != nil {
 				mu.Lock()
@@ -87,107 +93,113 @@ func (e *executor) executeArrow(ctx context.Context, qcx *Qcx, index string, c *
 		return nil, err
 	}
 	if len(tables) == 0 {
-		return &basicTable{name: "empty"}, nil
+		return &BasicTable{name: "empty"}, nil
 	}
 	tbl := Concat(tables[0].Schema(), tables, pool)
 	r := dataframe.NewChunkResolver(tbl.Column(0))
-	return &basicTable{resolver: &r, table: tbl}, nil
+	return &BasicTable{resolver: &r, table: tbl}, nil
 }
 
-type basicTable struct {
+type BasicTable struct {
 	resolver dataframe.Resolver
 	table    arrow.Table
 	filtered bool
 	name     string
 }
 
-func (st *basicTable) Name() string {
+func (st *BasicTable) Name() string {
 	return st.name
 }
 
-func (st *basicTable) Schema() *arrow.Schema {
+func (st *BasicTable) Schema() *arrow.Schema {
 	if st.table != nil {
 		return st.table.Schema()
 	}
 	return &arrow.Schema{}
 }
 
-func (st *basicTable) IsFiltered() bool {
+func (st *BasicTable) IsFiltered() bool {
 	return st.filtered
 }
 
-func (st *basicTable) NumRows() int64 {
+func (st *BasicTable) NumRows() int64 {
 	if st.resolver == nil {
 		return 0
 	}
 	return int64(st.resolver.NumRows())
 }
 
-func (st *basicTable) NumCols() int64 {
+func (st *BasicTable) NumCols() int64 {
 	if st.table != nil {
 		return st.table.NumCols()
 	}
 	return 0
 }
 
-func (st *basicTable) Column(i int) *arrow.Column {
+func (st *BasicTable) Column(i int) *arrow.Column {
 	if st.table != nil {
 		return st.table.Column(i)
 	}
 	return nil
 }
 
-func (st *basicTable) Retain() {
+func (st *BasicTable) Retain() {
 	if st.table != nil {
 		st.table.Retain()
 	}
 }
 
-func (st *basicTable) Release() {
+func (st *BasicTable) Release() {
 	if st.table != nil {
 		st.table.Retain()
 	}
 }
 
-func (st *basicTable) Get(column, row int) interface{} {
+func (st *BasicTable) Get(column, row int) interface{} {
 	field := st.Schema().Field(column)
 	c, i := st.resolver.Resolve(row)
+	nullable := field.Nullable
 
 	chunk := st.Column(column).Data().Chunk(c)
+	// TODO(twg) 2023/01/26 potential NULL support?
+	if nullable && chunk.IsNull(i) {
+		return nil
+	}
 	switch field.Type.(type) {
-	// case *arrow.BooleanType:
-	//	v := chunk.(*array.Boolean).BooleanValues()
-	//	return v[i]
+	case *arrow.BooleanType:
+		return chunk.(*array.Boolean).Value(i)
 	case *arrow.Int8Type:
 		v := chunk.(*array.Int8).Int8Values()
-		return v[i]
+		return int64(v[i])
 	case *arrow.Int16Type:
 		v := chunk.(*array.Int16).Int16Values()
-		return v[i]
+		return int64(v[i])
 	case *arrow.Int32Type:
 		v := chunk.(*array.Int32).Int32Values()
-		return v[i]
+		return int64(v[i])
 	case *arrow.Int64Type:
 		v := chunk.(*array.Int64).Int64Values()
-		return v[i]
+		return int64(v[i])
 	case *arrow.Uint8Type:
 		v := chunk.(*array.Uint8).Uint8Values()
-		return v[i]
+		return uint64(v[i])
 	case *arrow.Uint16Type:
 		v := chunk.(*array.Uint16).Uint16Values()
-		return v[i]
+		return uint64(v[i])
 	case *arrow.Uint32Type:
 		v := chunk.(*array.Uint32).Uint32Values()
-		return v[i]
+		return uint64(v[i])
 	case *arrow.Uint64Type:
 		v := chunk.(*array.Uint64).Uint64Values()
 		return v[i]
 	case *arrow.Float32Type:
 		v := chunk.(*array.Float32).Float32Values()
-		return v[i]
+		return float64(v[i])
 	case *arrow.Float64Type:
 		v := chunk.(*array.Float64).Float64Values()
 		return v[i]
+	case *arrow.StringType:
+		return chunk.(*array.String).Value(i)
 	}
 	return 0
 }
@@ -217,8 +229,10 @@ func builderFrom(mem memory.Allocator, dt arrow.DataType, size int64) array.Buil
 		bldr = array.NewFloat32Builder(mem)
 	case *arrow.Float64Type:
 		bldr = array.NewFloat64Builder(mem)
+	case *arrow.StringType:
+		bldr = array.NewStringBuilder(mem)
 	default:
-		panic(fmt.Errorf("npy2root: invalid Arrow type %v", dt))
+		panic(fmt.Errorf("builderFrom: invalid Arrow type %v", dt))
 	}
 	bldr.Reserve(int(size))
 	return bldr
@@ -248,12 +262,14 @@ func appendData(bldr array.Builder, v interface{}) {
 		bldr.Append(v.(float32))
 	case *array.Float64Builder:
 		bldr.Append(v.(float64))
+	case *array.StringBuilder:
+		bldr.Append(v.(string))
 	default:
-		panic(fmt.Errorf("npy2root: invalid Arrow builder type %T", bldr))
+		panic(fmt.Errorf("appendData: invalid Arrow builder type %T", bldr))
 	}
 }
 
-func Concat(schema *arrow.Schema, tables []*basicTable, mem memory.Allocator) arrow.Table {
+func Concat(schema *arrow.Schema, tables []*BasicTable, mem memory.Allocator) arrow.Table {
 	if len(tables) == 1 {
 		if !tables[0].IsFiltered() {
 			return tables[0]
@@ -295,7 +311,7 @@ func Concat(schema *arrow.Schema, tables []*basicTable, mem memory.Allocator) ar
 	return array.NewTable(schema, cols, -1)
 }
 
-func (st *basicTable) MarshalJSON() ([]byte, error) {
+func (st *BasicTable) MarshalJSON() ([]byte, error) {
 	results := make(map[string]interface{})
 	n := 0
 	if st.table != nil {
@@ -314,10 +330,10 @@ func (st *basicTable) MarshalJSON() ([]byte, error) {
 	return json.Marshal(results)
 }
 
-func BasicTableFromArrow(table arrow.Table, mem memory.Allocator) *basicTable {
+func BasicTableFromArrow(table arrow.Table, mem memory.Allocator) *BasicTable {
 	col := table.Column(0)
 	r := dataframe.NewChunkResolver(col)
-	return &basicTable{resolver: &r, table: table}
+	return &BasicTable{resolver: &r, table: table}
 }
 
 func filterColumns(filters []string, table arrow.Table) arrow.Table {
@@ -347,7 +363,7 @@ func filterColumns(filters []string, table arrow.Table) arrow.Table {
 	return array.NewTable(filterdSchema, cols, table.NumRows())
 }
 
-func (e *executor) executeArrowShard(ctx context.Context, qcx *Qcx, index string, c *pql.Call, shard uint64, pool memory.Allocator, columnFilter []string) (*basicTable, error) {
+func (e *executor) executeArrowShard(ctx context.Context, qcx *Qcx, index string, c *pql.Call, shard uint64, pool memory.Allocator, columnFilter []string) (*BasicTable, error) {
 	name := fmt.Sprintf("a. %v", shard)
 	span, _ := tracing.StartSpanFromContext(ctx, "Executor.executeArrowShard")
 	defer span.Finish()
@@ -361,7 +377,7 @@ func (e *executor) executeArrowShard(ctx context.Context, qcx *Qcx, index string
 		filter = row
 		if !filter.Any() {
 			// no need to actuall run the query for its not operating against any values
-			return &basicTable{name: name}, nil
+			return &BasicTable{name: name}, nil
 		}
 	}
 	//
@@ -371,12 +387,14 @@ func (e *executor) executeArrowShard(ctx context.Context, qcx *Qcx, index string
 	if idx == nil {
 		return nil, newNotFoundError(ErrIndexNotFound, index)
 	}
+
 	fname := idx.GetDataFramePath(shard)
-	if _, err := os.Stat(fname + ".parquet"); os.IsNotExist(err) {
-		return &basicTable{name: name}, nil
+
+	if !e.dataFrameExists(fname) {
+		return &BasicTable{name: name}, nil
 	}
 
-	table, err := readTableParquetCtx(context.TODO(), fname, pool)
+	table, err := e.getDataTable(ctx, fname, pool)
 	if err != nil {
 		return nil, errors.Wrap(err, "arrow readTableParquet")
 	}
@@ -393,7 +411,7 @@ func (e *executor) executeArrowShard(ctx context.Context, qcx *Qcx, index string
 	resolver = &p
 	if filter != nil {
 		if len(ids) == 0 {
-			return &basicTable{name: name}, nil
+			return &BasicTable{name: name}, nil
 		}
 		resolver, err = filterDataframe(resolver, pool, ids)
 		if err != nil {
@@ -401,5 +419,144 @@ func (e *executor) executeArrowShard(ctx context.Context, qcx *Qcx, index string
 		}
 	}
 	table.Retain()
-	return &basicTable{resolver: resolver, table: table, filtered: filter != nil, name: name}, nil
+	return &BasicTable{resolver: resolver, table: table, filtered: filter != nil, name: name}, nil
+}
+
+func (e *executor) dataFrameExists(fname string) bool {
+	if e.typeIsParquet() {
+		if _, err := os.Stat(fname + ".parquet"); os.IsNotExist(err) {
+			return false
+		}
+		return true
+	}
+	if _, err := os.Stat(fname + ".arrow"); os.IsNotExist(err) {
+		return false
+	}
+	return true
+}
+
+func (e *executor) getDataTable(ctx context.Context, fname string, mem memory.Allocator) (arrow.Table, error) {
+	if e.typeIsParquet() {
+		table, err := readTableParquetCtx(ctx, fname, mem)
+		return table, err
+	}
+	return readTableArrow(fname, mem)
+}
+
+func (e *executor) typeIsParquet() bool {
+	return e.datafameUseParquet
+}
+
+func (e *executor) IsDataframeFile(name string) bool {
+	if e.typeIsParquet() {
+		return strings.HasSuffix(name, ".parquet")
+	}
+	return strings.HasSuffix(name, ".arrow")
+}
+
+func (e *executor) SaveTable(name string, table arrow.Table, mem memory.Allocator) error {
+	if e.typeIsParquet() {
+		return writeTableParquet(table, name)
+	}
+	return writeTableArrow(table, name, mem)
+}
+
+func (e *executor) TableExtension() string {
+	if e.typeIsParquet() {
+		return ".parquet"
+	}
+	return ".arrow"
+}
+
+func readTableArrow(filename string, mem memory.Allocator) (arrow.Table, error) {
+	r, err := os.Open(filename + ".arrow")
+	if err != nil {
+		return nil, err
+	}
+	rr, err := ipc.NewFileReader(r, ipc.WithAllocator(mem))
+	if err != nil {
+		return nil, err
+	}
+	defer rr.Close()
+	records := make([]arrow.Record, rr.NumRecords())
+	i := 0
+	for {
+		rec, err := rr.Read()
+		if err == io.EOF {
+			break
+		} else if err != nil {
+			return nil, err
+		}
+		records[i] = rec
+		i++
+	}
+	records = records[:i]
+	table := array.NewTableFromRecords(rr.Schema(), records)
+	return table, nil
+}
+
+func readTableParquetCtx(ctx context.Context, filename string, mem memory.Allocator) (arrow.Table, error) {
+	r, err := os.Open(filename + ".parquet")
+	if err != nil {
+		return nil, err
+	}
+	defer r.Close()
+
+	pf, err := file.NewParquetReader(r)
+	if err != nil {
+		return nil, err
+	}
+
+	reader, err := pqarrow.NewFileReader(pf, pqarrow.ArrowReadProperties{}, mem)
+	if err != nil {
+		return nil, err
+	}
+	return reader.ReadTable(ctx)
+}
+
+func writeTableParquet(table arrow.Table, filename string) error {
+	f, err := os.Create(filename + ".parquet")
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+	props := parquet.NewWriterProperties(parquet.WithDictionaryDefault(false))
+	arrProps := pqarrow.DefaultWriterProps()
+	chunkSize := 10 * 1024 * 1024
+	err = pqarrow.WriteTable(table, f, int64(chunkSize), props, arrProps)
+	if err != nil {
+		return err
+	}
+	f.Sync()
+	return nil
+}
+
+func writeTableArrow(table arrow.Table, filename string, mem memory.Allocator) error {
+	f, err := os.Create(filename + ".arrow")
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+	writer, err := ipc.NewFileWriter(f, ipc.WithAllocator(mem), ipc.WithSchema(table.Schema()))
+	if err != nil {
+		panic(err)
+	}
+	chunkSize := int64(0)
+	tr := array.NewTableReader(table, chunkSize)
+	defer tr.Release()
+	n := 0
+	for tr.Next() {
+		arec := tr.Record()
+		err = writer.Write(arec)
+		if err != nil {
+			panic(err)
+		}
+		n++
+	}
+	err = writer.Close()
+	if err != nil {
+		panic(err)
+	}
+	f.Sync()
+	return nil
 }

arrow_test.go

@@ -0,0 +1,53 @@
+// Copyright 2021 Molecula Corp. All rights reserved.
+package pilosa
+
+import (
+	"context"
+	"encoding/hex"
+	"math/rand"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/apache/arrow/go/v10/arrow"
+	"github.com/apache/arrow/go/v10/arrow/array"
+	"github.com/apache/arrow/go/v10/arrow/memory"
+)
+
+func TempFileName(prefix string) string {
+	randBytes := make([]byte, 16)
+	rand.Read(randBytes)
+	return filepath.Join(os.TempDir(), prefix+hex.EncodeToString(randBytes))
+}
+
+func Test_TableParquet(t *testing.T) {
+	// create a arrow table
+	schema := arrow.NewSchema(
+		[]arrow.Field{
+			{Name: "num", Type: arrow.PrimitiveTypes.Float64},
+		},
+		nil, // no metadata
+	)
+	mem := memory.NewGoAllocator()
+	b := array.NewRecordBuilder(mem, schema)
+	defer b.Release()
+	b.Field(0).(*array.Float64Builder).AppendValues([]float64{1.0, 1.5, 2.0}, nil)
+	table := array.NewTableFromRecords(schema, []arrow.Record{b.NewRecord()})
+	defer table.Release()
+	fileName := TempFileName("pq-")
+	// save it as  a parquet file
+	err := writeTableParquet(table, fileName)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.Remove(fileName)
+
+	// read it back in and compare the result
+	got, err := readTableParquetCtx(context.Background(), fileName, mem)
+	if err != nil {
+		t.Fatalf("readTableParquetCtx() error = %v", err)
+	}
+	if got.NumCols() != table.NumCols() {
+		t.Errorf("got:%v expected:%v", got.NumCols(), table.NumCols())
+	}
+}

audit_test.go

@@ -7,7 +7,7 @@ import (
 	"os"
 	"reflect"
 
-	pilosa "github.com/featurebasedb/featurebase/v3"
+	"github.com/featurebasedb/featurebase/v3"
 	"github.com/featurebasedb/featurebase/v3/testhook"
 )
 

batch/Dockerfile-test

@@ -2,10 +2,10 @@ ARG GO_VERSION=1.19
 
 FROM golang:${GO_VERSION}
 
-WORKDIR /go/src/github.com/molecula/featurebase/
+WORKDIR /go/src/github.com/featurebasedb/featurebase/
 
 COPY . .
 
-WORKDIR /go/src/github.com/molecula/featurebase/batch/
+WORKDIR /go/src/github.com/featurebasedb/featurebase/batch/
 
 CMD ["go","test","-v","-mod=vendor","-tags=odbc,dynamic","./..."]

batch/batch.go

@@ -23,6 +23,7 @@ import (
 const (
 	DefaultKeyTranslateBatchSize = 100000
 	existenceFieldName           = "_exists"
+	existenceViewName            = "existence" // this should match top level featurebase viewExistence
 )
 
 // TODO if using column translation, column ids might get way out of
@@ -552,7 +553,6 @@ func (b *Batch) Add(rec Row) error {
 				// empty string is not a valid value at this point (Pilosa refuses to translate it)
 				if val == "" { //
 					b.rowIDs[i] = append(rowIDs, nilSentinel)
-
 				} else if rowID, ok := b.getRowTranslation(field.Name, val); ok {
 					b.rowIDs[i] = append(rowIDs, rowID)
 				} else {
@@ -574,7 +574,11 @@ func (b *Batch) Add(rec Row) error {
 		case int64:
 			b.values[field.Name] = append(b.values[field.Name], val)
 		case []string:
-			if len(val) == 0 {
+			// note that a length of 0 can be valid, and represents an
+			// empty set. an empty set counts as a non-NULL value for
+			// SQL purposes -- it means the existence view bit should
+			// get set.
+			if val == nil {
 				continue
 			}
 			rowIDSets, ok := b.rowIDSets[field.Name]
@@ -609,7 +613,11 @@ func (b *Batch) Add(rec Row) error {
 			}
 			b.rowIDSets[field.Name] = append(rowIDSets, rowIDs)
 		case []uint64:
-			if len(val) == 0 {
+			// note that a length of 0 can be valid, and represents an
+			// empty set. an empty set counts as a non-NULL value for
+			// SQL purposes -- it means the existence view bit should
+			// get set.
+			if val == nil {
 				continue
 			}
 			rowIDSets, ok := b.rowIDSets[field.Name]
@@ -664,6 +672,9 @@ func (b *Batch) Add(rec Row) error {
 
 	for i, uval := range rec.Clears {
 		field := b.header[i]
+		if field.Options.Type == featurebase.FieldTypeMutex && uval != nil {
+			return errors.Errorf("individual-bit clears not allowed on mutex fields; use nil to clear a mutex")
+		}
 		if _, ok := b.clearRowIDs[i]; !ok {
 			b.clearRowIDs[i] = make(map[int]uint64)
 		}
@@ -742,23 +753,27 @@ var ErrBatchNowStale = errors.New("batch is stale and needs to be imported (howe
 func (b *Batch) Import() error {
 	ctx := context.Background()
 	start := time.Now()
-	trns, err := b.importer.StartTransaction(ctx, "", b.prevDuration*10, false, time.Hour)
-	if err != nil {
-		return errors.Wrap(err, "starting transaction")
+	if !b.useShardTransactionalEndpoint {
+		trns, err := b.importer.StartTransaction(ctx, "", b.prevDuration*10, false, time.Hour)
+		if err != nil {
+			return errors.Wrap(err, "starting transaction")
+		}
+		defer func() {
+			if trns != nil {
+				if trnsl, err := b.importer.FinishTransaction(ctx, trns.ID); err != nil {
+					b.log.Errorf("error finishing transaction: %v. trns: %+v", err, trnsl)
+				}
+			}
+		}()
 	}
 	defer func() {
-		if trns != nil {
-			if trnsl, err := b.importer.FinishTransaction(ctx, trns.ID); err != nil {
-				b.log.Errorf("error finishing transaction: %v. trns: %+v", err, trnsl)
-			}
-		}
-		b.importer.StatsTiming(MetricBatchImportDurationSeconds, time.Since(start), 1.0)
+		featurebase.SummaryBatchImportDurationSeconds.Observe(time.Since(start).Seconds())
 	}()
 
 	size := len(b.ids)
 	transStart := time.Now()
 	// first we need to translate the toTranslate, then fill out the missing row IDs
-	err = b.doTranslation()
+	err := b.doTranslation()
 	if err != nil {
 		return errors.Wrap(err, "doing Translation")
 	}
@@ -825,7 +840,7 @@ func (b *Batch) Flush() error {
 		if err != nil {
 			b.log.Errorf("error finishing transaction: %v. trns: %+v", err, trnsl)
 		}
-		b.importer.StatsTiming(MetricBatchFlushDurationSeconds, time.Since(start), 1.0)
+		featurebase.SummaryBatchFlushDurationSeconds.Observe(time.Since(start).Seconds())
 	}()
 
 	importStart := time.Now()
@@ -1185,7 +1200,7 @@ func (b *Batch) doImportShardTransactional(frags, clearFrags fragments) error {
 		}
 	}
 
-	b.importer.StatsTiming(MetricBatchShardImportBuildRequestsSeconds, time.Since(start), 1.0)
+	featurebase.SummaryBatchShardImportBuildRequestsSeconds.Observe(time.Since(start).Seconds())
 	start = time.Now()
 	eg := egpool.Group{PoolSize: 20}
 	for shard, request := range requests {
@@ -1197,7 +1212,7 @@ func (b *Batch) doImportShardTransactional(frags, clearFrags fragments) error {
 	}
 	err := eg.Wait()
 	dur := time.Since(start)
-	b.importer.StatsTiming(MetricBatchShardImportDurationSeconds, dur, 1.0)
+	featurebase.SummaryBatchImportDurationSeconds.Observe(dur.Seconds())
 	b.log.Printf("import shard took: %v\n", dur)
 	return errors.Wrap(err, "doing shard-transactional imports")
 }
@@ -1242,7 +1257,7 @@ func (b *Batch) doImport(frags, clearFrags fragments) error {
 			}
 
 			ferr := b.importer.ImportRoaringBitmap(ctx, b.tbl.ID, fld, shard, viewMap, false)
-			b.log.Debugf("imp-roar     %s,shard:%d,views:%d %v", field, shard, len(clearViewMap), time.Since(starty))
+			b.log.Debugf("imp-roar    field: %s, shard:%d, views:%d %v", field, shard, len(viewMap), time.Since(starty))
 			return errors.Wrapf(ferr, "importing data for %s", field)
 		})
 	}
@@ -1313,20 +1328,15 @@ func (b *Batch) makeFragments(frags, clearFrags fragments) (fragments, fragments
 	shardWidth := b.shardWidth()
 	emptyClearRows := make(map[int]uint64)
 
-	// create _exists fragments if needed
-	// TODO(tlt): maybe make this a separate flag for backward compatibility?
-	// (because dax.Table doesn't have this).
-	//if b.index.Options.TrackExistence {
-	if true {
-		var curBM *roaring.Bitmap
-		curShard := ^uint64(0) // impossible sentinel value for shard.
-		for _, col := range b.ids {
-			if col/shardWidth != curShard {
-				curShard = col / shardWidth
-				curBM = frags.GetOrCreate(curShard, "_exists", "")
-			}
-			curBM.DirectAdd(col % shardWidth)
+	// create _exists fragments
+	var curBM *roaring.Bitmap
+	curShard := ^uint64(0) // impossible sentinel value for shard.
+	for _, col := range b.ids {
+		if col/shardWidth != curShard {
+			curShard = col / shardWidth
+			curBM = frags.GetOrCreate(curShard, "_exists", "")
 		}
+		curBM.DirectAdd(col % shardWidth)
 	}
 
 	for i, rowIDs := range b.rowIDs {
@@ -1345,6 +1355,7 @@ func (b *Batch) makeFragments(frags, clearFrags fragments) (fragments, fragments
 		curShard := ^uint64(0) // impossible sentinel value for shard.
 		var curBM *roaring.Bitmap
 		var clearBM *roaring.Bitmap
+		var existCurBM *roaring.Bitmap
 		for j := range b.ids {
 			col := b.ids[j]
 			row := nilSentinel
@@ -1357,8 +1368,12 @@ func (b *Batch) makeFragments(frags, clearFrags fragments) (fragments, fragments
 
 			if col/shardWidth != curShard {
 				curShard = col / shardWidth
+				// the API treats "" as standard
 				curBM = frags.GetOrCreate(curShard, field.Name, "")
 				clearBM = clearFrags.GetOrCreate(curShard, field.Name, "")
+				if opts.ActuallyTrackingExistence() {
+					existCurBM = frags.GetOrCreate(curShard, field.Name, existenceViewName)
+				}
 			}
 			if row != nilSentinel {
 				// TODO this is super ugly, but we want to avoid setting
@@ -1368,6 +1383,9 @@ func (b *Batch) makeFragments(frags, clearFrags fragments) (fragments, fragments
 				// the NoStandardView case would be great.
 				if !(opts.Type == featurebase.FieldTypeTime && opts.NoStandardView) {
 					curBM.DirectAdd(row*shardWidth + (col % shardWidth))
+					if opts.ActuallyTrackingExistence() {
+						existCurBM.DirectAdd(col % shardWidth)
+					}
 				}
 				if opts.Type == featurebase.FieldTypeTime {
 					views, err := b.times[j].views(opts.TimeQuantum)
@@ -1388,6 +1406,16 @@ func (b *Batch) makeFragments(frags, clearFrags fragments) (fragments, fragments
 				// we want to make sure that at this point, the "set"
 				// fragments don't contain the bit that we're clearing
 				curBM.DirectRemoveN(clearRow*shardWidth + (col % shardWidth))
+				// Because this is RowIDs, not RowIDSets, there's only one
+				// bit. We should not be setting the existence bit based on
+				// this value, if we're actually clearing it. This doesn't
+				// mean we will clear an existing existence bit, though.
+				// The case where we would clear an existence bit is the
+				// case where someone specified row[mutexField].Clears = nil,
+				// which is far from here.
+				if opts.ActuallyTrackingExistence() {
+					existCurBM.DirectRemoveN(col % shardWidth)
+				}
 			}
 		}
 	}
@@ -1406,14 +1434,23 @@ func (b *Batch) makeFragments(frags, clearFrags fragments) (fragments, fragments
 		opts := field.Options
 		curShard := ^uint64(0) // impossible sentinel value for shard.
 		var curBM *roaring.Bitmap
+		var existCurBM *roaring.Bitmap
 		for j := range b.ids {
 			col, rowIDs := b.ids[j], rowIDSets[j]
-			if len(rowIDs) == 0 {
-				continue
-			}
 			if col/shardWidth != curShard {
 				curShard = col / shardWidth
 				curBM = frags.GetOrCreate(curShard, fname, "")
+				if opts.ActuallyTrackingExistence() {
+					existCurBM = frags.GetOrCreate(curShard, fname, existenceViewName)
+				}
+			}
+			if len(rowIDs) == 0 {
+				// you can validly specify an empty set, which is not the same as a null,
+				// but which still ought to set the existence bit if we're tracking that.
+				if opts.ActuallyTrackingExistence() && rowIDs != nil {
+					existCurBM.DirectAdd(col % shardWidth)
+				}
+				continue
 			}
 			// TODO this is super ugly, but we want to avoid setting
 			// bits on the standard view in the specific case when
@@ -1424,6 +1461,9 @@ func (b *Batch) makeFragments(frags, clearFrags fragments) (fragments, fragments
 				for _, row := range rowIDs {
 					curBM.DirectAdd(row*shardWidth + (col % shardWidth))
 				}
+				if opts.ActuallyTrackingExistence() {
+					existCurBM.DirectAdd(col % shardWidth)
+				}
 			}
 			if opts.Type == featurebase.FieldTypeTime {
 				views, err := b.times[j].views(opts.TimeQuantum)
@@ -1551,6 +1591,11 @@ func (b *Batch) makeSingleValFragments(frags, clearFrags fragments) (fragments,
 		shard := ids[0] / shardWidth
 		bitmap := frags.GetOrCreate(shard, field.Name, "standard")
 		clearBM := clearFrags.GetOrCreate(shard, field.Name, "standard")
+		var existBM, existClearBM *roaring.Bitmap
+		if field.Options.ActuallyTrackingExistence() {
+			existBM = frags.GetOrCreate(shard, field.Name, existenceViewName)
+			existClearBM = clearFrags.GetOrCreate(shard, field.Name, existenceViewName)
+		}
 		for i, id := range ids {
 			if i+1 < len(ids) {
 				// we only want the last value set for each id
@@ -1563,6 +1608,10 @@ func (b *Batch) makeSingleValFragments(frags, clearFrags fragments) (fragments,
 				shard = id / shardWidth
 				bitmap = frags.GetOrCreate(shard, field.Name, "standard")
 				clearBM = clearFrags.GetOrCreate(shard, field.Name, "standard")
+				if field.Options.ActuallyTrackingExistence() {
+					existBM = frags.GetOrCreate(shard, field.Name, existenceViewName)
+					existClearBM = clearFrags.GetOrCreate(shard, field.Name, existenceViewName)
+				}
 			}
 			fragmentColumn := id % shardWidth
 			clearBM.Add(fragmentColumn) // Will use this to clear columns.
@@ -1570,6 +1619,11 @@ func (b *Batch) makeSingleValFragments(frags, clearFrags fragments) (fragments,
 				// clearSentinel is used for deletion
 				// so this value should only be added if its not clearSentinel
 				bitmap.Add(row*shardWidth + fragmentColumn)
+				if field.Options.ActuallyTrackingExistence() {
+					existBM.Add(fragmentColumn)
+				}
+			} else if field.Options.ActuallyTrackingExistence() {
+				existClearBM.Add(fragmentColumn)
 			}
 		}
 	}
@@ -1598,6 +1652,11 @@ func (b *Batch) makeSingleValFragments(frags, clearFrags fragments) (fragments,
 			fragmentColumn := recID % shardWidth
 
 			clearBM.Add(fragmentColumn)
+			if field.Options.ActuallyTrackingExistence() {
+				existClearBM := clearFrags.GetOrCreate(shard, field.Name, existenceViewName)
+
+				existClearBM.Add(fragmentColumn)
+			}
 		}
 	}
 
@@ -1620,6 +1679,10 @@ func (b *Batch) makeSingleValFragments(frags, clearFrags fragments) (fragments,
 
 			fragmentColumn := recID % shardWidth
 			clearBM.Add(fragmentColumn)
+			if field.Options.ActuallyTrackingExistence() {
+				exist := frags.GetOrCreate(shard, field.Name, existenceViewName)
+				exist.Add(fragmentColumn)
+			}
 
 			if boolVal {
 				bitmap.Add(trueRowOffset + fragmentColumn)
@@ -1705,7 +1768,7 @@ func (b *Batch) importValueData() error {
 					start := time.Now()
 					fld := featurebase.FieldInfoToField(field)
 					err := b.importer.DoImport(ctx, b.tbl.ID, fld, shard, path, data)
-					b.log.Debugf("imp-vals %s,shard:%d,data:%d %v", field, shard, len(data), time.Since(start))
+					b.log.Debugf("imp-vals    field: %s, shard: %d, data: %d %v", field.Name, shard, len(data), time.Since(start))
 					return errors.Wrapf(err, "importing values for field = %s", field.Name)
 				})
 				startIdx = i

batch/batch_test.go

@@ -103,6 +103,12 @@ func testStringSliceCombos(t *testing.T, importer featurebase.Importer, sapi fea
 		Index: idx.Name,
 		Query: "TopN(a1, n=10)",
 	})
+	if resp.Err != nil {
+		t.Fatalf("unexpected error from TopN query: %v", resp.Err)
+	}
+	if len(resp.Results) < 1 {
+		t.Fatalf("expected non-empty result set, got empty results")
+	}
 	pairsField, ok := resp.Results[0].(*featurebase.PairsField)
 	assert.True(t, ok, "wrong return type: %T", resp.Results[0])
 
@@ -508,10 +514,11 @@ func testStringSliceEmptyAndNil(t *testing.T, importer featurebase.Importer, sap
 			{
 				Name: "strslice",
 				Options: featurebase.FieldOptions{
-					Type:      featurebase.FieldTypeSet,
-					Keys:      true,
-					CacheType: featurebase.CacheTypeRanked,
-					CacheSize: 100,
+					Type:           featurebase.FieldTypeSet,
+					Keys:           true,
+					CacheType:      featurebase.CacheTypeRanked,
+					CacheSize:      100,
+					TrackExistence: true,
 				},
 			},
 		},
@@ -611,6 +618,14 @@ func testStringSliceEmptyAndNil(t *testing.T, importer featurebase.Importer, sap
 			pql: "Row(strslice='z')",
 			exp: []uint64{2},
 		},
+		{
+			pql: "Row(strslice==null)",
+			exp: []uint64{1},
+		},
+		{
+			pql: "Row(strslice!=null)",
+			exp: []uint64{0, 2, 3, 4},
+		},
 	}
 	for i, test := range tests {
 		t.Run(fmt.Sprintf("test-%d", i), func(t *testing.T) {
@@ -2045,7 +2060,7 @@ func mutexClearRegression(t *testing.T, importer featurebase.Importer, sapi feat
 	}
 
 	col := uint64(0)
-	row := uint64(1)
+	row := uint64(0)
 	for i := uint64(0); i <= 21; i++ {
 		col = (i%2+1)*featurebase.ShardWidth + i%5
 		row = i % 3
@@ -2126,7 +2141,7 @@ func mutexNilClearID(t *testing.T, importer featurebase.Importer, sapi featureba
 	}
 
 	col := uint64(0)
-	row := uint64(1)
+	row := uint64(0)
 	// populate mutex with some data
 	for i := uint64(0); i < 11; i++ {
 		col = (i%2+1)*featurebase.ShardWidth + i%5
@@ -2341,3 +2356,58 @@ func testImportBatchBools(t *testing.T, importer featurebase.Importer, sapi feat
 	assert.True(t, ok, "wrong return type: %T", resp.Results[0])
 	assert.Equal(t, uint64(2), count)
 }
+
+func TestConvert(t *testing.T) {
+	t.Run("timestampToInt", func(t *testing.T) {
+		tests := []struct {
+			unit TimeUnit
+			ts   string
+			exp  int64
+		}{
+			{unit: "s", ts: "2022-01-01T00:00:00Z", exp: 1640995200},
+			{unit: "ms", ts: "2022-01-01T00:00:00Z", exp: 1640995200000},
+			{unit: "us", ts: "2022-01-01T00:00:00Z", exp: 1640995200000000},
+			{unit: "ns", ts: "2022-01-01T00:00:00Z", exp: 1640995200000000000},
+			{unit: "x", ts: "2022-01-01T00:00:00Z", exp: 0},
+		}
+		for i, test := range tests {
+			t.Run(fmt.Sprintf("test-%d", i), func(t *testing.T) {
+				ts, err := time.Parse(time.RFC3339, test.ts)
+				assert.NoError(t, err)
+				v := timestampToInt(test.unit, ts)
+				assert.Equal(t, test.exp, v)
+			})
+		}
+	})
+
+	t.Run("Int64ToTimestamp", func(t *testing.T) {
+		tests := []struct {
+			unit  TimeUnit
+			epoch string
+			val   int64
+			exp   time.Time
+		}{
+			{
+				unit:  "ms",
+				epoch: "2022-01-01T00:00:00Z",
+				val:   0,
+				exp:   time.Date(2022, 1, 1, 0, 0, 0, 0, time.UTC),
+			},
+			{
+				unit:  "s",
+				epoch: "2022-01-01T00:00:00Z",
+				val:   86400,
+				exp:   time.Date(2022, 1, 2, 0, 0, 0, 0, time.UTC),
+			},
+		}
+		for i, test := range tests {
+			t.Run(fmt.Sprintf("test-%d", i), func(t *testing.T) {
+				epoch, err := time.Parse(time.RFC3339, test.epoch)
+				assert.NoError(t, err)
+				ts, err := Int64ToTimestamp(test.unit, epoch, test.val)
+				assert.NoError(t, err)
+				assert.Equal(t, test.exp, ts)
+			})
+		}
+	})
+}

batch/batcher.go

@@ -0,0 +1,20 @@
+package batch
+
+import (
+	"time"
+
+	"github.com/featurebasedb/featurebase/v3/dax"
+)
+
+// Batcher is an interface implemented by anything which can allocate new
+// batches.
+type Batcher interface {
+	NewBatch(cfg Config, tbl *dax.Table, fields []*dax.Field) (RecordBatch, error)
+}
+
+// Config is the configuration options passed to NewBatch for any implementation
+// of the Batcher interface.
+type Config struct {
+	Size         int
+	MaxStaleness time.Duration
+}

batch/egpool/egpool.go

@@ -58,11 +58,11 @@ func (eg *Group) err(err error) {
 	eg.errs = append(eg.errs, err)
 }
 
-type ErrPanic struct {
+type PanicError struct {
 	Value interface{}
 }
 
-func (p ErrPanic) Error() string {
+func (p PanicError) Error() string {
 	return fmt.Sprintf("panic: %v", p.Value)
 }
 
@@ -77,7 +77,7 @@ func (eg *Group) processJobs() {
 	defer func() {
 		if !finished {
 			if p := recover(); p != nil {
-				eg.err(ErrPanic{p})
+				eg.err(PanicError{p})
 			} else {
 				eg.err(ErrGoexit)
 			}

batch/metrics.go

@@ -1,28 +1,3 @@
 // Copyright 2022 Molecula Corp. (DBA FeatureBase).
 // SPDX-License-Identifier: Apache-2.0
 package batch
-
-const (
-	// MetricBatchImportDurationSeconds records the full time of the
-	// RecordBatch.Import call. This includes starting and finishing a
-	// transaction, doing key translation, building fragments locally,
-	// importing all data, and resetting internal structures.
-	MetricBatchImportDurationSeconds = "batch_import_duration_seconds"
-
-	// MetricBatchFlushDurationSeconds records the full time for
-	// RecordBatch.Flush (if splitBatchMode is in use). This includes
-	// starting and finishing a transaction, importing all data, and
-	// resetting internal structures.
-	MetricBatchFlushDurationSeconds = "batch_flush_duration_seconds"
-
-	// MetricBatchShardImportBuildRequestsSeconds is the time it takes
-	// after making fragments to build the shard-transactional request
-	// objects (but not actually import them or do any network activity).
-	MetricBatchShardImportBuildRequestsSeconds = "batch_shard_import_build_requests_seconds"
-
-	// MetricBatchShardImportDurationSeconds is the time it takes to
-	// import all data for all shards in the batch using the
-	// shard-transactional endpoint. This does not include the time it
-	// takes to build the requests locally.
-	MetricBatchShardImportDurationSeconds = "batch_shard_import_duration_seconds"
-)

buffer/filebuffer.go

@@ -0,0 +1,110 @@
+package buffer
+
+import (
+	"bytes"
+	"io"
+	"io/ioutil"
+	"os"
+	"sync"
+)
+
+// NewFileBuffer returns a file buffer which will use an in-memory buffer, until `max` bytes have been written, at which point it will write the contents of memory to a file, and continue writing future data to the file.
+// The file will be written to `temp` directory. The buffer fulfills the io.Reader and io.Writer interface
+func NewFileBuffer(max int, temp string) *FileBuffer {
+	return &FileBuffer{max: max, tempDir: temp}
+}
+
+type FileBuffer struct {
+	max     int
+	buf     bytes.Buffer
+	file    *os.File
+	tempDir string
+	reading bool
+	files   []*os.File
+	mu      sync.Mutex
+}
+
+func (fb *FileBuffer) Write(p []byte) (n int, err error) {
+	if fb.reading {
+		panic("cannot write after read")
+	}
+	if fb.file != nil {
+		return fb.file.Write(p)
+	}
+	n, err = fb.buf.Write(p)
+	if err != nil {
+		return
+	}
+	if fb.buf.Len() > fb.max {
+		fb.file, err = ioutil.TempFile(fb.tempDir, "filebuffer-")
+		if err != nil {
+			return
+		}
+		_, err = io.Copy(fb.file, &fb.buf)
+		fb.buf.Reset()
+	}
+	return
+}
+
+func (fb *FileBuffer) Len() (int64, error) {
+	if fb.file == nil {
+		return int64(fb.buf.Len()), nil
+	}
+	fi, err := fb.file.Stat()
+	if err != nil {
+		return 0, err
+	}
+
+	return fi.Size(), nil
+}
+
+func (fb *FileBuffer) Read(p []byte) (n int, err error) {
+	if fb.file != nil {
+		if !fb.reading {
+			fb.reading = true
+			_, err = fb.file.Seek(0, 0)
+			if err != nil {
+				return
+			}
+		}
+		return fb.file.Read(p)
+	}
+	fb.reading = true
+	return fb.buf.Read(p)
+}
+
+func (fb *FileBuffer) Close() error {
+	if fb.file != nil {
+		name := fb.file.Name()
+		if err := fb.file.Close(); err != nil {
+			return err
+		}
+		for _, f := range fb.files {
+			f.Close()
+		}
+		fb.files = fb.files[:0]
+		fb.file = nil
+		return os.Remove(name)
+	}
+	return nil
+}
+
+func (fb *FileBuffer) Reset() error {
+	fb.mu.Lock()
+	defer fb.mu.Unlock()
+	fb.reading = false
+	fb.buf.Reset()
+	return fb.Close()
+}
+
+func (fb *FileBuffer) NewReader() (io.Reader, error) {
+	fb.mu.Lock()
+	defer fb.mu.Unlock()
+	fb.reading = true
+	if fb.file == nil {
+		return bytes.NewReader(fb.buf.Bytes()), nil
+	}
+	f, err := os.OpenFile(fb.file.Name(), os.O_RDONLY, 0)
+	fb.files = append(fb.files, f)
+	return f, err
+}

bufferpool/bufferpool.go

@@ -0,0 +1,262 @@
+package bufferpool
+
+import (
+	"errors"
+	"fmt"
+	"sync"
+)
+
+// FrameID is the type for frame id
+type FrameID int
+
+// PageID is the type for page id
+type PageID int
+
+var pageSyncPool = sync.Pool{
+	New: func() any {
+		pg := new(Page)
+		pg.id = PageID(INVALID_PAGE)
+		pg.isDirty = false
+		pg.pinCount = 0
+		return pg
+	},
+}
+
+// BufferPool represents a buffer pool of pages
+type BufferPool struct {
+	// the underlying storage
+	diskManager DiskManager
+	// the actual pages in the buffer pool
+	pages []*Page
+	// the replacer that will elect replacements when buffer pool is full
+	replacer *ClockReplacer
+	// the list of free frames
+	freeList []FrameID
+	// the map of frames to page ids to frame ids
+	// frame ids are the offset into pages
+	// if you ask the pool for page 673, this will know at
+	// what offset in pages page 673 will exist
+	pageTable map[PageID]FrameID
+}
+
+// TODO(pok) implement a lazy writer
+//  * if free list is 'low' then
+//		* increase size of cache if there is physical memory available
+//		* write out old pages and boot them from the cache to increase free list
+
+// TODO(pok) implement a checkpoint that scans the pool and writes out dirty pages every
+// minute or so
+
+// NewBufferPool returns a buffer pool
+func NewBufferPool(maxSize int, diskManager DiskManager) *BufferPool {
+	freeList := make([]FrameID, 0)
+	pages := make([]*Page, maxSize)
+	for i := 0; i < maxSize; i++ {
+		frameNumber := FrameID(i)
+		freeList = append(freeList, frameNumber)
+	}
+	clockReplacer := NewClockReplacer(maxSize)
+	return &BufferPool{
+		diskManager: diskManager,
+		pages:       pages,
+		replacer:    clockReplacer,
+		freeList:    freeList,
+		pageTable:   make(map[PageID]FrameID),
+	}
+}
+
+// Dumps all the pages in the buffer pool
+func (b *BufferPool) Dump() {
+	fmt.Println()
+	fmt.Printf("------------------------------------------------------------------------------------------\n")
+	fmt.Printf("BUFFER POOL\n")
+	for _, p := range b.pages {
+		if p != nil {
+			p.Dump("")
+		}
+	}
+	fmt.Printf("------------------------------------------------------------------------------------------\n")
+	fmt.Println()
+}
+
+// FetchPage fetches the requested page from the buffer pool.
+func (b *BufferPool) FetchPage(pageID PageID) (*Page, error) {
+	// if it is in buffer pool already then just return it
+	if frameID, ok := b.pageTable[pageID]; ok {
+		page := b.pages[frameID]
+		page.pinCount++
+		b.replacer.Pin(frameID)
+		return page, nil
+	}
+
+	// not in the buffer pool so try the free list or
+	// the replacer will vote a page off the island
+	frameID, isFromFreeList, err := b.getFrameID()
+	if err != nil {
+		return nil, err
+	}
+
+	if !isFromFreeList {
+		// if it didn't come from the freelist then
+		// remove page from current frame, writing it out if dirty
+		currentPage := b.pages[frameID]
+		if currentPage != nil {
+			if currentPage.isDirty {
+				b.diskManager.WritePage(currentPage)
+			}
+
+			delete(b.pageTable, currentPage.id)
+		}
+	}
+
+	// if we got to here, sorry, have to do an I/O
+	page, err := b.diskManager.ReadPage(pageID)
+	if err != nil {
+		return nil, err
+	}
+	page.pinCount = 1
+	b.pageTable[pageID] = frameID
+	pageSyncPool.Put(b.pages[frameID])
+	b.pages[frameID] = page
+	b.replacer.Pin(frameID)
+
+	return page, nil
+}
+
+// UnpinPage unpins the target page from the buffer pool
+func (b *BufferPool) UnpinPage(pageID PageID) error {
+	if frameID, ok := b.pageTable[pageID]; ok {
+		page := b.pages[frameID]
+		page.DecPinCount()
+
+		if page.pinCount <= 0 {
+			b.replacer.Unpin(frameID)
+		}
+		return nil
+	}
+
+	return errors.New("could not find page")
+}
+
+// FlushPage Flushes the target page to disk
+func (b *BufferPool) FlushPage(pageID PageID) bool {
+	if frameID, ok := b.pageTable[pageID]; ok {
+		page := b.pages[frameID]
+		page.DecPinCount()
+
+		b.diskManager.WritePage(page)
+		page.isDirty = false
+
+		return true
+	}
+	return false
+}
+
+// NewPage allocates a new page in the buffer pool with the disk manager help
+func (b *BufferPool) NewPage() (*Page, error) {
+	// get a free frame
+	frameID, isFromFreeList, err := b.getFrameID()
+	if err != nil {
+		return nil, err
+	}
+
+	if !isFromFreeList {
+		// remove page from current frame
+		currentPage := b.pages[frameID]
+		if currentPage != nil {
+			if currentPage.isDirty {
+				b.diskManager.WritePage(currentPage)
+			}
+
+			delete(b.pageTable, currentPage.id)
+		}
+	}
+
+	// allocates new page
+	pageID, err := b.diskManager.AllocatePage()
+	if err != nil {
+		return nil, err
+	}
+	page := &Page{pageID, 1, false, [PAGE_SIZE]byte{}}
+	page.WritePageNumber(int32(pageID))
+	page.WriteFreeSpaceOffset(int16(PAGE_SIZE))
+	page.WriteNextPointer(int32(INVALID_PAGE))
+	page.WritePrevPointer(int32(INVALID_PAGE))
+
+	// update the frame table
+	b.pageTable[pageID] = frameID
+	pageSyncPool.Put(b.pages[frameID])
+	b.pages[frameID] = page
+
+	return page, nil
+}
+
+// ScratchPage returns a page outside the buffer pool - do not use if you intend the page
+// to be in the buffer pool (use NewPage() for that)
+// ScratchPage is intended to be used in cases where you need the Page primitives
+// and will copy the scratch page back over a real page later
+func (b *BufferPool) ScratchPage() *Page {
+	page := &Page{
+		id:       PageID(INVALID_PAGE),
+		pinCount: 0,
+		isDirty:  false,
+		data:     [PAGE_SIZE]byte{},
+	}
+	page.WritePageNumber(int32(INVALID_PAGE))
+	page.WriteFreeSpaceOffset(int16(PAGE_SIZE))
+	page.WriteNextPointer(int32(INVALID_PAGE))
+	page.WritePrevPointer(int32(INVALID_PAGE))
+	return page
+}
+
+// DeletePage deletes a page from the buffer pool
+func (b *BufferPool) DeletePage(pageID PageID) error {
+	var frameID FrameID
+	var ok bool
+	if frameID, ok = b.pageTable[pageID]; !ok {
+		return nil
+	}
+
+	page := b.pages[frameID]
+
+	if page.pinCount > 0 {
+		return errors.New("pin count greater than 0")
+	}
+	delete(b.pageTable, page.id)
+	b.replacer.Pin(frameID)
+	b.diskManager.DeallocatePage(pageID)
+
+	b.freeList = append(b.freeList, frameID)
+
+	return nil
+}
+
+// FlushAllpages flushes all the pages in the buffer pool to disk
+// Yeah, never call this unless you know what you are doing
+func (b *BufferPool) FlushAllpages() {
+	for pageID := range b.pageTable {
+		b.FlushPage(pageID)
+	}
+}
+
+func (b *BufferPool) getFrameID() (FrameID, bool, error) {
+	if len(b.freeList) > 0 {
+		frameID, newFreeList := b.freeList[0], b.freeList[1:]
+		b.freeList = newFreeList
+		return frameID, true, nil
+	}
+
+	victim, err := b.replacer.Victim()
+	return victim, false, err
+}
+
+// OnDiskSize exposes the on disk size of the backing store
+// behind this buffer pool
+func (b *BufferPool) OnDiskSize() int64 {
+	return b.diskManager.FileSize()
+}
+
+// Close closes the buffer pool
+func (b *BufferPool) Close() {
+	b.diskManager.Close()
+}

bufferpool/circularlist.go

@@ -0,0 +1,93 @@
+package bufferpool
+
+import (
+	"errors"
+)
+
+type circularListNode struct {
+	key   interface{}
+	value interface{}
+	next  *circularListNode
+	prev  *circularListNode
+}
+
+type circularList struct {
+	head     *circularListNode
+	tail     *circularListNode
+	size     int
+	capacity int
+}
+
+func newCircularList(maxSize int) *circularList {
+	return &circularList{nil, nil, 0, maxSize}
+}
+
+func (c *circularList) find(key interface{}) *circularListNode {
+	ptr := c.head
+	for i := 0; i < c.size; i++ {
+		if ptr.key == key {
+			return ptr
+		}
+		ptr = ptr.next
+	}
+	return nil
+}
+
+func (c *circularList) hasKey(key interface{}) bool {
+	return c.find(key) != nil
+}
+
+func (c *circularList) insert(key interface{}, value interface{}) error {
+	if c.size == c.capacity {
+		return errors.New("list is full")
+	}
+	newNode := &circularListNode{key, value, nil, nil}
+	if c.size == 0 {
+		newNode.next = newNode
+		newNode.prev = newNode
+		c.head = newNode
+		c.tail = newNode
+		c.size++
+		return nil
+	}
+
+	node := c.find(key)
+	if node != nil {
+		node.value = value
+		return nil
+	}
+
+	newNode.next = c.head
+	newNode.prev = c.tail
+	c.tail.next = newNode
+	if c.head == c.tail {
+		c.head.next = newNode
+	}
+	c.tail = newNode
+	c.head.prev = c.tail
+
+	c.size++
+	return nil
+}
+
+func (c *circularList) remove(key interface{}) {
+	node := c.find(key)
+	if node == nil {
+		return
+	}
+	if c.size == 1 {
+		c.head = nil
+		c.tail = nil
+		c.size--
+		return
+	}
+	if node == c.head {
+		c.head = c.head.next
+	}
+	if node == c.tail {
+		c.tail = c.tail.prev
+	}
+	node.next.prev = node.prev
+	node.prev.next = node.next
+	c.size--
+}

bufferpool/clockreplacer.go

@@ -0,0 +1,64 @@
+package bufferpool
+
+import "errors"
+
+// ClockReplacer implements a clock replacer algorithm
+type ClockReplacer struct {
+	cList     *circularList
+	clockHand **circularListNode
+}
+
+// NewClockReplacer instantiates a new clock replacer
+func NewClockReplacer(poolSize int) *ClockReplacer {
+	cList := newCircularList(poolSize)
+	return &ClockReplacer{cList, &cList.head}
+}
+
+// Victim removes the victim frame as defined by the replacement policy
+func (c *ClockReplacer) Victim() (FrameID, error) {
+	if c.cList.size == 0 {
+		return FrameID(INVALID_PAGE), errors.New("no victims available")
+	}
+	var victimFrameID FrameID
+	currentNode := (*c.clockHand)
+	for {
+
+		if currentNode.value.(bool) {
+			currentNode.value = false
+			c.clockHand = &currentNode.next
+		} else {
+			frameID := currentNode.key.(FrameID)
+			victimFrameID = frameID
+			c.clockHand = &currentNode.next
+			c.cList.remove(currentNode.key)
+			return victimFrameID, nil
+		}
+	}
+}
+
+// Unpin unpins a frame, indicating that it can now be victimized
+func (c *ClockReplacer) Unpin(id FrameID) {
+	if !c.cList.hasKey(id) {
+		c.cList.insert(id, true)
+		if c.cList.size == 1 {
+			c.clockHand = &c.cList.head
+		}
+	}
+}
+
+// Pin pins a frame, indicating that it should not be victimized until it is unpinned
+func (c *ClockReplacer) Pin(id FrameID) {
+	node := c.cList.find(id)
+	if node == nil {
+		return
+	}
+	if (*c.clockHand) == node {
+		c.clockHand = &(*c.clockHand).next
+	}
+	c.cList.remove(id)
+}
+
+// Size returns the size of the clock
+func (c *ClockReplacer) Size() int {
+	return c.cList.size
+}

bufferpool/diskmanager.go

@@ -0,0 +1,21 @@
+package bufferpool
+
+// DiskManager is responsible for interacting with disk
+type DiskManager interface {
+	// reads a page from the disk
+	ReadPage(PageID) (*Page, error)
+	// writes a page to the disk
+	WritePage(*Page) error
+
+	// allocates a page
+	AllocatePage() (PageID, error)
+
+	// deallocates a page
+	DeallocatePage(PageID) error
+
+	// returns on disk file size
+	FileSize() int64
+
+	// closes and does any clean up
+	Close()
+}

bufferpool/inmemdiskmanager.go

@@ -0,0 +1,162 @@
+package bufferpool
+
+import (
+	"errors"
+	"fmt"
+	"os"
+
+	uuid "github.com/satori/go.uuid"
+)
+
+// InMemDiskSpillingDiskManager is a memory implementation for a DiskManager interface
+// that can spill to disk when a threshold is reached
+type InMemDiskSpillingDiskManager struct {
+	// tracks the number of pages
+	numPages int
+
+	onDiskPages int
+
+	// tracks the number of pages we can consume before spilling
+	thresholdPages int
+	hasSpilled     *struct{}
+	fd             *os.File
+
+	// the data buffer
+	data []byte
+}
+
+// NewInMemDiskSpillingDiskManager returns a in-memory version of disk manager
+func NewInMemDiskSpillingDiskManager(thresholdPages int) *InMemDiskSpillingDiskManager {
+	dm := &InMemDiskSpillingDiskManager{
+		numPages:       0,
+		thresholdPages: thresholdPages,
+		data:           make([]byte, 0),
+	}
+	return dm
+}
+
+// ReadPage reads a page from pages
+func (d *InMemDiskSpillingDiskManager) ReadPage(pageID PageID) (*Page, error) {
+	// check we're not asking for page out of range
+	if pageID < 0 || int(pageID) >= d.numPages {
+		return nil, errors.New("page not found")
+	}
+	// check that the offset is within range
+	offset := int(pageID) * PAGE_SIZE
+
+	var page = pageSyncPool.Get().(*Page)
+	// we have to do this stupid check because if -cpuprofile is set for go test, this
+	// the previous line return a weird nil-ish thing...
+	if page == (*Page)(nil) {
+		page = pageSyncPool.New().(*Page)
+	}
+	page.id = pageID
+
+	// do the read
+	if d.hasSpilled == nil {
+		if offset+PAGE_SIZE > len(d.data) {
+			return nil, errors.New("offset out of range")
+		}
+		b := copy(page.data[:], d.data[offset:offset+PAGE_SIZE])
+		fmt.Printf("bytes read: %d", b)
+	} else {
+		var err error
+		if offset+PAGE_SIZE > d.numPages*PAGE_SIZE {
+			return nil, errors.New("offset out of range")
+		}
+		_, err = d.fd.ReadAt(page.data[:], int64(offset))
+		if err != nil {
+			return nil, err
+		}
+	}
+	return page, nil
+}
+
+// WritePage writes a page in memory to pages
+func (d *InMemDiskSpillingDiskManager) WritePage(page *Page) error {
+	// make sure the offset is sensible
+	offset := int(page.ID()) * PAGE_SIZE
+	// do the write
+	if d.hasSpilled == nil {
+		if offset+PAGE_SIZE > len(d.data) {
+			return errors.New("offset out of range")
+		}
+		copy(d.data[offset:], page.data[:])
+	} else {
+		var err error
+		if offset+PAGE_SIZE > d.numPages*PAGE_SIZE {
+			return errors.New("offset out of range")
+		}
+		_, err = d.fd.WriteAt(page.data[:], int64(offset))
+		if err != nil {
+			return err
+		}
+		// err = d.fd.Sync()
+		// if err != nil {
+		// 	return err
+		// }
+	}
+	return nil
+}
+
+// AllocatePage allocates a page and returns the page number
+func (d *InMemDiskSpillingDiskManager) AllocatePage() (PageID, error) {
+	d.numPages = d.numPages + 1
+	pageID := PageID(d.numPages - 1)
+
+	if d.hasSpilled == nil {
+		// we have not spilled (yet), so make storage bigger
+		newData := make([]byte, PAGE_SIZE)
+		d.data = append(d.data, newData...)
+
+		// check to see if we need to spill
+		if d.numPages > d.thresholdPages {
+			fileUUID, err := uuid.NewV4()
+			if err != nil {
+				return PageID(INVALID_PAGE), err
+			}
+			// TODO(pok) we should try to tell the OS not to cache this file
+			d.fd, err = os.CreateTemp("", fmt.Sprintf("fb-ehash-%s", fileUUID.String()))
+			if err != nil {
+				return PageID(INVALID_PAGE), err
+			}
+			_, err = d.fd.WriteAt(d.data, 0)
+			if err != nil {
+				return PageID(INVALID_PAGE), err
+			}
+			d.data = []byte{}
+			d.hasSpilled = &struct{}{}
+		}
+	} else {
+		if d.numPages >= d.onDiskPages {
+			// grow the file by a chunk - 512 pages
+			d.onDiskPages += 512
+			var err error
+			size := int64(d.onDiskPages * PAGE_SIZE)
+			_, err = d.fd.WriteAt([]byte{0}, size-1)
+			if err != nil {
+				return PageID(INVALID_PAGE), err
+			}
+		}
+	}
+
+	return pageID, nil
+}
+
+// DeallocatePage removes page from disk
+func (d *InMemDiskSpillingDiskManager) DeallocatePage(pageID PageID) error {
+	// nothing to do right now
+	return nil
+}
+
+func (d *InMemDiskSpillingDiskManager) FileSize() int64 {
+	return int64(len(d.data))
+}
+
+func (d *InMemDiskSpillingDiskManager) Close() {
+	// close and delete the file if we spilled
+	if d.fd != nil {
+		_ = d.fd.Close()
+		os.Remove(d.fd.Name())
+	}
+}

bufferpool/page.go

@@ -0,0 +1,371 @@
+package bufferpool
+
+import (
+	"encoding/binary"
+	"errors"
+	"fmt"
+)
+
+const PAGE_SIZE int = 8192
+
+const INVALID_PAGE int = -1
+
+const PAGE_TYPE_BTREE_INTERNAL = 10
+const PAGE_TYPE_BTREE_LEAF = 11
+const PAGE_TYPE_HASH_TABLE = 12
+
+// PAGE
+// page size 8192 bytes
+// byte aligned, big endian
+
+// |====================================================|
+// | offset | length        |                           |
+// |----------------------------------------------------|
+// | header                                             |
+// |====================================================|
+// | 0      | 4             |  pageNumber (int32)       |
+// | 4      | 2             |  pageType (int16)         |
+// | 6      | 2             |  slotCount (int16)        |
+// | 8      | 2             |  localDepth (int16)       |
+// | 10     | 2             |  freeSpaceOffset (int16)  |
+// | 12     | 4             |  prevPointer (int32)      |
+// | 16     | 4             |  nextPointer (int32)      |
+// |====================================================|
+// | <start of slot array 1..slotCount>                 |
+// |----------------------------------------------------|
+// | 20     | slotcount     |  slot entry is 2 int16    |
+// |        |  * slotwidth  |   values (payloadOffset,  |
+// |        |  * #slots     |   payloadLength)          |
+// |----------------------------------------------------|
+// | <free space>                                       |
+// |----------------------------------------------------|
+// | <payload starting at freeSpaceOffset>              |
+// |  payload entries are keylength (int16), key bytes, |
+// |  payload length (int32), payload bytes             |
+// |====================================================|
+
+const PAGE_NUMBER_OFFSET = 0        // offset 0, length 4, end 4
+const PAGE_TYPE_OFFSET = 4          // offset 4, length 2, end 6
+const PAGE_SLOT_COUNT_OFFSET = 6    // offset 6, length 2, end 8
+const PAGE_LOCAL_DEPTH_OFFSET = 8   // offset 8, length 2, end 10
+const PAGE_FREE_SPACE_OFFSET = 10   // offset 10, length 2, end 12
+const PAGE_PREV_POINTER_OFFSET = 12 // offset 12, length 4, end 16
+const PAGE_NEXT_POINTER_OFFSET = 16 // offset 16, length 4, end 20
+const PAGE_SLOTS_START_OFFSET = 20  // offset 20
+
+// PAGE_SLOT_LENGTH is the size of the page slot key/value.
+//
+// key offset int16    	//offset 0, length 2, end 2
+// value offset int16	    //offset 2, length 2, end 4
+const PAGE_SLOT_LENGTH = 4
+
+// Page represents a page on disk
+type Page struct {
+	id       PageID
+	pinCount int
+	isDirty  bool
+	data     [PAGE_SIZE]byte
+}
+
+type PageSlot struct {
+	KeyOffset   int16
+	ValueOffset int16
+}
+
+func (s *PageSlot) KeyBytes(page *Page) []byte {
+	offset := s.KeyOffset
+	keyLen := int16(binary.BigEndian.Uint16(page.data[offset:]))
+	offset += 2
+	result := make([]byte, keyLen)
+	copy(result, page.data[offset:offset+keyLen])
+	return result
+}
+
+func (s *PageSlot) KeyAsInt(page *Page) int32 {
+	return int32(binary.BigEndian.Uint32(page.data[s.KeyOffset+2:]))
+}
+
+func (s *PageSlot) ValueBytes(page *Page) []byte {
+	offset := s.ValueOffset
+	valueLen := int32(binary.BigEndian.Uint32(page.data[offset:]))
+	offset += 4
+	result := make([]byte, valueLen)
+	copy(result, page.data[offset:int32(offset)+valueLen])
+	return result
+}
+
+func (s *PageSlot) ValueAsPagePointer(page *Page) int32 {
+	return int32(binary.BigEndian.Uint32(page.data[s.ValueOffset+4:]))
+}
+
+type PageChunk struct {
+	KeyLength int16
+	KeyBytes  []byte
+	// TODO(pok) ValueBytes can be up to int32 long
+	// this requires an overflow page mechanism, that is not implemented
+	// yet, so be aware of this when storing stuff...
+	ValueLength int32
+	ValueBytes  []byte
+}
+
+func (pc *PageChunk) Length() int {
+	return 2 + len(pc.KeyBytes) + 4 + len(pc.ValueBytes)
+}
+
+func (pc *PageChunk) ComputeKeyOffset(pageOffset int) int {
+	return pageOffset
+}
+
+func (pc *PageChunk) ComputeValueOffset(pageOffset int) int {
+	return pageOffset + 2 + len(pc.KeyBytes)
+}
+
+func (p *Page) WritePageNumber(pageNumber int32) {
+	p.id = PageID(pageNumber)
+	binary.BigEndian.PutUint32(p.data[PAGE_NUMBER_OFFSET:], uint32(pageNumber))
+	p.isDirty = true
+}
+
+func (p *Page) ReadPageNumber() int {
+	return int(binary.BigEndian.Uint32(p.data[PAGE_NUMBER_OFFSET:]))
+}
+
+func (p *Page) WritePageType(pageType int16) {
+	binary.BigEndian.PutUint16(p.data[PAGE_TYPE_OFFSET:], uint16(pageType))
+	p.isDirty = true
+}
+
+func (p *Page) ReadPageType() int16 {
+	return int16(binary.BigEndian.Uint16(p.data[PAGE_TYPE_OFFSET:]))
+}
+
+func (p *Page) WriteSlotCount(slotCount int16) {
+	binary.BigEndian.PutUint16(p.data[PAGE_SLOT_COUNT_OFFSET:], uint16(slotCount))
+	p.isDirty = true
+}
+
+func (p *Page) ReadSlotCount() int16 {
+	return int16(binary.BigEndian.Uint16(p.data[PAGE_SLOT_COUNT_OFFSET:]))
+}
+
+func (p *Page) WriteLocalDepth(localDepth int16) {
+	binary.BigEndian.PutUint16(p.data[PAGE_LOCAL_DEPTH_OFFSET:], uint16(localDepth))
+	p.isDirty = true
+}
+
+func (p *Page) ReadLocalDepth() int16 {
+	return int16(binary.BigEndian.Uint16(p.data[PAGE_LOCAL_DEPTH_OFFSET:]))
+}
+
+func (p *Page) ReadSlot(slot int16) PageSlot {
+	offset := PAGE_SLOTS_START_OFFSET + PAGE_SLOT_LENGTH*slot
+	keyOffset := int16(binary.BigEndian.Uint16(p.data[offset:]))
+	offset += 2
+	valueOffset := int16(binary.BigEndian.Uint16(p.data[offset:]))
+	return PageSlot{
+		KeyOffset:   keyOffset,
+		ValueOffset: valueOffset,
+	}
+}
+
+func (p *Page) WriteSlot(slot int16, value PageSlot) {
+	offset := PAGE_SLOTS_START_OFFSET + PAGE_SLOT_LENGTH*slot
+	binary.BigEndian.PutUint16(p.data[offset:], uint16(value.KeyOffset))
+	offset += 2
+	binary.BigEndian.PutUint16(p.data[offset:], uint16(value.ValueOffset))
+}
+
+func (p *Page) WriteFreeSpaceOffset(offset int16) {
+	binary.BigEndian.PutUint16(p.data[PAGE_FREE_SPACE_OFFSET:], uint16(offset))
+	p.isDirty = true
+}
+
+func (p *Page) ReadFreeSpaceOffset() int16 {
+	return int16(binary.BigEndian.Uint16(p.data[PAGE_FREE_SPACE_OFFSET:]))
+}
+
+func (p *Page) WritePrevPointer(prevPointer int32) {
+	binary.BigEndian.PutUint32(p.data[PAGE_PREV_POINTER_OFFSET:], uint32(prevPointer))
+	p.isDirty = true
+}
+
+func (p *Page) ReadPrevPointer() int {
+	return int(binary.BigEndian.Uint32(p.data[PAGE_PREV_POINTER_OFFSET:]))
+}
+
+func (p *Page) WriteNextPointer(nextPointer int32) {
+	binary.BigEndian.PutUint32(p.data[PAGE_NEXT_POINTER_OFFSET:], uint32(nextPointer))
+	p.isDirty = true
+}
+
+func (p *Page) ReadNextPointer() int {
+	return int(binary.BigEndian.Uint32(p.data[PAGE_NEXT_POINTER_OFFSET:]))
+}
+
+func (p *Page) WriteChunk(offset int16, chunk PageChunk) {
+	binary.BigEndian.PutUint16(p.data[offset:], uint16(chunk.KeyLength))
+	offset += 2
+	copy(p.data[offset:], chunk.KeyBytes)
+	offset += int16(len(chunk.KeyBytes))
+	binary.BigEndian.PutUint32(p.data[offset:], uint32(chunk.ValueLength))
+	offset += 4
+	copy(p.data[offset:], chunk.ValueBytes)
+	p.isDirty = true
+}
+
+func (p *Page) ReadChunk(offset int16) PageChunk {
+	keyLen := int16(binary.BigEndian.Uint16(p.data[offset:]))
+	offset += 2
+	keyBytes := make([]byte, keyLen)
+	copy(keyBytes, p.data[offset:offset+keyLen])
+	offset += keyLen
+	valueLen := int32(binary.BigEndian.Uint32(p.data[offset:]))
+	offset += 4
+	valueBytes := make([]byte, valueLen)
+	copy(valueBytes, p.data[offset:int32(offset)+valueLen])
+	return PageChunk{
+		KeyLength:   keyLen,
+		KeyBytes:    keyBytes,
+		ValueLength: valueLen,
+		ValueBytes:  valueBytes,
+	}
+}
+
+func (p *Page) FreeSpace() int16 {
+	freeSpaceOffset := p.ReadFreeSpaceOffset()
+	freespace := freeSpaceOffset - (p.ReadSlotCount()*PAGE_SLOT_LENGTH + PAGE_SLOT_LENGTH + PAGE_SLOTS_START_OFFSET)
+	return freespace
+}
+
+func (p *Page) WriteKeyValueInSlot(slotNumber int16, key []byte, value []byte) error {
+	freeSpaceOffset := p.ReadFreeSpaceOffset()
+
+	// build a chunk
+	chunk := PageChunk{
+		KeyLength:   int16(len(key)),
+		KeyBytes:    key,
+		ValueLength: int32(len(value)),
+		ValueBytes:  value,
+	}
+
+	// compute the new free space offset
+	freeSpaceOffset -= int16(chunk.Length())
+
+	// check we won't blow free space on page
+	slotCount := p.ReadSlotCount()
+	slotEndOffset := slotCount*PAGE_SLOT_LENGTH + PAGE_SLOT_LENGTH + PAGE_SLOTS_START_OFFSET
+
+	// DEBUG!!
+	//fmt.Printf("freeSpaceOffset: %d, slotCount: %d, slotCount*4 + 4 + 20: %d, freeSpace: %d\n", freeSpaceOffset, slotCount, slotEndOffset, freeSpaceOffset-slotEndOffset)
+
+	if freeSpaceOffset-slotEndOffset <= 0 {
+		return errors.New("page is full")
+	}
+
+	keyOffset := chunk.ComputeKeyOffset(int(freeSpaceOffset))
+	valueOffset := chunk.ComputeValueOffset(int(freeSpaceOffset))
+
+	p.WriteChunk(freeSpaceOffset, chunk)
+
+	// update the free space offset
+	p.WriteFreeSpaceOffset(int16(freeSpaceOffset))
+
+	// make a slot
+	slot := PageSlot{
+		KeyOffset:   int16(keyOffset),
+		ValueOffset: int16(valueOffset),
+	}
+	// write the slot
+	p.WriteSlot(slotNumber, slot)
+
+	return nil
+}
+
+func (p *Page) WritePage(page *Page) {
+	// copy everything but pageNumber & pageType
+	offset := PAGE_SLOT_COUNT_OFFSET
+	copy(page.data[offset:], p.data[offset:offset+PAGE_SIZE-offset])
+}
+
+func (p *Page) PinCount() int {
+	return p.pinCount
+}
+
+func (p *Page) ID() PageID {
+	return p.id
+}
+
+func (p *Page) DecPinCount() {
+	if p.pinCount > 0 {
+		p.pinCount--
+	}
+}
+
+type PageSlotIterator struct {
+	page      *Page
+	slotCount int16
+	cursor    int16
+}
+
+func NewPageSlotIterator(page *Page, fromSlot int16) *PageSlotIterator {
+	i := &PageSlotIterator{
+		page:      page,
+		slotCount: page.ReadSlotCount(),
+		cursor:    fromSlot,
+	}
+	return i
+}
+
+func (i *PageSlotIterator) Next() *PageSlot {
+	if i.cursor < i.slotCount {
+		s := i.page.ReadSlot(i.cursor)
+		i.cursor++
+		return &s
+	}
+	return nil
+}
+
+func (i *PageSlotIterator) Cursor() int16 {
+	return i.cursor
+}
+
+func (pg *Page) Dump(label string) {
+	indent := 0
+	if len(label) > 0 {
+		fmt.Printf("%s%s:\n", fmt.Sprintf("%*s", indent, ""), label)
+		indent += 4
+	}
+	pageType := pg.ReadPageType()
+	fmt.Printf("%sPAGE(%d) pageType: %d slotCount: %d, prevPtr: %d, nextPtr: %d\n", fmt.Sprintf("%*s", indent, ""), pg.ID(), pageType, pg.ReadSlotCount(), pg.ReadPrevPointer(), pg.ReadNextPointer())
+	fmt.Printf("%sKEYS: -->\n", fmt.Sprintf("%*s", indent, ""))
+	indent += 4
+
+	// get the keys off the page
+	keys := make([]int, 0)
+	pointers := make([]int, 0)
+	iter := NewPageSlotIterator(pg, 0)
+	for {
+		ps := iter.Next()
+		if ps == nil {
+			break
+		}
+		keys = append(keys, int(ps.KeyAsInt(pg)))
+		if pageType == /*nodeTypeInternal*/ 10 {
+			pointers = append(pointers, int(ps.ValueAsPagePointer(pg)))
+		}
+	}
+
+	if pageType == /*nodeTypeLeaf*/ 11 {
+		for _, key := range keys {
+			fmt.Printf("%s(%d)\n", fmt.Sprintf("%*s", indent, ""), key)
+		}
+	} else {
+		for idx, key := range keys {
+			ptr := pointers[idx]
+			fmt.Printf("%s(%d, %d)\n", fmt.Sprintf("%*s", indent, ""), key, ptr)
+		}
+		ptr := pg.ReadNextPointer()
+		fmt.Printf("%s(-->, %d)\n", fmt.Sprintf("%*s", indent, ""), ptr)
+	}
+
+}

cache.go

@@ -13,7 +13,6 @@ import (
 
 	"github.com/featurebasedb/featurebase/v3/lru"
 	pb "github.com/featurebasedb/featurebase/v3/proto"
-	"github.com/featurebasedb/featurebase/v3/stats"
 	"github.com/pkg/errors"
 )
 
@@ -41,9 +40,6 @@ type cache interface {
 	// Returns an ordered list of the top ranked bitmaps.
 	Top() []bitmapPair
 
-	// SetStats defines the stats client used in the cache.
-	SetStats(s stats.StatsClient)
-
 	// Clear removes everything from the cache. If possible it should leave allocated structures in place to be reused.
 	Clear()
 }
@@ -52,7 +48,6 @@ type cache interface {
 type lruCache struct {
 	cache  *lru.Cache
 	counts map[uint64]uint64
-	stats  stats.StatsClient
 	// maxEntries is saved to support Clear which recreates the cache.
 	maxEntries uint32
 }
@@ -62,7 +57,6 @@ func newLRUCache(maxEntries uint32) *lruCache {
 	c := &lruCache{
 		cache:      lru.New(int(maxEntries)),
 		counts:     make(map[uint64]uint64),
-		stats:      stats.NopStatsClient,
 		maxEntries: maxEntries,
 	}
 	c.cache.OnEvicted = c.onEvicted
@@ -120,11 +114,6 @@ func (c *lruCache) Top() []bitmapPair {
 	return a
 }
 
-// SetStats defines the stats client used in the cache.
-func (c *lruCache) SetStats(s stats.StatsClient) {
-	c.stats = s
-}
-
 func (c *lruCache) Clear() {
 	for k := range c.counts {
 		delete(c.counts, k)
@@ -158,8 +147,6 @@ type rankCache struct {
 
 	// thresholdValue is the value of the last item in the cache
 	thresholdValue uint64
-
-	stats stats.StatsClient
 }
 
 // NewRankCache returns a new instance of RankCache.
@@ -168,7 +155,6 @@ func NewRankCache(maxEntries uint32) *rankCache {
 		maxEntries:      maxEntries,
 		thresholdBuffer: int(thresholdFactor * float64(maxEntries)),
 		entries:         make(map[uint64]uint64),
-		stats:           stats.NopStatsClient,
 	}
 }
 
@@ -229,7 +215,7 @@ func (c *rankCache) BulkAdd(id uint64, n uint64) {
 	// as this can take up an upbounded amount of memory. This is especially
 	// true when restoring shards as all rows will be added.
 	if len(c.entries) > int(2*c.maxEntries) {
-		c.stats.Count(MetricRecalculateCache, 1, 1.0)
+		CounterRecalculateCache.Inc()
 		c.recalculate()
 	}
 }
@@ -274,7 +260,7 @@ func (c *rankCache) Invalidate() {
 func (c *rankCache) Recalculate() {
 	c.mu.Lock()
 	defer c.mu.Unlock()
-	c.stats.Count(MetricRecalculateCache, 1, 1.0)
+	CounterRecalculateCache.Inc()
 	c.recalculate()
 }
 
@@ -286,12 +272,12 @@ func (c *rankCache) invalidate() {
 		// This is somewhat necessary for now since recalculation is not cheap.
 		// The cache will remain flagged as dirty and will be recalculated if Top is called.
 		// This may cause unexpected memory growth, so record it in metrics for debugging purposes.
-		c.stats.Count(MetricInvalidateCacheSkipped, 1, 1.0)
+		CounterInvalidateCacheSkipped.Inc()
 		// Ensure that we're marked as dirty even if we weren't otherwise.
 		c.dirty = true
 		return
 	}
-	c.stats.Count(MetricInvalidateCache, 1, 1.0)
+	CounterInvalidateCache.Inc()
 	c.recalculate()
 }
 
@@ -317,7 +303,7 @@ func (c *rankCache) recalculate() {
 
 	// Store the count of the item at the threshold index.
 	length := len(c.rankings)
-	c.stats.Gauge(MetricRankCacheLength, float64(length), 1.0)
+	GaugeRankCacheLength.Set(float64(length))
 
 	var removeItems []bitmapPair // cached, ordered list
 	if length > int(c.maxEntries) {
@@ -333,7 +319,7 @@ func (c *rankCache) recalculate() {
 
 	// If size is larger than the threshold then trim it.
 	if len(c.entries) > c.thresholdBuffer {
-		c.stats.Count(MetricCacheThresholdReached, 1, 1.0)
+		CounterCacheThresholdReached.Inc()
 		for _, pair := range removeItems {
 			delete(c.entries, pair.ID)
 		}
@@ -343,11 +329,6 @@ func (c *rankCache) recalculate() {
 	c.dirty = false
 }
 
-// SetStats defines the stats client used in the cache.
-func (c *rankCache) SetStats(s stats.StatsClient) {
-	c.stats = s
-}
-
 // Top returns an ordered list of pairs.
 func (c *rankCache) Top() []bitmapPair {
 	c.mu.Lock()
@@ -355,7 +336,7 @@ func (c *rankCache) Top() []bitmapPair {
 
 	if c.dirty {
 		// The cache is dirty, so we need to recalculate it to get a consistent view.
-		c.stats.Count(MetricReadDirtyCache, 1, 1.0)
+		CounterReadDirtyCache.Inc()
 		c.recalculate()
 	}
 
@@ -606,25 +587,21 @@ func (p uint64Slice) Len() int           { return len(p) }
 func (p uint64Slice) Less(i, j int) bool { return p[i] < p[j] }
 
 // nopCache represents a no-op Cache implementation.
-type nopCache struct {
-	stats stats.StatsClient
-}
+type nopCache struct{}
 
 // Ensure NopCache implements Cache.
-var globalNopCache cache = nopCache{
-	stats: stats.NopStatsClient,
-}
+var globalNopCache cache = nopCache{}
 
 func (c nopCache) Add(uint64, uint64)     {}
 func (c nopCache) BulkAdd(uint64, uint64) {}
 func (c nopCache) Get(uint64) uint64      { return 0 }
 func (c nopCache) IDs() []uint64          { return []uint64{} }
 
-func (c nopCache) Invalidate()                {}
-func (c nopCache) Len() int                   { return 0 }
-func (c nopCache) Recalculate()               {}
-func (c nopCache) SetStats(stats.StatsClient) {}
-func (c nopCache) Clear()                     {}
+func (c nopCache) Invalidate()  {}
+func (c nopCache) Len() int     { return 0 }
+func (c nopCache) Recalculate() {}
+
+func (c nopCache) Clear() {}
 
 func (c nopCache) Top() []bitmapPair {
 	return []bitmapPair{}

cache_test.go

@@ -62,7 +62,7 @@ func TestCache_Rank_Dirty(t *testing.T) {
 		cache.Add(v.ID, v.Count)
 	}
 
-	var got []pair
+	var got []pair //nolint:prealloc
 	for _, p := range cache.Top() {
 		got = append(got, pair(p))
 	}

catcher.go

@@ -124,6 +124,17 @@ func (c *catcherTx) Remove(index, field, view string, shard uint64, a ...uint64)
 	return c.b.Remove(index, field, view, shard, a...)
 }
 
+func (c *catcherTx) Removed(index, field, view string, shard uint64, a ...uint64) (changed []uint64, err error) {
+
+	defer func() {
+		if r := recover(); r != nil {
+			vprint.AlwaysPrintf("see Removed() PanicOn '%v' at '%v'", r, vprint.Stack())
+			vprint.PanicOn(r)
+		}
+	}()
+	return c.b.Removed(index, field, view, shard, a...)
+}
+
 func (c *catcherTx) Contains(index, field, view string, shard uint64, key uint64) (exists bool, err error) {
 
 	defer func() {

cli/Makefile

@@ -0,0 +1,15 @@
+.PHONY: test testv test-integration testv-integration
+
+GO=go
+
+test:
+	$(GO) test ./... -short
+
+testv:
+	$(GO) test -v ./... -short
+
+test-integration:
+	$(GO) test . -count 1 -timeout 20m -run TestCLIIntegration/$(RUN)
+
+testv-integration:
+	$(GO) test -v . -count 1 -timeout 20m -run TestCLIIntegration/$(RUN)

cli/batch/inserter.go

@@ -0,0 +1,8 @@
+package batch
+
+// Inserter can be implemented by anything which can handle a SQL statement
+// representing a write operation. An example is `BULK INSERT`. The Insert()
+// method on this interface does not return any results other than an error.
+type Inserter interface {
+	Insert(sql string) error
+}

cli/batch/sql.go

@@ -0,0 +1,215 @@
+package batch
+
+import (
+	"encoding/json"
+	"fmt"
+	"strings"
+	"time"
+
+	fbbatch "github.com/featurebasedb/featurebase/v3/batch"
+	"github.com/featurebasedb/featurebase/v3/dax"
+	"github.com/featurebasedb/featurebase/v3/errors"
+	"github.com/featurebasedb/featurebase/v3/pql"
+)
+
+// Ensure type implements interface.
+var _ fbbatch.Batcher = (*sqlBatcher)(nil)
+
+type sqlBatcher struct {
+	inserter Inserter
+	fields   []*dax.Field
+}
+
+func NewSQLBatcher(i Inserter, flds []*dax.Field) *sqlBatcher {
+	return &sqlBatcher{
+		inserter: i,
+		fields:   flds,
+	}
+}
+
+func (b *sqlBatcher) NewBatch(cfg fbbatch.Config, tbl *dax.Table, flds []*dax.Field) (fbbatch.RecordBatch, error) {
+	fields := flds
+	if b.fields != nil {
+		fields = b.fields
+	}
+	return &sqlBatch{
+		table:        tbl,
+		fields:       fields,
+		size:         cfg.Size,
+		maxStaleness: cfg.MaxStaleness,
+		ids:          make([]interface{}, 0, cfg.Size),
+		rows:         make([][]interface{}, 0, cfg.Size),
+		inserter:     b.inserter,
+	}, nil
+}
+
+// Ensure type implements interface.
+var _ fbbatch.RecordBatch = (*sqlBatch)(nil)
+
+type sqlBatch struct {
+	table  *dax.Table
+	fields []*dax.Field
+	size   int
+
+	ids  []interface{}
+	rows [][]interface{}
+
+	// staleTime tracks the time the first record of the batch was inserted
+	// plus the maxStaleness, in order to raise ErrBatchNowStale if the
+	// maxStaleness has elapsed
+	staleTime    time.Time
+	maxStaleness time.Duration
+
+	// inserter handles SQL INSERT statements generated for each batch.
+	inserter Inserter
+}
+
+func (b *sqlBatch) Add(rec fbbatch.Row) error {
+	// Clear rec.Values and rec.Clears upon return.
+	defer func() {
+		for i := range rec.Values {
+			rec.Values[i] = nil
+		}
+		for k := range rec.Clears {
+			delete(rec.Clears, k)
+		}
+	}()
+
+	if len(b.ids) == cap(b.ids) {
+		return fbbatch.ErrBatchAlreadyFull
+	}
+	if len(rec.Values) != len(b.fields) {
+		return errors.Errorf("record needs to match up with batch fields, got %d fields and %d record", len(b.fields), len(rec.Values))
+	}
+
+	// Append the ID to b.ids.
+	b.ids = append(b.ids, rec.ID)
+
+	// Convert decimal fields (which come in as int64, along with the scale in
+	// field) to pql.Decimal.
+	for i, fld := range b.fields {
+		switch b.fields[i].Type {
+		case dax.BaseTypeDecimal:
+			if val, ok := rec.Values[i].(int64); ok {
+				rec.Values[i] = pql.NewDecimal(val, fld.Options.Scale)
+			}
+		case dax.BaseTypeTimestamp:
+			if val, ok := rec.Values[i].(int64); ok {
+				ts := time.Unix(val, 0)
+				rec.Values[i] = ts.Format(time.RFC3339)
+			}
+		}
+	}
+
+	// Append the record to b.rows.
+	vals := make([]interface{}, 0, len(rec.Values))
+	vals = append(vals, rec.Values...)
+	b.rows = append(b.rows, vals)
+
+	// Check for batch full or stale.
+	if len(b.ids) == cap(b.ids) {
+		return fbbatch.ErrBatchNowFull
+	}
+	if b.maxStaleness != time.Duration(0) { // set maxStaleness to 0 to disable staleness checking
+		if len(b.ids) == 1 {
+			b.staleTime = time.Now().Add(b.maxStaleness)
+		} else if time.Now().After(b.staleTime) {
+			return fbbatch.ErrBatchNowStale
+		}
+	}
+	return nil
+}
+
+func (b *sqlBatch) Import() error {
+	if len(b.rows) == 0 {
+		return nil
+	}
+
+	// Construct the BULK INSERT statement based on the table and fields.
+	sql, err := buildBulkInsert(b.table, b.fields, b.ids, b.rows)
+	if err != nil {
+		return errors.Wrap(err, "building bulk insert statement")
+	}
+
+	// Reset batch data.
+	b.reset()
+
+	// Submit the SQL statement.
+	return b.inserter.Insert(sql)
+}
+
+func (b *sqlBatch) reset() {
+	b.ids = b.ids[:0]
+	b.rows = b.rows[:0]
+}
+
+func (b *sqlBatch) Len() int {
+	return len(b.rows)
+}
+
+func (b *sqlBatch) Flush() error {
+	return nil
+}
+
+func buildBulkInsert(tbl *dax.Table, fields []*dax.Field, ids []interface{}, rows [][]interface{}) (string, error) {
+	// Validation.
+	if tbl.Name == "" {
+		return "", errors.New(errors.ErrUncoded, "table name is required")
+	} else if len(fields) == 0 {
+		return "", errors.New(errors.ErrUncoded, "at least one field is required")
+	}
+
+	var sb strings.Builder
+
+	sb.WriteString(`BULK INSERT INTO `)
+	sb.WriteString(string(tbl.Name))
+	sb.WriteString(` (_id,`)
+
+	flds := make([]string, 0, len(fields))
+	maps := make([]string, 0, len(fields))
+	for i := range fields {
+		flds = append(flds, string(fields[i].Name))
+		maps = append(maps, fmt.Sprintf("'$.col_%d' %s", i, fields[i].FullType()))
+	}
+	// Fields
+	sb.WriteString(strings.Join(flds, ","))
+
+	// MAP
+	keyType := dax.BaseTypeID
+	if tbl.StringKeys() {
+		keyType = dax.BaseTypeString
+	}
+	sb.WriteString(`) MAP ('$._id' `)
+	sb.WriteString(keyType)
+	sb.WriteString(`,`)
+	sb.WriteString(strings.Join(maps, ","))
+	sb.WriteString(`) FROM x'`)
+
+	// Row values.
+
+	// m is a map representing a single row to be marshalled and added to the
+	// bulk insert as one line in the NDJSON payload. We re-use the map for each
+	// row.
+	m := make(map[string]interface{})
+	for i := range rows {
+		// Write the ID value.
+		m[string(dax.PrimaryKeyFieldName)] = ids[i]
+		// Write the rest of the data values.
+		for col := range rows[i] {
+			m[fmt.Sprintf("col_%d", col)] = rows[i][col]
+		}
+
+		// Marshal the map to json and add to the sql statement.
+		if j, err := json.Marshal(m); err != nil {
+			return "", errors.Wrap(err, "marshalling row to json")
+		} else {
+			sb.Write(j)
+			sb.WriteString("\n")
+		}
+	}
+
+	// WITH
+	sb.WriteString(fmt.Sprintf(`' WITH BATCHSIZE %d FORMAT 'NDJSON' INPUT 'STREAM'`, len(rows)))
+
+	return sb.String(), nil
+}

cli/batch/sql_test.go

@@ -0,0 +1,47 @@
+package batch
+
+import (
+	"testing"
+
+	"github.com/featurebasedb/featurebase/v3/dax"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestBatchSQL(t *testing.T) {
+	tbl := &dax.Table{
+		Name: "foo",
+	}
+	fields := []*dax.Field{
+		{
+			Name: "name",
+			Type: dax.BaseTypeString,
+		},
+		{
+			Name: "age",
+			Type: dax.BaseTypeInt,
+		},
+	}
+	ids := []interface{}{
+		0, 1, 2,
+	}
+	rows := [][]interface{}{
+		{
+			[]interface{}{"Alice", int64(11)},
+		},
+		{
+			[]interface{}{"Bob", int64(22)},
+		},
+		{
+			[]interface{}{"Carl,Comma", int64(33)},
+		},
+	}
+
+	s, err := buildBulkInsert(tbl, fields, ids, rows)
+	assert.NoError(t, err)
+
+	exp := `BULK INSERT INTO foo (_id,name,age) MAP ('$._id' id,'$.col_0' string,'$.col_1' int) FROM x'{"_id":0,"col_0":["Alice",11]}
+{"_id":1,"col_0":["Bob",22]}
+{"_id":2,"col_0":["Carl,Comma",33]}
+' WITH BATCHSIZE 3 FORMAT 'NDJSON' INPUT 'STREAM'`
+	assert.Equal(t, exp, s)
+}

cli/buffer.go

@@ -0,0 +1,91 @@
+package cli
+
+import (
+	"io"
+	"strings"
+
+	"github.com/featurebasedb/featurebase/v3/errors"
+)
+
+// buffer is a query buffer for SQL statements. Note that this is not a query
+// buffer as you would find on a database server (buffering query results).
+// Rather, this buffers the working SQL statement. The buffer has two
+// components: the buffer of query parts making up the working, incomplete SQL
+// statement, and the last completed SQL statement submitted to the Queryer.
+type buffer struct {
+	parts     []queryPart
+	lastQuery query
+
+	hasBatchFile bool
+}
+
+func newBuffer() *buffer {
+	return &buffer{}
+}
+
+// addPart adds the given queryPart to the buffer. If the part is of type
+// `partTerminator` (which is generally singified in the CLI by a ";"), the
+// buffer will finalize the query and return it. In all other cases, the
+// returned query is nil.
+func (b *buffer) addPart(part queryPart) (query, error) {
+	// Check for part type compatibility. For example, multiple batchFile parts
+	// are not allowed in the same query.
+	switch part.(type) {
+	case *partBatchFile:
+		if b.hasBatchFile {
+			return nil, errors.Errorf("multiple batch files in one query is not supported")
+		}
+		b.hasBatchFile = true
+	case *partTerminator:
+		return b.finalize(), nil
+	}
+
+	b.parts = append(b.parts, part)
+	return nil, nil
+}
+
+// finalize copies the contents (queryParts) of buffer to lastQuery and then
+// resets the buffer. It returns the query that was finalized.
+func (b *buffer) finalize() query {
+	q := make(query, len(b.parts))
+	copy(q, b.parts)
+	b.lastQuery = q
+	b.reset()
+	return q
+
+}
+
+// print returns the contents of the buffer as a string. This is generally used
+// to visually inspect the state of the buffer (for example, when a user issues
+// a `\p` meta-command in the CLI).
+func (b *buffer) print() string {
+	if len(b.parts) > 0 {
+		return query(b.parts).String()
+	} else if b.lastQuery != nil {
+		return b.lastQuery.String() + ";"
+	}
+	return "Query buffer is empty."
+}
+
+// reset clears the buffer. It returns a message which may optionally be used to
+// display to a user.
+func (b *buffer) reset() string {
+	b.parts = b.parts[:0]
+	b.hasBatchFile = false
+	return "Query buffer reset (cleared)."
+}
+
+func (b *buffer) Reader() io.Reader {
+	if len(b.parts) > 0 {
+		return query(b.parts).Reader()
+	} else if b.lastQuery != nil {
+		r := b.lastQuery.Reader()
+		// TODO(tlt): terminating the query here results in a line feed just
+		// before the semi-colon (for example, when you print out the query
+		// buffer using `\w [FILE]`). The removal and re-introduction of line
+		// feeds is kind of a mess.
+		term := strings.NewReader(";")
+		return io.MultiReader(r, term)
+	}
+	return strings.NewReader("")
+}

cli/cli.go

@@ -0,0 +1,816 @@
+// Package cli contains a FeatureBase command line interface.
+package cli
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"github.com/chzyer/readline"
+	featurebase "github.com/featurebasedb/featurebase/v3"
+	"github.com/featurebasedb/featurebase/v3/cli/batch"
+	"github.com/featurebasedb/featurebase/v3/cli/fbcloud"
+	"github.com/featurebasedb/featurebase/v3/errors"
+	"github.com/featurebasedb/featurebase/v3/logger"
+)
+
+const (
+	defaultHost     string = "localhost"
+	defaultClientID string = "6i2gs7mu215ab23cnvmshdoq6t" // production Cognito client ID
+	defaultRegion   string = "us-east-2"
+	terminationChar string = ";"
+	nullValue       string = "NULL"
+)
+
+var (
+	Stdin  io.ReadCloser = os.Stdin
+	Stdout io.Writer     = os.Stdout
+	Stderr io.Writer     = os.Stderr
+)
+
+var splash string = fmt.Sprintf(`FeatureBase CLI (%s)
+Type "\q" to quit.
+`, featurebase.Version)
+
+// Ensure type implments interfaces.
+var _ printer = (*Command)(nil)
+var _ batch.Inserter = (*Command)(nil)
+
+type Command struct {
+	host string
+	port string
+
+	splitter   *splitter
+	buffer     *buffer
+	workingDir *workingDir
+
+	organizationID string
+	database       string
+	databaseID     string
+	databaseName   string
+
+	Queryer Queryer `json:"-"`
+
+	stdin  io.ReadCloser `json:"-"`
+	stdout io.Writer     `json:"-"`
+	stderr io.Writer     `json:"-"`
+
+	// output is where actual results are written. This might point to stdout,
+	// or to a file, based on the current configuration.
+	output       io.Writer `json:"-"`
+	writeOptions *writeOptions
+
+	Config *Config `json:"config"`
+
+	historyPath string
+
+	// Commands contains optional commands provided via one or more `-c` (or
+	// `--command`) flags. If this is non-empty, the cli will run in
+	// non-interactive mode; i.e. it will quit after the command is complete.
+	Commands []string `json:"commands"`
+
+	// Files contains optional files provided via one or more `-f` (or `--file`)
+	// flags. If this is non-empty, the cli will run in non-interactive mode;
+	// i.e. it will quit after the command is complete.
+	Files []string `json:"files"`
+
+	// variables holds the variables created with the \set meta-command.
+	variables map[string]string
+
+	// nonInteractiveMode is set to true when fbsql is running in
+	// non-ineracative mode. And example of this is when the user has provided a
+	// `-c` flag in the command line.
+	nonInteractiveMode bool
+
+	// quit gets closed when Run should stop listening for input.
+	quit chan struct{}
+}
+
+func NewCommand(logdest logger.Logger) *Command {
+	variables := make(map[string]string)
+
+	return &Command{
+		Config: &Config{
+			Host: defaultHost,
+			Port: "",
+
+			OrganizationID: "",
+			Database:       "",
+
+			CloudAuth: CloudAuthConfig{
+				ClientID: defaultClientID,
+				Region:   defaultRegion,
+				Email:    "",
+				Password: "",
+			},
+
+			HistoryPath: "",
+
+			CSV: false,
+		},
+
+		buffer:     newBuffer(),
+		splitter:   newSplitter(newReplacer(variables)),
+		workingDir: newWorkingDir(),
+
+		stdin:  Stdin,
+		stdout: Stdout,
+		stderr: Stderr,
+
+		output:       Stdout,
+		writeOptions: defaultWriteOptions(),
+
+		variables: variables,
+
+		quit: make(chan struct{}),
+	}
+}
+
+// SetStdin sets stdin. This is useful for initial configuration in tests.
+func (cmd *Command) SetStdin(rc io.ReadCloser) {
+	cmd.stdin = rc
+}
+
+// SetStdout sets both stdout and output to the value provided. This is useful
+// for initial configuration in tests.
+func (cmd *Command) SetStdout(w io.Writer) {
+	cmd.stdout = w
+	cmd.output = w
+}
+
+// SetStderr sets stderr. This is useful for initial configuration in tests.
+func (cmd *Command) SetStderr(w io.Writer) {
+	cmd.stderr = w
+}
+
+// Run is the main entry-point to the CLI.
+func (cmd *Command) Run(ctx context.Context) error {
+	if err := cmd.run(ctx); err != nil {
+		cmd.Errorf(err.Error() + "\n")
+		return err
+	}
+	return nil
+}
+
+// run is effectively wrapped by the Run() method, but it's split out this way
+// so that run() can simply return errors, rather than worrying about how errors
+// should be printed; printing errors returned by run() is left up to the Run()
+// method.
+func (cmd *Command) run(ctx context.Context) error {
+	if err := cmd.setupConfig(); err != nil {
+		return errors.Wrap(err, "setting up config")
+	}
+
+	// Check to see if Command needs to run in non-interactive mode.
+	if len(cmd.Commands) > 0 ||
+		len(cmd.Files) > 0 ||
+		cmd.Config.KafkaConfig != "" ||
+		cmd.Config.CSV {
+		cmd.nonInteractiveMode = true
+	}
+
+	// Print the splash message.
+	if !cmd.nonInteractiveMode {
+		cmd.Printf(splash)
+	}
+
+	if err := cmd.setupClient(); err != nil {
+		return errors.Wrap(err, "setting up client")
+	}
+
+	// Print the connection info.
+	if !cmd.nonInteractiveMode {
+		cmd.printConnInfo()
+	}
+
+	if err := cmd.connectToDatabase(cmd.database); err != nil {
+		cmd.Errorf(errors.Wrap(err, "connecting to database").Error() + "\n")
+		// We intentionally do not return err here.
+	}
+
+	// Run in non-interactive mode based on flags and configuration.
+	// This includes either handling `-c` and/or `-f` flags, or handling a
+	// `--kafka-config` flag.
+	if len(cmd.Commands) > 0 || len(cmd.Files) > 0 {
+		// Run Commands.
+		for _, line := range cmd.Commands {
+			if err := cmd.handleLine(line); err != nil {
+				return errors.Wrapf(err, "handling line: %s", line)
+			}
+		}
+
+		// Run Files.
+		for _, fname := range cmd.Files {
+			if _, err := executeFile(cmd, fname); err != nil {
+				return errors.Wrapf(err, "executing file: %s", fname)
+			}
+		}
+
+		return nil
+	} else if cmd.Config.KafkaConfig != "" {
+		runner, err := cmd.newKafkaRunner(cmd.Config.KafkaConfig)
+		if err != nil {
+			return errors.Wrap(err, "getting new kafka runner")
+		}
+		if err := runner.Main.Run(); err != nil {
+			return errors.Wrap(err, "running kafka")
+		}
+		return nil
+	}
+
+	// From this point on, we should be in interactive mode.
+
+	// Set up history for saving user input.
+	cmd.setupHistory()
+
+	rl, err := readline.NewEx(&readline.Config{
+		Prompt:                 cmd.prompt(false),
+		HistoryFile:            cmd.historyPath,
+		HistoryLimit:           100000,
+		DisableAutoSaveHistory: true,
+
+		Stdin:  cmd.stdin,
+		Stdout: cmd.stdout,
+		Stderr: cmd.stderr,
+	})
+	if err != nil {
+		return errors.Wrap(err, "getting readline")
+	}
+	defer rl.Close()
+
+	// inMidCommand indicates whether a partial command has been received and
+	// we're still waiting for a termination character.
+	var inMidCommand bool
+
+	for {
+		rl.SetPrompt(cmd.prompt(inMidCommand))
+
+		// Read user provided input.
+		line, err := rl.Readline()
+		if err == readline.ErrInterrupt {
+			inMidCommand = false
+			cmd.buffer.reset()
+			continue
+		} else if err != nil {
+			return errors.Wrap(err, "reading line")
+		}
+
+		// We append a line feed at the end of each line because at this point
+		// we have effectively stripped any intentional line feeds (since we are
+		// reading a line at a time), and we don't want to do that. An example
+		// of an intentional line feed is in a BULK INSERT CSV STREAM like this
+		// example:
+		//
+		// bulk replace
+		// 	 into foo (_id, age)
+		// 	 map (0 id, 1 int)
+		// from
+		// 	 x'3,33
+		// 	 4,44
+		// 	 5,55'
+		// with
+		// 	 format 'CSV'
+		// 	 input 'STREAM';
+		//
+		// We want to preserve the line feeds that are contained in the x''
+		// block; those are intentional as they demarc records within the csv.
+		qps, mcs, err := cmd.splitter.split(line + "\n")
+		if err != nil {
+			cmd.Errorf("error splitting line: %s\n", err)
+			continue
+		}
+
+		// Save line in the history.
+		if err := rl.SaveHistory(line); err != nil {
+			cmd.Errorf("Couldn't save history: %v\n", err)
+		}
+
+		// This is wrapped in an anonymous function so we can capture any
+		// errors, ignore the rest of the line, and return back to a prompt.
+		if err := func() error {
+			for i := range qps {
+				if qry, err := cmd.buffer.addPart(qps[i]); err != nil {
+					return errors.Wrap(err, "adding part to buffer")
+				} else if qry != nil {
+					if err := cmd.executeAndWriteQuery(qry); err != nil {
+						return errors.Wrap(err, "executing query")
+					}
+					// In addition to saving each line in the history, we also
+					// save each successful query.
+					if err := rl.SaveHistory(qry.String() + ";"); err != nil {
+						cmd.Errorf("Couldn't save query in history: %v\n", err)
+					}
+					inMidCommand = false
+				} else {
+					inMidCommand = true
+				}
+			}
+			return nil
+		}(); err != nil {
+			cmd.Errorf(err.Error() + "\n")
+			inMidCommand = false
+			continue
+		}
+
+		// This is wrapped in an anonymous function so we can capture any
+		// errors, ignore the rest of the line, and return back to a prompt.
+		if err := func() error {
+			for i := range mcs {
+				action, err := mcs[i].execute(cmd)
+				if err != nil {
+					return errors.Wrap(err, "executing meta command")
+				}
+				switch action {
+				case actionQuit:
+					close(cmd.quit)
+					return nil
+				case actionReset:
+					inMidCommand = false
+				}
+			}
+			return nil
+		}(); err != nil {
+			cmd.Errorf(err.Error() + "\n")
+			inMidCommand = false
+			continue
+		}
+
+		select {
+		case <-cmd.quit:
+			if err := cmd.close(); err != nil {
+				cmd.Errorf("closing: %s\n", err)
+			}
+			return nil
+		default:
+			// pass
+		}
+	}
+}
+
+// prompt constructs the prompt that the user sees based on the currently
+// connected database and whether the user is in the middle of a sql statement.
+func (cmd *Command) prompt(mid bool) string {
+	db := "fbsql" // default prompt when a database is not set.
+	if cmd.databaseName != "" {
+		db = cmd.databaseName
+	}
+
+	if mid {
+		return strings.Repeat(" ", len(db)) + "-# "
+	}
+	return db + "=# "
+}
+
+// close is called upon quitting. It should close any remaining open file
+// handles used by the CLICommand.
+func (cmd *Command) close() error {
+	return cmd.closeOutput()
+}
+
+// setupConfig sets up private struct members based on values provided via the
+// configuration flags.
+func (cmd *Command) setupConfig() error {
+	if cmd.Config == nil {
+		return nil
+	}
+
+	cmd.host = cmd.Config.Host
+	cmd.port = cmd.Config.Port
+
+	cmd.organizationID = cmd.Config.OrganizationID
+	cmd.database = cmd.Config.Database
+
+	cmd.historyPath = cmd.Config.HistoryPath
+
+	// Apply any pset flag arguments.
+	for _, pset := range cmd.Config.PSets {
+		if err := cmd.applyPSet(pset); err != nil {
+			return errors.Wrapf(err, "applying pset: %s", pset)
+		}
+	}
+
+	// If running with the `--csv` flag, configure things to ensure the output
+	// is correct (i.e. that it's just the csv).
+	if cmd.Config.CSV {
+		cmd.writeOptions.format = formatCSV
+	}
+
+	return nil
+}
+
+// applyPSet takes a pset string of the form `arg` or `arg=val` and applies it
+// as if the user had run `\pset arg val`. The only difference is that applying
+// pset here suppresses any output to stdout.
+func (cmd *Command) applyPSet(pset string) error {
+	// We expect arg to be one of the folowing formats:
+	// arg
+	// arg=val
+	args := strings.SplitN(pset, "=", 2)
+
+	// This is kind of hacky, but until we re-think the metaCommand interface to
+	// take a printer interface somewhere (so we can pass in the nopPrinter
+	// here), we're just going to discard stdout for the duration of this apply,
+	// and then set stdout back to its previous writer after the apply.
+	hold := cmd.stdout
+	cmd.stdout = io.Discard
+	defer func() {
+		cmd.stdout = hold
+	}()
+
+	_, err := newMetaPSet(args).execute(cmd)
+	return err
+}
+
+func (cmd *Command) executeAndWriteQuery(qry query) error {
+	queryResponse, err := cmd.executeQuery(qry)
+	if err != nil {
+		if errors.Is(err, ErrOrganizationRequired) {
+			// Print an error message and return nil, effectively aborting any
+			// further writes for this query.
+			cmd.Errorf("Organization required. Use \\org to set an organization.\n")
+			return nil
+		}
+		return errors.Wrap(err, "making query")
+	}
+	if err := writeOutput(queryResponse, cmd.writeOptions, cmd.output, cmd.stdout, cmd.stderr); err != nil {
+		return errors.Wrap(err, "writing out response")
+	}
+
+	return nil
+}
+
+func (cmd *Command) executeQuery(qry query) (*featurebase.WireQueryResponse, error) {
+	wqr, err := cmd.Queryer.Query(cmd.organizationID, cmd.databaseID, qry.Reader())
+	if err != nil {
+		return nil, errors.Wrap(err, "executing query")
+	}
+
+	// If we're running in non-interactive mode, we need to check the error that
+	// comes back in the WireQueryResponse. If there's an error, we want to
+	// return it now (rather than just printing it later) so that we immediately
+	// stop any further execution of commands.
+	if cmd.nonInteractiveMode && wqr.Error != "" {
+		return nil, errors.Errorf(wqr.Error)
+	}
+
+	return wqr, nil
+}
+
+// printer is an interface which encapsulates the methods used to print output
+// to the various io.Writers.
+type printer interface {
+	Printf(format string, a ...any)
+	Outputf(format string, a ...any)
+	Errorf(format string, a ...any)
+}
+
+type nopPrinter struct{}
+
+func newNopPrinter() *nopPrinter {
+	return &nopPrinter{}
+}
+
+func (n *nopPrinter) Printf(format string, a ...any)  {}
+func (n *nopPrinter) Outputf(format string, a ...any) {}
+func (n *nopPrinter) Errorf(format string, a ...any)  {}
+
+// Printf is a helper method which sends the given payload to stdout.
+func (cmd *Command) Printf(format string, a ...any) {
+	out := fmt.Sprintf(format, a...)
+	cmd.stdout.Write([]byte(out))
+}
+
+// Outputf is a helper method which sends the given payload to output.
+func (cmd *Command) Outputf(format string, a ...any) {
+	out := fmt.Sprintf(format, a...)
+	cmd.output.Write([]byte(out))
+}
+
+// Errorf is a helper method which sends the given payload to stderr.
+func (cmd *Command) Errorf(format string, a ...any) {
+	out := fmt.Sprintf(format, a...)
+	cmd.stderr.Write([]byte(out))
+}
+
+func (cmd *Command) setupHistory() {
+	// If HistoryPath has already been configured (i.e. with a command flag),
+	// don't bother setting up the default in the home directory.
+	if cmd.historyPath != "" {
+		return
+	}
+
+	historyPath := ""
+	if home, err := os.UserHomeDir(); err != nil {
+		cmd.Errorf("Error getting home directory, command history persistence will be disabled: %v\n", err)
+	} else {
+		historyDir := filepath.Join(home, ".featurebase")
+		err := os.MkdirAll(historyDir, 0o750)
+		if err != nil {
+			cmd.Errorf("Creating directory for history: %v\n", err)
+		} else {
+			historyPath = filepath.Join(historyDir, "fbsql_history")
+		}
+	}
+	cmd.historyPath = historyPath
+}
+
+// printConnInfo displays the currently set host.
+// TODO(tlt): extend this to be the output of the /conninfo meta-command.
+func (cmd *Command) printConnInfo() {
+	cmd.Printf("Host: %s\n", hostPort(cmd.host, cmd.port))
+}
+
+func (cmd *Command) connectToDatabase(dbName string) error {
+	var p printer = cmd
+	if cmd.nonInteractiveMode {
+		p = newNopPrinter()
+	}
+
+	// Providing a blank ("") or hyphen ("-") dbName is the equivalent of
+	// disconnecting from the current database. We support the hyphen option
+	// because calling the `\c` meta-command without an argument is how you
+	// print the current connection.
+	switch dbName {
+	case "-", "":
+		cmd.databaseID = ""
+		cmd.databaseName = ""
+		p.Printf(cmd.connectionMessage())
+		return nil
+	}
+
+	// Look up dbID based on dbName.
+	wqr, err := cmd.executeQuery(newRawQuery("SHOW DATABASES"))
+	if err != nil {
+		return errors.Wrap(err, "executing query")
+	}
+
+	for _, db := range wqr.Data {
+		// 0: _id
+		// 1: name
+		if db[1] == dbName {
+			cmd.databaseName = dbName
+			cmd.databaseID = db[0].(string)
+			p.Printf(cmd.connectionMessage())
+			return nil
+		}
+	}
+	return errors.Errorf("invalid database: %s", dbName)
+}
+
+func (cmd *Command) orgMessage() string {
+	if cmd.organizationID == "" {
+		return "You have not set an organization.\n"
+	}
+	return fmt.Sprintf("You have set organization \"%s\".\n", cmd.organizationID)
+}
+
+func (cmd *Command) connectionMessage() string {
+	if cmd.databaseName == "" {
+		return "You are not connected to a database.\n"
+	}
+	return fmt.Sprintf("You are now connected to database \"%s\" (%s).\n", cmd.databaseName, cmd.databaseID)
+}
+
+func (cmd *Command) setupClient() error {
+	// If the Queryer has already been set (in tests for example), don't bother
+	// trying to detect it.
+	if cmd.Queryer != nil {
+		return nil
+	}
+
+	var p printer = cmd
+	if cmd.nonInteractiveMode {
+		p = newNopPrinter()
+	}
+
+	if strings.TrimSpace(cmd.host) == "" {
+		return errors.Errorf("no host provided\n")
+	}
+
+	if !strings.HasPrefix(cmd.host, "http") {
+		cmd.host = "http://" + cmd.host
+	}
+
+	typ, err := cmd.detectFBType()
+	if err != nil {
+		return errors.Wrap(err, "detecting FeatureBase deployment type")
+	}
+
+	switch typ {
+	case featurebaseTypeOnPremClassic:
+		p.Printf("Detected on-prem, classic deployment.\n")
+		cmd.Queryer = &standardQueryer{
+			Host: cmd.host,
+			Port: cmd.port,
+		}
+	case featurebaseTypeOnPremServerless:
+		p.Printf("Detected on-prem, serverless deployment.\n")
+		cmd.Queryer = &serverlessQueryer{
+			Host: cmd.host,
+			Port: cmd.port,
+		}
+	case featurebaseTypeCloud:
+		p.Printf("Detected cloud deployment.\n")
+
+		cmd.Queryer = &fbcloud.Queryer{
+			Host: hostPort(cmd.host, cmd.port),
+
+			ClientID: cmd.Config.CloudAuth.ClientID,
+			Region:   cmd.Config.CloudAuth.Region,
+			Email:    cmd.Config.CloudAuth.Email,
+			Password: cmd.Config.CloudAuth.Password,
+		}
+	case featurebaseTypeUnknown:
+		p.Printf("Could not detect deployment\n")
+		// cmd.Queryer = &nopQueryer{}
+		// Instead of using a no-op queryer when the type can't be detected, we
+		// default to using a cloud queryer.
+		cmd.Queryer = &fbcloud.Queryer{
+			Host: hostPort(cmd.host, cmd.port),
+
+			ClientID: cmd.Config.CloudAuth.ClientID,
+			Region:   cmd.Config.CloudAuth.Region,
+			Email:    cmd.Config.CloudAuth.Email,
+			Password: cmd.Config.CloudAuth.Password,
+		}
+	default:
+		return errors.Errorf("unknown type: %s", typ)
+	}
+	return nil
+}
+
+type featurebaseType string
+
+const (
+	featurebaseTypeUnknown          featurebaseType = "unknown"            // unknown
+	featurebaseTypeOnPremClassic    featurebaseType = "on-prem-standard"   // on-prem, classic
+	featurebaseTypeOnPremServerless featurebaseType = "on-prem-serverless" // on-prem, serverless
+	featurebaseTypeCloud            featurebaseType = "cloud"              // cloud, (both classic and serverless)?
+)
+
+func hostPort(host, port string) string {
+	if port == "" {
+		return host
+	}
+	return host + ":" + port
+}
+
+// detectFBType determines if we're talking to standalone FeatureBase
+// or FeatureBase Cloud
+func (cmd *Command) detectFBType() (featurebaseType, error) {
+	type trial struct {
+		port   string
+		health string
+		typ    featurebaseType
+	}
+
+	// trials is populated with the url/endpoints to try in order to detect if a
+	// process is running there which can support the cli requests.
+	trials := []trial{}
+
+	var clientTimeout time.Duration
+	if cmd.port != "" {
+		clientTimeout = 100 * time.Millisecond
+		trials = append(trials,
+			// on-prem, serverless
+			trial{
+				port:   cmd.port,
+				health: "/queryer/health",
+				typ:    featurebaseTypeOnPremServerless,
+			},
+			// on-prem, classic
+			trial{
+				port:   cmd.port,
+				health: "/status",
+				typ:    featurebaseTypeOnPremClassic,
+			},
+		)
+	} else if strings.HasPrefix(cmd.host, "https") {
+		// https suggesting we might be connecting to a cloud host
+		clientTimeout = 1 * time.Second
+		trials = append(trials,
+			// cloud
+			trial{
+				port:   "",
+				health: "/health",
+				typ:    featurebaseTypeCloud,
+			},
+		)
+	} else {
+		// Try default ports just in case.
+		clientTimeout = 100 * time.Millisecond
+		trials = append(trials,
+			// on-prem, serverless
+			trial{
+				port:   "8080",
+				health: "/queryer/health",
+				typ:    featurebaseTypeOnPremServerless,
+			},
+			// on-prem, classic
+			trial{
+				port:   "10101",
+				health: "/status",
+				typ:    featurebaseTypeOnPremClassic,
+			},
+		)
+	}
+
+	client := http.Client{
+		Timeout: clientTimeout,
+	}
+	for _, trial := range trials {
+		url := hostPort(cmd.host, trial.port) + trial.health
+		if resp, err := client.Get(url); err != nil {
+			continue
+		} else if resp.StatusCode/100 == 2 {
+			cmd.port = trial.port
+			return trial.typ, nil
+		}
+	}
+
+	return featurebaseTypeUnknown, nil
+}
+
+func (cmd *Command) closeOutput() error {
+	if cmd.output == nil {
+		return nil
+	}
+
+	if closer, ok := cmd.output.(io.Closer); ok {
+		return closer.Close()
+	}
+
+	return nil
+}
+
+func (cmd *Command) handleLine(line string) error {
+	// For single-line command handling, we handle either a meta-command, or
+	// query parts, but not both. The logic is that any line which begins with
+	// "\" will be handled as a meta-command, otherwise it will be handled as a
+	// query.
+	if len(line) == 0 {
+		return nil
+	} else if line[0] == byte('\\') {
+		return cmd.handleLineAsMetaCommand(line)
+	} else {
+		return cmd.handleLineAsQueryParts(line)
+	}
+}
+
+func (cmd *Command) handleLineAsMetaCommand(line string) error {
+	_, mcs, err := cmd.splitter.split(line)
+	if err != nil {
+		return errors.Wrapf(err, "splitting line")
+	}
+
+	for i := range mcs {
+		_, err := mcs[i].execute(cmd)
+		if err != nil {
+			return errors.Wrap(err, "executing meta command")
+		}
+	}
+
+	return nil
+}
+
+func (cmd *Command) handleLineAsQueryParts(line string) error {
+	qps, mcs, err := cmd.splitter.split(line)
+	if err != nil {
+		return errors.Wrapf(err, "splitting line")
+	} else if len(mcs) > 0 {
+		return errors.Errorf("--command does not support meta-commands")
+	}
+
+	// Add a termintor part to the end of []queryPart. We do this because the
+	// command is coming in from the --command flag, it may not end with a
+	// semi-colon, but we still want to execute it.
+	if len(qps) > 0 {
+		if _, ok := qps[len(qps)-1].(*partTerminator); !ok {
+			qps = append(qps, newPartTerminator())
+		}
+	}
+
+	for i := range qps {
+		if qry, err := cmd.buffer.addPart(qps[i]); err != nil {
+			return errors.Wrap(err, "adding part to buffer")
+		} else if qry != nil {
+			if err := cmd.executeAndWriteQuery(qry); err != nil {
+				return errors.Wrap(err, "executing query")
+			}
+		}
+	}
+	return nil
+}
+
+func (cmd *Command) Insert(sql string) error {
+	wqr, err := cmd.executeQuery(newRawQuery(sql))
+	if wqr.Error != "" {
+		return errors.Errorf(wqr.Error)
+	}
+	return err
+}

cli/cli_integration_test.go

@@ -0,0 +1,257 @@
+package cli_test
+
+import (
+	"bufio"
+	"context"
+	"fmt"
+	"os"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/featurebasedb/featurebase/v3/cli"
+	"github.com/featurebasedb/featurebase/v3/dax/server/test"
+	"github.com/featurebasedb/featurebase/v3/logger"
+	"github.com/stretchr/testify/require"
+)
+
+func TestCLIIntegration(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test")
+	}
+
+	ctx := context.Background()
+
+	t.Run("Stubbed Framework", func(t *testing.T) {
+		mc := test.MustRunManagedCommand(t)
+		defer mc.Close()
+
+		addr := mc.Address()
+
+		capture := newCapture(t)
+
+		comparer := newComparer(t)
+		comparer.run()
+
+		fbsql := cli.NewCommand(logger.StderrLogger)
+		fbsql.SetStdin(capture)
+		fbsql.SetStdout(comparer)
+		fbsql.SetStderr(comparer)
+
+		fbsql.Config = &cli.Config{
+			Host: addr.Host(),
+			Port: fmt.Sprintf("%d", addr.Port()),
+		}
+
+		// Run fbsql in a goroutine so we can continue to send it commands
+		// below.
+		didQuit := make(chan struct{})
+		go func() {
+			require.NoError(t, fbsql.Run(ctx))
+			close(didQuit)
+		}()
+
+		// testFiles reference files located in the cli/testdata directory. All
+		// tests should be placed there; other than adding another test file to
+		// this list, you probably shouldn't be editing this file unless you are
+		// trying to modify the way the test framework itself works.
+		testFiles := []string{
+			"setup",
+			"database",
+			"table",
+			// the tests below may be dependent on the previous tests, which do
+			// setup and some shared database and table creation.
+			"query_buffer",
+			// meta commands
+			"meta_bang",
+			"meta_cd",
+			"meta_echo",
+			"meta_describe",
+			"meta_file",
+			"meta_pset_border",
+			"meta_pset_expanded",
+			"meta_pset_format_csv",
+			"meta_pset_tuples_only",
+			"meta_include",
+			"meta_output",
+			"meta_set",
+			"meta_timing",
+			"meta_write",
+		}
+
+		for _, testFile := range testFiles {
+			t.Run(testFile, func(t *testing.T) {
+				f, err := os.Open("testdata/" + testFile)
+				require.NoError(t, err)
+
+				scanner := bufio.NewScanner(f)
+				var lineNo int
+				for scanner.Scan() {
+					line := scanner.Text()
+					lineNo++
+
+					// Empty lines and comments (//) are ignored.
+					if line == "" {
+						continue
+					} else if strings.HasPrefix(line, "//") {
+						continue
+					}
+
+					parts := strings.SplitN(line, ":", 2)
+
+					switch parts[0] {
+					case "SEND":
+						v := ""
+						if len(parts) == 2 {
+							v = parts[1]
+						}
+						capture.sendLine(v)
+					case "EXPECT":
+						v := ""
+						if len(parts) == 2 {
+							v = parts[1]
+						}
+						comparer.expectLine(v, testFile, lineNo)
+					case "EXPECTCOMP":
+						if len(parts) == 2 {
+							comps := strings.SplitN(parts[1], ":", 2)
+							v := ""
+							if len(comps) == 2 {
+								v = comps[1]
+							}
+							comparer.expectLineComp(comparator(comps[0]), v, testFile, lineNo)
+						} else {
+							t.Errorf("unexpected line: %s[%d]:%s", testFile, lineNo, line)
+						}
+					default:
+						t.Errorf("unexpected line: %s[%d]:%s", testFile, lineNo, line)
+					}
+				}
+				require.NoError(t, scanner.Err())
+			})
+		}
+
+		// End with quit to ensure that fbsql closes without error.
+		capture.sendLine(`\q`)
+
+		// Ensure fbsql quits cleanly.
+		select {
+		case <-didQuit:
+		case <-time.After(time.Second):
+			t.Fatalf("expected fbsql to quit")
+		}
+	})
+}
+
+// compare is used to compare fbsql output written to its Stdout with expected
+// lines.
+type comparer struct {
+	t       *testing.T
+	out     chan byte
+	outline chan []byte
+	exp     chan []byte
+}
+
+func newComparer(t *testing.T) *comparer {
+	return &comparer{
+		t:       t,
+		out:     make(chan byte, 1024),
+		outline: make(chan []byte, 128),
+		exp:     make(chan []byte, 1024),
+	}
+}
+
+func (c *comparer) run() {
+	// Read bytes off output, and for every line (designated by a line feed "\n"),
+	// push the line onto the outline channel.
+	go func() {
+		var line []byte
+		for {
+			b := <-c.out
+			if b == byte('\n') {
+				c.outline <- line
+				line = []byte{}
+				continue
+			}
+			line = append(line, b)
+		}
+	}()
+}
+
+type comparator string
+
+const (
+	compEquals     = "Equals"
+	compHasPrefix  = "HasPrefix"
+	compWithFormat = "WithFormat"
+)
+
+// expectLine is a convenience method which calls expectLineComp with the compEq
+// comparator and the given line.
+func (c *comparer) expectLine(line string, fileName string, lineNo int) {
+	c.expectLineComp(compEquals, line, fileName, lineNo)
+}
+
+// expectLineComp reads the next line from the outline channel and compares it
+// with the given `line`. A comparator can be provided to inform how the lines
+// should be compared (for example, the compHasPrefix comparator will just
+// compare the beginning part of the outline).
+func (c *comparer) expectLineComp(comp comparator, line string, fileName string, lineNo int) {
+	var outline []byte
+	select {
+	case outline = <-c.outline:
+	case <-time.After(10 * time.Second):
+		// TODO(tlt): this is 10 seconds to account for the fb_views creation on
+		// a local mac. This should really be something like 2 seconds. Put this
+		// back to 2 once fb_views issue is addressed.
+		c.t.Fatalf("expected output line %s[%d]: >%s<", fileName, lineNo, line)
+	}
+
+	// msg is included in any require which fails.
+	msg := []interface{}{"exp: %s[%d], got: >%s<", fileName, lineNo, outline}
+
+	switch comp {
+	case compEquals:
+		require.Equal(c.t, []byte(line), outline, msg...)
+	case compHasPrefix:
+		require.True(c.t, strings.HasPrefix(string(outline), line), msg...)
+	case compWithFormat:
+		require.True(c.t, compareByteSlices(outline, []byte(line)), msg...)
+	default:
+		c.t.Fatalf("invalid comparator: %s", comp)
+	}
+}
+
+func (c *comparer) Write(b []byte) (n int, err error) {
+	for i := range b {
+		c.out <- b[i]
+	}
+	return len(b), err
+}
+
+// compareByteSlices compares a byte slice s with another byte slice format and
+// returns true if they are the same. It will accept underscore as a
+// single-character wildcard anywhere in slice format.
+func compareByteSlices(s, format []byte) bool {
+	// Replace some helpers in format before comparing.
+	f := string(format)
+	f = strings.ReplaceAll(f, `{uuid}`, `________-____-____-____-____________`)
+	f = strings.ReplaceAll(f, `{timestamp}`, `____-__-__T__:__:__Z`)
+	format = []byte(f)
+
+	if len(s) != len(format) {
+		return false
+	}
+
+	for i := range s {
+		if format[i] == '_' {
+			continue
+		}
+		if s[i] != format[i] {
+			// log.Printf("DEBUG: characters differ: (%d): '%v' != '%v'", i, s[i], format[i])
+			return false
+		}
+	}
+
+	return true
+}

cli/cli_test.go

@@ -0,0 +1,199 @@
+package cli_test
+
+import (
+	"context"
+	"io"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	featurebase "github.com/featurebasedb/featurebase/v3"
+	"github.com/featurebasedb/featurebase/v3/cli"
+	"github.com/featurebasedb/featurebase/v3/logger"
+	"github.com/pkg/errors"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestCLI(t *testing.T) {
+	t.Run("Input", func(t *testing.T) {
+		ctx := context.Background()
+
+		capture := newCapture(t)
+
+		cli := cli.NewCommand(logger.StderrLogger)
+		cli.SetStdin(capture)
+		cli.SetStdout(capture)
+		cli.Queryer = capture
+
+		go func() {
+			assert.NoError(t, cli.Run(ctx))
+		}()
+
+		none := []string{}
+
+		// One statement, one line.
+		capture.Assert("one;", []string{"one\n"})
+
+		// One statement, multiple lines.
+		capture.Assert("one", none)
+		capture.Assert(" two ", none)
+		capture.Assert("three;", []string{"one\ntwo\nthree\n"})
+
+		// Multiple statements, one line.
+		capture.Assert("foo; bar;", []string{"foo\n", "bar\n"})
+
+		// Multiple statements, multiple lines.
+		capture.Assert("a1", none)
+		capture.Assert("a2; b1", []string{"a1\na2\n"})
+		capture.Assert("b2;", []string{"b1\nb2\n"})
+
+		// Blank lines.
+		capture.Assert("one", none)
+		capture.Assert("", none)
+		capture.Assert("three;", []string{"one\nthree\n"})
+
+		// Just a semi-colon.
+		capture.Assert(";", []string{""})
+
+		// Multi-line with just a semi-colon.
+		capture.Assert("one", none)
+		capture.Assert(";", []string{"one\n"})
+
+		// Ensure a clean exit with no errors.
+		assert.NoError(t, capture.Exit())
+	})
+}
+
+////////////////////////////////////////////////////////
+
+// Ensure type implementes interface.
+var _ io.ReadCloser = (*capture)(nil)
+var _ io.Writer = (*capture)(nil)
+var _ cli.Queryer = (*capture)(nil)
+
+// capture implements the various CLI interfaces in order to capture test input
+// and submit it as though that input were being read from the command line. It
+// also captures calls made to the Queryer.Query method and ensures the sql they
+// contain is expected.
+type capture struct {
+	t *testing.T
+
+	// ch is a channel of strings (one line at a time) of CLI input.
+	ch chan string
+
+	mu   sync.RWMutex
+	sqls []string
+
+	// queryDone will receive an event any time the Query method is called and
+	// has completed. This is to tell the Assert method that it's safe to
+	// compare the sqls slice.
+	queryDone chan struct{}
+
+	asserting chan struct{}
+
+	err error
+}
+
+func newCapture(t *testing.T) *capture {
+	return &capture{
+		t:         t,
+		ch:        make(chan string),
+		sqls:      make([]string, 0),
+		queryDone: make(chan struct{}),
+	}
+}
+
+func (c *capture) Exit() error {
+	c.sendLine(`\q`)
+	c.mu.RLock()
+	defer c.mu.RUnlock()
+	return c.err
+}
+
+func (c *capture) Assert(in string, out []string) {
+	c.asserting = make(chan struct{})
+
+	c.sendLine(in)
+
+	// Wait for the CLI command to complete processing the input and send the
+	// sql to Query() by blocking on the queryDone channel. Because Query gets
+	// called for every sql statement in the input, an input resulting in
+	// multiple sql statements needs to wait for all expected queries to
+	// complete. A timeout is included to this so it doesn't deadlock in the
+	// case where Query is expected to be called, but isn't; after the timeout,
+	// the test should fail completely. In summary: we wait on queryDone the
+	// number of sql statements we expect. If we receive fewer than expected,
+	// the timeout will occur. If we receive more than expected, the Query()
+	// method will effectively deadlock, reach its own timout, then write to
+	// capture.err, which will be reported upon Exit().
+	for range out {
+		select {
+		case <-c.queryDone:
+		case <-time.After(2 * time.Second):
+			c.t.Fatalf("expected Query() to be called")
+		}
+	}
+
+	close(c.asserting)
+
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	assert.Equal(c.t, out, c.sqls)
+
+	// Reset the slice.
+	c.sqls = c.sqls[:0]
+
+}
+
+// sendLine sends the given string as a line input to the CLI command. It
+// appends a line feed to the end of string in order to mimic the user hitting
+// the return key.
+func (c *capture) sendLine(s string) {
+	// Add a line feed before putting s on the channel in order to mimic the
+	// user hitting the return key.
+	c.ch <- s + "\n"
+}
+
+// Read is read by the CLI in place of user input. It effectively sends lines of
+// input to the CLI, getting each line to be sent off the channel.
+func (c *capture) Read(b []byte) (n int, err error) {
+	s := <-c.ch
+	return strings.NewReader(s).Read(b)
+}
+
+func (c *capture) Close() error {
+	close(c.ch)
+	return nil
+}
+
+// Write is called with anything written to output. This would included results
+// from calling Query() under normal, non-testing conditions, as well as other
+// informational text sent to output, such as the splash message.
+func (c *capture) Write(b []byte) (n int, err error) {
+	return 0, nil
+}
+
+// Query is called by the CLI command once a full SQL statement is received
+// (signified by the terminator: `;`).
+func (c *capture) Query(org string, db string, sql io.Reader) (*featurebase.WireQueryResponse, error) {
+	tmpBuf := new(strings.Builder)
+	_, err := io.Copy(tmpBuf, sql)
+	if err != nil {
+		return nil, err
+	}
+
+	c.mu.Lock()
+	c.sqls = append(c.sqls, tmpBuf.String())
+	c.mu.Unlock()
+
+	select {
+	case c.queryDone <- struct{}{}:
+	case <-c.asserting:
+		c.mu.Lock()
+		c.err = errors.Errorf("unexpected query: %s", sql)
+		c.mu.Unlock()
+	}
+	return &featurebase.WireQueryResponse{}, nil
+}

cli/config.go

@@ -0,0 +1,31 @@
+package cli
+
+// Config represents the configuration for the command.
+type Config struct {
+	Host string `json:"host"`
+	Port string `json:"port"`
+
+	OrganizationID string `json:"org-id"`
+	Database       string `json:"db"`
+
+	// CloudAuth
+	CloudAuth CloudAuthConfig `json:"cloud-auth"`
+
+	// Kafka
+	KafkaConfig string `json:"kafka-config"`
+
+	HistoryPath string `json:"history-path"`
+
+	// CSV (Comma-Separated Values) table output mode.
+	CSV bool `json:"csv"`
+
+	// PSet takes one or more pset arguments of the form: `--pset=VAR[=ARG]`.
+	PSets []string `json:"pset"`
+}
+
+type CloudAuthConfig struct {
+	ClientID string `json:"client-id"`
+	Region   string `json:"region"`
+	Email    string `json:"email"`
+	Password string `json:"password"`
+}

cli/errors.go

@@ -0,0 +1,16 @@
+package cli
+
+import (
+	"github.com/featurebasedb/featurebase/v3/errors"
+)
+
+const (
+	ErrOrganizationRequired errors.Code = "OrganizationRequired"
+)
+
+func NewErrOrganizationRequired() error {
+	return errors.New(
+		ErrOrganizationRequired,
+		"organization required",
+	)
+}

cli/fbcloud/auth.go

@@ -23,11 +23,11 @@ type cognitoParameters struct {
 type cognitoAuthRequest struct {
 	AuthParameters cognitoParameters `json:"AuthParameters"`
 	AuthFlow       string            `json:"AuthFlow"`
-	AppClientId    string            `json:"ClientId"`
+	AppClientID    string            `json:"ClientId"`
 }
 
 type cognitoAuthResult struct {
-	IdToken string `json:"IdToken"`
+	IDToken string `json:"IdToken"`
 }
 
 type cognitoAuthResponse struct {
@@ -41,7 +41,7 @@ func authenticate(clientID, region, email, password string) (string, error) {
 			Password: password,
 		},
 		AuthFlow:    authFlow,
-		AppClientId: clientID,
+		AppClientID: clientID,
 	}
 
 	data, err := json.Marshal(authPayload)
@@ -75,5 +75,5 @@ func authenticate(clientID, region, email, password string) (string, error) {
 		return "", errors.Wrap(err, "decoding cognito auth response")
 	}
 
-	return auth.Result.IdToken, nil
+	return auth.Result.IDToken, nil
 }

cli/fbcloud/client.go

@@ -1,7 +1,6 @@
 package fbcloud
 
 import (
-	"bytes"
 	"encoding/json"
 	"fmt"
 	"io"
@@ -37,40 +36,29 @@ func (cq *Queryer) tokenRefresh() error {
 	}
 	cq.token = token
 	cq.lastRefresh = time.Now()
-	fmt.Println("refreshed auth token")
 	return nil
 }
 
-type tokenizedSQL struct {
-	Language  string `json:"language"`
-	Statement string `json:"statement"`
-}
-
 // Query issues a SQL query formatted for the FeatureBase cloud query endpoint.
-func (cq *Queryer) Query(org, db, sql string) (*featurebase.WireQueryResponse, error) {
+func (cq *Queryer) Query(org string, db string, sql io.Reader) (*featurebase.WireQueryResponse, error) {
 	if time.Since(cq.lastRefresh) > TokenRefreshTimeout {
 		if err := cq.tokenRefresh(); err != nil {
 			return nil, errors.Wrap(err, "refreshing token")
 		}
 	}
-	url := fmt.Sprintf("%s/v2/databases/%s/query", cq.Host, db)
-
-	sqlReq := &tokenizedSQL{
-		Language:  "sql",
-		Statement: sql,
-	}
-	var buf bytes.Buffer
-	if err := json.NewEncoder(&buf).Encode(sqlReq); err != nil {
-		return nil, errors.Wrapf(err, "encoding sql request: %s", sql)
+	url := fmt.Sprintf("%s/databases/%s/sql", cq.Host, db)
+	if db == "" {
+		url = fmt.Sprintf("%s/sql", cq.Host)
 	}
 
 	client := &http.Client{
 		Timeout: time.Second * 30,
 	}
-	req, err := http.NewRequest(http.MethodPost, url, &buf)
+	req, err := http.NewRequest(http.MethodPost, url, sql)
 	if err != nil {
 		return nil, errors.Wrap(err, "creating new post request")
 	}
+	req.Header.Add("Content-Type", "text/plain")
 	req.Header.Add("Authorization", cq.token)
 
 	var resp *http.Response
@@ -85,11 +73,11 @@ func (cq *Queryer) Query(org, db, sql string) (*featurebase.WireQueryResponse, e
 	if resp.StatusCode/100 != 2 {
 		return nil, errors.Errorf("unexpected status: %s, full body: '%s'", resp.Status, fullbod)
 	}
-	var cloudResp cloudResponse
-	if err := json.Unmarshal(fullbod, &cloudResp); err != nil {
+
+	var sqlResponse featurebase.WireQueryResponse
+	if err := json.Unmarshal(fullbod, &sqlResponse); err != nil {
 		return nil, errors.Wrapf(err, "decoding cloud response, body:\n%s", fullbod)
 	}
-	sqlResponse := cloudResp.Results
 	return &sqlResponse, nil
 }
 
@@ -141,7 +129,3 @@ func (cq *Queryer) HTTPRequest(method, path, body string, v interface{}) ([]byte
 
 	return bodbytes, nil
 }
-
-type cloudResponse struct {
-	Results featurebase.WireQueryResponse `json:"results"`
-}

cli/kafka.go

@@ -0,0 +1,70 @@
+package cli
+
+import (
+	"fmt"
+
+	"github.com/featurebasedb/featurebase/v3/cli/batch"
+	"github.com/featurebasedb/featurebase/v3/cli/kafka"
+	"github.com/featurebasedb/featurebase/v3/errors"
+	"github.com/spf13/viper"
+)
+
+func (cmd *Command) newKafkaRunner(cfgFile string) (*kafka.Runner, error) {
+	// Read the kafka config file.
+	v := viper.New()
+	v.SetConfigFile(cfgFile)
+	v.SetConfigType("toml")
+	err := v.ReadInConfig()
+	if err != nil {
+		return nil, fmt.Errorf("error reading configuration file '%s': %v", cfgFile, err)
+	}
+
+	cfg := kafka.Config{}
+	if err := v.Unmarshal(&cfg); err != nil {
+		return nil, errors.Wrap(err, "unmarshalling config")
+	}
+
+	if err := kafka.ValidateConfig(cfg); err != nil {
+		return nil, errors.Wrap(err, "validating config")
+	}
+
+	// Create a new config with defaults.
+
+	// Look up fields based on table provided in the config.
+	wqr, err := cmd.executeQuery(newRawQuery("SHOW COLUMNS FROM " + cfg.Table))
+	if err != nil {
+		return nil, errors.Wrap(err, "executing query")
+	}
+
+	scr, err := wqr.ShowColumnsResponse()
+	if err != nil {
+		return nil, errors.Wrap(err, "getting show columns from wire query response")
+	}
+
+	// If no fields were provided in the config, use the fields defined on the
+	// table and assume a 1-to-1 mapping of source to destination.
+	if len(cfg.Fields) == 0 {
+		cfg.Fields = kafka.FieldsToConfig(scr.Fields)
+	} else {
+		cfg.Fields, err = kafka.CheckFieldCompatibility(cfg.Fields, scr)
+		if err != nil {
+			return nil, errors.Wrap(err, "validating config fields")
+		}
+	}
+
+	idkCfg, err := kafka.ConvertConfig(cfg)
+	if err != nil {
+		return nil, errors.Wrap(err, "cleaning config")
+	}
+
+	flds, err := kafka.ConfigToFields(cfg)
+	if err != nil {
+		return nil, errors.Wrap(err, "getting fields from config")
+	}
+
+	return kafka.NewRunner(
+		idkCfg,
+		batch.NewSQLBatcher(cmd, flds),
+		cmd.stderr,
+	), nil
+}

cli/kafka/config.go

@@ -0,0 +1,251 @@
+package kafka
+
+import (
+	"fmt"
+	"time"
+
+	featurebase "github.com/featurebasedb/featurebase/v3"
+	"github.com/featurebasedb/featurebase/v3/dax"
+	"github.com/featurebasedb/featurebase/v3/idk"
+	"github.com/pkg/errors"
+)
+
+// Config is the user-facing configuration for kafka support in the CLI. This is
+// unmarshalled from the the toml config file supplied by the user.
+type Config struct {
+	Hosts  []string `mapstructure:"hosts" help:"Kafka hosts."`
+	Group  string   `mapstructure:"group" help:"Kafka group."`
+	Topics []string `mapstructure:"topics" help:"Kafka topics to read from."`
+
+	BatchSize         int           `mapstructure:"batch-size" help:"Batch size."`
+	BatchMaxStaleness time.Duration `mapstructure:"batch-max-staleness" help:"Maximum length of time that the oldest record in a batch can exist before flushing the batch. Note that this can potentially stack with timeouts waiting for the source."`
+	Timeout           time.Duration `mapstructure:"timeout" help:"Time to wait for more records from Kafka before flushing a batch. 0 to disable."`
+
+	Table  string  `mapstructure:"table" help:"Destination table name."`
+	Fields []Field `mapstructure:"fields"`
+}
+
+// Field is a user-facing configuration field.
+type Field struct {
+	Name       string   `mapstructure:"name"`
+	SourceType string   `mapstructure:"source-type"`
+	SourcePath []string `mapstructure:"source-path"`
+	PrimaryKey bool     `mapstructure:"primary-key"`
+}
+
+// ConfigForIDK represents Config converted to values suitable for IDK. In
+// particular, the idk.RawField is used in parsing the schema in IDK.
+type ConfigForIDK struct {
+	Hosts  []string
+	Group  string
+	Topics []string
+
+	BatchSize         int
+	BatchMaxStaleness time.Duration
+	Timeout           time.Duration
+
+	Table   string
+	IDField string
+	Fields  []idk.RawField
+}
+
+// ValidateConfig validates the config is usable.
+func ValidateConfig(c Config) error {
+	if c.Table == "" {
+		return errors.Errorf("table is required")
+	} else if len(c.Topics) == 0 {
+		return errors.Errorf("at least one topic is required")
+	} else if len(c.Fields) > 0 {
+		// We only need to do these checks if any fields are specified at all.
+		// If no fields are specified, that's ok because then we default to
+		// using fields based off the existing table.
+		if len(c.Fields) < 2 {
+			return errors.Errorf("at least two fields are required (one should be a primary key)")
+		} else {
+			var found int
+			for i := range c.Fields {
+				if c.Fields[i].PrimaryKey {
+					found++
+				}
+				if c.Fields[i].Name == "" {
+					return errors.Errorf("a name attribute (which isn't equal to \"\") should exist for all fields")
+				}
+			}
+			if found != 1 {
+				return errors.Errorf("exactly one primary key field is required")
+			}
+		}
+	}
+	return nil
+}
+
+// ConvertConfig converts a Config to one that suitable for IDK.
+func ConvertConfig(c Config) (ConfigForIDK, error) {
+	// Set a default kafka host in case one isn't provided.
+	hosts := []string{"localhost:9092"}
+	if len(c.Hosts) > 0 {
+		hosts = c.Hosts
+	}
+
+	// Copy all the shared members from Config to ConfigForIDK.
+	out := ConfigForIDK{
+		Hosts:             hosts,
+		Group:             c.Group,
+		Topics:            c.Topics,
+		BatchSize:         c.BatchSize,
+		BatchMaxStaleness: c.BatchMaxStaleness,
+		Timeout:           c.Timeout,
+		Table:             c.Table,
+	}
+
+	if len(c.Fields) == 0 {
+		return out, errors.New("fields cannot be empty")
+	}
+
+	// rawFields wil be the same as c.Fields, but possibly enhanced.
+	rawFields := make([]idk.RawField, 0, len(c.Fields))
+
+	var foundPK bool
+	for _, fld := range c.Fields {
+		if fld.PrimaryKey {
+			out.IDField = fld.Name
+			foundPK = true
+		}
+
+		typ, quals, err := dax.SplitFieldType(fld.SourceType)
+		if err != nil {
+			return out, errors.Wrap(err, "getting base type")
+		}
+
+		rawFld := idk.RawField{
+			Name: fld.Name,
+			Type: string(typ),
+			Path: fld.SourcePath,
+		}
+		// If a SourcePath wasn't provided, default to using the field name.
+		if len(rawFld.Path) == 0 {
+			rawFld.Path = []string{fld.Name}
+		}
+
+		switch typ {
+		case dax.BaseTypeInt:
+			// We don't have to handle min/max because we don't create the table.
+		case dax.BaseTypeDecimal:
+			if len(quals) != 1 {
+				return out, errors.Errorf("expected decimal scale")
+			}
+			rawFld.Config = []byte(fmt.Sprintf(`{"scale":%d}`, quals[0]))
+		case dax.BaseTypeID:
+			rawFld.Config = []byte("{\"mutex\":true}")
+		case dax.BaseTypeIDSet:
+			rawFld.Type = "ids"
+		case dax.BaseTypeString:
+			rawFld.Config = []byte("{\"mutex\":true}")
+		case dax.BaseTypeStringSet:
+			rawFld.Type = "strings"
+		case dax.BaseTypeTimestamp:
+			// No timestamp options are handled for now.
+		}
+
+		rawFields = append(rawFields, rawFld)
+	}
+	if !foundPK {
+		return out, errors.New("primary-key not found in fields")
+	}
+
+	out.Fields = rawFields
+
+	return out, nil
+}
+
+// ConfigToFields returns a list of *dax.Field based on the IDField and Fields
+// in the Config.
+func ConfigToFields(c Config) ([]*dax.Field, error) {
+	// We don't know if a primary key will be found, so we can't set the
+	// capacity to `len(c.Fields)-1`.
+	out := make([]*dax.Field, 0, len(c.Fields))
+
+	for _, fld := range c.Fields {
+		if fld.PrimaryKey {
+			continue
+		}
+		typ, quals, err := dax.SplitFieldType(fld.SourceType)
+		if err != nil {
+			return nil, errors.Wrap(err, "splitting field type")
+		}
+		dfld := &dax.Field{
+			Name: dax.FieldName(fld.Name),
+			Type: typ,
+		}
+		switch typ {
+		case dax.BaseTypeDecimal:
+			if len(quals) != 1 {
+				return nil, errors.Errorf("expected decimal scale")
+			}
+			scale, ok := quals[0].(int64)
+			if !ok {
+				return nil, errors.Errorf("invalid decimal scale: %v", quals[0])
+			}
+			dfld.Options.Scale = scale
+		}
+		out = append(out, dfld)
+	}
+
+	return out, nil
+}
+
+// FieldsToConfig returns a Config.Fields based on a list of *dax.Field.
+func FieldsToConfig(flds []*dax.Field) []Field {
+	out := make([]Field, 0, len(flds))
+	for _, fld := range flds {
+		out = append(out, Field{
+			Name:       string(fld.Name),
+			SourceType: fld.FullType(),
+			PrimaryKey: fld.IsPrimaryKey(),
+		})
+	}
+	return out
+}
+
+// CheckFieldCompatibility ensures that the fields provided in the kafka config
+// are compatible with the fields in the existing table. It returns a copy of
+// the kafka config fields with empty values defaulted to the table field
+// configuration.
+func CheckFieldCompatibility(cflds []Field, scr *featurebase.ShowColumnsResponse) ([]Field, error) {
+	out := make([]Field, len(cflds))
+	for i, cfld := range cflds {
+		out[i] = cfld
+		cfldName := dax.FieldName(cfld.Name)
+
+		// Primary key field.
+		if cfld.PrimaryKey {
+			f := scr.Field(dax.PrimaryKeyFieldName)
+			if f == nil {
+				return nil, dax.NewErrFieldDoesNotExist(dax.PrimaryKeyFieldName) // It should be impossible to hit this.
+			}
+			if out[i].SourceType == "" {
+				if f.StringKeys() {
+					out[i].SourceType = dax.BaseTypeString
+				} else {
+					out[i].SourceType = dax.BaseTypeID
+				}
+			}
+			continue
+		}
+
+		// Non primary key fields.
+		if cfldName == dax.PrimaryKeyFieldName {
+			return nil, errors.Errorf("field named '%s' must be a primary key", dax.PrimaryKeyFieldName)
+		}
+
+		f := scr.Field(cfldName)
+		if f == nil {
+			return nil, dax.NewErrFieldDoesNotExist(cfldName)
+		}
+
+		if out[i].SourceType == "" {
+			out[i].SourceType = f.FullType()
+		}
+	}
+	return out, nil
+}

cli/kafka/runner.go

@@ -0,0 +1,67 @@
+package kafka
+
+import (
+	"io"
+	"time"
+
+	fbbatch "github.com/featurebasedb/featurebase/v3/batch"
+	"github.com/featurebasedb/featurebase/v3/errors"
+	"github.com/featurebasedb/featurebase/v3/idk"
+	"github.com/featurebasedb/featurebase/v3/idk/kafka_static"
+	"github.com/featurebasedb/featurebase/v3/logger"
+)
+
+// Runner is a CLI-specific kafka consumer. It's similar to
+// idk.kafka_static.Main in that it embeds idk.Main and contains additional
+// functionality specific to its use case.
+type Runner struct {
+	idk.Main   `flag:"!embed"`
+	KafkaHosts []string       `help:"Comma separated list of host:port pairs for Kafka."`
+	Group      string         `help:"Kafka group."`
+	Topics     []string       `help:"Kafka topics to read from."`
+	Timeout    time.Duration  `help:"Time to wait for more records from Kafka before flushing a batch. 0 to disable."`
+	Header     []idk.RawField `help:"Header configuration."`
+}
+
+func NewRunner(cfg ConfigForIDK, batcher fbbatch.Batcher, logWriter io.Writer) *Runner {
+	idkMain := idk.NewMain()
+	idkMain.IDField = cfg.IDField
+	idkMain.Index = cfg.Table
+	idkMain.Batcher = batcher
+	idkMain.BatchSize = cfg.BatchSize
+	idkMain.BatchMaxStaleness = cfg.BatchMaxStaleness
+	idkMain.SetBasic()
+	idkMain.SetLog(logger.NewStandardLogger(logWriter))
+
+	kr := &Runner{
+		Main:       *idkMain,
+		KafkaHosts: cfg.Hosts,
+		Group:      cfg.Group,
+		Topics:     cfg.Topics,
+		Header:     cfg.Fields,
+		Timeout:    cfg.Timeout,
+	}
+	kr.OffsetMode = true
+	kr.Main.Namespace = "cli_kafka_runner"
+	kr.Main.Pprof = "" // don't initialize pprof until we actually use it in tests
+	kr.NewSource = func() (idk.Source, error) {
+		source := kafka_static.NewSource()
+		source.Hosts = kr.KafkaHosts
+		source.Group = kr.Group
+		source.Topics = kr.Topics
+		source.Log = kr.Main.Log()
+		// source.TLS = m.KafkaTLS
+		source.Timeout = kr.Timeout
+		// source.SkipOld = m.SkipOld
+		source.HeaderFields = kr.Header
+		// source.S3Region = m.S3Region
+		// source.AllowMissingFields = m.AllowMissingFields
+
+		err := source.Open()
+		if err != nil {
+			return nil, errors.Wrap(err, "opening source")
+		}
+		return source, nil
+	}
+	return kr
+}

cli/parts.go

@@ -0,0 +1,136 @@
+package cli
+
+import (
+	"fmt"
+	"io"
+	"os"
+	"strings"
+)
+
+// query is a collection of queryParts which, when applied together, make up an
+// executable SQL query.
+type query []queryPart
+
+func (q query) String() string {
+	var sb strings.Builder
+	for i := range q {
+		sb.WriteString(q[i].String())
+		if i < len(q)-1 {
+			sb.WriteRune('\n')
+		}
+	}
+	return sb.String()
+}
+
+// Reader returns the query as an io.Reader so that it can be passed to, for
+// example, http.Post().
+func (q query) Reader() io.Reader {
+	readers := make([]io.Reader, 0, len(q))
+	for i := range q {
+		readers = append(readers, q[i].Reader())
+	}
+	return io.MultiReader(readers...)
+}
+
+// queryPart is an interface representing anything which can use to build up a
+// query.
+type queryPart interface {
+	fmt.Stringer
+	Reader() io.Reader
+}
+
+func newRawQuery(s string) query {
+	return []queryPart{
+		newPartRaw(s),
+	}
+}
+
+// ////////////////////////////////////////////////////////////////////////////
+// raw
+// ////////////////////////////////////////////////////////////////////////////
+
+// Ensure type implements interface.
+var _ queryPart = (*partRaw)(nil)
+
+type partRaw struct {
+	raw string
+}
+
+func newPartRaw(s string) *partRaw {
+	return &partRaw{
+		raw: s,
+	}
+}
+
+func (p *partRaw) Reader() io.Reader {
+	return strings.NewReader(p.raw + "\n")
+}
+
+func (p *partRaw) String() string {
+	return p.raw
+}
+
+// ////////////////////////////////////////////////////////////////////////////
+// file
+// ////////////////////////////////////////////////////////////////////////////
+
+// Ensure type implements interface.
+var _ queryPart = (*partFile)(nil)
+
+type partFile struct {
+	file *os.File
+}
+
+func newPartFile(f *os.File) *partFile {
+	return &partFile{
+		file: f,
+	}
+}
+
+func (p *partFile) Reader() io.Reader {
+	return p.file
+}
+
+func (p *partFile) String() string {
+	return fmt.Sprintf("[file: %s]", p.file.Name())
+}
+
+// ////////////////////////////////////////////////////////////////////////////
+// batch file
+// ////////////////////////////////////////////////////////////////////////////
+
+// Ensure type implements interface.
+var _ queryPart = (*partBatchFile)(nil)
+
+type partBatchFile struct {
+	file *os.File
+}
+
+func (p *partBatchFile) Reader() io.Reader {
+	return p.file
+}
+
+func (p *partBatchFile) String() string {
+	return p.file.Name()
+}
+
+// ////////////////////////////////////////////////////////////////////////////
+// terminator (i.e. ";")
+// ////////////////////////////////////////////////////////////////////////////
+
+// Ensure type implements interface.
+var _ queryPart = (*partTerminator)(nil)
+
+type partTerminator struct{}
+
+func newPartTerminator() *partTerminator {
+	return &partTerminator{}
+}
+
+func (p *partTerminator) Reader() io.Reader {
+	return nil
+}
+
+func (p *partTerminator) String() string {
+	return terminationChar
+}

cli/queryer.go

@@ -0,0 +1,108 @@
+package cli
+
+import (
+	"fmt"
+	"io"
+	"net/http"
+	"time"
+
+	featurebase "github.com/featurebasedb/featurebase/v3"
+	"github.com/pkg/errors"
+)
+
+type Queryer interface {
+	Query(org string, db string, sql io.Reader) (*featurebase.WireQueryResponse, error)
+}
+
+// Ensure type implements interface.
+var _ Queryer = (*nopQueryer)(nil)
+
+type nopQueryer struct{}
+
+func (qryr *nopQueryer) Query(org string, db string, sql io.Reader) (*featurebase.WireQueryResponse, error) {
+	return nil, errors.Errorf("no-op queryer")
+}
+
+// Ensure type implements interface.
+var _ Queryer = (*standardQueryer)(nil)
+
+// standardQueryer supports a standard featurebase deployment hitting the /sql
+// endpoint with a payload containing only the sql statement.
+type standardQueryer struct {
+	Host string
+	Port string
+}
+
+func (qryr *standardQueryer) Query(org string, db string, sql io.Reader) (*featurebase.WireQueryResponse, error) {
+	url := fmt.Sprintf("%s/sql", hostPort(qryr.Host, qryr.Port))
+
+	resp, err := http.Post(url, "application/json", sql)
+	if err != nil {
+		return nil, errors.Wrapf(err, "posting query")
+	}
+
+	fullbod, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return nil, errors.Wrap(err, "reading response")
+	}
+
+	sqlResponse := &featurebase.WireQueryResponse{}
+	// TODO(tlt): switch this back once all responses are typed
+	// TODO(twg) 2023/03/01 using json.Number to decode large ints so care must be made
+	// if err := json.Unmarshal(fullbod, sqlResponse); err != nil {
+	if err := sqlResponse.UnmarshalJSONTyped(fullbod, true); err != nil {
+		return nil, errors.Wrapf(err, "unmarshaling query response, body:\n'%s'\n", fullbod)
+	}
+
+	return sqlResponse, nil
+}
+
+// Ensure type implements interface.
+var _ Queryer = (*serverlessQueryer)(nil)
+
+// serverlessQueryer is similar to the standardQueryer except that it hits a
+// different endpoint, and its payload is database-aware.
+type serverlessQueryer struct {
+	Host string
+	Port string
+}
+
+func (qryr *serverlessQueryer) Query(org string, db string, sql io.Reader) (*featurebase.WireQueryResponse, error) {
+	if org == "" {
+		return nil, NewErrOrganizationRequired()
+	}
+
+	url := fmt.Sprintf("%s/queryer/databases/%s/sql", hostPort(qryr.Host, qryr.Port), db)
+	if db == "" {
+		url = fmt.Sprintf("%s/queryer/sql", hostPort(qryr.Host, qryr.Port))
+	}
+
+	client := &http.Client{
+		Timeout: time.Second * 30,
+	}
+
+	req, err := http.NewRequest(http.MethodPost, url, sql)
+	if err != nil {
+		return nil, errors.Wrap(err, "creating new post request")
+	}
+	req.Header.Add("Content-Type", "text/plain")
+	req.Header.Add("OrganizationID", org)
+
+	var resp *http.Response
+	if resp, err = client.Do(req); err != nil {
+		return nil, errors.Wrap(err, "executing post request")
+	}
+
+	fullbod, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return nil, errors.Wrap(err, "reading response")
+	}
+	sqlResponse := &featurebase.WireQueryResponse{}
+	// TODO(tlt): switch this back once all responses are typed
+	// if err := json.Unmarshal(fullbod, sqlResponse); err != nil {
+	if err := sqlResponse.UnmarshalJSONTyped(fullbod, true); err != nil {
+		return nil, errors.Wrapf(err, "unmarshaling query response, body:\n'%s'\n", fullbod)
+	}
+
+	return sqlResponse, nil
+}

cli/replacer.go

@@ -0,0 +1,109 @@
+package cli
+
+import (
+	"strings"
+
+	"github.com/benhoyt/goawk/lexer"
+)
+
+// replacer can replace parts of a string based on some rules and the provided
+// map[string]string. For example, the Command can replace strings with values
+// in its `variables` map.
+type replacer struct {
+	m map[string]string
+}
+
+func newReplacer(m map[string]string) *replacer {
+	return &replacer{
+		m: m,
+	}
+}
+
+// replace replaces all instances of the string pattern `:key` with the value at
+// m[key]. For example we want something like this:
+//
+// GIVEN: `start :one,:'two', :"three" ::four ::`
+//
+// with map
+//
+//	map[string]string{
+//		"one":   "repl1",
+//		"three": "repl3",
+//	}
+//
+// WANT: `start repl1,:'two', "repl3" ::four ::`
+func (r *replacer) replace(s string) string {
+	// If no variables have been added to the map, there's no need to parse the
+	// string for variable replacement.
+	if len(r.m) == 0 {
+		return s
+	}
+
+	line := []byte(s)
+	lex := lexer.NewLexer(line)
+
+	// finger contains the index into line at the start of non-variable text
+	// that we want to include, as-is in the output.
+	var finger int
+
+	// sb builds the string which will be the final output.
+	var sb strings.Builder
+	for {
+		pos, tok, _ := lex.Scan()
+
+		switch tok {
+		case lexer.COLON:
+			// last is the last normal character position before the colon.
+			last := pos.Column - 1
+
+			// Get the next byte to see if the colon value is quoted, and if so,
+			// whether its has single or double quotes.
+			b := lex.PeekByte()
+
+			// padding is the amount of padding we have to consider around the
+			// variable name. If the variable is not quoted, it doesn't require
+			// any padding. But if it has quotes, it needs 2 characters of
+			// paddings to accomodate the quotes.
+			padding := 0
+
+			// quote holds the character to use to quote the final, replaced
+			// output value. Because the lexer doesn't tell us how a certain
+			// `string` token was quoted, we need to keep track of that here so
+			// we can put them back.
+			quote := ""
+			switch b {
+			case byte('\''): // single quote
+				quote = `'`
+				padding = 2
+			case byte('"'): // double quote
+				quote = `"`
+				padding = 2
+			}
+
+			pos, tok, key := lex.Scan()
+			switch tok {
+			case lexer.NAME, lexer.STRING:
+				// Write the normal text up to the variable replacement
+				// position.
+				sb.Write(line[finger:last])
+
+				if v, ok := r.m[key]; ok {
+					// Write replaced variable with the quotes it had.
+					sb.WriteString(quote + v + quote)
+				} else {
+					// Since the variable was not found in the map, just write
+					// back what was already there.
+					sb.WriteString(":" + quote + key + quote)
+				}
+
+				// Reset finger to point to the next position after the
+				// variable.
+				finger = pos.Column + len(key) + padding - 1
+			}
+		case lexer.EOF:
+			// Write the remainder of the string and return.
+			sb.Write(line[finger:])
+			return sb.String()
+		}
+	}
+}

cli/replacer_test.go

@@ -0,0 +1,109 @@
+package cli
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestReplacer(t *testing.T) {
+	t.Run("general replace function", func(t *testing.T) {
+
+		m := map[string]string{
+			"v1": "newVone",
+			"v2": "newVtwo",
+		}
+
+		tests := []struct {
+			s   string
+			m   map[string]string
+			exp string
+		}{
+			{
+				// no variables present
+				s:   "foo",
+				m:   m,
+				exp: "foo",
+			},
+			{
+				// variable prefix, but not in map
+				s:   ":foo",
+				m:   m,
+				exp: ":foo",
+			},
+			{
+				// variable name match, but missing prefix
+				s:   "v1",
+				m:   m,
+				exp: "v1",
+			},
+			{
+				// variable name match
+				s:   ":v1",
+				m:   m,
+				exp: "newVone",
+			},
+			{
+				// two variables, the same, no space
+				s:   ":v1:v1",
+				m:   m,
+				exp: "newVonenewVone",
+			},
+			{
+				// two variables, different, no space
+				s:   ":v1:v2",
+				m:   m,
+				exp: "newVonenewVtwo",
+			},
+			{
+				// two variables, different, spaces
+				s:   ":v1  :v2",
+				m:   m,
+				exp: "newVone  newVtwo",
+			},
+			{
+				// one variable, one non-variable, no space
+				s:   ":v1:foo",
+				m:   m,
+				exp: "newVone:foo",
+			},
+			{
+				// one non-variable, one variable, no space
+				s:   "foo:v1",
+				m:   m,
+				exp: "foonewVone",
+			},
+			{
+				// two variables, different, comma
+				s:   ":v1, :v2",
+				m:   m,
+				exp: "newVone, newVtwo",
+			},
+			{
+				// single quotes
+				s:   ":'v1'",
+				m:   m,
+				exp: "'newVone'",
+			},
+			{
+				// double quotes
+				s:   `:"v2"`,
+				m:   m,
+				exp: `"newVtwo"`,
+			},
+			{
+				// more quotes
+				s:   `start :v1,:'two', :"v2" ::four :: `,
+				m:   m,
+				exp: `start newVone,:'two', "newVtwo" ::four :: `,
+			},
+		}
+		for i, test := range tests {
+			t.Run(fmt.Sprintf("test-%d", i), func(t *testing.T) {
+				replacer := newReplacer(test.m)
+				assert.Equal(t, test.exp, replacer.replace(test.s))
+			})
+		}
+	})
+}

cli/splitter.go

@@ -0,0 +1,133 @@
+package cli
+
+import (
+	"strings"
+
+	"github.com/pkg/errors"
+)
+
+// splitter is a line splitter which splits a line into queryParts and
+// metaCommands. It may not be necessary to have this be a separate struct since
+// it contains no members and just has the one `split()` method, but here we
+// are.
+type splitter struct {
+	replacer *replacer
+}
+
+func newSplitter(r *replacer) *splitter {
+	return &splitter{
+		replacer: r,
+	}
+}
+
+// split splits the given line into queryParts and metaCommands.
+// If a metaCommand is found, everything after that is considered either arguments to that
+// metaCommand, or additional metaCommands. In other words, queryParts can not follow
+// metaCommands in the same line.
+//
+// A line can contain any of the following patterns:
+// 1- [queryParts...]: "select * from tbl; select"
+// 2- [metaCommands...]: "\! pwd \q"
+// 3- [queryParts...][metaCommands...]: "select * from \i file.sql"
+func (s *splitter) split(line string) ([]queryPart, []metaCommand, error) {
+	// Look for a comment line.
+	if strings.HasPrefix(line, "--") {
+		return nil, nil, nil
+	}
+
+	// Look for a meta command.
+	parts := strings.SplitN(line, `\`, 2)
+
+	switch len(parts) {
+	case 1:
+		// slice of queryParts (pattern 1)
+		if qps, err := s.splitQueryParts(strings.TrimSpace(parts[0])); err != nil {
+			return nil, nil, errors.Wrap(err, "splitting query parts")
+		} else {
+			return qps, nil, nil
+		}
+	case 2:
+		// slice of parts + slice of meta commands (pattern 3)
+		// or
+		// slice of meta commands (pattern 2)
+		qps, err := s.splitQueryParts(strings.TrimSpace(parts[0]))
+		if err != nil {
+			return nil, nil, errors.Wrap(err, "splitting query parts")
+		}
+
+		mcs, err := s.splitMetaCommands(strings.TrimSpace(parts[1]))
+		if err != nil {
+			return nil, nil, errors.Wrap(err, "splitting meta commands")
+		}
+
+		return qps, mcs, nil
+	}
+
+	return nil, nil, nil
+}
+
+func (s *splitter) splitQueryParts(line string) ([]queryPart, error) {
+	if line == "" {
+		return nil, nil
+	}
+
+	// Look for a termination character;
+	parts := strings.Split(line, terminationChar)
+
+	// Do variable replacement.
+	for i := range parts {
+		parts[i] = s.replacer.replace(parts[i])
+	}
+
+	if len(parts) == 1 {
+		part0 := strings.TrimSpace(parts[0])
+		return []queryPart{
+			newPartRaw(part0),
+		}, nil
+	}
+
+	qps := make([]queryPart, 0)
+	for i := range parts {
+		part := strings.TrimSpace(parts[i])
+		if part == "" {
+			// If the line starts with a ";", treat it as a terminator for a
+			// previous line.
+			if i == 0 {
+				qps = append(qps, &partTerminator{})
+			}
+			continue
+		}
+		qps = append(qps, newPartRaw(part))
+		if i < len(parts)-1 {
+			qps = append(qps, &partTerminator{})
+		}
+	}
+
+	return qps, nil
+}
+
+func (s *splitter) splitMetaCommands(in string) ([]metaCommand, error) {
+	parts := strings.Split(in, `\`)
+	if len(parts) == 1 {
+		mc, err := splitMetaCommand(parts[0], s.replacer)
+		if err != nil {
+			return nil, errors.Wrapf(err, "splitting meta command: %s", parts[0])
+		}
+		return []metaCommand{mc}, nil
+	}
+
+	mcs := make([]metaCommand, 0)
+	for i := range parts {
+		part := strings.TrimSpace(parts[i])
+		if part == "" {
+			continue
+		}
+		mc, err := splitMetaCommand(part, s.replacer)
+		if err != nil {
+			return nil, errors.Wrapf(err, "splitting meta command: %s", part)
+		}
+		mcs = append(mcs, mc)
+	}
+
+	return mcs, nil
+}

cli/splitter_test.go

@@ -0,0 +1,146 @@
+package cli
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestSplitter(t *testing.T) {
+	s := newSplitter(newReplacer(nil))
+	t.Run("Split", func(t *testing.T) {
+		tests := []struct {
+			line            string
+			expQueryParts   []queryPart
+			expMetaCommands []metaCommand
+			expError        string
+		}{
+			{
+				line: `foo`,
+				expQueryParts: []queryPart{
+					newPartRaw("foo"),
+				},
+			},
+			{
+				line: `foo;`,
+				expQueryParts: []queryPart{
+					newPartRaw("foo"),
+					newPartTerminator(),
+				},
+			},
+			{
+				line: `foo; `,
+				expQueryParts: []queryPart{
+					newPartRaw("foo"),
+					newPartTerminator(),
+				},
+			},
+			{
+				line: `foo; ; `,
+				expQueryParts: []queryPart{
+					newPartRaw("foo"),
+					newPartTerminator(),
+				},
+			},
+			{
+				line: `foo; bar`,
+				expQueryParts: []queryPart{
+					newPartRaw("foo"),
+					newPartTerminator(),
+					newPartRaw("bar"),
+				},
+			},
+			{
+				line: `foo; bar;`,
+				expQueryParts: []queryPart{
+					newPartRaw("foo"),
+					newPartTerminator(),
+					newPartRaw("bar"),
+					newPartTerminator(),
+				},
+			},
+			{
+				line: `\q`,
+				expMetaCommands: []metaCommand{
+					&metaQuit{},
+				},
+			},
+			{
+				line: ` \p`,
+				expMetaCommands: []metaCommand{
+					&metaPrint{},
+				},
+			},
+			{
+				line: `\q \p`,
+				expMetaCommands: []metaCommand{
+					&metaQuit{},
+					&metaPrint{},
+				},
+			},
+			{
+				line: `\q \p arg1 arg2`,
+				expMetaCommands: []metaCommand{
+					&metaQuit{},
+					&metaPrint{},
+				},
+			},
+			{
+				line: `\set`,
+				expMetaCommands: []metaCommand{
+					&metaSet{
+						args: []string{},
+					},
+				},
+			},
+			{
+				line: `\set arg1 arg2`,
+				expMetaCommands: []metaCommand{
+					&metaSet{
+						args: []string{"arg1", "arg2"},
+					},
+				},
+			},
+			{
+				line: `\set 'arg1' 'arg2'`,
+				expMetaCommands: []metaCommand{
+					&metaSet{
+						args: []string{"arg1", "arg2"},
+					},
+				},
+			},
+			{
+				line: `\set 'arg1' '"arg2"'`,
+				expMetaCommands: []metaCommand{
+					&metaSet{
+						args: []string{"arg1", "\"arg2\""},
+					},
+				},
+			},
+			{
+				line:     `\`,
+				expError: "unsupported meta-command:",
+			},
+			{
+				line:     `\xyzxyz`,
+				expError: "unsupported meta-command:",
+			},
+		}
+		for i, tt := range tests {
+			t.Run(fmt.Sprintf("test-%d-%s", i, tt.line), func(t *testing.T) {
+				qps, mcs, err := s.split(tt.line)
+				if tt.expError != "" {
+					if assert.Error(t, err) {
+						assert.Contains(t, err.Error(), tt.expError)
+					}
+					return
+				}
+
+				assert.NoError(t, err)
+				assert.ElementsMatch(t, tt.expQueryParts, qps)
+				assert.ElementsMatch(t, tt.expMetaCommands, mcs)
+			})
+		}
+	})
+}

cli/testdata/database

@@ -0,0 +1,53 @@
+// Show databases now that we have set org.
+SEND:SHOW DATABASES;
+EXPECT:+-----+------+-------+------------+------------+------------+-------+-------------+
+EXPECT:| _id | name | owner | updated_by | created_at | updated_at | units | description |
+EXPECT:+-----+------+-------+------------+------------+------------+-------+-------------+
+EXPECT:+-----+------+-------+------------+------------+------------+-------+-------------+
+EXPECT:
+
+// Create db1.
+SEND:CREATE DATABASE db1 WITH UNITS 1;
+EXPECT:
+
+// List databases via SHOW DATABASES.
+SEND:SHOW DATABASES;
+EXPECT:+--------------------------------------+------+-------+------------+----------------------+----------------------+-------+-------------+
+EXPECT:| _id                                  | name | owner | updated_by | created_at           | updated_at           | units | description |
+EXPECT:+--------------------------------------+------+-------+------------+----------------------+----------------------+-------+-------------+
+EXPECTCOMP:WithFormat:| {uuid} | db1  |       |            | {timestamp} | {timestamp} |     1 |             |
+EXPECT:+--------------------------------------+------+-------+------------+----------------------+----------------------+-------+-------------+
+EXPECT:
+
+// List databases via SHOW DATABASES.
+SEND:\l
+EXPECT:+--------------------------------------+------+-------+------------+----------------------+----------------------+-------+-------------+
+EXPECT:| _id                                  | name | owner | updated_by | created_at           | updated_at           | units | description |
+EXPECT:+--------------------------------------+------+-------+------------+----------------------+----------------------+-------+-------------+
+EXPECTCOMP:WithFormat:| {uuid} | db1  |       |            | {timestamp} | {timestamp} |     1 |             |
+EXPECT:+--------------------------------------+------+-------+------------+----------------------+----------------------+-------+-------------+
+EXPECT:
+
+// Check database connection.
+SEND:\c
+EXPECT:You are not connected to a database.
+
+// Try connecting to an invalid database.
+SEND:\c invalid
+EXPECT:executing meta command: invalid database: invalid
+
+// Try connecting with too many arguments.
+SEND:\c db1 extra
+EXPECT:executing meta command: meta command 'connect' takes zero or one argument
+
+// Connect to a database.
+SEND:\c db1
+EXPECTCOMP:WithFormat:You are now connected to database "db1" ({uuid}).
+
+// Disconnect from the current database.
+SEND:\c -
+EXPECT:You are not connected to a database.
+
+// Connect to a database again.
+SEND:\c db1
+EXPECTCOMP:WithFormat:You are now connected to database "db1" ({uuid}).

cli/testdata/famous.csv

@@ -0,0 +1,10 @@
+
+"Id", "Name", "Short description", "Gender", "Country", "Occupation", "Birth year", "Death year", "Manner of death", "Age of death"
+1, "George Washington", "1st president of the United States (1732–1799)", "Male", "United States of America; Kingdom of Great Britain", "Politician", "1732", "1799", "natural causes", "67"
+2, "Douglas Adams", "English writer and humorist", "Male", "United Kingdom", "Artist", "1952", "2001", "natural causes", "49"
+3, "Abraham Lincoln", "16th president of the United States (1809-1865)", "Male", "United States of America", "Politician", "1809", "1865", "homicide", "56"
+4, "Wolfgang Amadeus Mozart", "Austrian composer of the Classical period", "Male", "Archduchy of Austria; Archbishopric of Salzburg", "Artist", "1756", "1791", "0", "35"
+5, "Ludwig van Beethoven", "German classical and romantic composer", "Male", "Holy Roman Empire; Austrian Empire", "Artist", "1770", "1827", "0", "57"
+6, "Jean-François Champollion", "French classical scholar", "Male", "Kingdom of France; First French Empire", "Egyptologist", "1790", "1832", "natural causes", "42"
+7, "Paul Morand", "French writer", "Male", "France", "Artist", "1888", "1976", "0", "88"
+8, "Claude Monet", "French impressionist painter (1840-1926)", "Male", "France", "Artist", "1840", "1926", "natural causes", "86"

cli/testdata/meta_bang

@@ -0,0 +1,8 @@
+SEND:\! echo 'foo'
+EXPECT:foo
+
+SEND:\! echo "foo"
+EXPECT:"foo"
+
+SEND:\!
+EXPECT:executing meta command: meta command '!' requires at least one argument

cli/testdata/meta_cd

@@ -0,0 +1,17 @@
+// Make a directory so we can test \cd'ing into it.
+SEND:\! mkdir cli-test-dir
+SEND:\cd cli-test-dir
+SEND:\cd ..
+SEND:\! rmdir cli-test-dir
+
+// TODO(tlt): before we do this, we should implement the ability to execute
+// commands in a \set like:
+// \set homedir `pwd`
+// then we can store what directory we're in so we can move back to it 
+// at the end of the test
+// Switch to home directory.
+// SEND:\cd
+
+// Expect error on extra argument to \cd.
+SEND:\cd dir extra
+EXPECT:executing meta command: meta command 'cd' takes zero or one argument

cli/testdata/meta_describe

@@ -0,0 +1,33 @@
+// TODO(tlt): we can't run this test until we get the system tables under control (i.e. sorted). Currently, fb_views is in a map with users, so the following can fail 50% of the time.
+// Show tables for database by calling describe with no args.
+// SEND:\d
+// EXPECT:+-------------------------+-------------------------+-------+------------+----------------------+----------------------+-------+------------+------------------------+
+// EXPECT:| _id                     | name                    | owner | updated_by | created_at           | updated_at           | keys  | space_used | description            |
+// EXPECT:+-------------------------+-------------------------+-------+------------+----------------------+----------------------+-------+------------+------------------------+
+// EXPECTCOMP:WithFormat:| fb_veiws                | fb_views                |       |            | {timestamp} | {timestamp} | true  |          0 | system table for views |
+// EXPECTCOMP:WithFormat:| users                   | users                   |       |            | {timestamp} | {timestamp} | false |          0 |                        |
+// EXPECTCOMP:WithFormat:| fb_____________________ | fb_____________________ |       |            | {timestamp} | {timestamp} | false |          0 |                        |
+// EXPECTCOMP:WithFormat:| fb_____________________ | fb_____________________ |       |            | {timestamp} | {timestamp} | false |          0 |                        |
+// EXPECTCOMP:WithFormat:| fb_____________________ | fb_____________________ |       |            | {timestamp} | {timestamp} | false |          0 |                        |
+// EXPECTCOMP:WithFormat:| fb_____________________ | fb_____________________ |       |            | {timestamp} | {timestamp} | false |          0 |                        |
+// EXPECTCOMP:WithFormat:| fb_____________________ | fb_____________________ |       |            | {timestamp} | {timestamp} | false |          0 |                        |
+// EXPECT:+-------------------------+-------------------------+-------+------------+----------------------+----------------------+-------+------------+------------------------+
+// EXPECT:
+
+// Show columns for table.
+SEND:\d users
+EXPECT:+------+------+--------+----------------------+-------+------------+------------+-------+----------------------+---------------------+----------+-------+-------------+-----+
+EXPECT:| _id  | name | type   | created_at           | keys  | cache_type | cache_size | scale |                  min |                 max | timeunit | epoch | timequantum | ttl |
+EXPECT:+------+------+--------+----------------------+-------+------------+------------+-------+----------------------+---------------------+----------+-------+-------------+-----+
+EXPECTCOMP:WithFormat:| _id  | _id  | id     | {timestamp} | false |            |          0 |     0 |                    0 |                   0 |          |     0 |             | 0s  |
+EXPECTCOMP:WithFormat:| name | name | string | {timestamp} | true  | ranked     |      50000 |     0 |                    0 |                   0 |          |     0 |             | 0s  |
+EXPECTCOMP:WithFormat:| age  | age  | int    | {timestamp} | false |            |          0 |     0 | -9223372036854775808 | 9223372036854775807 |          |     0 |             | 0s  |
+EXPECT:+------+------+--------+----------------------+-------+------------+------------+-------+----------------------+---------------------+----------+-------+-------------+-----+
+EXPECT:
+
+// Show columns for an invalid table.
+SEND:\d invalid
+EXPECT:Error: compiling plan: [1:19] table 'invalid' not found
+
+SEND:\d users extra
+EXPECT:executing meta command: meta command 'describe' takes zero or one argument

cli/testdata/meta_echo

@@ -0,0 +1,6 @@
+SEND:\echo
+EXPECT:
+
+// Simple \echo.
+SEND:\echo foo bar
+EXPECT:foo bar

cli/testdata/meta_file

@@ -0,0 +1,70 @@
+// Create a table.
+SEND:CREATE TABLE famous (
+SEND:    _id ID,
+SEND:    name STRING,
+SEND:    description STRING,
+SEND:    gender STRING,
+SEND:    country STRING,
+SEND:    occupation STRING,
+SEND:    birth_year INT min -32767 max 32767,
+SEND:    death_year INT min -32767 max 32767,
+SEND:    death_manner STRING,
+SEND:    birth_age INT min -32767 max 32767
+SEND:);
+EXPECT:
+
+// Open bulk insert.
+SEND:BULK INSERT
+SEND:INTO famous (_id, name, description, gender, country, occupation,
+SEND:    birth_year, death_year, death_manner, birth_age )
+SEND:MAP(0 INT,
+SEND:1 STRING,
+SEND:2 STRING,
+SEND:3 STRING,
+SEND:4 STRING,
+SEND:5 STRING,
+SEND:6 INT,
+SEND:7 INT,
+SEND:8 STRING,
+SEND:9 INT )
+SEND:FROM
+SEND:    x'
+
+// Call \file
+SEND:\file testdata/famous.csv
+
+// Close bulk insert.
+SEND:'
+SEND:WITH
+SEND:    BATCHSIZE 100000
+SEND:    FORMAT 'CSV'
+SEND:    INPUT 'STREAM'
+SEND:    HEADER_ROW;
+EXPECT:
+
+// Query table to ensure we have data.
+SEND:SELECT * FROM famous;
+EXPECT:+-----+---------------------------+-------------------------------------------------+--------+----------------------------------------------------+--------------+------------+------------+----------------+-----------+
+EXPECT:| _id | name                      | description                                     | gender | country                                            | occupation   | birth_year | death_year | death_manner   | birth_age |
+EXPECT:+-----+---------------------------+-------------------------------------------------+--------+----------------------------------------------------+--------------+------------+------------+----------------+-----------+
+EXPECT:|   1 | George Washington         | 1st president of the United States (1732–1799)  | Male   | United States of America; Kingdom of Great Britain | Politician   |       1732 |       1799 | natural causes |        67 |
+EXPECT:|   2 | Douglas Adams             | English writer and humorist                     | Male   | United Kingdom                                     | Artist       |       1952 |       2001 | natural causes |        49 |
+EXPECT:|   3 | Abraham Lincoln           | 16th president of the United States (1809-1865) | Male   | United States of America                           | Politician   |       1809 |       1865 | homicide       |        56 |
+EXPECT:|   4 | Wolfgang Amadeus Mozart   | Austrian composer of the Classical period       | Male   | Archduchy of Austria; Archbishopric of Salzburg    | Artist       |       1756 |       1791 | 0              |        35 |
+EXPECT:|   5 | Ludwig van Beethoven      | German classical and romantic composer          | Male   | Holy Roman Empire; Austrian Empire                 | Artist       |       1770 |       1827 | 0              |        57 |
+EXPECT:|   6 | Jean-François Champollion | French classical scholar                        | Male   | Kingdom of France; First French Empire             | Egyptologist |       1790 |       1832 | natural causes |        42 |
+EXPECT:|   7 | Paul Morand               | French writer                                   | Male   | France                                             | Artist       |       1888 |       1976 | 0              |        88 |
+EXPECT:|   8 | Claude Monet              | French impressionist painter (1840-1926)        | Male   | France                                             | Artist       |       1840 |       1926 | natural causes |        86 |
+EXPECT:+-----+---------------------------+-------------------------------------------------+--------+----------------------------------------------------+--------------+------------+------------+----------------+-----------+
+EXPECT:
+
+// TODO(tlt): dropping the table seems to cause problems.
+// Drop the table.
+//SEND:DROP TABLE famous;
+
+// Ensure that invalid aruments (none or too many) return an error.
+SEND:\file
+EXPECT:executing meta command: meta command 'file' requires exactly one argument
+SEND:\file filename extra
+EXPECT:executing meta command: meta command 'file' requires exactly one argument
+

cli/testdata/meta_include

@@ -0,0 +1,24 @@
+// Include with no argument should error.
+SEND:\i
+EXPECT:executing meta command: meta command 'include' requires exactly one argument
+
+// Include with too many arguments should error.
+SEND:\include testdata/people.sql extra
+EXPECT:executing meta command: meta command 'include' requires exactly one argument
+
+// Invalid file should error.
+SEND:\include invalid.file
+EXPECT:executing meta command: opening file: invalid.file: open invalid.file: no such file or directory
+
+SEND:\include testdata/people.sql
+EXPECT:
+EXPECT:
+EXPECT:+-----+------+-----+
+EXPECT:| _id | name | age |
+EXPECT:+-----+------+-----+
+EXPECT:|   1 | Amy  |  42 |
+EXPECT:|   2 | Bob  |  27 |
+EXPECT:|   3 | Carl |  33 |
+EXPECT:+-----+------+-----+
+EXPECT:
+EXPECT:mix in a meta command

cli/testdata/meta_output

@@ -0,0 +1,67 @@
+SEND:SELECT * FROM users;
+EXPECT:+-----+-------+-----+
+EXPECT:| _id | name  | age |
+EXPECT:+-----+-------+-----+
+EXPECT:|   1 | Anne  |  38 |
+EXPECT:|   2 | Bill  |  23 |
+EXPECT:|   3 | Cindy |  64 |
+EXPECT:+-----+-------+-----+
+EXPECT:
+
+// Redirect output to a file.
+SEND:\o test-output-file
+SEND:SELECT * FROM users;
+
+// Ensure the output went to the file.
+SEND:\! cat test-output-file
+EXPECT:+-----+-------+-----+
+EXPECT:| _id | name  | age |
+EXPECT:+-----+-------+-----+
+EXPECT:|   1 | Anne  |  38 |
+EXPECT:|   2 | Bill  |  23 |
+EXPECT:|   3 | Cindy |  64 |
+EXPECT:+-----+-------+-----+
+EXPECT:
+
+// Let's test some qecho stuff here while we're at it.
+SEND:\qecho string with "double quotes"
+SEND:\qecho -n one
+SEND:\qecho -n two
+SEND:\qecho three
+SEND:\qecho four
+
+SEND:\! cat test-output-file
+EXPECT:+-----+-------+-----+
+EXPECT:| _id | name  | age |
+EXPECT:+-----+-------+-----+
+EXPECT:|   1 | Anne  |  38 |
+EXPECT:|   2 | Bill  |  23 |
+EXPECT:|   3 | Cindy |  64 |
+EXPECT:+-----+-------+-----+
+EXPECT:
+EXPECT:string with "double quotes"
+EXPECT:onetwothree
+EXPECT:four
+
+// And \warn messages should still go to stderr, not the file.
+SEND:\warn a warning string
+EXPECT:a warning string
+
+// Remove the file.
+SEND:\! rm test-output-file
+
+// Set the output back to stdout.
+SEND:\o
+SEND:SELECT * FROM users;
+EXPECT:+-----+-------+-----+
+EXPECT:| _id | name  | age |
+EXPECT:+-----+-------+-----+
+EXPECT:|   1 | Anne  |  38 |
+EXPECT:|   2 | Bill  |  23 |
+EXPECT:|   3 | Cindy |  64 |
+EXPECT:+-----+-------+-----+
+EXPECT:
+
+// Ensure extra arguments to \output causes an error.
+SEND:\o filename extra
+EXPECT:executing meta command: meta command 'output' takes zero or one argument

cli/testdata/meta_pset_border

@@ -0,0 +1,52 @@
+SEND:\pset border 2
+EXPECT:Border style is 2.
+
+SEND:SELECT 1 as foo, 'baz' as bar;
+EXPECT:+-----+-----+
+EXPECT:| foo | bar |
+EXPECT:+-----+-----+
+EXPECT:|   1 | baz |
+EXPECT:+-----+-----+
+EXPECT:
+
+SEND:\pset border
+EXPECT:Border style is 2.
+
+SEND:\pset border 999
+EXPECT:Border style is 0.
+
+SEND:\pset border 1
+EXPECT:Border style is 1.
+
+SEND:SELECT 1 as foo, 'baz' as bar;
+EXPECT: foo | bar 
+EXPECT:-----+-----
+EXPECT:   1 | baz 
+EXPECT:
+
+SEND:\pset border 2
+EXPECT:Border style is 2.
+
+SEND:SELECT 1 as foo, 'baz' as bar;
+EXPECT:+-----+-----+
+EXPECT:| foo | bar |
+EXPECT:+-----+-----+
+EXPECT:|   1 | baz |
+EXPECT:+-----+-----+
+EXPECT:
+
+SEND:\pset border 0
+EXPECT:Border style is 0.
+
+SEND:SELECT 1 as foo, 'baz' as bar;
+EXPECT:foo bar
+EXPECT:--- ---
+EXPECT:  1 baz
+EXPECT:
+
+SEND:\pset border 1 extra
+EXPECT:executing meta command: meta command 'pset' takes zero, one, or two arguments
+
+// Set border back to the testing default.
+SEND:\pset border 2
+EXPECT:Border style is 2.

cli/testdata/meta_pset_expanded

@@ -0,0 +1,35 @@
+// Set to off.
+SEND:\pset expanded off
+EXPECT:Expanded display is off.
+
+// Set to on.
+SEND:\pset expanded on
+EXPECT:Expanded display is on.
+
+// Toggle to off.
+SEND:\pset expanded
+EXPECT:Expanded display is off.
+
+// Toggle to on.
+SEND:\pset expanded
+EXPECT:Expanded display is on.
+
+// Set to something invalid.
+SEND:\pset expanded invalid
+EXPECT:executing meta command: unrecognized value "invalid" for "expanded": Boolean expected
+
+// Ensure expanded shows results vertically.
+SEND:SELECT 1 as foo, 'baz' as bar;
+EXPECT:+-----+-----+
+EXPECT:| foo | 1   |
+EXPECT:| bar | baz |
+EXPECT:+-----+-----+
+EXPECT:
+
+// Set back to off as we started.
+SEND:\pset expanded off
+EXPECT:Expanded display is off.
+
+// make sure the \x meta-command returns expected errors
+SEND:\x on extra
+EXPECT:executing meta command: meta command 'expanded' takes zero or one argument

cli/testdata/meta_pset_format_csv

@@ -0,0 +1,47 @@
+SEND:\pset format csv
+EXPECT:Output format is csv.
+
+SEND:SELECT * FROM users;
+EXPECT:_id,name,age
+EXPECT:1,Anne,38
+EXPECT:2,Bill,23
+EXPECT:3,Cindy,64
+
+// Exclude headers.
+SEND:\t on
+EXPECT:Tuples only is on.
+
+SEND:SELECT * FROM users;
+EXPECT:1,Anne,38
+EXPECT:2,Bill,23
+EXPECT:3,Cindy,64
+
+// Reset headers.
+SEND:\t off
+EXPECT:Tuples only is off.
+
+// Set expanded to on.
+SEND:\x on
+EXPECT:Expanded display is on.
+
+SEND:SELECT * FROM users;
+EXPECT:_id,1
+EXPECT:name,Anne
+EXPECT:age,38
+EXPECT:_id,2
+EXPECT:name,Bill
+EXPECT:age,23
+EXPECT:_id,3
+EXPECT:name,Cindy
+EXPECT:age,64
+
+// Set expanded back to off.
+SEND:\x off
+EXPECT:Expanded display is off.
+
+// Set format back to aligned as we started.
+SEND:\pset format aligned
+EXPECT:Output format is aligned.
+
+SEND:\pset format invalid
+EXPECT:executing meta command: \pset: allowed formats are aligned, csv

cli/testdata/meta_pset_tuples_only

@@ -0,0 +1,34 @@
+// Set to off.
+SEND:\pset tuples_only off
+EXPECT:Tuples only is off.
+
+// Set to on.
+SEND:\pset tuples_only on
+EXPECT:Tuples only is on.
+
+// Toggle to off.
+SEND:\pset tuples_only
+EXPECT:Tuples only is off.
+
+// Toggle to on.
+SEND:\pset tuples_only
+EXPECT:Tuples only is on.
+
+// Set to something invalid.
+SEND:\pset tuples_only invalid
+EXPECT:executing meta command: unrecognized value "invalid" for "tuples_only": Boolean expected
+
+// Ensure tuples_only shows only tuples.
+SEND:SELECT 1 as foo, 'baz' as bar;
+EXPECT:+---+-----+
+EXPECT:| 1 | baz |
+EXPECT:+---+-----+
+EXPECT:
+
+// Set back to off as we started.
+SEND:\pset tuples_only off
+EXPECT:Tuples only is off.
+
+// make sure the \t meta-command returns expected errors
+SEND:\t off extra
+EXPECT:executing meta command: meta command 'tuples_only' takes zero or one argument

cli/testdata/meta_set

@@ -0,0 +1,31 @@
+SEND:\set
+
+SEND:\set var1 foo
+SEND:\set
+EXPECT:var1 = 'foo'
+
+SEND:\set var2 bar
+SEND:\set
+EXPECT:var1 = 'foo'
+EXPECT:var2 = 'bar'
+
+SEND:\set var3 zoo
+SEND:\set
+EXPECT:var1 = 'foo'
+EXPECT:var2 = 'bar'
+EXPECT:var3 = 'zoo'
+
+SEND:\unset
+EXPECT:\unset: missing required argument
+
+SEND:\unset non-existent-key
+
+SEND:\unset var1
+SEND:\set
+EXPECT:var2 = 'bar'
+EXPECT:var3 = 'zoo'
+
+SEND:\unset var2 extra
+EXPECT:\unset: extra argument "extra" ignored
+SEND:\set
+EXPECT:var3 = 'zoo'

cli/testdata/meta_timing

@@ -0,0 +1,50 @@
+// Start by ensuring timing is off.
+SEND:\timing off
+EXPECT:Timing is off.
+
+// Set timing on.
+SEND:\timing on
+EXPECT:Timing is on.
+
+// Toggle timing.
+SEND:\timing
+EXPECT:Timing is off.
+
+// Toggle timing again.
+SEND:\timing
+EXPECT:Timing is on.
+
+// Send extra argument to \timing.
+SEND:\timing on extra
+EXPECT:executing meta command: meta command 'timing' takes zero or one argument
+
+
+SEND:SELECT * FROM users;
+EXPECT:+-----+-------+-----+
+EXPECT:| _id | name  | age |
+EXPECT:+-----+-------+-----+
+EXPECT:|   1 | Anne  |  38 |
+EXPECT:|   2 | Bill  |  23 |
+EXPECT:|   3 | Cindy |  64 |
+EXPECT:+-----+-------+-----+
+EXPECT:
+EXPECTCOMP:HasPrefix:Execution time:
+
+// Turn timing back off.
+SEND:\timing off
+EXPECT:Timing is off.
+
+// Ensure we don't get timing.
+SEND:SELECT * FROM users;
+EXPECT:+-----+-------+-----+
+EXPECT:| _id | name  | age |
+EXPECT:+-----+-------+-----+
+EXPECT:|   1 | Anne  |  38 |
+EXPECT:|   2 | Bill  |  23 |
+EXPECT:|   3 | Cindy |  64 |
+EXPECT:+-----+-------+-----+
+EXPECT:
+
+// Ensure an invalid timing value returns an error.
+SEND:\timing invalid
+EXPECT:executing meta command: unrecognized value "invalid" for "\timing": Boolean expected

cli/testdata/meta_write

@@ -0,0 +1,24 @@
+// Make sure there's something in the query buffer.
+// This is left unterminated because we don't need to execute the query;
+// we just need there to be something in the buffer.
+SEND:SELECT * FROM invalid-table
+SEND:\write query-buffer-contents
+
+// Reset the buffer.
+SEND:\r
+EXPECT:Query buffer reset (cleared).
+
+// Read from the file.
+SEND:\! cat query-buffer-contents
+EXPECT:SELECT * FROM invalid-table
+
+// Remove the file.
+SEND:\! rm query-buffer-contents
+
+// Send \write with no arguments.
+SEND:\write
+EXPECT:\w: missing required argument
+
+// Send \write with extra arguments.
+SEND:\write filename extra
+EXPECT:executing meta command: meta command 'w' exactly one argument

cli/testdata/people.sql

@@ -0,0 +1,12 @@
+-- Create a table.
+create table people (_id id, name string, age int);
+
+-- Insert some values.
+insert into people values (1, 'Amy', 42), (2, 'Bob', 27), (3, 'Carl', 33);
+
+-- Get all rows from the table.
+select * from people;
+
+-- Mix in a meta-command to show that both are supported
+-- in the include file.
+\echo mix in a meta command

cli/testdata/query_buffer

@@ -0,0 +1,40 @@
+SEND:select 1 as foo;
+EXPECT:+-----+
+EXPECT:| foo |
+EXPECT:+-----+
+EXPECT:|   1 |
+EXPECT:+-----+
+EXPECT:
+
+SEND:\p
+EXPECT:select 1 as foo;
+
+SEND:select 2
+SEND:\p
+EXPECT:select 2
+SEND:\r
+EXPECT:Query buffer reset (cleared).
+
+SEND:\p
+EXPECT:select 1 as foo;
+
+SEND:select 3
+SEND:\p
+EXPECT:select 3
+
+SEND:as foo
+SEND:\p
+EXPECT:select 3
+EXPECT:as foo
+
+SEND:;
+EXPECT:+-----+
+EXPECT:| foo |
+EXPECT:+-----+
+EXPECT:|   3 |
+EXPECT:+-----+
+EXPECT:
+
+SEND:\p
+EXPECT:select 3
+EXPECT:as foo;

cli/testdata/setup

@@ -0,0 +1,51 @@
+// Startup splash.
+EXPECT:FeatureBase CLI ()
+EXPECT:Type "\q" to quit.
+EXPECT:Detected on-prem, serverless deployment.
+EXPECTCOMP:HasPrefix:Host: http://localhost:
+EXPECT:You are not connected to a database.
+
+// Show databases.
+SEND:SHOW DATABASES;
+EXPECT:Organization required. Use \org to set an organization.
+
+// Get current org.
+SEND:\org
+EXPECT:You have not set an organization.
+
+// Set org.
+SEND:\org acme
+EXPECT:You have set organization "acme".
+
+// Try to set org with too many arguments.
+SEND:\org acme extra
+EXPECT:executing meta command: meta command 'org' takes zero or one argument
+
+// Set location to UTC so that expected timestamp size is consistent.
+// Without this, a test running locally in may have a timestamp that
+// ends in a timezone offset such as `-06:00`, while one running as UTC
+// will have `Z`. Since these string lengths differ, our generic
+// {timestamp} comparison will fail.
+SEND:\pset location UTC
+EXPECT:Location is UTC.
+
+// Set an invalid location.
+SEND:\pset location invalid
+EXPECT:executing meta command: loading location: invalid: unknown time zone invalid
+
+// Try to set location with too many arguments.
+SEND:\pset location UTC extra
+EXPECT:executing meta command: meta command 'pset' takes zero, one, or two arguments
+
+// Set border to 2 for testing because it makes it easier to visually see
+// what the tests are expecting (because lines don't end in spaces).
+SEND:\pset border 2
+EXPECT:Border style is 2.
+
+// Check the state of pset.
+SEND:\pset
+EXPECT:border      2
+EXPECT:expanded    off
+EXPECT:format      aligned
+EXPECT:location    UTC
+EXPECT:tuples_only off

cli/testdata/table

@@ -0,0 +1,72 @@
+// Show tables for database using SHOW TABLES WITH SYSTEM.
+SEND:SHOW TABLES WITH SYSTEM;
+EXPECT:+-------------------------+-------------------------+-------+------------+----------------------+----------------------+-------+------------+-------------+
+EXPECT:| _id                     | name                    | owner | updated_by | created_at           | updated_at           | keys  | space_used | description |
+EXPECT:+-------------------------+-------------------------+-------+------------+----------------------+----------------------+-------+------------+-------------+
+EXPECTCOMP:WithFormat:| fb_____________________ | fb_____________________ |       |            | {timestamp} | {timestamp} | false |          0 |             |
+EXPECTCOMP:WithFormat:| fb_____________________ | fb_____________________ |       |            | {timestamp} | {timestamp} | false |          0 |             |
+EXPECTCOMP:WithFormat:| fb_____________________ | fb_____________________ |       |            | {timestamp} | {timestamp} | false |          0 |             |
+EXPECTCOMP:WithFormat:| fb_____________________ | fb_____________________ |       |            | {timestamp} | {timestamp} | false |          0 |             |
+EXPECTCOMP:WithFormat:| fb_____________________ | fb_____________________ |       |            | {timestamp} | {timestamp} | false |          0 |             |
+EXPECT:+-------------------------+-------------------------+-------+------------+----------------------+----------------------+-------+------------+-------------+
+EXPECT:
+
+// Show tables for database using \d.
+SEND:\d
+EXPECT:+-------------------------+-------------------------+-------+------------+----------------------+----------------------+-------+------------+-------------+
+EXPECT:| _id                     | name                    | owner | updated_by | created_at           | updated_at           | keys  | space_used | description |
+EXPECT:+-------------------------+-------------------------+-------+------------+----------------------+----------------------+-------+------------+-------------+
+EXPECTCOMP:WithFormat:| fb_____________________ | fb_____________________ |       |            | {timestamp} | {timestamp} | false |          0 |             |
+EXPECTCOMP:WithFormat:| fb_____________________ | fb_____________________ |       |            | {timestamp} | {timestamp} | false |          0 |             |
+EXPECTCOMP:WithFormat:| fb_____________________ | fb_____________________ |       |            | {timestamp} | {timestamp} | false |          0 |             |
+EXPECTCOMP:WithFormat:| fb_____________________ | fb_____________________ |       |            | {timestamp} | {timestamp} | false |          0 |             |
+EXPECTCOMP:WithFormat:| fb_____________________ | fb_____________________ |       |            | {timestamp} | {timestamp} | false |          0 |             |
+EXPECT:+-------------------------+-------------------------+-------+------------+----------------------+----------------------+-------+------------+-------------+
+EXPECT:
+
+// Show tables for database using SHOW TABLES.
+SEND:SHOW TABLES;
+EXPECT:+-----+------+-------+------------+------------+------------+------+------------+-------------+
+EXPECT:| _id | name | owner | updated_by | created_at | updated_at | keys | space_used | description |
+EXPECT:+-----+------+-------+------------+------------+------------+------+------------+-------------+
+EXPECT:+-----+------+-------+------------+------------+------------+------+------------+-------------+
+EXPECT:
+
+
+// Show tables for database using \dt.
+SEND:\dt
+EXPECT:+-----+------+-------+------------+------------+------------+------+------------+-------------+
+EXPECT:| _id | name | owner | updated_by | created_at | updated_at | keys | space_used | description |
+EXPECT:+-----+------+-------+------------+------------+------------+------+------------+-------------+
+EXPECT:+-----+------+-------+------------+------------+------------+------+------------+-------------+
+EXPECT:
+
+// Create a table. That can be used for general testing.
+SEND:CREATE TABLE users (_id id, name string, age int);
+EXPECT:
+SEND:INSERT INTO users VALUES (1, 'Anne', 38), (2, 'Bill', 23), (3, 'Cindy', 64);
+EXPECT:
+
+
+// Show tables for database to get the newly created table.
+SEND:\dt
+EXPECT:+-------+-------+-------+------------+----------------------+----------------------+-------+------------+-------------+
+EXPECT:| _id   | name  | owner | updated_by | created_at           | updated_at           | keys  | space_used | description |
+EXPECT:+-------+-------+-------+------------+----------------------+----------------------+-------+------------+-------------+
+EXPECTCOMP:WithFormat:| users | users |       |            | {timestamp} | {timestamp} | false |          0 |             |
+EXPECT:+-------+-------+-------+------------+----------------------+----------------------+-------+------------+-------------+
+EXPECT:
+
+// We don't select from users until AFTER we check SHOW TABLES above because
+// running this creates the fb_views sytem table which has a description.
+// And it's annoying to mask out all of the description fields because we
+// don't know which row fb_views will fall into.
+SEND:SELECT * FROM users;
+EXPECT:+-----+-------+-----+
+EXPECT:| _id | name  | age |
+EXPECT:+-----+-------+-----+
+EXPECT:|   1 | Anne  |  38 |
+EXPECT:|   2 | Bill  |  23 |
+EXPECT:|   3 | Cindy |  64 |
+EXPECT:+-----+-------+-----+
+EXPECT:

cli/workingdir.go

@@ -0,0 +1,21 @@
+package cli
+
+import (
+	"os"
+)
+
+// workingDir was originally set up with the intention of using it to maintain a
+// reference to the current working directory. But it turns out we haven't
+// really needed that so far. The `cd()` method is unsed in one of the meta
+// commands, but we could probably just call `os.Chdir()` directly there. With
+// that said, I'm leaving it here for now until we're abosolutely sure we don't
+// need to use this for other directory/file handling functionality.
+type workingDir struct{}
+
+func newWorkingDir() *workingDir {
+	return &workingDir{}
+}
+
+func (wd *workingDir) cd(dir string) error {
+	return os.Chdir(dir)
+}

cli/writer.go

@@ -0,0 +1,294 @@
+package cli
+
+import (
+	"encoding/csv"
+	"fmt"
+	"io"
+	"log"
+	"time"
+
+	featurebase "github.com/featurebasedb/featurebase/v3"
+	"github.com/jedib0t/go-pretty/table"
+	"github.com/jedib0t/go-pretty/text"
+	"github.com/pkg/errors"
+)
+
+// writeOptions contains user configuration options which describe how to write
+// the query output.
+type writeOptions struct {
+	border     int
+	expanded   bool
+	format     string
+	location   *time.Location
+	timing     bool
+	tuplesOnly bool
+}
+
+const (
+	formatAligned = "aligned"
+	formatCSV     = "csv"
+)
+
+func defaultWriteOptions() *writeOptions {
+	return &writeOptions{
+		border:     1,
+		expanded:   false,
+		format:     formatAligned,
+		location:   time.Local,
+		timing:     false,
+		tuplesOnly: false,
+	}
+}
+
+// writeOutput writes the query response, taking the format into consideration.
+// It sends query output to qOut, non-error informational output (such as query
+// timing) to wOut, and errors to wErr.
+func writeOutput(r *featurebase.WireQueryResponse, opts *writeOptions, qOut io.Writer, wOut io.Writer, wErr io.Writer) error {
+	if r == nil {
+		return errors.New("attempt to write out nil response")
+	}
+	if r.Error != "" {
+		if _, err := wErr.Write([]byte("Error: " + r.Error + "\n")); err != nil {
+			return errors.Wrapf(err, "writing error: %s", r.Error)
+		}
+		return writeWarnings(r, wErr)
+	}
+
+	switch opts.format {
+	case formatAligned:
+		if err := writeTable(r, opts, qOut); err != nil {
+			return errors.Wrap(err, "writing table")
+		}
+
+		// Add some white space after query results.
+		qOut.Write([]byte("\n"))
+
+	case formatCSV:
+		if err := writeCSV(r, opts, qOut); err != nil {
+			return errors.Wrap(err, "writing csv")
+		}
+
+	default:
+		return errors.Errorf("invalid format: %s", opts.format)
+	}
+
+	if err := writeWarnings(r, wErr); err != nil {
+		return err
+	}
+
+	// Timing.
+	if opts.timing {
+		if _, err := wOut.Write([]byte(fmt.Sprintf("Execution time: %dμs\n", r.ExecutionTime))); err != nil {
+			return errors.Wrapf(err, "writing execution time: %s", r.Error)
+		}
+	}
+
+	return nil
+}
+
+// writeCSV writes the WireQueryResponse to qOut as csv.
+func writeCSV(r *featurebase.WireQueryResponse, opts *writeOptions, qOut io.Writer) error {
+	w := csv.NewWriter(qOut)
+
+	if opts.expanded {
+		// Expanded csv
+
+		// rec is used to write the row as a slice of strings. It is reused to
+		// avoid unnecessary memory allocation.
+		rec := make([]string, 2)
+
+		for _, row := range r.Data {
+			cleanRow(row, opts)
+			for i, col := range r.Schema.Fields {
+				rec[0] = string(col.Name)
+				rec[1] = fmt.Sprintf("%v", row[i])
+
+				// Write the record.
+				if err := w.Write(rec); err != nil {
+					log.Fatalln("error writing expanded record to csv:", err)
+				}
+			}
+		}
+
+	} else {
+		// Normal csv (i.e. NOT expanded)
+
+		// Write the schema.
+		if !opts.tuplesOnly {
+			header := make([]string, 0, len(r.Schema.Fields))
+			for i := range r.Schema.Fields {
+				header = append(header, string(r.Schema.Fields[i].Name))
+			}
+			if err := w.Write(header); err != nil {
+				return errors.Wrapf(err, "error writing header to csv")
+			}
+		}
+
+		// Write the records.
+
+		// rec is used to write the row as a slice of strings. It is reused to
+		// avoid unnecessary memory allocation.
+		rec := make([]string, len(r.Schema.Fields))
+
+		for _, row := range r.Data {
+			cleanRow(row, opts)
+			for i := range row {
+				rec[i] = fmt.Sprintf("%v", row[i])
+			}
+			if err := w.Write(rec); err != nil {
+				log.Fatalln("error writing record to csv:", err)
+			}
+		}
+	}
+
+	// Write any buffered data to the underlying writer (standard output).
+	w.Flush()
+
+	return w.Error()
+}
+
+// writeTable writes the WireQueryResponse to qOut in a tabular format.
+func writeTable(r *featurebase.WireQueryResponse, opts *writeOptions, qOut io.Writer) error {
+	t := table.NewWriter()
+	t.SetOutputMirror(qOut)
+	switch opts.border {
+	case 0:
+		t.SetStyle(styleBorder0)
+	case 1:
+		t.SetStyle(styleBorder1)
+	default:
+		t.SetStyle(styleBorder2)
+		// In expanded mode with a border, we need borders between each record.
+		if opts.expanded {
+			t.Style().Options.SeparateRows = true
+		}
+	}
+
+	// Don't uppercase the header values.
+	t.Style().Format.Header = text.FormatDefault
+
+	if opts.expanded {
+		// Expanded table
+		for _, row := range r.Data {
+			cleanRow(row, opts)
+			colRow := make([]interface{}, 2)
+			scolRow := make([]string, 2)
+			div := "\n"
+			for i, col := range r.Schema.Fields {
+				if i == len(r.Schema.Fields)-1 {
+					div = ""
+				}
+				scolRow[0] += fmt.Sprintf("%s%s", col.Name, div)
+				scolRow[1] += fmt.Sprintf("%v%s", row[i], div)
+			}
+			colRow[0] = scolRow[0]
+			colRow[1] = scolRow[1]
+			t.AppendRow(table.Row(colRow[:]))
+		}
+	} else {
+		// Normal table (i.e. NOT expanded)
+		if !opts.tuplesOnly {
+			t.AppendHeader(schemaToRow(r.Schema))
+		}
+		for _, row := range r.Data {
+			cleanRow(row, opts)
+			t.AppendRow(table.Row(row))
+		}
+	}
+	t.Render()
+
+	return nil
+}
+
+// cleanRow loops through all the columns of row and modifies its value based on
+// type.
+//
+// If the value is nil, replace it with a null string; go-pretty doesn't expect
+// nil pointers in the data values.
+//
+// If the value is a time.Time, we want to print it using RFC3339Nano to be
+// consistent with everything else.
+func cleanRow(row []interface{}, opts *writeOptions) {
+	for i := range row {
+		switch v := row[i].(type) {
+		case nil:
+			row[i] = nullValue
+		case time.Time:
+			row[i] = v.In(opts.location).Format(time.RFC3339Nano)
+		}
+	}
+}
+
+func schemaToRow(schema featurebase.WireQuerySchema) []interface{} {
+	ret := make([]interface{}, len(schema.Fields))
+	for i, field := range schema.Fields {
+		ret[i] = field.Name
+	}
+	return ret
+}
+
+func writeWarnings(r *featurebase.WireQueryResponse, w io.Writer) error {
+	if len(r.Warnings) == 0 {
+		return nil
+	}
+
+	if _, err := w.Write([]byte("\n")); err != nil {
+		return errors.Wrapf(err, "writing line feed")
+	}
+	for _, warning := range r.Warnings {
+		if _, err := w.Write([]byte("Warning: " + warning + "\n")); err != nil {
+			return errors.Wrapf(err, "writing warning: %s", warning)
+		}
+	}
+	return nil
+}
+
+var styleBorder2 table.Style = table.StyleDefault
+
+var styleBorder1 table.Style = table.Style{
+	Name:   "StyleBorder1",
+	Box:    table.StyleBoxDefault,
+	Color:  table.ColorOptionsDefault,
+	Format: table.FormatOptionsDefault,
+	Options: table.Options{
+		DrawBorder:      false,
+		SeparateColumns: true,
+		SeparateFooter:  true,
+		SeparateHeader:  true,
+		SeparateRows:    false,
+	},
+	Title: table.TitleOptionsDefault,
+}
+
+var styleBorder0 table.Style = table.Style{
+	Name: "StyleBorder0",
+	Box: table.BoxStyle{
+		BottomLeft:       "+",
+		BottomRight:      "+",
+		BottomSeparator:  "+",
+		Left:             "|",
+		LeftSeparator:    "+",
+		MiddleHorizontal: "-",
+		MiddleSeparator:  " ",
+		MiddleVertical:   " ",
+		PaddingLeft:      "",
+		PaddingRight:     "",
+		PageSeparator:    "\n",
+		Right:            "|",
+		RightSeparator:   "+",
+		TopLeft:          "+",
+		TopRight:         "+",
+		TopSeparator:     "+",
+		UnfinishedRow:    " ~",
+	},
+	Color:  table.ColorOptionsDefault,
+	Format: table.FormatOptionsDefault,
+	Options: table.Options{
+		DrawBorder:      false,
+		SeparateColumns: true,
+		SeparateFooter:  true,
+		SeparateHeader:  true,
+		SeparateRows:    false,
+	},
+	Title: table.TitleOptionsDefault,
+}

cli/writer_test.go

@@ -0,0 +1,188 @@
+package cli
+
+import (
+	"bytes"
+	"fmt"
+	"strings"
+	"testing"
+
+	featurebase "github.com/featurebasedb/featurebase/v3"
+	dax "github.com/featurebasedb/featurebase/v3/dax"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestWriter(t *testing.T) {
+	t.Run("writeTable", func(t *testing.T) {
+		wqr := &featurebase.WireQueryResponse{
+			Schema: featurebase.WireQuerySchema{
+				Fields: []*featurebase.WireQueryField{
+					{Name: "_id", Type: dax.BaseTypeID},
+					{Name: "name", Type: dax.BaseTypeString},
+					{Name: "age", Type: dax.BaseTypeInt},
+				},
+			},
+			Data: [][]interface{}{
+				{1, "Amy", 44},
+				{2, "Bob", 32},
+				{3, "Cindy", 28},
+			},
+		}
+
+		// TODO(tlt): used for debugging
+		// format := defaultWriteOptions()
+		// assert.NoError(t, writeTable(wqr, format, os.Stdout, os.Stdout, os.Stdout))
+		// return
+
+		tests := []struct {
+			format  *writeOptions
+			expQOut string
+			expOut  string
+			expErr  string
+		}{
+			{
+				// default format
+				format: defaultWriteOptions(),
+				expQOut: stringOfLines(
+					" _id | name  | age ",
+					"-----+-------+-----",
+					"   1 | Amy   |  44 ",
+					"   2 | Bob   |  32 ",
+					"   3 | Cindy |  28 ",
+					"",
+				),
+				expOut: "",
+				expErr: "",
+			},
+			{
+				// timing on
+				format: &writeOptions{
+					border:     1,
+					expanded:   false,
+					format:     formatAligned,
+					timing:     true,
+					tuplesOnly: false,
+				},
+				expQOut: stringOfLines(
+					" _id | name  | age ",
+					"-----+-------+-----",
+					"   1 | Amy   |  44 ",
+					"   2 | Bob   |  32 ",
+					"   3 | Cindy |  28 ",
+					"",
+				),
+				expOut: "Execution time: 0μs\n",
+				expErr: "",
+			},
+			{
+				// format.border = 2 (or higher)
+				format: &writeOptions{
+					border:     2,
+					expanded:   false,
+					format:     formatAligned,
+					timing:     false,
+					tuplesOnly: false,
+				},
+				expQOut: stringOfLines(
+					"+-----+-------+-----+",
+					"| _id | name  | age |",
+					"+-----+-------+-----+",
+					"|   1 | Amy   |  44 |",
+					"|   2 | Bob   |  32 |",
+					"|   3 | Cindy |  28 |",
+					"+-----+-------+-----+",
+					"",
+				),
+				expOut: "",
+				expErr: "",
+			},
+			{
+				// format.border = 0
+				format: &writeOptions{
+					border:     0,
+					expanded:   false,
+					format:     formatAligned,
+					timing:     false,
+					tuplesOnly: false,
+				},
+				expQOut: stringOfLines(
+					"_id name  age",
+					"--- ----- ---",
+					"  1 Amy    44",
+					"  2 Bob    32",
+					"  3 Cindy  28",
+					"",
+				),
+				expOut: "",
+				expErr: "",
+			},
+			{
+				// format.tuplesOnly = true
+				format: &writeOptions{
+					border:     1,
+					expanded:   false,
+					format:     formatAligned,
+					timing:     false,
+					tuplesOnly: true,
+				},
+				expQOut: stringOfLines(
+					" 1 | Amy   | 44 ",
+					" 2 | Bob   | 32 ",
+					" 3 | Cindy | 28 ",
+					"",
+				),
+				expOut: "",
+				expErr: "",
+			},
+			{
+				// format.border = 2, expanded
+				format: &writeOptions{
+					border:     2,
+					expanded:   true,
+					format:     formatAligned,
+					timing:     false,
+					tuplesOnly: false,
+				},
+				expQOut: stringOfLines(
+					"+------+-------+",
+					"| _id  | 1     |",
+					"| name | Amy   |",
+					"| age  | 44    |",
+					"+------+-------+",
+					"| _id  | 2     |",
+					"| name | Bob   |",
+					"| age  | 32    |",
+					"+------+-------+",
+					"| _id  | 3     |",
+					"| name | Cindy |",
+					"| age  | 28    |",
+					"+------+-------+",
+					"",
+				),
+				expOut: "",
+				expErr: "",
+			},
+		}
+		for i, test := range tests {
+			t.Run(fmt.Sprintf("test-%d", i), func(t *testing.T) {
+				// Set up buffers to capture the output.
+				qOut := bytes.NewBuffer(make([]byte, 0, 100000))
+				wOut := bytes.NewBuffer(make([]byte, 0, 100000))
+				wErr := bytes.NewBuffer(make([]byte, 0, 100000))
+
+				assert.NoError(t, writeOutput(wqr, test.format, qOut, wOut, wErr))
+
+				assert.Equal(t, test.expQOut, qOut.String())
+				assert.Equal(t, test.expOut, wOut.String())
+				assert.Equal(t, test.expErr, wErr.String())
+			})
+		}
+	})
+}
+
+func stringOfLines(lines ...string) string {
+	var sb strings.Builder
+	for _, line := range lines {
+		sb.WriteString(line + "\n")
+	}
+	return sb.String()
+}

client/api.go

@@ -26,6 +26,26 @@ func NewSchemaAPI(c *Client) *schemaAPI {
 	}
 }
 
+func (s *schemaAPI) CreateDatabase(context.Context, *dax.Database) error {
+	return errors.Errorf("unimplemented: schemaAPI.CreateDatabase()")
+}
+func (s *schemaAPI) DropDatabase(context.Context, dax.DatabaseID) error {
+	return errors.Errorf("unimplemented: schemaAPI.DropDatabase()")
+}
+
+func (s *schemaAPI) DatabaseByName(ctx context.Context, dbname dax.DatabaseName) (*dax.Database, error) {
+	return nil, errors.Errorf("unimplemented: schemaAPI.DatabaseByName()")
+}
+func (s *schemaAPI) DatabaseByID(ctx context.Context, dbid dax.DatabaseID) (*dax.Database, error) {
+	return nil, errors.Errorf("unimplemented: schemaAPI.DatabaseByID()")
+}
+func (s *schemaAPI) SetDatabaseOption(ctx context.Context, dbid dax.DatabaseID, option string, value string) error {
+	return nil
+}
+func (s *schemaAPI) Databases(context.Context, ...dax.DatabaseID) ([]*dax.Database, error) {
+	return nil, errors.Errorf("unimplemented: schemaAPI.Databases()")
+}
+
 func (s *schemaAPI) TableByName(ctx context.Context, tname dax.TableName) (*dax.Table, error) {
 	return nil, errors.New(errors.ErrUncoded, "schemaAPI.TableByName not implemented")
 }

client/client.go

@@ -20,7 +20,6 @@ import (
 	"sync"
 	"time"
 
-	"github.com/golang/protobuf/proto" //nolint:staticcheck
 	pilosa "github.com/featurebasedb/featurebase/v3"
 	"github.com/featurebasedb/featurebase/v3/client/types"
 	fbproto "github.com/featurebasedb/featurebase/v3/encoding/proto" // TODO use this everywhere and get rid of proto import
@@ -29,8 +28,8 @@ import (
 	"github.com/featurebasedb/featurebase/v3/pb"
 	"github.com/featurebasedb/featurebase/v3/pql"
 	"github.com/featurebasedb/featurebase/v3/roaring"
-	"github.com/featurebasedb/featurebase/v3/stats"
 	"github.com/featurebasedb/featurebase/v3/vprint"
+	"github.com/golang/protobuf/proto" //nolint:staticcheck
 	"github.com/opentracing/opentracing-go"
 	"github.com/pkg/errors"
 	"golang.org/x/sync/errgroup"
@@ -52,7 +51,6 @@ type Client struct {
 	manualFragmentNode *fragmentNode
 	manualServerURI    *pnet.URI
 	tracer             opentracing.Tracer
-	Stats              stats.StatsClient
 	// An exponential backoff algorithm retries requests exponentially (if an HTTP request fails),
 	// increasing the waiting time between retries up to a maximum backoff time.
 	maxBackoff time.Duration
@@ -211,11 +209,6 @@ func newClientWithOptions(options *ClientOptions) *Client {
 	} else {
 		c.tracer = options.tracer
 	}
-	if options.stats == nil {
-		c.Stats = stats.NopStatsClient
-	} else {
-		c.Stats = options.stats
-	}
 
 	c.maxRetries = *options.retries
 	c.maxBackoff = 2 * time.Minute
@@ -1359,7 +1352,6 @@ type ClientOptions struct {
 	manualServerAddress bool
 	tracer              opentracing.Tracer
 	retries             *int
-	stats               stats.StatsClient
 	nat                 map[pnet.URI]pnet.URI
 	pathPrefix          string
 }
@@ -1445,14 +1437,6 @@ func OptClientRetries(retries int) ClientOption {
 	}
 }
 
-// OptClientStatsClient sets a stats client, such as Prometheus
-func OptClientStatsClient(stats stats.StatsClient) ClientOption {
-	return func(options *ClientOptions) error {
-		options.stats = stats
-		return nil
-	}
-}
-
 // OptClientNAT sets a NAT map used to translate the advertised URI to something
 // else (for example, when accessing pilosa running in docker).
 func OptClientNAT(nat map[string]string) ClientOption {
@@ -1842,15 +1826,5 @@ func (c *Client) ApplyDataframeChangeset(indexName string, cr *pilosa.ChangesetR
 		})
 	}
 	err = eg.Wait()
-
-	//	status, body, err := c.HTTPRequest(http.MethodPost, path, buffer.Bytes(), headers)
-	/*
-		var result map[string]interface{}
-		err = json.Unmarshal(body, &result)
-		if err != nil {
-			return nil, errors.Wrap(err, "unmarshalling response")
-		}
-	*/
-
 	return nil, err
 }

client/importer.go

@@ -328,7 +328,3 @@ func (i *importer) EncodeImport(ctx context.Context, tid dax.TableID, fld *dax.F
 func (i *importer) DoImport(ctx context.Context, tid dax.TableID, fld *dax.Field, shard uint64, path string, data []byte) error {
 	return i.client.DoImport(string(tid), shard, path, data)
 }
-
-func (i *importer) StatsTiming(name string, value time.Duration, rate float64) {
-	i.client.Stats.Timing(name, value, rate)
-}

cluster.go

@@ -4,6 +4,7 @@ package pilosa
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
 	"log"
 	"sync"
@@ -11,6 +12,7 @@ import (
 
 	"github.com/featurebasedb/featurebase/v3/dax"
 	"github.com/featurebasedb/featurebase/v3/dax/computer"
+	"github.com/featurebasedb/featurebase/v3/dax/storage"
 	"github.com/featurebasedb/featurebase/v3/disco"
 	"github.com/featurebasedb/featurebase/v3/logger"
 	"github.com/featurebasedb/featurebase/v3/roaring"
@@ -24,7 +26,7 @@ const (
 )
 
 // cluster represents a collection of nodes.
-type cluster struct { // nolint: maligned
+type cluster struct { //nolint: maligned
 	noder disco.Noder
 
 	id   string
@@ -72,8 +74,7 @@ type cluster struct { // nolint: maligned
 
 	partitionAssigner string
 
-	writeLogWriter computer.WriteLogWriter
-	versionStore   dax.VersionStore
+	serverlessStorage *storage.ResourceManager
 
 	// isComputeNode is set to true if this node is running as a DAX compute
 	// node.
@@ -100,8 +101,6 @@ func newCluster() *cluster {
 
 		disCo: disco.NopDisCo,
 		noder: disco.NewEmptyLocalNoder(),
-
-		writeLogWriter: computer.NewNopWriteLogWriter(),
 	}
 }
 
@@ -318,6 +317,44 @@ func (c *cluster) findFieldKeys(ctx context.Context, field *Field, keys ...strin
 	return translations, nil
 }
 
+func (c *cluster) appendFieldKeysWriteLog(ctx context.Context, qtid dax.QualifiedTableID, fieldName dax.FieldName, translations map[string]uint64) error {
+	// TODO move marshaling somewhere more centralized and less... explicitly json-y
+	msg := computer.FieldKeyMap{
+		TableKey:   qtid.Key(),
+		Field:      fieldName,
+		StringToID: translations,
+	}
+
+	b, err := json.Marshal(msg)
+	if err != nil {
+		return errors.Wrap(err, "marshalling field key map to json")
+	}
+	resource := c.serverlessStorage.GetFieldKeyResource(qtid, fieldName)
+	err = resource.Append(b)
+	if err != nil {
+		return errors.Wrap(err, "appending field keys")
+	}
+	return nil
+
+}
+
+func (c *cluster) appendTableKeysWriteLog(ctx context.Context, qtid dax.QualifiedTableID, partition dax.PartitionNum, translations map[string]uint64) error {
+	msg := computer.PartitionKeyMap{
+		TableKey:   qtid.Key(),
+		Partition:  partition,
+		StringToID: translations,
+	}
+
+	b, err := json.Marshal(msg)
+	if err != nil {
+		return errors.Wrap(err, "marshalling partition key map to json")
+	}
+
+	resource := c.serverlessStorage.GetTableKeyResource(qtid, partition)
+	return errors.Wrap(resource.Append(b), "appending table keys")
+
+}
+
 func (c *cluster) createFieldKeys(ctx context.Context, field *Field, keys ...string) (map[string]uint64, error) {
 	if idx := field.ForeignIndex(); idx != "" {
 		// The field uses foreign index keys.
@@ -350,17 +387,9 @@ func (c *cluster) createFieldKeys(ctx context.Context, field *Field, keys ...str
 		tkey := dax.TableKey(field.Index())
 		qtid := tkey.QualifiedTableID()
 		fieldName := dax.FieldName(field.Name())
-
-		// Get the current version for field.
-		version, found, err := c.versionStore.FieldVersion(ctx, qtid, fieldName)
+		err = c.appendFieldKeysWriteLog(ctx, qtid, fieldName, translations)
 		if err != nil {
-			return nil, errors.Wrap(err, "getting field version")
-		} else if !found {
-			return nil, errors.Errorf("no version found for table(%s) field(%s)", qtid, fieldName)
-		}
-
-		if err := c.writeLogWriter.CreateFieldKeys(ctx, qtid, fieldName, version, translations); err != nil {
-			return nil, errors.Errorf("logging field(%s/%s) keys(%v)", field.Index(), field.Name(), keys)
+			return nil, errors.Wrap(err, "appending to write log")
 		}
 
 		return translations, nil
@@ -754,16 +783,7 @@ func (c *cluster) createIndexKeys(ctx context.Context, indexName string, keys ..
 			tkey := dax.TableKey(idx.Name())
 			qtid := tkey.QualifiedTableID()
 			partitionNum := dax.PartitionNum(partitionID)
-
-			// Get the current version for partition.
-			version, found, err := c.versionStore.PartitionVersion(ctx, qtid, partitionNum)
-			if err != nil {
-				return errors.Wrap(err, "getting partition version")
-			} else if !found {
-				return errors.Errorf("no version found for table(%s) partition(%d)", qtid, partitionNum)
-			}
-
-			return c.writeLogWriter.CreateTableKeys(ctx, qtid, partitionNum, version, translations)
+			return c.appendTableKeysWriteLog(ctx, qtid, partitionNum, translations)
 		})
 	}
 
@@ -993,9 +1013,9 @@ type TransactionMessage struct {
 	Action      string
 }
 
-func intInPartitions(i int, s dax.VersionedPartitions) bool {
+func intInPartitions(i int, s dax.PartitionNums) bool {
 	for _, a := range s {
-		if int(a.Num) == i {
+		if int(a) == i {
 			return true
 		}
 	}

cmd/auth_token.go

@@ -16,7 +16,7 @@ func newAuthTokenCommand(logdest logger.Logger) *cobra.Command {
 		Long: `
 Retrieves an auth-token for use in authenticating with FeatureBase from the configured identity provider.
 `,
-		RunE: usageErrorWrapper(cmd),
+		RunE: UsageErrorWrapper(cmd),
 	}
 
 	flags := ccmd.Flags()

cmd/backup.go

@@ -16,7 +16,7 @@ func newBackupCommand(logdest logger.Logger) *cobra.Command {
 		Long: `
 Backs up a FeatureBase server to a local, tar-formatted snapshot file.
 `,
-		RunE: usageErrorWrapper(cmd),
+		RunE: UsageErrorWrapper(cmd),
 	}
 
 	flags := ccmd.Flags()

cmd/backup_tar.go

@@ -2,12 +2,13 @@
 package cmd
 
 import (
+	"io"
+
 	"github.com/featurebasedb/featurebase/v3/ctl"
-	"github.com/featurebasedb/featurebase/v3/logger"
 	"github.com/spf13/cobra"
 )
 
-func newBackupTarCommand(logdest logger.Logger) *cobra.Command {
+func newBackupTarCommand(logdest io.Writer) *cobra.Command {
 	cmd := ctl.NewBackupTarCommand(logdest)
 	ccmd := &cobra.Command{
 		Use:   "backuptar",
@@ -15,7 +16,7 @@ func newBackupTarCommand(logdest logger.Logger) *cobra.Command {
 		Long: `
 Backs up a FeatureBase server to a local, tar-formatted snapshot file.
 `,
-		RunE: usageErrorWrapper(cmd),
+		RunE: UsageErrorWrapper(cmd),
 	}
 
 	flags := ccmd.Flags()
@@ -27,6 +28,7 @@ Backs up a FeatureBase server to a local, tar-formatted snapshot file.
 	ctl.SetTLSConfig(flags, "", &cmd.TLS.CertificatePath, &cmd.TLS.CertificateKeyPath, &cmd.TLS.CACertPath, &cmd.TLS.SkipVerify, &cmd.TLS.EnableClientVerification)
 	flags.StringVar(&cmd.AuthToken, "auth-token", "", "Authentication token")
 	flags.StringVar(&cmd.HeaderTimeoutStr, "header-timeout", cmd.HeaderTimeoutStr, "Length of time to wait for initial HTTP response before giving up.")
+	flags.StringVar(&cmd.TempDir, "temp-dir", cmd.TempDir, "Location of temporary spillover files. The default is the system's default (usually /tmp)")
 
 	return ccmd
 }

cmd/chksum.go

@@ -3,13 +3,15 @@
 package cmd
 
 import (
+	"os"
+
 	"github.com/featurebasedb/featurebase/v3/ctl"
 	"github.com/featurebasedb/featurebase/v3/logger"
 	"github.com/spf13/cobra"
 )
 
 func newChkSumCommand(logdest logger.Logger) *cobra.Command {
-	cmd := ctl.NewChkSumCommand(logdest)
+	cmd := ctl.NewChkSumCommand(logdest, os.Stdout)
 	ccmd := &cobra.Command{
 		Use:   "chksum",
 		Short: "Digital signature of FeatureBase data",
@@ -17,7 +19,7 @@ func newChkSumCommand(logdest logger.Logger) *cobra.Command {
 			Generates a digital signature of all the data associated with a provided FeatureBase server
 			WARNING: could be slow if high cardinality fields exist
 `,
-		RunE: usageErrorWrapper(cmd),
+		RunE: UsageErrorWrapper(cmd),
 	}
 
 	flags := ccmd.Flags()

cmd/cli.go

@@ -1,35 +0,0 @@
-// Copyright 2021 Molecula Corp. All rights reserved.
-package cmd
-
-import (
-	"github.com/featurebasedb/featurebase/v3/ctl"
-	"github.com/featurebasedb/featurebase/v3/logger"
-	"github.com/spf13/cobra"
-)
-
-var cli *ctl.CLICommand
-
-// newCLICommand runs the FeatureBase CLI subcommand for ingesting bulk data.
-func newCLICommand(logdest logger.Logger) *cobra.Command {
-	cli = ctl.NewCLICommand(logdest)
-	cliCmd := &cobra.Command{
-		Use:   "cli",
-		Short: "Query FB with SQL3 from the command line",
-		Long:  ``,
-		RunE:  usageErrorWrapper(cli),
-	}
-
-	flags := cliCmd.Flags()
-	flags.StringVarP(&cli.Host, "host", "", cli.Host, "hostname of FeatureBase.")
-	flags.StringVarP(&cli.Port, "port", "", cli.Port, "port of FeatureBase.")
-	flags.StringVar(&cli.HistoryPath, "history-path", cli.HistoryPath, "path for history files.")
-	flags.StringVar(&cli.OrganizationID, "org-id", cli.OrganizationID, "OrganizationID.")
-	flags.StringVar(&cli.DatabaseID, "db-id", cli.DatabaseID, "DatabaseID.")
-
-	flags.StringVar(&cli.ClientID, "client-id", cli.ClientID, "Cognito Client ID for FeatureBase Cloud access.")
-	flags.StringVar(&cli.Region, "region", cli.Region, "Cloud region for FeatureBase Cloud access (e.g. us-east-2).")
-	flags.StringVar(&cli.Email, "email", cli.Email, "Email address for FeatureBase Cloud access.")
-	flags.StringVar(&cli.Password, "password", cli.Password, "Password for FeatureBase Cloud access.")
-
-	return cliCmd
-}

cmd/dataframe-csv-loader.go

@@ -0,0 +1,38 @@
+// Copyright 2021 Molecula Corp. All rights reserved.
+package cmd
+
+import (
+	"github.com/featurebasedb/featurebase/v3/ctl"
+	"github.com/featurebasedb/featurebase/v3/logger"
+	"github.com/spf13/cobra"
+)
+
+// newImportCommand runs the FeatureBase import subcommand for ingesting bulk data.
+func newDataframeCsvLoaderCommand(logdest logger.Logger) *cobra.Command {
+	cmd := ctl.NewDataframeCsvLoaderCommand(logdest)
+	loaderCmd := &cobra.Command{
+		Use:   "dataframe-csv-loader",
+		Short: "load dataframe integer and floating point values into featurebase",
+		Long: `
+`,
+		RunE: UsageErrorWrapper(cmd),
+	}
+	flags := loaderCmd.Flags()
+	flags.StringVar(&cmd.Path, "csv", "", "path to csv input file")
+	flags.StringVar(&cmd.Host, "host", "localhost:10101", "host:port of FeatureBase.")
+	flags.StringVar(&cmd.Pprof, "pprof", cmd.Pprof, "host:port to listen for profiling requests at /debug/pprof and /debug/fgprof.")
+	flags.StringVar(&cmd.AuthToken, "auth-token", "", "Authentication token")
+	flags.StringVar(&cmd.Index, "index", "", "Destination Index. ")
+	flags.IntVar(&cmd.MaxCapacity, "buffer", 0, "Maximum size of of the line buffer defaults to go bufio default ")
+	flags.IntVar(&cmd.BatchSize, "batch-size", 1048576, "Maximum number of records to send in a single batch ")
+	ctl.SetTLSConfig(
+		flags, "",
+		&cmd.TLS.CertificatePath,
+		&cmd.TLS.CertificateKeyPath,
+		&cmd.TLS.CACertPath,
+		&cmd.TLS.SkipVerify,
+		&cmd.TLS.EnableClientVerification,
+	)
+
+	return loaderCmd
+}

cmd/export.go

@@ -26,7 +26,7 @@ The format of the CSV file is:
 
 The file does not contain any headers.
 `,
-		RunE: usageErrorWrapper(Exporter),
+		RunE: UsageErrorWrapper(Exporter),
 	}
 	flags := exportCmd.Flags()
 

cmd/fbsql/main.go

@@ -0,0 +1,66 @@
+// Copyright 2021 Molecula Corp. All rights reserved.
+package main
+
+import (
+	"io"
+	"os"
+
+	"github.com/featurebasedb/featurebase/v3/cli"
+	"github.com/featurebasedb/featurebase/v3/cmd"
+	"github.com/featurebasedb/featurebase/v3/logger"
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+)
+
+func main() {
+	command := newCLICommand(os.Stderr)
+	command.Execute()
+}
+
+// newCLICommand runs the FeatureBase CLI subcommand.
+func newCLICommand(stderr io.Writer) *cobra.Command {
+	logdest := logger.NewStandardLogger(stderr)
+	cliCmd := cli.NewCommand(logdest)
+	cobraCmd := &cobra.Command{
+		Use:   "fbsql",
+		Short: "Query FeatureBase with SQL from the command line",
+		Long:  ``,
+		RunE:  cmd.UsageErrorWrapper(cliCmd),
+		PersistentPreRunE: func(cobraCmd *cobra.Command, args []string) error {
+			v := viper.New()
+			return cmd.SetAllConfig(v, cobraCmd.Flags(), "FBSQL")
+		},
+		SilenceErrors: true,
+	}
+
+	// Attach flags to the command.
+	buildFlags(cobraCmd, cliCmd)
+	return cobraCmd
+}
+
+// buildFlags attaches a set of flags to the command for a cli instance.
+func buildFlags(cmd *cobra.Command, cliCmd *cli.Command) {
+	flags := cmd.Flags()
+
+	// Base struct flags.
+	flags.StringSliceVarP(&cliCmd.Commands, "command", "c", cliCmd.Commands, "Command to run in non-interactive mode. Provide multiple flags to execute more than one command. All `--command` flags run before all `--file` flags.")
+	flags.StringSliceVarP(&cliCmd.Files, "file", "f", cliCmd.Files, "File to run in non-interactive mode. Provide multiple flags to execute more than one file. All `--command` flags run before all `--file` flags.")
+
+	// Config flags.
+	flags.StringVarP(&cliCmd.Config.Host, "host", "", cliCmd.Config.Host, "hostname of FeatureBase.")
+	flags.StringVarP(&cliCmd.Config.Port, "port", "p", cliCmd.Config.Port, "port of FeatureBase.")
+	flags.StringVar(&cliCmd.Config.HistoryPath, "history-path", cliCmd.Config.HistoryPath, "path for history files.")
+	flags.StringVar(&cliCmd.Config.OrganizationID, "org-id", cliCmd.Config.OrganizationID, "OrganizationID.")
+	flags.StringVarP(&cliCmd.Config.Database, "dbname", "d", cliCmd.Config.Database, "Name of the database to connect to.")
+
+	flags.StringVar(&cliCmd.Config.CloudAuth.ClientID, "client-id", cliCmd.Config.CloudAuth.ClientID, "Cognito Client ID for FeatureBase Cloud access.")
+	flags.StringVar(&cliCmd.Config.CloudAuth.Region, "region", cliCmd.Config.CloudAuth.Region, "Cloud region for FeatureBase Cloud access (e.g. us-east-2).")
+	flags.StringVar(&cliCmd.Config.CloudAuth.Email, "email", cliCmd.Config.CloudAuth.Email, "Email address for FeatureBase Cloud access.")
+	flags.StringVar(&cliCmd.Config.CloudAuth.Password, "password", cliCmd.Config.CloudAuth.Password, "Password for FeatureBase Cloud access.")
+
+	flags.StringVar(&cliCmd.Config.KafkaConfig, "kafka-config", cliCmd.Config.KafkaConfig, "Kafka configuration file to read from.")
+	flags.BoolVar(&cliCmd.Config.CSV, "csv", cliCmd.Config.CSV, "CSV (Comma-Separated Values) table output mode.")
+	flags.StringSliceVar(&cliCmd.Config.PSets, "pset", cliCmd.Config.PSets, "Set printing option VAR to ARG (see \\pset command). Use form: --pset=VAR[=ARG]")
+
+	flags.String("config", "", "Configuration file to read from.")
+}

cmd/generate_config.go

@@ -18,7 +18,7 @@ func newGenerateConfigCommand(logdest logger.Logger) *cobra.Command {
 		Short: "Print the default configuration.",
 		Long: `generate-config prints the default configuration to stdout
 `,
-		RunE: usageErrorWrapper(generateConf),
+		RunE: UsageErrorWrapper(generateConf),
 	}
 
 	return confCmd

cmd/keygen.go

@@ -16,7 +16,7 @@ func newKeygenCommand(logdest logger.Logger) *cobra.Command {
 		Long: `
 Generate secret key to configure FeatureBase for Authentication.
 `,
-		RunE: usageErrorWrapper(cmd),
+		RunE: UsageErrorWrapper(cmd),
 	}
 
 	flags := ccmd.Flags()

cmd/parquet-info.go

@@ -0,0 +1,33 @@
+// Copyright 2022 Molecula Corp. (DBA FeatureBase).
+// SPDX-License-Identifier: Apache-2.0
+package cmd
+
+import (
+	"fmt"
+
+	"github.com/featurebasedb/featurebase/v3/ctl"
+	"github.com/featurebasedb/featurebase/v3/logger"
+	"github.com/spf13/cobra"
+)
+
+func newParquetInfoCommand(logdest logger.Logger) *cobra.Command {
+	c := ctl.NewParquetInfoCommand(logdest)
+	cmd := &cobra.Command{
+		Use:   "parquet-info PATH|URL",
+		Short: "Inspect Parquet Files.",
+		Long: `
+Displays schema and sample data from the specified file
+`,
+		Args: func(cmd *cobra.Command, args []string) error {
+			if len(args) == 0 {
+				return fmt.Errorf("data directory path required")
+			} else if len(args) > 1 {
+				return fmt.Errorf("too many command line arguments")
+			}
+			c.Path = args[0]
+			return nil
+		},
+		RunE: UsageErrorWrapper(c),
+	}
+	return cmd
+}

cmd/presort.go

@@ -0,0 +1,32 @@
+// Copyright 2021 Molecula Corp. All rights reserved.
+package cmd
+
+import (
+	"github.com/featurebasedb/featurebase/v3/ctl"
+	"github.com/featurebasedb/featurebase/v3/logger"
+	"github.com/spf13/cobra"
+)
+
+func newPreSortCommand(logdest logger.Logger) *cobra.Command {
+	cmd := ctl.NewPreSortCommand(logdest)
+	ccmd := &cobra.Command{
+		Use:   "presort",
+		Short: "Sort records within files into files by FB partition for more efficient ingest",
+		Long: `
+Takes all input files and writes PartitionN numbered files to a directory, where each file contains only records that will go into the partition it is named for.
+`,
+		RunE: UsageErrorWrapper(cmd),
+	}
+
+	flags := ccmd.Flags()
+	flags.StringVarP(&cmd.File, "file", "", "", "Input file or directory.")
+	flags.StringVarP(&cmd.Table, "table", "", "", "Name of table (used to hash keys to determine partition).")
+	flags.StringVarP(&cmd.Type, "type", "", cmd.Type, "Input file type (csv or ndjson).")
+	flags.StringSliceVar(&cmd.PrimaryKeyFields, "primary-key-fields", []string{}, "Names of primary key fields. For CSV there must be a header row and these are pulled from there.")
+	flags.IntVar(&cmd.PartitionN, "partition-n", cmd.PartitionN, "Number of partitions.")
+	flags.StringVarP(&cmd.OutputDir, "output-dir", "", cmd.OutputDir, "Directory name to write output to.")
+	flags.StringVarP(&cmd.PrimaryKeySeparator, "primary-key-separator", "", cmd.PrimaryKeySeparator, "Separator to write in between primary key fields, can be empty.")
+	flags.IntVar(&cmd.JobSize, "job-size", cmd.JobSize, "Number of lines to put into each job (purely a performance tuning parameter, only supported by ndjson mode).")
+	flags.IntVar(&cmd.NumWorkers, "num-workers", cmd.NumWorkers, "Number of parallel worker routines doing decode->hash->encode. Only supported by ndjson mode.")
+	return ccmd
+}

cmd/rbf.go

@@ -44,7 +44,7 @@ Executes a consistency check on an RBF data directory.
 			c.Path = args[0]
 			return nil
 		},
-		RunE: usageErrorWrapper(c),
+		RunE: UsageErrorWrapper(c),
 	}
 	return cmd
 }
@@ -76,7 +76,7 @@ Dumps the raw hex data for one or more RBF pages.
 
 			return nil
 		},
-		RunE: usageErrorWrapper(c),
+		RunE: UsageErrorWrapper(c),
 	}
 	return cmd
 }
@@ -98,7 +98,7 @@ Prints a line for every page in the database with its type/status.
 			c.Path = args[0]
 			return nil
 		},
-		RunE: usageErrorWrapper(c),
+		RunE: UsageErrorWrapper(c),
 	}
 
 	flags := cmd.Flags()
@@ -133,7 +133,7 @@ Prints the header & cell data for one or more pages.
 
 			return nil
 		},
-		RunE: usageErrorWrapper(c),
+		RunE: UsageErrorWrapper(c),
 	}
 	return cmd
 }