Update README.md

changed links to the community help repo

.circleci/config.yml

@@ -1,305 +0,0 @@
-# version: 2.1
-
-# executors:
-#   golang:
-#     parameters:
-#       version:
-#         type: string
-#         default: "1.15.8"
-#       resource_class:
-#         type: string
-#         default: medium
-#     docker:
-#       - image: circleci/golang:<< parameters.version >>
-#     resource_class: << parameters.resource_class >>
-#     working_directory: /go/src/github.com/molecula/featurebase
-
-# commands:
-#   add-github-auth:
-#     steps:
-#       - run: git config --global url."https://${GITHUB_USER}:${GITHUB_PERSONAL_ACCESS_TOKEN}@github.com/".insteadOf "https://github.com/"
-#       - run: git config --global url."https://${GITHUB_USER}:${GITHUB_PERSONAL_ACCESS_TOKEN}@github.com/".insteadOf "git@github.com:"
-#   restore-mod-cache:
-#     steps:
-#       - restore_cache:
-#           key: mod-cache-{{ checksum "go.sum" }}
-#   save-mod-cache:
-#     steps:
-#       - save_cache:
-#           key: mod-cache-{{ checksum "go.sum" }}
-#           paths:
-#             - /go/pkg/mod/
-#   checkout-plus:
-#     steps:
-#       - add-github-auth
-#       - checkout
-#       - restore-mod-cache
-#   skip-if-root-unchanged:
-#     description: "skips the parent job if the PR includes no changes to featurebase"
-#     steps:
-#       - run: |
-#           ROOT_CHANGED_FILES="$(git diff --name-only HEAD $(git merge-base master HEAD) | grep -v '^lattice/')" || true
-#           echo "ROOT_CHANGED_FILES = $ROOT_CHANGED_FILES"
-#           if [ -z "$ROOT_CHANGED_FILES" ] ; then
-#             echo "halting step"
-#             circleci step halt
-#           fi
-#   skip-if-lattice-unchanged:
-#     description: "skips the parent job if the PR includes no changes to lattice"
-#     steps:
-#       - run: |
-#           LATTICE_CHANGED_FILES="$(git diff --name-only HEAD $(git merge-base master HEAD) | grep '^lattice/')" || true
-#           echo "LATTICE_CHANGED_FILES = $LATTICE_CHANGED_FILES"
-#           if [ -z "$LATTICE_CHANGED_FILES" ] ; then
-#             echo "halting step"
-#             circleci step halt
-#           fi
-
-# jobs:
-#   setup:
-#     executor:
-#       name: golang
-#     steps:
-#       - checkout-plus
-#       - run: go mod download
-#       - save-mod-cache
-#   linter:
-#     executor:
-#       name: golang
-#     steps:
-#       - checkout-plus
-#       - skip-if-root-unchanged
-#       - run: curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sudo sh -s -- -b /usr/local/bin v1.31.0
-#       - run: make golangci-lint
-#   go-mod-tidy:
-#     executor:
-#       name: golang
-#     steps:
-#       - checkout-plus
-#       - skip-if-root-unchanged
-#       - run: go mod tidy
-#       - run: git diff --exit-code -- go.mod go.sum
-#   check-changelog-label:
-#     executor:
-#       name: golang
-#     steps:
-#       - run: '[[ -n $CIRCLE_PULL_REQUEST ]] || circleci step halt || true' # Skip if this is not a pull request
-#       - run: curl https://$GITHUB_USER:$GITHUB_PERSONAL_ACCESS_TOKEN@api.github.com/repos/molecula/featurebase/pulls/$(basename $CIRCLE_PULL_REQUEST) | jq "[.labels[] | .name | startswith(\"changelog\")] | any" -e
-#   test-build-arm:
-#     executor:
-#       name: golang
-#     steps:
-#       - checkout-plus
-#       - skip-if-root-unchanged
-#       - run: make build GOOS=linux GOARCH=arm GOARM=5
-#       - run: make build GOOS=linux GOARCH=arm GOARM=6
-#       - run: make build GOOS=linux GOARCH=arm GOARM=7
-#       - run: make build GOOS=linux GOARCH=arm64
-#   test:
-#     parameters:
-#       resource_class:
-#         type: string
-#         default: medium
-#       golang_version:
-#         type: string
-#         default: "1.15.8"
-#       shard_width:
-#         type: string
-#         default: "20"
-#       test_make_target:
-#         type: string
-#         default: "test"
-#       test_flags:
-#         type: string
-#         default: ""
-#       goarch:
-#         type: string
-#         default: amd64
-#     executor:
-#       name: golang
-#       version: << parameters.golang_version >>
-#       resource_class: << parameters.resource_class >>
-#     environment:
-#       TMPDIR: /mnt/ramdisk
-#     steps:
-#       - checkout-plus
-#       - skip-if-root-unchanged
-#       - run: sudo apt-get update --allow-releaseinfo-change -y
-#       - run: sudo apt-get install lsof
-#       - run:
-#           command: make << parameters.test_make_target >> SHARD_WIDTH=<< parameters.shard_width >> GOARCH=<< parameters.goarch >>
-#           no_output_timeout: 30m
-#   test-external-lookup:
-#     docker:
-#       - image: circleci/golang:1.15.8
-#       - image: circleci/postgres:13.2-ram
-#         environment:
-#           POSTGRES_PASSWORD=password
-#     steps:
-#       - checkout-plus
-#       - skip-if-root-unchanged
-#       - run: sudo apt-get update --allow-releaseinfo-change -y
-#       - run: sudo apt-get install postgresql-client
-#       - run: (for i in `seq 1 20`; do pg_isready -h localhost && exit 0 || sleep 1; done; exit 1)
-#       - run:
-#           command: make test-external-lookup EXTERNAL_LOOKUP_DSN=postgresql://postgres:password@localhost/circle_test?sslmode=disable
-#           no_output_timeout: 30m
-#   cluster-tests:
-#     executor:
-#       name: golang
-#     steps:
-#       - checkout-plus
-#       - skip-if-root-unchanged
-#       - setup_remote_docker
-#       - run: make clustertests
-#   release:
-#     executor:
-#       name: golang
-#     steps:
-#       - checkout-plus
-#       - attach_workspace:
-#           at: .
-#       - setup_remote_docker:
-#           version: 19.03.13 # see https://support.circleci.com/hc/en-us/articles/360050934711
-#       - run: echo -n $DOCKER_PASS | docker login -u $DOCKER_USER --password-stdin
-#       - run: make docker-release
-#       - store_artifacts:
-#           path: build
-#       - persist_to_workspace:
-#           root: .
-#           paths: build
-#   publish_release:
-#     executor:
-#       name: golang
-#     steps:
-#       - attach_workspace:
-#           at: .
-#       - run: go get github.com/tcnksm/ghr
-#       - run: ghr -t ${GITHUB_PERSONAL_ACCESS_TOKEN} -u ${CIRCLE_PROJECT_USERNAME} -r ${CIRCLE_PROJECT_REPONAME} -c ${CIRCLE_SHA1} -delete ${CIRCLE_TAG} ./build/
-#   docker-build:
-#     executor:
-#       name: golang
-#     steps:
-#       - checkout-plus
-#       - setup_remote_docker:
-#           version: 19.03.13 # see https://support.circleci.com/hc/en-us/articles/360050934711
-#       - run: echo -n $DOCKER_PASS | docker login -u $DOCKER_USER --password-stdin
-#       - run: make docker GO_VERSION=1.15.8
-#       - run: docker run featurebase:$(git describe --tags) help
-#   dockerhub-upload-unstable:
-#     executor:
-#       name: golang
-#     steps:
-#       - checkout-plus
-#       - setup_remote_docker:
-#           version: 19.03.13 # see https://support.circleci.com/hc/en-us/articles/360050934711
-#       - run: echo -n $DOCKER_PASS | docker login -u $DOCKER_USER --password-stdin
-#       - run: make docker
-#       - run: docker run featurebase:$(git describe --tags) help
-#       - run: make docker-tag-push DOCKER_TARGET=moleculacorp/featurebase:<< pipeline.git.branch >>
-#   dockerhub-upload-stable:
-#     executor:
-#       name: golang
-#     steps:
-#       - checkout-plus
-#       - setup_remote_docker:
-#           version: 19.03.13 # see https://support.circleci.com/hc/en-us/articles/360050934711
-#       - run: echo -n $DOCKER_PASS | docker login -u $DOCKER_USER --password-stdin
-#       - run: make docker
-#       - run: docker run featurebase:$(git describe --tags) help
-#       - run: make docker-tag-push DOCKER_TARGET=moleculacorp/featurebase:<< pipeline.git.tag >>
-#       - run: make docker-tag-push DOCKER_TARGET=moleculacorp/featurebase:latest
-
-# workflows:
-#   build:
-#     jobs:
-#       - setup:
-#           context: molecula
-#           filters:
-#             tags:
-#               only: /^v.*/
-#       - linter:
-#           context: molecula
-#           requires:
-#             - setup
-#       - go-mod-tidy:
-#           context: molecula
-#           requires:
-#             - setup
-#       - check-changelog-label:
-#           context: molecula
-#           requires:
-#             - setup
-#       - test-build-arm:
-#           context: molecula
-#           requires:
-#             - setup
-#       - test:
-#           name: test-golang-<< matrix.golang_version >>
-#           resource_class: large
-#           context: molecula
-#           requires:
-#             - setup
-#           matrix:
-#             parameters:
-#               golang_version: ["1.15.8", "1.16.10"]
-#       - test:
-#           name: << matrix.test_make_target >>
-#           resource_class: xlarge
-#           context: molecula
-#           requires:
-#             - setup
-#           matrix:
-#             parameters:
-#               test_make_target: ["test-race"]
-#       - test:
-#           name: test-shardwidth-22
-#           context: molecula
-#           shard_width: "22"
-#           resource_class: large
-#           requires:
-#             - setup
-#       - test-external-lookup:
-#           context: molecula
-#           requires:
-#             - setup
-#       - cluster-tests:
-#           context: molecula
-#           requires:
-#             - setup
-#       - docker-build:
-#           context: molecula
-#           requires:
-#             - setup
-#       - release:
-#           context: molecula
-#           requires:
-#             - setup
-#           filters:
-#             tags:
-#               only: /^v.*/
-#       - publish_release:
-#           context: molecula
-#           requires:
-#             - release
-#           filters:
-#             tags:
-#               only: /^v.*/
-#             branches:
-#               ignore: /.*/
-#       - dockerhub-upload-unstable:
-#           context: molecula
-#           requires:
-#             - setup
-#           filters:
-#             branches:
-#               only: master
-#       - dockerhub-upload-stable:
-#           context: molecula
-#           requires:
-#             - setup
-#           filters:
-#             tags:
-#               only: /^v.*/
-#             branches:
-#               ignore: /.*/

.dockerignore

@@ -1,5 +0,0 @@
-lattice/.git
-lattice/node_modules
-lattice/build
-statik/statik.go
-build

.github/ISSUE_TEMPLATE.md

@@ -1,16 +0,0 @@
-For bugs, please provide the following:
-
-### What's going wrong?
-
-### What was expected?
-
-### Steps to reproduce the behavior
-
-### Information about your environment (OS/architecture, CPU, RAM, cluster/solo, configuration, etc.)
-
-
-For feature requests, please provide the following:
-
-### Description
-
-### Success criteria (What criteria will consider this ticket closeable?)

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,24 +0,0 @@
-## Overview
-
-[Describe what this pull request addresses.]
-
-Fixes #
-
-## Pull request checklist
-
-- [ ] I have updated the [documentation](https://github.com/molecula/docs).
-- [ ] I have resolved any merge conflicts.
-- [ ] I have included tests that cover my changes.
-- [ ] All new and existing tests pass.
-- [ ] Add appropriate changelog label to PR (if applicable).
-
-## Code review checklist
-This is the checklist that the reviewer will follow while reviewing your pull request. You do not need to do anything with this checklist, but be aware of what the reviewer will be looking for.
-
-- [ ] Ensure that any changes to external docs have been included in this pull request.
-- [ ] If the changes require that minor/major versions need to be updated, tag the PR appropriately.
-- [ ] Ensure the new code is [properly commented](https://github.com/golang/go/wiki/CodeReviewComments#doc-comments) and follows [Idiomatic Go](https://dmitri.shuralyov.com/idiomatic-go).
-- [ ] Check that tests have been written and that they cover the new functionality.
-- [ ] Run tests and ensure they pass.
-- [ ] Build and run the code, performing any applicable integration testing.
-- [ ] Make sure PR is tagged with appropriate changelog label.

.github/workflows/cd.yml

@@ -0,0 +1,77 @@
+name: CD
+
+on:
+  push:
+    branches: ["master"]
+
+  workflow_dispatch:
+
+jobs:
+  validate:
+    runs-on: ubuntu-latest
+
+    steps:
+      # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
+      - uses: actions/checkout@v3
+
+      - uses: actions/setup-go@v3
+        with:
+          go-version: "^1.19.1"
+
+      - run: go version
+
+      - name: golangci-lint
+        uses: golangci/golangci-lint-action@v3
+        with:
+          args: --timeout=5m
+
+      - name: go vet
+        run: go vet ./...
+
+      - name: test
+        run: go test ./...
+
+  release:
+    needs: validate
+    runs-on: ubuntu-latest
+    outputs:
+      version: ${{ steps.semrel.outputs.version }}
+
+    steps:
+      - name: go-semantic-release
+        id: semrel
+        uses: go-semantic-release/action@v1.17.0
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+
+  build:
+    needs: release
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        goos:
+          - "darwin"
+          - "linux"
+          - "windows"
+        goarch:
+          - "amd64"
+          - "arm64"
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - uses: actions/setup-go@v3
+        with:
+          go-version: "^1.19.1"
+
+      - name: build
+        run: GOOS=${{ matrix.goos }} GOARCH=${{ matrix.goarch }} go build -o ./build/${{ matrix.goos }}-${{ matrix.goarch }}
+
+      - name: Upload binaries to release
+        uses: svenstaro/upload-release-action@v2
+        with:
+          repo_token: ${{ secrets.GITHUB_TOKEN }}
+          file: ./build/${{ matrix.goos }}-${{ matrix.goarch }}
+          asset_name: ${{ matrix.goos }}-${{ matrix.goarch }}
+          tag: ${{ github.ref }}
+

.github/workflows/ci.yml

@@ -0,0 +1,57 @@
+# This is a basic workflow to help you get started with Actions
+
+name: CI
+
+# Controls when the workflow will run
+on:
+  pull_request:
+    branches: ["master"]
+
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+
+# A workflow run is made up of one or more jobs that can run sequentially or in parallel
+jobs:
+  validate-title:
+    name: Validate PR title
+    runs-on: ubuntu-latest
+    steps:
+      - uses: go-semantic-release/action@v1
+        id: semrel
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          dry: true
+
+  # golangci-lint must be run separately from "validate" as there are go mod issues if you run it after the vet
+  golangci:
+    name: lint
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/setup-go@v3
+        with:
+          go-version: 1.19
+      - uses: actions/checkout@v3
+      - name: golangci-lint
+        uses: golangci/golangci-lint-action@v3
+        with:
+          # Optional: version of golangci-lint to use in form of v1.2 or v1.2.3 or `latest` to use the latest version
+          # version: v1.29
+          args: --timeout=8m
+
+  validate:
+    name: Code Checks
+    runs-on: ubuntu-latest
+
+    # Steps represent a sequence of tasks that will be executed as part of the job
+    steps:
+      # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
+      - uses: actions/checkout@v3
+
+      - uses: actions/setup-go@v3
+        with:
+          go-version: "^1.19.1"
+
+      - run: go version
+
+      - name: go vet
+        run: go vet ./...

.github/workflows/release.yml

@@ -0,0 +1,25 @@
+name: Release
+
+on: workflow_dispatch
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        goos:
+          - "darwin"
+          - "linux"
+        goarch:
+          - "amd64"
+          - "arm64"
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - uses: actions/setup-go@v3
+        with:
+          go-version: "^1.19.1"
+
+      - name: build
+        run: GOOS=${{ matrix.goos }} GOARCH=${{ matrix.goarch }} go build -o ./build/${{ matrix.goos }}-${{ matrix.goarch }}

.gitignore

@@ -20,4 +20,68 @@ launch.json
 __pycache__/
 report.xml
 outputs.json
-builds/
+builds/
+*.tfstate.backup
+.vscode
+
+batch/testdata/batch*.out
+
+idk/testdata/idk*.out
+idk/testenv/certs/*
+
+# copy of .gitignore from archived idk repo
+# Compiled Object files, Static and Dynamic libs (Shared Objects)
+*.o
+*.a
+*.so
+
+# Folders
+_obj
+_test
+
+# Architecture specific extensions/prefixes
+*.[568vq]
+[568vq].out
+
+*.cgo1.go
+*.cgo2.c
+_cgo_defun.c
+_cgo_gotypes.go
+_cgo_export.*
+
+_testmain.go
+
+*.exe
+*.test
+*.prof
+
+vendor
+
+.terraform
+terraform.tfstate*
+
+bin
+build
+testenv
+.pulled
+
+pilosa-sec-data-idk
+
+.idea/
+tags.dot
+*.log
+*.swp
+*__debug_bin
+
+# SQL3
+/sql3/sql3.html
+
+staticcheck.conf
+
+
+.quick
+dax/dax-data
+
+coverage-from-docker
+*.client_id.txt
+

.gitlab/.perf-able-gitlab-ci.yml

@@ -1,11 +0,0 @@
-stages:
-  - loadtest
-
-loadtest:
-  image:
-    name: loadimpact/k6:latest
-    entrypoint: ['']
-  stage: loadtest
-  script:
-    - echo "executing local k6 in k6 container..."
-    - k6 run ./qa/scripts/perf/able/script.js

.gitlab/batch-ci.yml

@@ -0,0 +1,28 @@
+run go tests batch:
+  variables:
+    PROJECT: batch_${CI_CONCURRENT_ID}
+  # this test relies on stuff that happens after build-lattice, which
+  # makes it pause the entire CI run waiting for this. we accept the
+  # small risk of wasting a build against the near certainty of spending
+  # five minutes running only one job.
+  stage: nonblocking
+  retry: 1
+  script:
+    - echo "Running test-all"
+    - cd ./batch/
+    - echo $PROJECT
+    - make build-featurebase
+    - make test-all
+  after_script:
+    - cd ./batch/
+    - make save-featurebase-logs
+    - make shutdown
+  artifacts:
+    paths:
+      - ./batch/testdata/*_coverage.out
+      - ./batch/testdata/*_logs.txt
+  tags:
+    - shell
+    - aws
+  needs:
+    - job: build amd container fb

.golangci.yml

@@ -1,8 +1,8 @@
 run:
-  #skip the protobuf generated files
   deadline: 5m
   timeout: 5m
   skip-dirs-use-default: true
+  #skip the protobuf generated files
   skip-dirs:
     - pb
     - proto
@@ -10,8 +10,26 @@ run:
     - pql/pql.peg.go
 linters:
   enable:
+    # Recommended to be enabled by default (https://golangci-lint.run).
+    # - errcheck (lots to fix)
+    - gosimple
     - govet
+    - ineffassign
+    - staticcheck
+    - typecheck
+    # - unused (about 20 to fix)
+
+    # Additional linters we choose to enable.
+    # - bodyclose (lots to fix, but we should)
+    - errchkjson
+    - errname
     - gofmt
+    # - misspell (lots to fix, but we should)
+    - prealloc
+    # - predeclared (20 to fix)
+    # - stylecheck (quite a lot to fix, but we should definitely work on this)
+    - stylecheck
+    # - unconvert (not at all critical, but makes for cleaner code)
   enable-all: false
   disable-all: true
 
@@ -49,6 +67,12 @@ linters-settings:
       - shadow
     disable-all: false
 
+  stylecheck:
+    # ST1000: at least one file in a package should have a package comment
+    # ST1003: golang naming standards
+    # ST1016: methods on the same type should have the same receiver name
+    # ST1020: comment on exported function
+    checks: ["all", "-ST1000", "-ST1003", "-ST1016", "-ST1020"]
 
 issues:
   exclude-use-default: false
@@ -57,4 +81,3 @@ issues:
   exclude:
     - 'declaration of "(err|ctx)" shadows declaration at'
     - 'Error return value of .(.*\.Help|.*\.MarkFlagRequired|(os\.)?std(out|err)\..*|.*Close|.*Flush|os\.Remove(All)?|.*printf?|os\.(Un)?Setenv). is not checked'
-

CODE_OF_CONDUCT.md

@@ -0,0 +1,133 @@
+
+# Contributor Covenant Code of Conduct
+
+## Our Pledge
+
+We as members, contributors, and leaders pledge to make participation in our
+community a harassment-free experience for everyone, regardless of age, body
+size, visible or invisible disability, ethnicity, sex characteristics, gender
+identity and expression, level of experience, education, socio-economic status,
+nationality, personal appearance, race, caste, color, religion, or sexual
+identity and orientation.
+
+We pledge to act and interact in ways that contribute to an open, welcoming,
+diverse, inclusive, and healthy community.
+
+## Our Standards
+
+Examples of behavior that contributes to a positive environment for our
+community include:
+
+* Demonstrating empathy and kindness toward other people
+* Being respectful of differing opinions, viewpoints, and experiences
+* Giving and gracefully accepting constructive feedback
+* Accepting responsibility and apologizing to those affected by our mistakes,
+  and learning from the experience
+* Focusing on what is best not just for us as individuals, but for the overall
+  community
+
+Examples of unacceptable behavior include:
+
+* The use of sexualized language or imagery, and sexual attention or advances of
+  any kind
+* Trolling, insulting or derogatory comments, and personal or political attacks
+* Public or private harassment
+* Publishing others' private information, such as a physical or email address,
+  without their explicit permission
+* Other conduct which could reasonably be considered inappropriate in a
+  professional setting
+
+## Enforcement Responsibilities
+
+Community leaders are responsible for clarifying and enforcing our standards of
+acceptable behavior and will take appropriate and fair corrective action in
+response to any behavior that they deem inappropriate, threatening, offensive,
+or harmful.
+
+Community leaders have the right and responsibility to remove, edit, or reject
+comments, commits, code, wiki edits, issues, and other contributions that are
+not aligned to this Code of Conduct, and will communicate reasons for moderation
+decisions when appropriate.
+
+## Scope
+
+This Code of Conduct applies within all community spaces, and also applies when
+an individual is officially representing the community in public spaces.
+Examples of representing our community include using an official e-mail address,
+posting via an official social media account, or acting as an appointed
+representative at an online or offline event.
+
+## Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be
+reported to the community leaders responsible for enforcement at
+community@featurebase.com.
+All complaints will be reviewed and investigated promptly and fairly.
+
+All community leaders are obligated to respect the privacy and security of the
+reporter of any incident.
+
+## Enforcement Guidelines
+
+Community leaders will follow these Community Impact Guidelines in determining
+the consequences for any action they deem in violation of this Code of Conduct:
+
+### 1. Correction
+
+**Community Impact**: Use of inappropriate language or other behavior deemed
+unprofessional or unwelcome in the community.
+
+**Consequence**: A private, written warning from community leaders, providing
+clarity around the nature of the violation and an explanation of why the
+behavior was inappropriate. A public apology may be requested.
+
+### 2. Warning
+
+**Community Impact**: A violation through a single incident or series of
+actions.
+
+**Consequence**: A warning with consequences for continued behavior. No
+interaction with the people involved, including unsolicited interaction with
+those enforcing the Code of Conduct, for a specified period of time. This
+includes avoiding interactions in community spaces as well as external channels
+like social media. Violating these terms may lead to a temporary or permanent
+ban.
+
+### 3. Temporary Ban
+
+**Community Impact**: A serious violation of community standards, including
+sustained inappropriate behavior.
+
+**Consequence**: A temporary ban from any sort of interaction or public
+communication with the community for a specified period of time. No public or
+private interaction with the people involved, including unsolicited interaction
+with those enforcing the Code of Conduct, is allowed during this period.
+Violating these terms may lead to a permanent ban.
+
+### 4. Permanent Ban
+
+**Community Impact**: Demonstrating a pattern of violation of community
+standards, including sustained inappropriate behavior, harassment of an
+individual, or aggression toward or disparagement of classes of individuals.
+
+**Consequence**: A permanent ban from any sort of public interaction within the
+community.
+
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage],
+version 2.1, available at
+[https://www.contributor-covenant.org/version/2/1/code_of_conduct.html][v2.1].
+
+Community Impact Guidelines were inspired by
+[Mozilla's code of conduct enforcement ladder][Mozilla CoC].
+
+For answers to common questions about this code of conduct, see the FAQ at
+[https://www.contributor-covenant.org/faq][FAQ]. Translations are available at
+[https://www.contributor-covenant.org/translations][translations].
+
+[homepage]: https://www.contributor-covenant.org
+[v2.1]: https://www.contributor-covenant.org/version/2/1/code_of_conduct.html
+[Mozilla CoC]: https://github.com/mozilla/diversity
+[FAQ]: https://www.contributor-covenant.org/faq
+[translations]: https://www.contributor-covenant.org/translations

Dockerfile

@@ -4,7 +4,7 @@ ARG GO_VERSION=latest
 ### Lattice builder ###
 #######################
 
-FROM moleculacorp/nodejs:latest as lattice-builder
+FROM ghcr.io/featurebasedb/nodejs:0.0.1 as lattice-builder
 WORKDIR /lattice
 
 COPY lattice/package.json ./
@@ -20,14 +20,17 @@ RUN yarn build
 
 FROM golang:${GO_VERSION} as pilosa-builder
 ARG MAKE_FLAGS
+ARG SOURCE_DATE_EPOCH
+
 WORKDIR /pilosa
 
-RUN go get github.com/rakyll/statik
+RUN go install github.com/rakyll/statik@v0.1.7
 
 COPY . ./
 COPY --from=lattice-builder /lattice/build /lattice
 RUN /go/bin/statik -src=/lattice -dest=/pilosa
 
+ENV SOURCE_DATE_EPOCH=${SOURCE_DATE_EPOCH}
 RUN make build FLAGS="-o build/featurebase" ${MAKE_FLAGS}
 
 #####################
@@ -38,7 +41,7 @@ FROM alpine:3.13.2 as runner
 
 LABEL maintainer "dev@molecula.com"
 
-RUN apk add --no-cache curl jq
+RUN apk add --no-cache curl jq tree
 
 COPY --from=pilosa-builder /pilosa/build/featurebase /
 

Dockerfile-clustertests

@@ -1,11 +1,11 @@
 # This Dockerfile is used for cluster testing - it produces a much larger image
 # and includes all of Go as well as some utilities.
 
-FROM golang:1.16
+FROM golang:1.19
 
 LABEL maintainer "dev@pilosa.com"
 
-COPY . /go/src/github.com/molecula/featurebase/
+COPY . /go/src/github.com/featurebasedb/featurebase/
 
 
 # download pumba for fault injection
@@ -14,22 +14,25 @@ RUN chmod +x /pumba
 
 # add docker client to pause/unpause nodes
 RUN apt update
-RUN apt install -y docker.io 
+RUN apt install -y docker.io
 
 # add docker-compose so tests can use it for stuff
 ADD https://github.com/docker/compose/releases/latest/download/docker-compose-Linux-x86_64 /usr/local/bin/docker-compose
 RUN chmod +x /usr/local/bin/docker-compose
 
+WORKDIR /go/src/github.com/featurebasedb/featurebase/cmd/featurebase
+
 # generate an instrumented binary to allow for calculating code coverage for clustertests
-# the entrypoint for the binary is TestRunMain, which is wrapper for main 
-RUN cd /go/src/github.com/molecula/featurebase/cmd/featurebase && \
-    go test -covermode=atomic -coverpkg=../../... -c -tags testrunmain -o featurebase && \
-    cp /go/src/github.com/molecula/featurebase/cmd/featurebase/featurebase /featurebase
+# the entrypoint for the binary is TestRunMain, which is wrapper for main
+RUN go test -covermode=atomic -coverpkg=../../... -c -tags testrunmain -o featurebase
+RUN cp /go/src/github.com/featurebasedb/featurebase/cmd/featurebase/featurebase /featurebase
 
 COPY NOTICE /NOTICE
 
 EXPOSE 10101
 VOLUME /data
 
-ENTRYPOINT ["bash", "-c"]
-CMD ["/featurebase", "-test.run=TestRunMain", "-test.coverprofile=/results/coverage.out", "server", "--data-dir", "/data", "--bind", "http://0.0.0.0:10101"]
+
+# use e.g. "-test.coverprofile=/results/coverage.out"
+CMD ["/featurebase", "-test.run=TestRunMain", "server"]
+

Dockerfile-clustertests-client

@@ -1,11 +1,11 @@
 # This Dockerfile is used for cluster testing - it produces a much larger image
 # and includes all of Go as well as some utilities.
 
-FROM golang:1.16
+FROM golang:1.19
 
 LABEL maintainer "dev@pilosa.com"
 
-COPY . /go/src/github.com/molecula/featurebase/
+COPY . /go/src/github.com/featurebasedb/featurebase/
 
 # download pumba for fault injection
 ADD https://github.com/alexei-led/pumba/releases/download/0.6.0/pumba_linux_amd64 /pumba
@@ -13,23 +13,25 @@ RUN chmod +x /pumba
 
 # add docker client to pause/unpause nodes
 RUN apt update
-RUN apt install -y docker.io 
+RUN apt install -y docker.io
 
 # add docker-compose so tests can use it for stuff
 ADD https://github.com/docker/compose/releases/latest/download/docker-compose-Linux-x86_64 /usr/local/bin/docker-compose
 RUN chmod +x /usr/local/bin/docker-compose
 
-RUN cd /go/src/github.com/molecula/featurebase/cmd/featurebase && \
-    go test -covermode=atomic -coverpkg=../../... -c -tags testrunmain -o featurebase && \
-    cp /go/src/github.com/molecula/featurebase/cmd/featurebase/featurebase /featurebase
+WORKDIR /go/src/github.com/featurebasedb/featurebase/cmd/featurebase
+
+RUN go test -covermode=atomic -coverpkg=../../... -c -tags testrunmain -o featurebase
+RUN cp /go/src/github.com/featurebasedb/featurebase/cmd/featurebase/featurebase /featurebase
 
 
 COPY NOTICE /NOTICE
 
-COPY ./internal/clustertests /go/src/github.com/molecula/featurebase/internal/clustertests
+COPY ./internal/clustertests /go/src/github.com/featurebasedb/featurebase/internal/clustertests
 
 EXPOSE 10101
 VOLUME /data
 
-ENTRYPOINT ["bash", "-c"]
+WORKDIR /go/src/github.com/featurebasedb/featurebase
+
 CMD ["/featurebase", "-test.run=TestRunMain", "-test.coverprofile=/results/coverage.out", "server", "--data-dir", "/data", "--bind", "http://0.0.0.0:10101"]

Dockerfile-datagen

@@ -0,0 +1,47 @@
+# syntax=docker/dockerfile:1
+
+##########################
+### datagen builder    ###
+##########################
+
+FROM golang:alpine as builder
+
+WORKDIR /featurebase
+
+COPY . ./
+
+RUN apk add --no-cache build-base bash git make librdkafka pkgconfig
+
+# install librdkafka
+RUN git clone https://github.com/edenhill/librdkafka.git
+RUN cd librdkafka && ./configure --prefix /usr && make && make install
+
+ENV PKG_CONFIG_PATH=/usr/lib/pkgconfig/
+
+RUN cd idk && make build-datagen
+
+# ENTRYPOINT ["tail", "-f", "/dev/null"]
+
+#########################
+### datagen runner    ###
+#########################
+
+FROM alpine:3.15.3 as runner
+
+WORKDIR /
+
+LABEL maintainer "dev@molecula.com"
+
+RUN apk add --no-cache curl jq
+
+COPY --from=builder /featurebase/idk/build/datagen /bin/
+COPY --from=builder /usr/lib/librdkafka* /usr/lib/
+COPY idk/datagen/testdata/* /testdata/
+
+EXPOSE 8080
+
+# VOLUME /data
+# ENV ADDR 0.0.0.0:8080
+
+#ENTRYPOINT ["sleep", "infinity"]
+ENTRYPOINT ["datagen"]

Dockerfile-dax

@@ -0,0 +1,32 @@
+ARG GO_VERSION=latest
+
+
+###########################
+### FeatureBase Builder ###
+###########################
+
+FROM golang:${GO_VERSION} as featurebase-builder
+ARG MAKE_FLAGS
+WORKDIR /fb
+
+COPY . ./
+RUN make build FLAGS="-o build/featurebase" ${MAKE_FLAGS}
+
+##########################
+### FeatureBase runner ###
+##########################
+
+FROM golang:alpine as runner
+
+LABEL maintainer "dev@featurebase.com"
+
+RUN apk add --no-cache curl jq tree
+
+COPY --from=featurebase-builder /fb/build/featurebase /
+
+COPY NOTICE /NOTICE
+
+EXPOSE 8080
+
+ENTRYPOINT ["/featurebase"]
+CMD ["dax"]

Dockerfile-dax-quick

@@ -0,0 +1,18 @@
+ARG GO_VERSION=latest
+
+##########################
+### FeatureBase runner ###
+##########################
+
+FROM alpine:3.13.2 as runner
+
+LABEL maintainer "dev@featurebase.com"
+
+RUN apk add --no-cache curl jq tree
+
+COPY ./fb_linux /featurebase
+
+EXPOSE 8080
+
+ENTRYPOINT ["/featurebase"]
+CMD ["dax"]

Dockerfile-fbsql

@@ -0,0 +1,48 @@
+ARG GO_VERSION=1.19
+
+FROM golang:1.19-buster as builder
+
+WORKDIR /
+RUN apt-get update -y -qq && apt-get install -y -qq \
+  build-essential \
+  git \
+  musl-tools \
+  netcat \
+  unixodbc \
+  unixodbc-dev \
+  && rm -rf /var/lib/apt/lists/*
+RUN ["git", "clone", "https://github.com/edenhill/librdkafka.git"]
+WORKDIR /librdkafka
+RUN ./configure --prefix /usr && \
+  make && \
+  make install
+
+WORKDIR /featurebase
+
+COPY . .
+
+ARG MAKE_FLAGS
+ARG GO_BUILD_FLAGS
+ARG SOURCE_DATE_EPOCH
+
+WORKDIR /featurebase/
+
+ENV SOURCE_DATE_EPOCH=${SOURCE_DATE_EPOCH}
+RUN make build-fbsql GO_BUILD_FLAGS="-mod=vendor ${GO_BUILD_FLAGS}" ${MAKE_FLAGS}
+
+FROM ubuntu:20.04 as runner
+
+RUN apt-get update -y -qq && apt-get install -y -qq \
+  ca-certificates \
+  musl-tools \
+  netcat \
+  unixodbc-dev \
+  && rm -rf /var/lib/apt/lists/*
+
+COPY --from=builder /featurebase/fbsql /usr/local/bin/
+
+# Verify that the linker can find everything.
+FROM runner as linkcheck
+RUN if [ -e /usr/local/bin/fbsql ] ; then ldd /usr/local/bin/fbsql; fi
+
+FROM runner

LICENSE

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright 2023 Molecula Corp. All rights reserved.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

LICENSE-2.0.txt

@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

Makefile

@@ -1,34 +1,38 @@
-.PHONY: build check-clean clean build-lattice cover cover-viz default docker docker-build docker-test docker-tag-push generate generate-protoc generate-pql generate-statik gometalinter install install-build-deps install-golangci-lint install-gometalinter install-protoc install-protoc-gen-gofast install-peg install-statik release release-build test testv testv-race testvsub testvsub-race test-txstore-rbf
+.PHONY: build clean build-lattice cover cover-viz default docker docker-build docker-build-fbsql docker-tag-push generate generate-protoc generate-pql generate-statik generate-stringer install install-protoc-gen-gofast install-protoc install-statik install-peg test docker-login
 
-CLONE_URL=github.com/pilosa/pilosa
+SHELL := /bin/bash
 VERSION := $(shell git describe --tags 2> /dev/null || echo unknown)
 VARIANT = Molecula
 GO=go
 GOOS=$(shell $(GO) env GOOS)
 GOARCH=$(shell $(GO) env GOARCH)
 VERSION_ID=$(if $(TRIAL_DEADLINE),trial-$(TRIAL_DEADLINE)-,)$(VERSION)-$(GOOS)-$(GOARCH)
-BRANCH := $(if $(CIRCLE_BRANCH),$(CIRCLE_BRANCH),$(shell git rev-parse --abbrev-ref HEAD))
-BRANCH_ID := $(BRANCH)-$(GOOS)-$(GOARCH)
-BUILD_TIME := $(shell date -u +%FT%T%z)
+DATE_FMT="+%FT%T%z"
+# set SOURCE_DATE_EPOCH like this to use the last git commit timestamp
+# export SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct) instead of the current time from running `date`
+ifdef SOURCE_DATE_EPOCH
+	BUILD_TIME ?= $(shell date -u -d "@$(SOURCE_DATE_EPOCH)" "$(DATE_FMT)" 2>/dev/null || date -u -r "$(SOURCE_DATE_EPOCH)" "$(DATE_FMT)" 2>/dev/null || date -u "$(DATE_FMT)")
+else
+	BUILD_TIME ?= $(shell date -u "$(DATE_FMT)")
+endif
 SHARD_WIDTH = 20
 COMMIT := $(shell git describe --exact-match >/dev/null 2>&1 || git rev-parse --short HEAD)
-LDFLAGS="-X github.com/molecula/featurebase/v3.Version=$(VERSION) -X github.com/molecula/featurebase/v3.BuildTime=$(BUILD_TIME) -X github.com/molecula/featurebase/v3.Variant=$(VARIANT) -X github.com/molecula/featurebase/v3.Commit=$(COMMIT) -X github.com/molecula/featurebase/v3.TrialDeadline=$(TRIAL_DEADLINE)"
-GO_VERSION=1.16.10
+LDFLAGS="-X github.com/featurebasedb/featurebase/v3.Version=$(VERSION) -X github.com/featurebasedb/featurebase/v3.BuildTime=$(BUILD_TIME) -X github.com/featurebasedb/featurebase/v3.Variant=$(VARIANT) -X github.com/featurebasedb/featurebase/v3.Commit=$(COMMIT) -X github.com/featurebasedb/featurebase/v3.TrialDeadline=$(TRIAL_DEADLINE)"
+GO_VERSION=1.19
+GO_BUILD_FLAGS=
 DOCKER_BUILD= # set to 1 to use `docker-build` instead of `build` when creating a release
-BUILD_TAGS += shardwidth$(SHARD_WIDTH)
+BUILD_TAGS += 
 TEST_TAGS = roaringparanoia
-UNAME := $(shell uname -s)
-TEST_TIMEOUT=30m
-RACE_TEST_TIMEOUT=90m
-ifeq ($(UNAME), Darwin)
-    IS_MACOS:=1
-else
-    IS_MACOS:=0
-endif
+TEST_TIMEOUT=10m
+RACE_TEST_TIMEOUT=10m
+# size in GB to use for ramdisk, ?= so you can override it with env
+# 4GB is not enough for `make test`, 8GB usually is.
+RAMDISK_SIZE ?= 8
 
 export GO111MODULE=on
 export GOPRIVATE=github.com/molecula
 export CGO_ENABLED=0
+AWS_ACCOUNTID ?= undefined
 
 # Run tests and compile Pilosa
 default: test build
@@ -36,6 +40,7 @@ default: test build
 # Remove build directories
 clean:
 	rm -rf vendor build
+	rm -f *.rpm *.deb
 
 # Set up vendor directory using `go mod vendor`
 vendor: go.mod
@@ -44,17 +49,21 @@ vendor: go.mod
 version:
 	@echo $(VERSION)
 
+# We build a list of packages that omits the IDK and batch packages because
+# those packages require fancy environment setup.
+GOPACKAGES := $(shell $(GO) list ./... | grep -v "/v3/idk" | grep -v "/v3/batch")
+
 # Run test suite
 test:
-	$(GO) test ./... -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) -v -timeout $(TEST_TIMEOUT)
+	$(GO) test $(GOPACKAGES) -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) -v -timeout $(TEST_TIMEOUT) -count=1
 
 # Run test suite with race flag
 test-race:
-	CGO_ENABLED=1 $(GO) test ./... -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) -race -timeout $(RACE_TEST_TIMEOUT) -v
+	CGO_ENABLED=1 $(GO) test $(GOPACKAGES) -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) -race -timeout $(RACE_TEST_TIMEOUT) -v
 
-testv: topt testvsub
+testv: testvsub
 
-testv-race: topt-race testvsub-race
+testv-race: testvsub-race
 
 # testvsub: run go test -v in sub-directories in "local mode" with incremental output,
 #            avoiding go -test ./... "package list mode" which doesn't give output
@@ -62,25 +71,42 @@ testv-race: topt-race testvsub-race
 #            find which test is hung/deadlocked.
 #
 testvsub:
-	set -e; for i in boltdb client ctl http pg pql rbf roaring server sql txkey; do \
-           echo; echo "___ testing subpkg $$i"; \
-           cd $$i; pwd; \
-           $(GO) test -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) -v -timeout $(RACE_TEST_TIMEOUT) || break; \
-           echo; echo "999 done testing subpkg $$i"; \
-           cd ..; \
-        done
+	@set -e; for pkg in $(GOPACKAGES); do \
+			if [ $${pkg:0:38} == "github.com/featurebasedb/featurebase/v3/idk" ]; then \
+				echo; echo "___ skipping subpkg $$pkg"; \
+				continue; \
+			fi; \
+			echo; echo "___ testing subpkg $$pkg"; \
+			$(GO) test -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) -v -timeout $(RACE_TEST_TIMEOUT) $$pkg || break; \
+			echo; echo "999 done testing subpkg $$pkg"; \
+		done
+
+# make a $(RAMDISK_SIZE)GB RAMDisk. Speed up tests by running
+# them with TMPDIR=/mnt/ramdisk.
+ramdisk-linux:
+	mount -o size=$(RAMDISK__SIZE)G -t tmpfs none /mnt/ramdisk
+
+# make a $(RAMDISK_SIZE)GB RAMDisk. Speed up tests by running
+# them with TMPDIR=/Volumes/RAMDisk. This is more important on
+# OS X than it is on Linux, because there's performance issues
+# with fsync on OS X that can make the SSD slow down to moving-platters
+# drive speeds. Oops.
+ramdisk-osx:
+	diskutil erasevolume HFS+ 'RAMDisk' $$(hdiutil attach -nobrowse -nomount ram://$$(expr 2097152 \* $(RAMDISK_SIZE)))
+
+detach-ramdisk-osx:
+	hdiutil detach /Volumes/RAMDisk
 
 testvsub-race:
-	set -e; for i in boltdb client ctl http pg pql rbf roaring server sql txkey; do \
-           echo; echo "___ testing subpkg $$i -race"; \
-           cd $$i; pwd; \
-           CGO_ENABLED=1 $(GO) test -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) -v -race -timeout $(RACE_TEST_TIMEOUT) || break; \
-           echo; echo "999 done testing subpkg $$i -race"; \
-           cd ..; \
+	@set -e; for pkg in $(GOPACKAGES); do \
+           echo; echo "___ testing subpkg $$pkg"; \
+           CGO_ENABLED=1 $(GO) test -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) -v -race -timeout $(RACE_TEST_TIMEOUT) $$pkg || break; \
+           echo; echo "999 done testing subpkg $$pkg"; \
         done
 
+
 bench:
-	$(GO) test ./... -bench=. -run=NoneZ -timeout=127m $(TESTFLAGS)
+	$(GO) test $(GOPACKAGES) -bench=. -run=NoneZ -timeout=127m $(TESTFLAGS)
 
 # Run test suite with coverage enabled
 cover:
@@ -91,52 +117,15 @@ cover:
 cover-viz: cover
 	$(GO) tool cover -html=build/coverage.out
 
-# Compile Pilosa
+# Build featurebase
 build:
 	$(GO) build -tags='$(BUILD_TAGS)' -ldflags $(LDFLAGS) $(FLAGS) ./cmd/featurebase
 
-# Create a single release build under the build directory
-release-build:
-	$(MAKE) $(if $(DOCKER_BUILD),docker-)build FLAGS="-o build/featurebase-$(VERSION_ID)/featurebase"
-	cp NOTICE install/featurebase.conf install/featurebase*.service build/featurebase-$(VERSION_ID)
-	tar -cvz -C build -f build/featurebase-$(VERSION_ID).tar.gz featurebase-$(VERSION_ID)/
-	@echo Created release build: build/featurebase-$(VERSION_ID).tar.gz
-
-test-release-build: docker-build
-	mv build/featurebase-$(VERSION_ID).tar.gz install/
-	cd install && docker build -t featurebase:test_installation \
-		-f test_installation.Dockerfile \
-		--build-arg release_tarball=featurebase-$(VERSION_ID).tar.gz .
-	mv install/featurebase-$(VERSION_ID).tar.gz build/
-	docker run -it -v /sys/fs/cgroup:/sys/fs/cgroup:ro \
-		featurebase:test_installation
-
-# Error out if there are untracked changes in Git
-check-clean:
-ifndef SKIP_CHECK_CLEAN
-	$(if $(shell git status --porcelain),$(error Git status is not clean! Please commit or checkout/reset changes.))
-endif
-
-# Create release build tarballs for all supported platforms. DEPRECATED: Use `docker-release`
-release: check-clean generate-statik-docker
-	$(MAKE) release-build GOOS=darwin GOARCH=amd64
-	$(MAKE) release-build GOOS=darwin GOARCH=arm64
-	$(MAKE) release-build GOOS=linux GOARCH=amd64
-	$(MAKE) release-build GOOS=linux GOARCH=arm64
-
-# Create release build tarballs for all supported platforms. Same as `release`, but without embedded Lattice UI.
-release-sans-ui: check-clean
-	rm -f statik/statik.go
-	$(MAKE) release-build GOOS=darwin GOARCH=amd64
-	$(MAKE) release-build GOOS=darwin GOARCH=arm64
-	$(MAKE) release-build GOOS=linux GOARCH=amd64
-	$(MAKE) release-build GOOS=linux GOARCH=arm64
-
 package:
-	go build -o featurebase ./cmd/featurebase
-	nfpm package --packager deb --target featurebase_$(VERSION_ID).deb
-	nfpm package --packager rpm --target featurebase_$(VERSION_ID).rpm
-
+	GOOS=$(GOOS) GOARCH=$(GOARCH) $(MAKE) build
+	GOOS=$(GOOS) GOARCH=$(GOARCH) $(MAKE) build-fbsql
+	GOARCH=$(GOARCH) VERSION=$(VERSION) nfpm package --packager deb --target featurebase.$(VERSION).$(GOARCH).deb
+	GOARCH=$(GOARCH) VERSION=$(VERSION) nfpm package --packager rpm --target featurebase.$(VERSION).$(GOARCH).rpm
 
 # We allow setting a custom docker-compose "project". Multiple of the
 # same docker-compose environment can exist simultaneously as long as
@@ -155,22 +144,26 @@ clustertests: vendor
 	PROJECT=$(PROJECT) $(DOCKER_COMPOSE) -f internal/clustertests/docker-compose.yml run client1
 	$(DOCKER_COMPOSE) -f internal/clustertests/docker-compose.yml down
 
-# Run the cluster tests with authentication enabled 
-AUTH_ARGS="-c /go/src/github.com/molecula/featurebase/internal/clustertests/testdata/featurebase.conf"
+# Run the cluster tests with authentication enabled
+AUTH_ARGS="-c /go/src/github.com/featurebasedb/featurebase/internal/clustertests/testdata/featurebase.conf"
 authclustertests: vendor
-	$(eval PROJECT=authclustertests)
 	CLUSTERTESTS_FB_ARGS=$(AUTH_ARGS) $(DOCKER_COMPOSE) -f internal/clustertests/docker-compose.yml down
 	CLUSTERTESTS_FB_ARGS=$(AUTH_ARGS) $(DOCKER_COMPOSE) -f internal/clustertests/docker-compose.yml build
 	CLUSTERTESTS_FB_ARGS=$(AUTH_ARGS) $(DOCKER_COMPOSE) -f internal/clustertests/docker-compose.yml up -d pilosa1 pilosa2 pilosa3
 	PROJECT=$(PROJECT) ENABLE_AUTH=1 $(DOCKER_COMPOSE) -f internal/clustertests/docker-compose.yml run client1
 	CLUSTERTESTS_FB_ARGS=$(AUTH_ARGS) $(DOCKER_COMPOSE) -f internal/clustertests/docker-compose.yml down
 
-# Install Pilosa
-install:
+# Install FeatureBase and IDK
+install: install-featurebase install-idk install-fbsql
+
+install-featurebase:
 	$(GO) install -tags='$(BUILD_TAGS)' -ldflags $(LDFLAGS) $(FLAGS) ./cmd/featurebase
 
-install-bench:
-	$(GO) install -tags='$(BUILD_TAGS)' -ldflags $(LDFLAGS) $(FLAGS) ./cmd/pilosa-bench
+install-idk:
+	$(MAKE) -C ./idk install
+
+install-fbsql:
+	CGO_ENABLED=1 $(GO) install ./cmd/fbsql
 
 # Build the lattice assets
 build-lattice:
@@ -179,11 +172,11 @@ build-lattice:
 
 # `go generate` protocol buffers
 generate-protoc: require-protoc require-protoc-gen-gofast
-	$(GO) generate github.com/molecula/featurebase/v3/pb
+	$(GO) generate github.com/featurebasedb/featurebase/v3/pb
 
 # `go generate` statik assets (lattice UI)
 generate-statik: build-lattice require-statik
-	$(GO) generate github.com/molecula/featurebase/v3/statik
+	$(GO) generate github.com/featurebasedb/featurebase/v3/statik
 
 # `go generate` statik assets (lattice UI) in Docker
 generate-statik-docker: build-lattice
@@ -191,19 +184,15 @@ generate-statik-docker: build-lattice
 
 # `go generate` stringers
 generate-stringer:
-	$(GO) generate github.com/molecula/featurebase/v3
+	$(GO) generate github.com/featurebasedb/featurebase/v3
 
 generate-pql: require-peg
 	cd pql && peg -inline pql.peg && cd ..
 
 generate-proto-grpc: require-protoc require-protoc-gen-go
 	protoc -I proto proto/pilosa.proto --go_out=plugins=grpc:proto
-	protoc -I proto proto/vdsm/vdsm.proto --go_out=plugins=grpc:proto
-	# TODO: Modify above commands and remove the below mv if possible.
-	# See https://go-review.googlesource.com/c/protobuf/+/219298/ for info on --go-opt
-	# I couldn't get it to work during development - Cody
-	cp -r proto/github.com/molecula/featurebase/v3/proto/ proto/
-	rm -rf proto/github.com
+#	address re-generation here only if we need to	
+#	protoc -I proto proto/vdsm.proto --go_out=plugins=grpc:proto
 
 # `go generate` all needed packages
 generate: generate-protoc generate-statik generate-stringer generate-pql
@@ -220,6 +209,7 @@ docker-build: vendor
 	docker build \
 	    --build-arg GO_VERSION=$(GO_VERSION) \
 	    --build-arg MAKE_FLAGS="TRIAL_DEADLINE=$(TRIAL_DEADLINE) GOOS=$(GOOS) GOARCH=$(GOARCH)" \
+	    --build-arg SOURCE_DATE_EPOCH=$(SOURCE_DATE_EPOCH) \
 	    --target pilosa-builder \
 	    --tag featurebase:build .
 	docker create --name featurebase-build featurebase:build
@@ -237,6 +227,61 @@ docker-image: vendor
 	    --tag featurebase:$(VERSION) .
 	@echo Created docker image: featurebase:$(VERSION)
 
+docker-image-featurebase: vendor
+	docker build \
+	    --build-arg GO_VERSION=$(GO_VERSION) \
+		--file Dockerfile-dax \
+		--tag dax/featurebase .
+
+docker-image-featurebase-linux-amd64: vendor
+	docker build \
+	    --build-arg GO_VERSION=$(GO_VERSION) \
+	    --platform linux/amd64 \
+	    --file Dockerfile-dax \
+	    --tag dax/featurebase .
+
+docker-image-featurebase-test: vendor
+	docker build \
+	    --build-arg GO_VERSION=$(GO_VERSION) \
+		--file Dockerfile-clustertests \
+		--tag dax/featurebase-test .
+
+
+# build-for-quick builds a linux featurebase binary outside of docker
+# (which is much faster for some reason), and places it in the .quick
+# subdirectory.
+build-for-quick:
+	GOOS=linux $(MAKE) build FLAGS="-o .quick/fb_linux"
+
+# docker-image-featurebase-quick uses a pre-built featurebase binary
+# to quickly create a fresh docker image without needing to send the
+# context of the featurebase top level directory.
+docker-image-featurebase-quick: build-for-quick
+	docker build \
+	    --build-arg GO_VERSION=$(GO_VERSION) \
+	    --file Dockerfile-dax-quick \
+	    --tag dax/featurebase ./.quick/
+
+
+docker-image-datagen: vendor
+	docker build --tag dax/datagen --file Dockerfile-datagen .
+
+get-account-id:
+	$(eval AWS_ACCOUNTID := $(shell aws sts get-caller-identity --output=json | jq -r .Account))
+
+
+ecr-push-featurebase: docker-login
+	echo "Pushing to account $(AWS_ACCOUNTID), profile $(AWS_PROFILE)"
+	docker tag dax/featurebase:latest $(AWS_ACCOUNTID).dkr.ecr.us-east-2.amazonaws.com/dax/featurebase:latest
+	docker push $(AWS_ACCOUNTID).dkr.ecr.us-east-2.amazonaws.com/dax/featurebase:latest
+
+ecr-push-datagen: docker-login
+	docker tag dax/datagen:latest $(AWS_ACCOUNTID).dkr.ecr.us-east-2.amazonaws.com/dax/datagen:latest
+	docker push $(AWS_ACCOUNTID).dkr.ecr.us-east-2.amazonaws.com/dax/datagen:latest
+
+docker-login: get-account-id
+	aws ecr get-login-password --region us-east-2 | docker login --username AWS --password-stdin $(AWS_ACCOUNTID).dkr.ecr.us-east-2.amazonaws.com
+
 # Create docker image (alias)
 docker: docker-image # alias
 
@@ -246,35 +291,21 @@ docker-tag-push: vendor
 	docker push $(DOCKER_TARGET)
 	@echo Pushed docker image: $(DOCKER_TARGET)
 
-# Install diagnostic pilosa-keydump tool. Allows viewing the keys in a transaction-engine directory.
-pilosa-keydump:
-	$(GO) install -tags='$(BUILD_TAGS)' -ldflags $(LDFLAGS) $(FLAGS) ./cmd/pilosa-keydump
-
-# Install diagnostic pilosa-chk tool for string translations and fragment checksums.
-pilosa-chk:
-	$(GO) install -tags='$(BUILD_TAGS)' -ldflags $(LDFLAGS) $(FLAGS) ./cmd/pilosa-chk
-
-pilosa-fsck:
-	cd ./cmd/pilosa-fsck && make install && make release
-
-# Run Pilosa tests inside Docker container
-docker-test:
-	docker run --rm -v $(PWD):/go/src/$(CLONE_URL) -w /go/src/$(CLONE_URL) golang:$(GO_VERSION) go test -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) -timeout $(TEST_TIMEOUT) ./...
-
-# Must use bash in order to -o pipefail; otherwise the tee will hide red tests.
-# run top tests, not subdirs. print summary red/green after.
-# The \-\-\- FAIL avoids counting the extra two FAIL strings at then bottom of log.topt.
-topt:
-	mv log.topt.roar log.topt.roar.prev || true
-	$(eval SHELL:=/bin/bash) set -o pipefail; $(GO) test -v -timeout $(RACE_TEST_TIMEOUT) -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) 2>&1 | tee log.topt.roar
-	@echo "   log.topt.roar green: \c"; cat log.topt.roar | grep PASS |wc -l
-	@echo "   log.topt.roar   red: \c"; cat log.topt.roar | grep '\-\-\- FAIL' | wc -l
-
-topt-race:
-	mv log.topt.race log.topt.race.prev || true
-	$(eval SHELL:=/bin/bash) set -o pipefail; CGO_ENABLED=1 $(GO) test -race -timeout $(RACE_TEST_TIMEOUT) -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) 2>&1 | tee log.topt.race
-	@echo "   log.topt.race green: \c"; cat log.topt.race | grep PASS |wc -l
-	@echo "   log.topt.race   red: \c"; cat log.topt.race | grep '\-\-\- FAIL' | wc -l
+# These commands (docker-idk and docker-idk-tag-push)
+# are designed to be used in CI.
+# docker-idk builds idk docker images and tags them - intended for use in CI.
+docker-idk: vendor
+	docker build \
+		-f idk/Dockerfile \
+		--build-arg GO_VERSION=$(GO_VERSION) \
+		--build-arg MAKE_FLAGS="GOOS=$(GOOS) GOARCH=$(GOARCH) BUILD_CGO=$(BUILD_CGO)" \
+		--tag registry.gitlab.com/molecula/featurebase/idk:$(VERSION_ID) .
+	@echo Created docker image: registry.gitlab.com/molecula/featurebase/idk:$(VERSION_ID)
+# docker-idk-tag-push pushes tagged docker images to the GitLab container
+# registry - intended for use in CI.
+docker-idk-tag-push:
+	docker push registry.gitlab.com/molecula/featurebase/idk:$(VERSION_ID)
+	@echo Pushed docker image: registry.gitlab.com/molecula/featurebase/idk:$(VERSION_ID)
 
 # Run golangci-lint
 golangci-lint: require-golangci-lint
@@ -286,29 +317,6 @@ linter: golangci-lint
 # Better alias
 ocd: golangci-lint
 
-# Run gometalinter with custom flags
-gometalinter: require-gometalinter vendor
-	GO111MODULE=off gometalinter --vendor --disable-all \
-	    --deadline=300s \
-	    --enable=deadcode \
-	    --enable=gochecknoinits \
-	    --enable=gofmt \
-	    --enable=goimports \
-	    --enable=gotype \
-	    --enable=gotypex \
-	    --enable=ineffassign \
-	    --enable=interfacer \
-	    --enable=maligned \
-	    --enable=misspell \
-	    --enable=nakedret \
-	    --enable=staticcheck \
-	    --enable=unconvert \
-	    --enable=unparam \
-	    --enable=vet \
-	    --exclude "^internal/.*\.pb\.go" \
-	    --exclude "^pql/pql.peg.go" \
-	    ./...
-
 ######################
 # Build dependencies #
 ######################
@@ -319,22 +327,20 @@ require-%:
 		$(info Verified build dependency "$*" is installed.),\
 		$(error Build dependency "$*" not installed. To install, try `make install-$*`))
 
-install-build-deps: install-protoc-gen-gofast install-protoc install-statik install-stringer install-peg
+install-build-deps: install-protoc-gen-gofast install-protoc install-statik install-peg
 
 install-statik:
 	go install github.com/rakyll/statik@latest
 
-install-stringer:
-	GO111MODULE=off $(GO) get -u golang.org/x/tools/cmd/stringer
-
 install-protoc-gen-gofast:
 	GO111MODULE=off $(GO) get -u github.com/gogo/protobuf/protoc-gen-gofast
 
-install-protoc-gen-go:
-	GO111MODULE=off $(GO) get -u github.com/golang/protobuf/protoc-gen-go
-
 install-protoc:
 	@echo This tool cannot automatically install protoc. Please download and install protoc from https://google.github.io/proto-lens/installing-protoc.html
+	@echo On mac, brew install protobuf seems to work.
+	@echo As of the commit that added this line, protoc-gen-gofast was at 226206f39bd7, and the protoc version in use was:
+	@echo $$ protoc --version
+	@echo libprotoc 3.19.4
 
 install-peg:
 	GO111MODULE=off $(GO) get github.com/pointlander/peg
@@ -342,10 +348,58 @@ install-peg:
 install-golangci-lint:
 	GO111MODULE=off $(GO) get github.com/golangci/golangci-lint/cmd/golangci-lint
 
-install-gometalinter:
-	GO111MODULE=off $(GO) get -u github.com/alecthomas/gometalinter
-	GO111MODULE=off gometalinter --install
-	GO111MODULE=off $(GO) get github.com/remyoudompheng/go-misc/deadcode
-
 test-external-lookup:
 	$(GO) test . -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) -run ^TestExternalLookup$$ -externalLookupDSN $(EXTERNAL_LOOKUP_DSN)
+
+bnf:
+	ebnf2railroad --no-overview-diagram  --no-optimizations ./sql3/sql3.ebnf
+
+#################################
+# fbsql builds in docker
+#################################
+
+# This allows multiple concurrent builds to happen in CI without
+# creating container name conflicts and such. (different BUILD_NAMEs
+# are passed in from gitlab-ci.yml)
+BUILD_NAME ?= fbsql-build
+
+LDFLAGS_STATIC="-linkmode external -extldflags \"-static\" -X 'github.com/featurebasedb/featurebase/v3/fbsql.Version=$(VERSION)' -X 'github.com/featurebasedb/featurebase/v3/fbsql.BuildTime=$(BUILD_TIME)' "
+
+UNAME_P := $(shell uname -p)
+BUILD_CGO ?= 0
+
+# Build fbsql
+build-fbsql:
+	@echo GOOS=$(GOOS) GOARCH=$(GOARCH) uname -p=$(UNAME_P) build_cgo=$(BUILD_CGO)
+ifeq ($(BUILD_CGO), 0)
+	make build-fbsql-non-cgo
+endif
+ifeq ($(BUILD_CGO), 1)
+	make build-fbsql-cgo
+endif
+
+build-fbsql-non-cgo:
+	CGO_ENABLED=0 $(GO) build -ldflags $(LDFLAGS) $(GO_BUILD_FLAGS) -o fbsql ./cmd/fbsql
+
+build-fbsql-cgo:
+ifeq ($(GOARCH), arm64)
+	CGO_ENABLED=1 $(GO) build -tags dynamic $(GO_BUILD_FLAGS) -o fbsql ./cmd/fbsql
+endif
+ifeq ($(GOARCH), amd64)
+	CC=/usr/bin/musl-gcc CGO_ENABLED=1 $(GO) build -tags "musl static" -ldflags $(LDFLAGS_STATIC) $(GO_BUILD_FLAGS) -o fbsql ./cmd/fbsql
+endif
+
+docker-build-fbsql: vendor
+	DOCKER_BUILDKIT=0 docker build \
+		--file Dockerfile-fbsql \
+	    --build-arg GO_VERSION=$(GO_VERSION) \
+	    --build-arg MAKE_FLAGS="GOOS=$(GOOS) GOARCH=$(GOARCH) BUILD_CGO=$(BUILD_CGO)" \
+	    --build-arg GO_BUILD_FLAGS=$(GO_BUILD_FLAGS) \
+		--build-arg SOURCE_DATE_EPOCH=$(SOURCE_DATE_EPOCH) \
+	    --target builder \
+	    --tag fbsql:$(BUILD_NAME) .
+	mkdir -p build
+	docker create --name $(BUILD_NAME) fbsql:$(BUILD_NAME)
+	docker cp $(BUILD_NAME):/featurebase/fbsql ./build/fbsql_$(GOOS)_$(GOARCH)
+	docker rm $(BUILD_NAME)
+

OPENSOURCE.md

@@ -0,0 +1,45 @@
+## User Contribution Guidelines for FeatureBase
+
+Thank you for your interest in contributing to FeatureBase! We appreciate your support in making this open-source project even better. Here are some guidelines to help you get started with contributing to FeatureBase:
+
+1. Familiarize Yourself with the Project:
+   - Visit the FeatureBase website at www.featurebase.com to understand the project's goals, capabilities, and features.
+   - Read the documentation available on the website, including the installation guide, configuration options, and data modeling concepts.
+   - Explore the codebase by cloning the repository and reviewing the source code.
+
+2. Join the Community:
+   - Visit the FeatureBase community page at https://www.featurebase.com/community to learn more about the project's community and how to get involved.
+   - Join the Discord server at https://discord.gg/FBn2vEp7Na to chat with other contributors and users, ask questions, and share your ideas.
+
+3. Set Up Your Development Environment:
+   - Ensure you have Go installed on your machine. Make sure your shell's search path includes the go/bin directory.
+   - Clone the FeatureBase repository or download it as a zip file from the repository's page.
+   - Follow the "Build FeatureBase Server from source" instructions in the README file to compile the server binary and the ingester binaries.
+
+4. Choose a Contribution Area:
+   - Identify the area you'd like to contribute to, such as bug fixes, new features, performance improvements, documentation updates, or community support.
+   - Check the issue tracker on the repository or the FeatureBase community for open issues or feature requests that align with your interests and skills. Alternatively, propose your own idea by creating a new issue.
+
+5. Create a New Branch:
+   - Before making any changes, create a new branch in the repository's Git repository. This branch will contain your contributions.
+   - Give your branch a descriptive name that reflects the nature of your contribution.
+
+6. Make Your Changes:
+   - Follow the coding style and conventions used in the existing codebase.
+   - Write clear and concise commit messages for each logical change.
+   - If you're introducing new features or modifying existing behavior, make sure to update the documentation to reflect the changes.
+
+7. Test Your Changes:
+   - Run the existing test suite to ensure that your modifications do not introduce any regressions.
+   - If applicable, write additional tests to cover the changes you made.
+   - Document any new testing procedures required for your contribution.
+
+8. Submitting Your Contribution:
+   - Push your branch to the main repository or create a fork and submit a pull request to the main repository.
+   - Provide a detailed description of your changes, including the problem you solved and the approach you took.
+   - Be responsive to any feedback or suggestions provided by the project maintainers or other contributors.
+   - Once your contribution is approved, it will be reviewed and merged into the main codebase.
+
+Please note that by contributing to FeatureBase, you agree that your contributions will be licensed under the Apache 2.0 license, which governs the project.
+
+Thank you for considering contributing to FeatureBase! Your contributions are valuable and help improve the project for everyone.

README.md

@@ -1,6 +1,72 @@
-# FeatureBase, a distributed bitmap index
+# FeatureBase Community
 
-See our [internal documentation](https://internal-docs.molecula.cloud), which includes all [external documentation](https://docs.molecula.cloud), plus many internal-only pages, listed under the "Internal" heading in the main navigation bar.
+FeatureBase Community is now archived and no longer maintained.
 
-Follow along with the [Sample Project](https://internal-docs.molecula.cloud/tutorials/getting-started) to get a better understanding of FeatureBase's capabilities.
+* [FeatureBase Community Help](https://github.com/FeatureBaseDB/FB-community-help)
 
+
+
+## Pilosa is now FeatureBase
+
+As of September 7, 2022, the Pilosa project is now FeatureBase. The core of the project remains the same: FeatureBase is the first real-time distributed database built entirely on bitmaps. (More information about updated capabilities and improvements below.)
+
+FeatureBase delivers low-latency query results, regardless of throughput or query volumes, on fresh data with extreme efficiency. It works because bitmaps are faster, simpler, and far more I/O efficient than traditional column-oriented data formats. With FeatureBase, you can ingest data from batch data sources (e.g. S3, CSV, Snowflake, BigQuery, etc.) and/or streaming data sources (e.g. Kafka/Confluent, Kinesis, Pulsar).
+
+For more information about FeatureBase, please visit [www.featurebase.com][HomePage].
+
+## Getting Started
+
+* [Learn how to install FeatureBase Community](https://github.com/FeatureBaseDB/FB-community-help/blob/main/docs/community/com-getstart/com-getstart-home.md)
+
+### Build FeatureBase Server from source
+
+0. Install go. Ensure that your shell's search path includes the go/bin directory.
+1. Clone the FeatureBase repository (or download as zip).
+2. In the featurebase directory, run `make install` to compile the FeatureBase server binary. By default, it will be installed in the go/bin directory.
+3. In the idk directory, run `make install` to compile the ingester binaries. By default, they will be installed in the go/bin directory.
+4. Run `featurebase server --handler.allowed-origins=http://localhost:3000` to run FeatureBase server with default settings (learn more about configuring FeatureBase at the link below). The `--handler.allowed-origins` parameter allows the standalone web UI to talk to the server; this can be omitted if the web UI is not needed.
+5. Run `curl localhost:10101/status` to verify the server is running and accessible.
+
+### Data Model
+
+Because FeatureBase is built on bitmaps, there is bit of a learning curve to grasp how your data is represented. 
+
+* [Learn about Data Modeling](https://github.com/FeatureBaseDB/FB-community-help/blob/main/docs/concepts/concepts-home.md)
+
+
+### Ingest Data and Query
+
+* [Learn how to ingest data from multiple data sources](https://github.com/FeatureBaseDB/FB-community-help/blob/main/docs/community/com-ingest/com-ingest-manage.md)
+
+## Community
+
+You can email us at community@featurebase.com and [learn more about contributing](https://github.com/FeatureBaseDB/featurebase/blob/master/OPENSOURCE.md).
+
+Chat with us: [https://discord.gg/FBn2vEp7Na][Discord]
+
+## What's Changed Since the Pilosa Days? 
+
+A lot has changed since the days of Pilosa. This list highlights some new capabilites included in FeatureBase. We have also made signficant improvements to the performance, scalability, and stability of the FeatureBase product. 
+
+* Query Languages: FeatureBase supports Pilosa Query Language (PQL), as well as SQL
+* Stream and Batch Ingest: Combine real-time data streams with batch historical data and act on it within milliseconds.
+* Mutable: Perform inserts, updates, and deletes at scale, in real time and on-the-fly. This is key for meeting data compliance requirements, and for reflecting the constantly-changing nature of high-volume data.
+* Multi-Valued Set Fields: Store multiple comma-delimited values within a single field while *increasing* query performance of counts, TopKs, etc.
+* Time Quantums: Setting a time quantum on a field creates extra views which allow ranged Row queries down to the time interval specified. For example, if the time quantum is set to YMD, ranged Row queries down to the granularity of a day are supported.
+* RBF storage backend: this is a new compressed bitmap format which improves performance in a number of ways: ACID support on a per shard basis, prevents issues with the number of open files, reduces memory allocation and lock contention for reads, provides more consistent garbage collection, and allows backups to run concurrently with writes. However, because of this change, Pilosa backup files cannot be restored into FeatureBase.
+
+## License
+
+FeatureBase is licensed under the [Apache License, Version 2.0][License]
+
+[Community]: https://github.com/FeatureBaseDB/FB-community-help/tree/main
+[Install]:https://github.com/FeatureBaseDB/FB-community-help/blob/main/docs/community/com-getstart/com-getstart-home.md
+[Config]: https://github.com/FeatureBaseDB/FB-community-help/tree/main/docs/community/com-config
+[DataModel]: https://github.com/FeatureBaseDB/FB-community-help/blob/main/docs/concepts/concepts-home.md
+[Discord]: https://discord.gg/FBn2vEp7Na
+[HomePage]: http://featurebase.com?utm_campaign=Open%20Source&utm_source=GitHub
+[Ingest]: https://github.com/FeatureBaseDB/FB-community-help/blob/main/docs/community/com-ingest/com-ingest-manage.md
+ 
+[License]: http://www.apache.org/licenses/LICENSE-2.0
+[PQL]: https://docs.featurebase.com/docs/pql-guide/pql-home/?utm_campaign=Open%20Source&utm_source=GitHub
+[SQL]: https://docs.featurebase.com/docs/sql-guide/sql-guide-home/?utm_campaign=Open%20Source&utm_source=GitHub

api/client/grpc.go

@@ -1,4 +1,5 @@
-// Copyright 2021 Molecula Corp. All rights reserved.
+// Copyright 2022 Molecula Corp. (DBA FeatureBase).
+// SPDX-License-Identifier: Apache-2.0
 package client
 
 import (
@@ -6,12 +7,13 @@ import (
 	"crypto/tls"
 	"sync"
 
-	"github.com/molecula/featurebase/v3/logger"
-	pb "github.com/molecula/featurebase/v3/proto"
+	"github.com/featurebasedb/featurebase/v3/logger"
+	pb "github.com/featurebasedb/featurebase/v3/proto"
 	"github.com/pkg/errors"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/connectivity"
 	"google.golang.org/grpc/credentials"
+	"google.golang.org/grpc/credentials/insecure"
 )
 
 const maxMsgSize = 1024 * 1024 * 100 // 100 megs ought to be enough for anybody!
@@ -63,7 +65,7 @@ func (c *GRPCClient) resetConn() error {
 		creds := credentials.NewTLS(c.tlsConfig)
 		opts = append(opts, grpc.WithTransportCredentials(creds))
 	} else {
-		opts = append(opts, grpc.WithInsecure())
+		opts = append(opts, grpc.WithTransportCredentials(insecure.NewCredentials()))
 	}
 
 	opts = append(opts, grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(maxMsgSize)))

api_directive.go

@@ -0,0 +1,987 @@
+// Copyright 2021 Molecula Corp. All rights reserved.
+package pilosa
+
+import (
+	"context"
+	"io"
+	"log"
+	"sync"
+
+	"github.com/featurebasedb/featurebase/v3/dax"
+	"github.com/featurebasedb/featurebase/v3/dax/computer"
+	"github.com/featurebasedb/featurebase/v3/dax/storage"
+	"github.com/featurebasedb/featurebase/v3/disco"
+	"github.com/pkg/errors"
+)
+
+// ApplyDirective applies a Directive received, from the Controller, at the
+// /directive endpoint.
+func (api *API) ApplyDirective(ctx context.Context, d *dax.Directive) error {
+	// Get the current directive for comparison.
+	previousDirective := api.holder.Directive()
+
+	// Check that incoming version is newer.
+	// Note: 0 is an invalid Directive version. This decision was made because
+	// previousDirective is not a pointer to a directive, but a concrete
+	// Directive. Which means we can't check for nil, and by default it has a
+	// version of 0. So in order to ensure the version has increased, we need to
+	// require that incoming directive versions are greater than 0.
+	if d.Version == 0 {
+		return errors.Errorf("directive version cannot be 0")
+	} else if previousDirective.Version >= d.Version {
+		return errors.Errorf("directive version mismatch, got %d, but already have %d", d.Version, previousDirective.Version)
+	}
+
+	// Handle the operations based on the directive method.
+	switch d.Method {
+	case dax.DirectiveMethodDiff:
+		// In order to prevent adding too much code specific to handling a diff
+		// directive (e.g. adding something like an `enactDirectiveDiff()`
+		// method), we are instead going to build a full Directive based on the
+		// diff, and then proceed normally as if we had received a full
+		// Directive. We do that by copying the previous Directive and then
+		// applying the diffs to the copy.
+		newD := previousDirective.Copy()
+
+		// Apply the diffs from the incoming Directive to the new, copied
+		// Directive.
+		newD.ApplyDiff(d)
+
+		// Now proceed with the new diff as if we had received it as a full diff.
+		d = newD
+
+	case dax.DirectiveMethodFull:
+		// pass: normal operation
+
+	case dax.DirectiveMethodReset:
+		// Delete all tables.
+		if err := api.deleteAllIndexes(ctx); err != nil {
+			return errors.Wrap(err, "deleting all indexes")
+		}
+		// Set previousDirective to empty so the diff handles everything as new.
+		previousDirective = dax.Directive{}
+
+	case dax.DirectiveMethodSnapshot:
+		// TODO(tlt): this was the existing logic, but we should really diff the
+		// directive and ensure that overwriting the value in the cache doesn't
+		// have a negative effect.
+		api.holder.SetDirective(d)
+		return nil
+
+	default:
+		return errors.Errorf("invalid directive method: %s", d.Method)
+	}
+
+	// Cache this directive as the latest applied. There is functionality within
+	// the "enactDirective" stage of ApplyDirective which validates against this
+	// cached Directive, so it's important that it be set before calling
+	// enactDirective(). An example: when loading partition data from the
+	// Writelogger, there are validations to ensure that the partition being
+	// loaded is meant to be handled by this node; that validation is done
+	// against the cached Directive.
+	// TODO(tlt): despite what this comment says, this logic is not sound; we
+	// shouldn't be setting the directive until enactiveDirective() succeeds.
+	api.holder.SetDirective(d)
+	defer api.holder.SetDirectiveApplied(true)
+
+	return api.enactDirective(ctx, &previousDirective, d)
+}
+
+// deleteAllIndexes deletes all indexes handled by this node.
+func (api *API) deleteAllIndexes(ctx context.Context) error {
+	indexes, err := api.Schema(ctx, false)
+	if err != nil {
+		return errors.Wrap(err, "getting schema")
+	}
+
+	for i := range indexes {
+		if err := api.DeleteIndex(ctx, indexes[i].Name); err != nil {
+			return errors.Wrapf(err, "deleting index: %s", indexes[i].Name)
+		}
+	}
+
+	return nil
+}
+
+// directiveJobType allows us to switch on jobType in the directiveWorker in
+// order to use a single worker pool for all job types (as opposed to having a
+// separate worker pool for each job type).
+type directiveJobType interface {
+	// We have this method just to prevent *any* struct from implementing this
+	// interface automatically. But, interestingly enough, we don't actually
+	// have to have this method on the implementation because we embed the
+	// interface.
+	isJobType() bool
+}
+
+type directiveJobTableKeys struct {
+	directiveJobType
+	idx       *Index
+	tkey      dax.TableKey
+	partition dax.PartitionNum
+}
+
+type directiveJobFieldKeys struct {
+	directiveJobType
+	tkey  dax.TableKey
+	field dax.FieldName
+}
+
+type directiveJobShards struct {
+	directiveJobType
+	tkey  dax.TableKey
+	shard dax.ShardNum
+}
+
+// directiveWorker is a worker in a worker pool which handles portions of a
+// directive. Multiple instances of directiveWorker run in goroutines in order
+// to load data from snapshotter and writelogger concurrently. Note: unlike the
+// api.ingestWorkerPool, of which one pool is always running, the
+// directiveWorker pool is only running during the life of the
+// api.ApplyDirective call. Technically, this means that multiple
+// directiveWorker pools could be active at the same time, but we should never
+// be running more than once instance of ApplyDirective concurrently.
+func (api *API) directiveWorker(ctx context.Context, jobs <-chan directiveJobType, errs chan<- error) {
+	for j := range jobs {
+		switch job := j.(type) {
+		case directiveJobTableKeys:
+			if err := api.loadTableKeys(ctx, job.idx, job.tkey, job.partition); err != nil {
+				errs <- errors.Wrapf(err, "loading table keys: %s, %s", job.tkey, job.partition)
+			}
+		case directiveJobFieldKeys:
+			if err := api.loadFieldKeys(ctx, job.tkey, job.field); err != nil {
+				errs <- errors.Wrapf(err, "loading field keys: %s, %s", job.tkey, job.field)
+			}
+		case directiveJobShards:
+			if err := api.loadShard(ctx, job.tkey, job.shard); err != nil {
+				errs <- errors.Wrapf(err, "loading shard: %s, %s", job.tkey, job.shard)
+			}
+		default:
+			errs <- errors.Errorf("unsupported job type: %T %[1]v", job)
+		}
+
+		select {
+		case <-ctx.Done():
+			return
+		default:
+			// continue pulling jobs off the channel
+		}
+	}
+}
+
+func (api *API) enactDirective(ctx context.Context, fromD, toD *dax.Directive) error {
+	// enactTables is called before the jobs that run in the worker pool because
+	// it probably makes sense to apply the schema before trying to load data
+	// concurrently.
+	if err := api.enactTables(ctx, fromD, toD); err != nil {
+		return errors.Wrap(err, "enactTables")
+	}
+
+	// The following types use a shared pool of workers to run each
+	// directiveJobType.
+
+	var wg sync.WaitGroup
+
+	// open job channel
+	jobs := make(chan directiveJobType, api.directiveWorkerPoolSize)
+	errs := make(chan error)
+	done := make(chan struct{})
+
+	// Spin up n workers in goroutines that pull jobs from the jobs channel.
+	for i := 0; i < api.directiveWorkerPoolSize; i++ {
+		wg.Add(1)
+		go func() {
+			api.directiveWorker(ctx, jobs, errs)
+			defer wg.Done()
+		}()
+	}
+
+	// Wait for the WaitGroup counter to reach 0. When it has, indicate that
+	// we're done processing all jobs by closing the done channel.
+	go func() {
+		wg.Wait()
+		close(done)
+	}()
+
+	// Run through all the "enact" methods. These push jobs onto the jobs
+	// channel. Once all the jobs have been queued to the channel, we close the
+	// jobs channel. This allows the directiveWorkers to exit out of the
+	// function, which will then decrement the WaitGroup counter.
+	go func() {
+		api.pushJobsTableKeys(ctx, jobs, fromD, toD)
+		api.pushJobsFieldKeys(ctx, jobs, fromD, toD)
+		api.pushJobsShards(ctx, jobs, fromD, toD)
+		close(jobs)
+	}()
+
+	// Keep running until we get an error or until the done channel is closed.
+	// Note: the code is written such that only non-nil errors are pushed to the
+	// errs channel.
+	for {
+		select {
+		case err := <-errs:
+			return err
+		case <-done:
+			return nil
+		}
+	}
+}
+
+func (api *API) enactTables(ctx context.Context, fromD, toD *dax.Directive) error {
+	currentIndexes := api.holder.Indexes()
+
+	// Make a list of indexes that currently exist (from).
+	from := make(dax.TableKeys, 0, len(currentIndexes))
+	for _, idx := range currentIndexes {
+		qtid, err := dax.QualifiedTableIDFromKey(idx.Name())
+		if err != nil {
+			return errors.Wrap(err, "converting index name to qualified table id")
+		}
+		from = append(from, qtid.Key())
+	}
+
+	// TODO sanity check holder against fromD. We're getting existing
+	// indexes from holder, but in theory fromD should be
+	// identical. If we have an error in our directive-caching logic
+	// (it has happened before (just now, in fact!) and we'd be
+	// foolish to think it won't happen again), or we have schema
+	// mutations that are not going through the directive path, we
+	// could potentially catch them here.
+
+	// Make a list of tables that are in the directive (to) along with a map of
+	// tableKey to table (m).
+	m := make(map[dax.TableKey]*dax.QualifiedTable, len(toD.Tables))
+	to := make(dax.TableKeys, 0, len(toD.Tables))
+	for _, t := range toD.Tables {
+		m[t.Key()] = t
+		to = append(to, t.Key())
+	}
+
+	sc := newSliceComparer(from, to)
+
+	// Remove all indexes that are no longer part of the directive.
+	for _, tkey := range sc.removed() {
+		idx := string(tkey)
+		if err := api.DeleteIndex(ctx, idx); err != nil {
+			return errors.Wrapf(err, "deleting index: %s", tkey)
+		}
+	}
+
+	// Put partitions into a map by table.
+	partitionMap := toD.TranslatePartitionsMap()
+
+	// Add all indexes that weren't previously (but now are) a part of the
+	// directive.
+	for _, tkey := range sc.added() {
+		if qtbl, found := m[tkey]; !found {
+			return errors.Errorf("table '%s' was not in map", tkey)
+		} else if err := api.createTableAndFields(qtbl, partitionMap[tkey]); err != nil {
+			return err
+		}
+	}
+
+	// Check fields on all indexes present in both from and to.
+	for _, tkey := range sc.same() {
+		if err := api.enactFieldsForTable(ctx, tkey, fromD, toD); err != nil {
+			return errors.Wrapf(err, "enacting fields for table: '%s'", tkey)
+		}
+	}
+
+	return nil
+}
+
+func (api *API) enactFieldsForTable(ctx context.Context, tkey dax.TableKey, fromD, toD *dax.Directive) error {
+	qtid := tkey.QualifiedTableID()
+
+	fromT, err := fromD.Table(qtid)
+	if err != nil {
+		return errors.Wrap(err, "getting from table")
+	}
+	toT, err := toD.Table(qtid)
+	if err != nil {
+		return errors.Wrap(err, "getting to table")
+	}
+
+	// Get the index for tkey.
+	idx := api.holder.Index(string(tkey))
+	if idx == nil {
+		return errors.Errorf("index not found: %s", tkey)
+	}
+
+	sc := newSliceComparer(fromT.FieldNames(), toT.FieldNames())
+
+	// Add fields new to toT.
+	for _, fldName := range sc.added() {
+		if field, found := toT.Field(fldName); !found {
+			return dax.NewErrFieldDoesNotExist(fldName)
+		} else if err := createField(idx, field); err != nil {
+			return errors.Wrapf(err, "creating field: %s/%s", tkey, fldName)
+		}
+	}
+
+	// Remove fields which don't exist in toT.
+	for _, fldName := range sc.removed() {
+		if err := api.DeleteField(ctx, string(tkey), string(fldName)); err != nil {
+			return errors.Wrapf(err, "deleting field: %s/%s", tkey, fldName)
+		}
+	}
+
+	// // Update any field options which have changed for existing fields.
+	// for _, fldName := range sc.same() {
+	// 	// handle changed field options??
+	// }
+
+	return nil
+}
+
+func (api *API) pushJobsTableKeys(ctx context.Context, jobs chan<- directiveJobType, fromD, toD *dax.Directive) {
+	toPartitionsMap := toD.TranslatePartitionsMap()
+
+	// Get the diff between from/to directive.partitions.
+	partComp := newPartitionsComparer(fromD.TranslatePartitionsMap(), toPartitionsMap)
+
+	// Remove any partitions which are no longer assigned to this worker.
+	// TODO(tlt): currently, this is just removing the file lock on the
+	// resource; it's not actually removing the resource from the local
+	// computer. We should do that.
+	for tkey, partitions := range partComp.removed() {
+		qtid := tkey.QualifiedTableID()
+		for _, partition := range partitions {
+			api.serverlessStorage.RemoveTableKeyResource(qtid, partition)
+		}
+	}
+
+	// Loop over the partition map and load from Writelogger.
+	for tkey, partitions := range partComp.added() {
+		// Get index in order to find the translate stores (by partition) for
+		// the table.
+		idx := api.holder.Index(string(tkey))
+		if idx == nil {
+			log.Printf("index not found in holder: %s", tkey)
+			continue
+		}
+
+		// Update the cached version of translate partitions that we keep on the
+		// Index.
+		idx.SetTranslatePartitions(toPartitionsMap[tkey])
+
+		for _, partition := range partitions {
+			jobs <- directiveJobTableKeys{
+				idx:       idx,
+				tkey:      tkey,
+				partition: partition,
+			}
+		}
+	}
+}
+
+func (api *API) loadTableKeys(ctx context.Context, idx *Index, tkey dax.TableKey, partition dax.PartitionNum) error {
+	qtid := tkey.QualifiedTableID()
+
+	resource := api.serverlessStorage.GetTableKeyResource(qtid, partition)
+	if resource.IsLocked() {
+		api.logger().Warnf("skipping loadTableKeys (already held) %s %d", tkey, partition)
+		return nil
+	}
+
+	// load latest snapshot
+	if rc, err := resource.LoadLatestSnapshot(); err != nil {
+		return errors.Wrap(err, "loading table key snapshot")
+	} else if rc != nil {
+		defer rc.Close()
+		if err := api.TranslateIndexDB(ctx, string(tkey), int(partition), rc); err != nil {
+			return errors.Wrap(err, "restoring table keys")
+		}
+	}
+
+	// define write log loading in a function since we have to do it
+	// before and after locking
+	loadWriteLog := func() error {
+		writelog, err := resource.LoadWriteLog()
+		if err != nil {
+			return errors.Wrap(err, "getting write log reader for table keys")
+		}
+		if writelog == nil {
+			return nil
+		}
+		reader := storage.NewTableKeyReader(qtid, partition, writelog)
+		defer reader.Close()
+		store := idx.TranslateStore(int(partition))
+		for msg, err := reader.Read(); err != io.EOF; msg, err = reader.Read() {
+			if err != nil {
+				return errors.Wrap(err, "reading from log reader")
+			}
+			for key, id := range msg.StringToID {
+				if err := store.ForceSet(id, key); err != nil {
+					return errors.Wrapf(err, "forcing set id, key: %d, %s", id, key)
+				}
+			}
+		}
+		return nil
+	}
+	// 1st write log load
+	if err := loadWriteLog(); err != nil {
+		return err
+	}
+
+	// acquire lock on this partition's keys
+	if err := resource.Lock(); err != nil {
+		return errors.Wrap(err, "locking table key partition")
+	}
+
+	// reload writelog in case of changes between last load and
+	// lock. The resource object takes care of only loading new data.
+	return loadWriteLog()
+}
+
+func (api *API) pushJobsFieldKeys(ctx context.Context, jobs chan<- directiveJobType, fromD, toD *dax.Directive) {
+	// Get the diff between from/to directive.fields.
+	fieldComp := newFieldsComparer(fromD.TranslateFieldsMap(), toD.TranslateFieldsMap())
+
+	// Remove any field keys which are no longer assigned to this worker.
+	// TODO(tlt): currently, this is just removing the file lock on the
+	// resource; it's not actually removing the resource from the local
+	// computer. We should do that.
+	for tkey, fields := range fieldComp.removed() {
+		qtid := tkey.QualifiedTableID()
+		for _, field := range fields {
+			api.serverlessStorage.RemoveFieldKeyResource(qtid, field)
+		}
+	}
+
+	// Loop over the field map and load from Writelogger.
+	for tkey, fields := range fieldComp.added() {
+		for _, field := range fields {
+			jobs <- directiveJobFieldKeys{
+				tkey:  tkey,
+				field: field,
+			}
+		}
+	}
+}
+
+func (api *API) loadFieldKeys(ctx context.Context, tkey dax.TableKey, field dax.FieldName) error {
+	qtid := tkey.QualifiedTableID()
+
+	resource := api.serverlessStorage.GetFieldKeyResource(qtid, field)
+	if resource.IsLocked() {
+		api.logger().Warnf("skipping loadFieldKeys (already held) %s %s", tkey, field)
+		return nil
+	}
+
+	// load latest snapshot
+	if rc, err := resource.LoadLatestSnapshot(); err != nil {
+		return errors.Wrap(err, "loading field key snapshot")
+	} else if rc != nil {
+		defer rc.Close()
+		if err := api.TranslateFieldDB(ctx, string(tkey), string(field), rc); err != nil {
+			return errors.Wrap(err, "restoring field keys")
+		}
+	}
+
+	// define write log loading in a function since we have to do it
+	// before and after locking
+	loadWriteLog := func() error {
+		writelog, err := resource.LoadWriteLog()
+		if err != nil {
+			return errors.Wrap(err, "getting write log reader for field keys")
+		}
+		if writelog == nil {
+			return nil
+		}
+		reader := storage.NewFieldKeyReader(qtid, field, writelog)
+		defer reader.Close()
+		// Get field in order to find the translate store.
+		fld := api.holder.Field(string(tkey), string(field))
+		if fld == nil {
+			log.Printf("field not found in holder: %s", field)
+			return nil
+		}
+		store := fld.TranslateStore()
+
+		for msg, err := reader.Read(); err != io.EOF; msg, err = reader.Read() {
+			if err != nil {
+				return errors.Wrap(err, "reading from log reader")
+			}
+			for key, id := range msg.StringToID {
+				if err := store.ForceSet(id, key); err != nil {
+					return errors.Wrapf(err, "forcing set id, key: %d, %s", id, key)
+				}
+			}
+		}
+		return nil
+	}
+	// 1st write log load
+	if err := loadWriteLog(); err != nil {
+		return err
+	}
+
+	// acquire lock on this partition's keys
+	if err := resource.Lock(); err != nil {
+		return errors.Wrap(err, "locking field key partition")
+	}
+
+	// reload writelog in case of changes between last load and
+	// lock. The resource object takes care of only loading new data.
+	return loadWriteLog()
+}
+
+func (api *API) pushJobsShards(ctx context.Context, jobs chan<- directiveJobType, fromD, toD *dax.Directive) {
+	// Put shards into a map by table.
+	shardMap := toD.ComputeShardsMap()
+
+	// Get the diff between from/to directive shards.
+	shardComp := newShardsComparer(fromD.ComputeShardsMap(), shardMap)
+
+	// Remove any shards which are no longer assigned to this worker.
+	// TODO(tlt): currently, this is just removing the file lock on the
+	// resource; it's not actually removing the resource from the local
+	// computer. We should do that.
+	for tkey, shards := range shardComp.removed() {
+		qtid := tkey.QualifiedTableID()
+		for _, shard := range shards {
+			partition := dax.PartitionNum(disco.ShardToShardPartition(string(tkey), uint64(shard), disco.DefaultPartitionN))
+			api.serverlessStorage.RemoveShardResource(qtid, partition, shard)
+		}
+	}
+
+	// Loop over the shard map and load from Writelogger.
+	for tkey, shards := range shardComp.added() {
+		for _, shard := range shards {
+			jobs <- directiveJobShards{
+				tkey:  tkey,
+				shard: shard,
+			}
+		}
+	}
+}
+
+func (api *API) loadShard(ctx context.Context, tkey dax.TableKey, shard dax.ShardNum) error {
+	qtid := tkey.QualifiedTableID()
+
+	partition := dax.PartitionNum(disco.ShardToShardPartition(string(tkey), uint64(shard), disco.DefaultPartitionN))
+
+	resource := api.serverlessStorage.GetShardResource(qtid, partition, shard)
+	if resource.IsLocked() {
+		api.logger().Warnf("skipping loadShard (already held) %s %d", tkey, shard)
+		return nil
+	}
+
+	if rc, err := resource.LoadLatestSnapshot(); err != nil {
+		return errors.Wrap(err, "reading latest snapshot for shard")
+	} else if rc != nil {
+		defer rc.Close()
+		if err := api.RestoreShard(ctx, string(tkey), uint64(shard), rc); err != nil {
+			return errors.Wrap(err, "restoring shard data")
+		}
+	}
+
+	// define write log loading in a func because we do it twice.
+	loadWriteLog := func() error {
+		writelog, err := resource.LoadWriteLog()
+		if err != nil {
+			return errors.Wrap(err, "")
+		}
+		if writelog == nil {
+			return nil
+		}
+
+		reader := storage.NewShardReader(qtid, partition, shard, writelog)
+		defer reader.Close()
+		for logMsg, err := reader.Read(); err != io.EOF; logMsg, err = reader.Read() {
+			if err != nil {
+				return errors.Wrap(err, "reading from log reader")
+			}
+
+			switch msg := logMsg.(type) {
+			case *computer.ImportRoaringMessage:
+				req := &ImportRoaringRequest{
+					Clear:           msg.Clear,
+					Action:          msg.Action,
+					Block:           msg.Block,
+					Views:           msg.Views,
+					UpdateExistence: msg.UpdateExistence,
+					SuppressLog:     true,
+				}
+				if err := api.ImportRoaring(ctx, msg.Table, msg.Field, msg.Shard, true, req); err != nil {
+					return errors.Wrapf(err, "import roaring, table: %s, field: %s, shard: %d", msg.Table, msg.Field, msg.Shard)
+				}
+
+			case *computer.ImportMessage:
+				req := &ImportRequest{
+					Index:      msg.Table,
+					Field:      msg.Field,
+					Shard:      msg.Shard,
+					RowIDs:     msg.RowIDs,
+					ColumnIDs:  msg.ColumnIDs,
+					RowKeys:    msg.RowKeys,
+					ColumnKeys: msg.ColumnKeys,
+					Timestamps: msg.Timestamps,
+					Clear:      msg.Clear,
+				}
+
+				qcx := api.Txf().NewQcx()
+				defer qcx.Abort()
+
+				opts := []ImportOption{
+					OptImportOptionsClear(msg.Clear),
+					OptImportOptionsIgnoreKeyCheck(msg.IgnoreKeyCheck),
+					OptImportOptionsPresorted(msg.Presorted),
+					OptImportOptionsSuppressLog(true),
+				}
+				if err := api.Import(ctx, qcx, req, opts...); err != nil {
+					return errors.Wrapf(err, "import, table: %s, field: %s, shard: %d", msg.Table, msg.Field, msg.Shard)
+				}
+
+			case *computer.ImportValueMessage:
+				req := &ImportValueRequest{
+					Index:           msg.Table,
+					Field:           msg.Field,
+					Shard:           msg.Shard,
+					ColumnIDs:       msg.ColumnIDs,
+					ColumnKeys:      msg.ColumnKeys,
+					Values:          msg.Values,
+					FloatValues:     msg.FloatValues,
+					TimestampValues: msg.TimestampValues,
+					StringValues:    msg.StringValues,
+					Clear:           msg.Clear,
+				}
+
+				qcx := api.Txf().NewQcx()
+				defer qcx.Abort()
+
+				opts := []ImportOption{
+					OptImportOptionsClear(msg.Clear),
+					OptImportOptionsIgnoreKeyCheck(msg.IgnoreKeyCheck),
+					OptImportOptionsPresorted(msg.Presorted),
+					OptImportOptionsSuppressLog(true),
+				}
+				if err := api.ImportValue(ctx, qcx, req, opts...); err != nil {
+					return errors.Wrapf(err, "import value, table: %s, field: %s, shard: %d", msg.Table, msg.Field, msg.Shard)
+				}
+			case *computer.ImportRoaringShardMessage:
+				req := &ImportRoaringShardRequest{
+					Remote:      true,
+					Views:       make([]RoaringUpdate, len(msg.Views)),
+					SuppressLog: true,
+				}
+				for i, view := range msg.Views {
+					req.Views[i] = RoaringUpdate{
+						Field:        view.Field,
+						View:         view.View,
+						Clear:        view.Clear,
+						Set:          view.Set,
+						ClearRecords: view.ClearRecords,
+					}
+				}
+				if err := api.ImportRoaringShard(ctx, msg.Table, msg.Shard, req); err != nil {
+					return errors.Wrapf(err, "import roaring shard table: %s, shard: %d", msg.Table, msg.Shard)
+				}
+			}
+		}
+		return nil
+	}
+	// 1st write log load
+	if err := loadWriteLog(); err != nil {
+		return err
+	}
+
+	// acquire lock on this partition's keys
+	if err := resource.Lock(); err != nil {
+		return errors.Wrap(err, "locking field key partition")
+	}
+
+	// reload writelog in case of changes between last load and
+	// lock. The resource object takes care of only loading new data.
+	return loadWriteLog()
+}
+
+//////////////////////////////////////////////////////////////
+
+// sliceComparer is used to compare the differences between two slices of comparables.
+type sliceComparer[K comparable] struct {
+	from []K
+	to   []K
+}
+
+func newSliceComparer[K comparable](from []K, to []K) *sliceComparer[K] {
+	return &sliceComparer[K]{
+		from: from,
+		to:   to,
+	}
+}
+
+// added returns the items which are present in `to` but not in `from`.
+func (s *sliceComparer[K]) added() []K {
+	return thingsAdded(s.from, s.to)
+}
+
+// removed returns the items which are present in `from` but not in `to`.
+func (s *sliceComparer[K]) removed() []K {
+	return thingsAdded(s.to, s.from)
+}
+
+// same returns the items which are in both `to` and `from`.
+func (s *sliceComparer[K]) same() []K {
+	var same []K
+	for _, fromThing := range s.from {
+		for _, toThing := range s.to {
+			if fromThing == toThing {
+				same = append(same, fromThing)
+				break
+			}
+		}
+	}
+	return same
+}
+
+// thingsAdded returns the comparable things which are present in `to` but not
+// in `from`.
+func thingsAdded[K comparable](from []K, to []K) []K {
+	var added []K
+	for i := range to {
+		var found bool
+		for j := range from {
+			if from[j] == to[i] {
+				found = true
+				break
+			}
+		}
+		if !found {
+			added = append(added, to[i])
+		}
+	}
+	return added
+}
+
+// partitionsComparer is used to compare the differences between two maps of
+// table:[]partition.
+type partitionsComparer struct {
+	from map[dax.TableKey]dax.PartitionNums
+	to   map[dax.TableKey]dax.PartitionNums
+}
+
+func newPartitionsComparer(from map[dax.TableKey]dax.PartitionNums, to map[dax.TableKey]dax.PartitionNums) *partitionsComparer {
+	return &partitionsComparer{
+		from: from,
+		to:   to,
+	}
+}
+
+// added returns the partitions which are present in `to` but not in `from`. The
+// results remain in the format of a map of table:[]partition.
+func (p *partitionsComparer) added() map[dax.TableKey]dax.PartitionNums {
+	return partitionsAdded(p.from, p.to)
+}
+
+// removed returns the partitions which are present in `from` but not in `to`.
+// The results remain in the format of a map of table:[]partition.
+func (p *partitionsComparer) removed() map[dax.TableKey]dax.PartitionNums {
+	return partitionsAdded(p.to, p.from)
+}
+
+// partitionsAdded returns the partitions which are present in `to` but not in `from`.
+func partitionsAdded(from map[dax.TableKey]dax.PartitionNums, to map[dax.TableKey]dax.PartitionNums) map[dax.TableKey]dax.PartitionNums {
+	if from == nil {
+		return to
+	}
+
+	added := make(map[dax.TableKey]dax.PartitionNums)
+	for tt, tps := range to {
+		fps, found := from[tt]
+		if !found {
+			added[tt] = tps
+			continue
+		}
+
+		addedPartitions := dax.PartitionNums{}
+		for i := range tps {
+			var found bool
+			for j := range fps {
+				if fps[j] == tps[i] {
+					found = true
+					break
+				}
+			}
+			if !found {
+				addedPartitions = append(addedPartitions, tps[i])
+			}
+		}
+
+		if len(addedPartitions) > 0 {
+			added[tt] = addedPartitions
+		}
+	}
+	return added
+}
+
+// fieldsComparer is used to compare the differences between two maps of
+// table:[]fieldVersion.
+type fieldsComparer struct {
+	from map[dax.TableKey][]dax.FieldName
+	to   map[dax.TableKey][]dax.FieldName
+}
+
+func newFieldsComparer(from map[dax.TableKey][]dax.FieldName, to map[dax.TableKey][]dax.FieldName) *fieldsComparer {
+	return &fieldsComparer{
+		from: from,
+		to:   to,
+	}
+}
+
+// added returns the fields which are present in `to` but not in `from`. The
+// results remain in the format of a map of table:[]field.
+func (f *fieldsComparer) added() map[dax.TableKey][]dax.FieldName {
+	return fieldsAdded(f.from, f.to)
+}
+
+// removed returns the fields which are present in `from` but not in `to`.
+// The results remain in the format of a map of table:[]field.
+func (f *fieldsComparer) removed() map[dax.TableKey][]dax.FieldName {
+	return fieldsAdded(f.to, f.from)
+}
+
+// fieldsAdded returns the fields which are present in `to` but not in `from`.
+func fieldsAdded(from map[dax.TableKey][]dax.FieldName, to map[dax.TableKey][]dax.FieldName) map[dax.TableKey][]dax.FieldName {
+	if from == nil {
+		return to
+	}
+
+	added := make(map[dax.TableKey][]dax.FieldName)
+	for tt, tps := range to {
+		fps, found := from[tt]
+		if !found {
+			added[tt] = tps
+			continue
+		}
+
+		addedFieldVersions := []dax.FieldName{}
+		for i := range tps {
+			var found bool
+			for j := range fps {
+				if fps[j] == tps[i] {
+					found = true
+					break
+				}
+			}
+			if !found {
+				addedFieldVersions = append(addedFieldVersions, tps[i])
+			}
+		}
+
+		if len(addedFieldVersions) > 0 {
+			added[tt] = addedFieldVersions
+		}
+	}
+	return added
+}
+
+// shardsComparer is used to compare the differences between two maps of
+// table:[]shardV.
+type shardsComparer struct {
+	from map[dax.TableKey]dax.ShardNums
+	to   map[dax.TableKey]dax.ShardNums
+}
+
+func newShardsComparer(from map[dax.TableKey]dax.ShardNums, to map[dax.TableKey]dax.ShardNums) *shardsComparer {
+	return &shardsComparer{
+		from: from,
+		to:   to,
+	}
+}
+
+// added returns the shards which are present in `to` but not in `from`. The
+// results remain in the format of a map of table:[]shard.
+func (s *shardsComparer) added() map[dax.TableKey]dax.ShardNums {
+	return shardsAdded(s.from, s.to)
+}
+
+// removed returns the shards which are present in `from` but not in `to`. The
+// results remain in the format of a map of table:[]shard.
+func (s *shardsComparer) removed() map[dax.TableKey]dax.ShardNums {
+	return shardsAdded(s.to, s.from)
+}
+
+// shardsAdded returns the shards which are present in `to` but not in `from`.
+func shardsAdded(from map[dax.TableKey]dax.ShardNums, to map[dax.TableKey]dax.ShardNums) map[dax.TableKey]dax.ShardNums {
+	if from == nil {
+		return to
+	}
+
+	added := make(map[dax.TableKey]dax.ShardNums)
+	for tt, tss := range to {
+		fss, found := from[tt]
+		if !found {
+			added[tt] = tss
+			continue
+		}
+
+		addedShards := dax.ShardNums{}
+		for i := range tss {
+			var found bool
+			for j := range fss {
+				if fss[j] == tss[i] {
+					found = true
+					break
+				}
+			}
+			if !found {
+				addedShards = append(addedShards, tss[i])
+			}
+		}
+
+		if len(addedShards) > 0 {
+			added[tt] = addedShards
+		}
+	}
+	return added
+}
+
+// createTableAndFields creates the FeatureBase Tables and Fields provided in
+// the dax.Directive format.
+func (api *API) createTableAndFields(tbl *dax.QualifiedTable, partitions dax.PartitionNums) error {
+	cim := &CreateIndexMessage{
+		Index:     string(tbl.Key()),
+		CreatedAt: 0,
+		Meta: IndexOptions{
+			Keys:           tbl.StringKeys(),
+			TrackExistence: true,
+		},
+	}
+
+	// Create the index in etcd as the system of record.
+	if err := api.holder.persistIndex(context.Background(), cim); err != nil {
+		return errors.Wrap(err, "persisting index")
+	}
+
+	idx, err := api.holder.createIndexWithPartitions(cim, partitions)
+	if err != nil {
+		return errors.Wrapf(err, "adding index: %s", tbl.Name)
+	}
+
+	// Add the fields
+	for _, fld := range tbl.Fields {
+		if fld.IsPrimaryKey() {
+			continue
+		}
+		if err := createField(idx, fld); err != nil {
+			return errors.Wrapf(err, "creating field: %s", fld.Name)
+		}
+	}
+
+	return nil
+}
+
+// createField creates a FeatureBase Field in the provided FeatureBase Index
+// based on the provided field's type.
+func createField(idx *Index, fld *dax.Field) error {
+	opts, err := FieldOptionsFromField(fld)
+	if err != nil {
+		return errors.Wrapf(err, "creating field options from field: %s", fld.Name)
+	}
+
+	if _, err := idx.createNullableField(string(fld.Name), "", opts...); err != nil {
+		return errors.Wrapf(err, "creating field on index: %s", fld.Name)
+	}
+	return nil
+}

api_directive_internal_test.go

@@ -0,0 +1,25 @@
+package pilosa
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestThingsAddedGeneric(t *testing.T) {
+	from := []string{"a", "b", "c"}
+	to := []string{"b", "c", "d"}
+
+	added := thingsAdded(from, to)
+	assert.Equal(t, added, []string{"d"})
+}
+
+func TestSliceComparer(t *testing.T) {
+	from := []string{"a", "b", "c"}
+	to := []string{"b", "c", "d"}
+
+	sc := newSliceComparer(from, to)
+
+	added := sc.added()
+	assert.Equal(t, added, []string{"d"})
+}

api_directive_test.go

@@ -0,0 +1,97 @@
+package pilosa_test
+
+import (
+	"context"
+	"testing"
+
+	pilosa "github.com/featurebasedb/featurebase/v3"
+	"github.com/featurebasedb/featurebase/v3/dax"
+	daxtest "github.com/featurebasedb/featurebase/v3/dax/test"
+	"github.com/featurebasedb/featurebase/v3/test"
+	"github.com/stretchr/testify/assert"
+)
+
+// Ensure holder can handle an incoming directive.
+func TestAPI_Directive(t *testing.T) {
+
+	c := test.MustRunCluster(t, 1)
+	defer c.Close()
+
+	api := c.GetPrimary().API
+	ctx := context.Background()
+
+	qdbid := dax.NewQualifiedDatabaseID("acme", "db1")
+	tbl1 := daxtest.TestQualifiedTableWithID(t, qdbid, "1", "tbl1", 12, false)
+	tbl2 := daxtest.TestQualifiedTableWithID(t, qdbid, "2", "tbl2", 12, false)
+	tbl3 := daxtest.TestQualifiedTableWithID(t, qdbid, "3", "tbl3", 12, false)
+
+	t.Run("Schema", func(t *testing.T) {
+
+		// Empty directive (and empty holder).
+		{
+			d := &dax.Directive{
+				Method:  dax.DirectiveMethodFull,
+				Version: 1,
+			}
+			err := api.ApplyDirective(ctx, d)
+			assert.NoError(t, err)
+			assertTablesMatch(t, []string{}, api.Holder().Indexes())
+		}
+
+		// Add a new table.
+		{
+			d := &dax.Directive{
+				Method: dax.DirectiveMethodFull,
+				Tables: []*dax.QualifiedTable{
+					tbl1,
+				},
+				Version: 2,
+			}
+			err := api.ApplyDirective(ctx, d)
+			assert.NoError(t, err)
+			assertTablesMatch(t, []string{"tbl__acme__db1__1"}, api.Holder().Indexes())
+		}
+
+		// Add a new table, and keep the existing table.
+		{
+			d := &dax.Directive{
+				Method: dax.DirectiveMethodFull,
+				Tables: []*dax.QualifiedTable{
+					tbl1,
+					tbl2,
+				},
+				Version: 3,
+			}
+			err := api.ApplyDirective(ctx, d)
+			assert.NoError(t, err)
+			assertTablesMatch(t, []string{"tbl__acme__db1__1", "tbl__acme__db1__2"}, api.Holder().Indexes())
+		}
+
+		// Add a new table and remove one of the existing tables.
+		{
+			d := &dax.Directive{
+				Method: dax.DirectiveMethodFull,
+				Tables: []*dax.QualifiedTable{
+					tbl2,
+					tbl3,
+				},
+				Version: 4,
+			}
+			err := api.ApplyDirective(ctx, d)
+			assert.NoError(t, err)
+			assertTablesMatch(t, []string{"tbl__acme__db1__2", "tbl__acme__db1__3"}, api.Holder().Indexes())
+		}
+	})
+}
+
+// assertTablesMatch is a helper function which asserts that the list of index
+// names in `actual` match those provided in `expected`.
+func assertTablesMatch(t *testing.T, expected []string, actual []*pilosa.Index) {
+	t.Helper()
+
+	act := make([]string, len(actual))
+	for i := range actual {
+		act[i] = actual[i].Name()
+	}
+	assert.ElementsMatch(t, expected, act)
+}

api_test.go

@@ -1,4 +1,5 @@
-// Copyright 2021 Molecula Corp. All rights reserved.
+// Copyright 2022 Molecula Corp. (DBA FeatureBase).
+// SPDX-License-Identifier: Apache-2.0
 package pilosa_test
 
 import (
@@ -21,46 +22,26 @@ import (
 	"testing"
 	"time"
 
+	pilosa "github.com/featurebasedb/featurebase/v3"
+	"github.com/featurebasedb/featurebase/v3/authn"
+	"github.com/featurebasedb/featurebase/v3/roaring"
+	"github.com/featurebasedb/featurebase/v3/server"
+	"github.com/featurebasedb/featurebase/v3/shardwidth"
+	"github.com/featurebasedb/featurebase/v3/test"
+	. "github.com/featurebasedb/featurebase/v3/vprint" // nolint:staticcheck
 	"github.com/golang-jwt/jwt"
-	pilosa "github.com/molecula/featurebase/v3"
-	"github.com/molecula/featurebase/v3/authn"
-	"github.com/molecula/featurebase/v3/boltdb"
-	"github.com/molecula/featurebase/v3/server"
-	"github.com/molecula/featurebase/v3/shardwidth"
-	"github.com/molecula/featurebase/v3/test"
-	. "github.com/molecula/featurebase/v3/vprint" // nolint:staticcheck
+
+	"golang.org/x/sync/errgroup"
 )
 
 func TestAPI_Import(t *testing.T) {
-	c := test.MustRunCluster(t, 3,
-		[]server.CommandOption{
-			server.OptCommandServerOptions(
-				pilosa.OptServerNodeID("node0"),
-				pilosa.OptServerClusterHasher(&offsetModHasher{}),
-				pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore),
-				pilosa.OptServerOpenTranslateReader(pilosa.GetOpenTranslateReaderFunc(nil)),
-			)},
-		[]server.CommandOption{
-			server.OptCommandServerOptions(
-				pilosa.OptServerNodeID("node1"),
-				pilosa.OptServerClusterHasher(&offsetModHasher{}),
-				pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore),
-				pilosa.OptServerOpenTranslateReader(pilosa.GetOpenTranslateReaderFunc(nil)),
-			)},
-		[]server.CommandOption{
-			server.OptCommandServerOptions(
-				pilosa.OptServerNodeID("node2"),
-				pilosa.OptServerClusterHasher(&offsetModHasher{}),
-				pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore),
-				pilosa.OptServerOpenTranslateReader(pilosa.GetOpenTranslateReaderFunc(nil)),
-			)},
-	)
+	c := test.MustRunCluster(t, 3)
 	defer c.Close()
 
 	m0 := c.GetNode(0)
 	m1 := c.GetNode(1)
 
-	indexNames := map[bool]string{false: "i", true: "ki"}
+	indexNames := map[bool]string{false: c.Idx("u"), true: c.Idx("k")}
 	fieldNames := map[bool]string{false: "f", true: "kf"}
 
 	ctx := context.Background()
@@ -100,7 +81,7 @@ func TestAPI_Import(t *testing.T) {
 
 	t.Run("RowIDColumnKey", func(t *testing.T) {
 		// Import data with keys to the primary and verify that it gets
-		// translated and forwarded to the owner of shard 0 (node1; because of offsetModHasher)
+		// translated and forwarded to the owner of shard 0
 		req := &pilosa.ImportRequest{
 			Index:      indexNames[true],
 			Field:      fieldNames[false],
@@ -148,6 +129,7 @@ func TestAPI_Import(t *testing.T) {
 		}
 	})
 	t.Run("ExpectedErrors", func(t *testing.T) {
+		t.Skip("partitioning strategy changed, test not supported") // skipping due to change partitioning strategy
 		ctx := context.Background()
 		for ik, indexName := range indexNames {
 			for fk, fieldName := range fieldNames {
@@ -219,26 +201,7 @@ func TestAPI_Import(t *testing.T) {
 }
 
 func TestAPI_ImportValue(t *testing.T) {
-	c := test.MustRunCluster(t, 3,
-		[]server.CommandOption{
-			server.OptCommandServerOptions(
-				pilosa.OptServerNodeID("node0"),
-				pilosa.OptServerClusterHasher(&offsetModHasher{}),
-				pilosa.OptServerOpenTranslateReader(pilosa.GetOpenTranslateReaderFunc(nil)),
-			)},
-		[]server.CommandOption{
-			server.OptCommandServerOptions(
-				pilosa.OptServerNodeID("node1"),
-				pilosa.OptServerClusterHasher(&offsetModHasher{}),
-				pilosa.OptServerOpenTranslateReader(pilosa.GetOpenTranslateReaderFunc(nil)),
-			)},
-		[]server.CommandOption{
-			server.OptCommandServerOptions(
-				pilosa.OptServerNodeID("node2"),
-				pilosa.OptServerClusterHasher(&offsetModHasher{}),
-				pilosa.OptServerOpenTranslateReader(pilosa.GetOpenTranslateReaderFunc(nil)),
-			)},
-	)
+	c := test.MustRunCluster(t, 3)
 	defer c.Close()
 
 	coord := c.GetPrimary()
@@ -248,7 +211,7 @@ func TestAPI_ImportValue(t *testing.T) {
 
 	t.Run("ValColumnKey", func(t *testing.T) {
 		ctx := context.Background()
-		index := "valck"
+		index := c.Idx("valck")
 		field := "f"
 
 		_, err := coord.API.CreateIndex(ctx, index, pilosa.IndexOptions{Keys: true})
@@ -270,7 +233,7 @@ func TestAPI_ImportValue(t *testing.T) {
 		colKeys := []string{"col10", "col8", "col9", "col6", "col7", "col4", "col5", "col2", "col3", "col1"}
 
 		// Import data with keys to the primary and verify that it gets
-		// translated and forwarded to the owner of shard 0 (node1; because of offsetModHasher)
+		// translated and forwarded to the owner of shard 0
 		req := &pilosa.ImportValueRequest{
 			Index:      index,
 			Field:      field,
@@ -288,18 +251,24 @@ func TestAPI_ImportValue(t *testing.T) {
 		pql := fmt.Sprintf("Row(%s>0)", field)
 
 		// Query node0.
-		if res, err := m0.API.Query(ctx, &pilosa.QueryRequest{Index: index, Query: pql}); err != nil {
+		res, err := m0.API.Query(ctx, &pilosa.QueryRequest{Index: index, Query: pql})
+		if err != nil {
 			t.Fatal(err)
-		} else if keys := res.Results[0].(*pilosa.Row).Keys; !reflect.DeepEqual(keys, colKeys) {
+		}
+		keys := res.Results[0].(*pilosa.Row).Keys
+		if !sameStringSlice(keys, colKeys) {
 			t.Fatalf("unexpected column keys: %+v", keys)
 		}
 
 		// Query node1.
 		if err := test.RetryUntil(5*time.Second, func() error {
-			if res, err := m1.API.Query(ctx, &pilosa.QueryRequest{Index: index, Query: pql}); err != nil {
-				return err
-			} else if keys := res.Results[0].(*pilosa.Row).Keys; !reflect.DeepEqual(keys, colKeys) {
-				return fmt.Errorf("unexpected column keys: %+v", keys)
+			res, err := m1.API.Query(ctx, &pilosa.QueryRequest{Index: index, Query: pql})
+			if err != nil {
+				t.Fatal(err)
+			}
+			keys := res.Results[0].(*pilosa.Row).Keys
+			if !sameStringSlice(keys, colKeys) {
+				t.Fatalf("unexpected column keys: %+v", keys)
 			}
 			return nil
 		}); err != nil {
@@ -309,7 +278,7 @@ func TestAPI_ImportValue(t *testing.T) {
 
 	t.Run("ValIntEmpty", func(t *testing.T) {
 		ctx := context.Background()
-		index := "valintempty"
+		index := c.Idx("valintempty")
 		field := "fld"
 		createIndexForTest(index, coord, t)
 		createFieldForTest(index, field, coord, t)
@@ -362,6 +331,7 @@ func TestAPI_ImportValue(t *testing.T) {
 	})
 
 	t.Run("ValDecimalField", func(t *testing.T) {
+		t.Skip() // skipping due to change partitioning strategy
 		ctx := context.Background()
 		index := "valdec"
 		field := "fdec"
@@ -381,7 +351,7 @@ func TestAPI_ImportValue(t *testing.T) {
 			colIDs = append(colIDs, uint64(i))
 		}
 		// Import data with keys to node1 and verify that it gets translated and
-		// forwarded to the owner of shard 0 (node0; because of offsetModHasher)
+		// forwarded to the owner of shard 0
 		req := &pilosa.ImportValueRequest{
 			Index:       index,
 			Field:       field,
@@ -404,7 +374,7 @@ func TestAPI_ImportValue(t *testing.T) {
 
 	t.Run("ValDecimalFieldNegativeScale", func(t *testing.T) {
 		ctx := context.Background()
-		index := "valdecneg"
+		index := c.Idx("valdecneg")
 		field := "fdecneg"
 
 		_, err := m0.API.CreateIndex(ctx, index, pilosa.IndexOptions{})
@@ -418,8 +388,9 @@ func TestAPI_ImportValue(t *testing.T) {
 	})
 
 	t.Run("ValTimestampField", func(t *testing.T) {
+		t.Skip("partition strategy change invalidated") // skipping due to change partitioning strategy
 		ctx := context.Background()
-		index := "valts"
+		index := c.Idx("valts")
 		field := "fts"
 
 		_, err := m1.API.CreateIndex(ctx, index, pilosa.IndexOptions{})
@@ -440,7 +411,7 @@ func TestAPI_ImportValue(t *testing.T) {
 		}
 
 		// Import data with keys to node1 and verify that it gets translated and
-		// forwarded to the owner of shard 0 (node0; because of offsetModHasher)
+		// forwarded to the owner of shard 0
 		req := &pilosa.ImportValueRequest{
 			Index:           index,
 			Field:           field,
@@ -465,8 +436,9 @@ func TestAPI_ImportValue(t *testing.T) {
 	})
 
 	t.Run("ValStringField", func(t *testing.T) {
+		t.Skip("partition strategy change invalidated") // skipping due to change partitioning strategy
 		ctx := context.Background()
-		index := "valstr"
+		index := c.Idx("valstr")
 		field := "fstr"
 
 		fgnIndex := "fgnvalstr"
@@ -498,8 +470,7 @@ func TestAPI_ImportValue(t *testing.T) {
 		}
 
 		// Import data with keys to the node0 and verify that it gets translated
-		// and forwarded to the owner of shard 0 (node1; because of
-		// offsetModHasher)
+		// and forwarded to the owner of shard 0
 		req := &pilosa.ImportValueRequest{
 			Index:        index,
 			Field:        field,
@@ -526,26 +497,15 @@ func TestAPI_ImportValue(t *testing.T) {
 func TestAPI_Ingest(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
-	c := test.MustRunCluster(t, 1,
-		[]server.CommandOption{
-			server.OptCommandServerOptions(
-				pilosa.OptServerNodeID("node0"),
-				pilosa.OptServerClusterHasher(&offsetModHasher{}),
-				pilosa.OptServerOpenTranslateReader(pilosa.GetOpenTranslateReaderFunc(nil)),
-			)},
-	)
+	c := test.MustRunCluster(t, 1)
 	defer c.Close()
-
 	coord := c.GetPrimary()
-	// m0 := c.GetNode(0)
-	// m1 := c.GetNode(1)
-	// m2 := c.GetNode(2)
-
-	index := "ingest"
+	index := c.Idx()
 	setField := "set"
 	timeField := "tq"
+	intField := "int"
 
-	_, err := coord.API.CreateIndex(ctx, index, pilosa.IndexOptions{Keys: false})
+	_, err := coord.API.CreateIndex(ctx, index, pilosa.IndexOptions{Keys: false, TrackExistence: true})
 	if err != nil {
 		t.Fatalf("creating index: %v", err)
 	}
@@ -553,167 +513,109 @@ func TestAPI_Ingest(t *testing.T) {
 	if err != nil {
 		t.Fatalf("creating field: %v", err)
 	}
-	_, err = coord.API.CreateField(ctx, index, timeField, pilosa.OptFieldTypeTime("YMD"))
+	_, err = coord.API.CreateField(ctx, index, timeField, pilosa.OptFieldTypeTime("YMD", "0"))
 	if err != nil {
 		t.Fatalf("creating field: %v", err)
 	}
-	sampleJson := []byte(`
-[
-  {
-    "action": "set",
-    "records": {
-      "2": {
-	"set": [2],
-	"tq": { "time": "2006-01-02T15:04:05.999999999Z", "values": [6] }
-      },
-      "5": { "set": [3] },
-      "8": { "set": [3] },
-      "1": {
-	"set": [2],
-	"tq": { "time": "2006-01-02T15:04:05.999999999Z", "values": [3, 4] }
-      },
-      "4": { "set": [3, 7] }
-    }
-  },
-  {
-    "action": "clear",
-    "record_ids": [ 5, 6, 7 ],
-    "fields": [ "tq", "set" ]
-  },
-  {
-    "action": "write",
-    "records": {
-      "8": { "tq": { "time": "2006-01-02T15:04:05.999999999Z", "values": [3, 4] } },
-      "9": { "set": [7, 3] }
-    }
-  },
-  {
-    "action": "delete",
-    "record_ids": [ 9 ]
-  }
-]
-`)
-	// just for set row 3:
-	// first operation should set it for 4, 5, and 8.
-	// clear operation should clear it for 5, 6, and 7, leaving it still set for 4 and 8.
-	// the write operation should clear set for record 8, even though record 8 doesn't
-	// contain that field in that op, because set is present in record 9, which also
-	// gets row 3 set. but then we delete 9.
-	// so after all that we expect Row(set=3) to be 4...
-	sampleBuf := bytes.NewBuffer(sampleJson)
-	qcx := coord.API.Txf().NewQcx()
-	defer func() {
-		if err := qcx.Finish(); err != nil {
-			t.Fatalf("finishing qcx: %v", err)
+	_, err = coord.API.CreateField(ctx, index, intField, pilosa.OptFieldTypeInt(0, 100000))
+	if err != nil {
+		t.Fatalf("creating field: %v", err)
+	}
+
+	t.Run("ImportRoaringShard", func(t *testing.T) {
+		setBuf := &bytes.Buffer{}
+		setBits := roaring.NewBitmap(7, pilosa.ShardWidth+7)
+		_, _ = setBits.WriteTo(setBuf) // bytes.Buffer never errors
+		intBuf := &bytes.Buffer{}
+		intBits := roaring.NewBitmap(7, pilosa.ShardWidth*2+7)
+		_, _ = intBits.WriteTo(intBuf) // bytes.Buffer never errors
+		request := &pilosa.ImportRoaringShardRequest{
+			Remote: true,
+			Views: []pilosa.RoaringUpdate{
+				{
+					Field: setField,
+					View:  "standard",
+					Set:   setBuf.Bytes(),
+				},
+				{
+					Field: intField,
+					View:  "bsig_" + intField,
+					Set:   intBuf.Bytes(),
+				},
+			},
 		}
-	}()
-	err = coord.API.IngestOperations(ctx, qcx, index, sampleBuf)
-	if err != nil {
-		t.Fatalf("importing data: %v", err)
-	}
-	query := "Row(set=3)"
-	res, err := coord.API.Query(context.Background(), &pilosa.QueryRequest{Index: index, Query: query})
-	if err != nil {
-		t.Errorf("query: %v", err)
-	}
-	r := res.Results[0].(*pilosa.Row).Columns()
-	if len(r) != 1 || r[0] != 4 {
-		t.Fatalf("expected row with 4 set, got %d", r)
-	}
-}
-
-// ingestBenchmarkHelper makes it easier to exclude this from benchmark computations
-// and profiles.
-func ingestBenchmarkHelper() []byte {
-	buf := &bytes.Buffer{}
-	buf.WriteString(`[{"action": "write", "records": {`)
-	comma := ""
-	now := time.Now().Add(-3840000 * time.Second)
-	for i := 0; i < 1000000; i++ {
-		then := now.Add(time.Duration(rand.Int63n(1234567)) * time.Second)
-		fmt.Fprintf(buf, `%s"%d": { "set": [%d, %d], "int": %d, "tq": { "time": "%s", "values": %d } }`, comma, i, i%2, (i%4)+2, rand.Int63n(163840),
-			then.Format(time.RFC3339), rand.Int63n(25))
-		comma = ", "
-	}
-	buf.WriteString(`}}]`)
-	data := buf.Bytes()
-	return data
-}
-
-func BenchmarkIngest(b *testing.B) {
-	b.StopTimer()
-	data := ingestBenchmarkHelper()
-	ctx, cancel := context.WithCancel(context.Background())
-	defer cancel()
-	c := test.MustRunCluster(b, 1,
-		[]server.CommandOption{
-			server.OptCommandServerOptions(
-				pilosa.OptServerNodeID("node0"),
-				pilosa.OptServerClusterHasher(&offsetModHasher{}),
-				pilosa.OptServerOpenTranslateReader(pilosa.GetOpenTranslateReaderFunc(nil)),
-			)},
-	)
-	defer c.Close()
-
-	coord := c.GetPrimary()
-	m0 := c.GetNode(0)
-	// m1 := c.GetNode(1)
-	// m2 := c.GetNode(2)
-
-	index := "ingest"
-	setField := "set"
-	intField := "int"
-	tqField := "tq"
-	_, err := coord.API.CreateIndex(ctx, index, pilosa.IndexOptions{Keys: false})
-	if err != nil {
-		b.Fatalf("creating index: %v", err)
-	}
-	_, err = coord.API.CreateField(ctx, index, setField, pilosa.OptFieldTypeSet("none", 0))
-	if err != nil {
-		b.Fatalf("creating field: %v", err)
-	}
-	_, err = coord.API.CreateField(ctx, index, intField, pilosa.OptFieldTypeInt(0, 163840))
-	if err != nil {
-		b.Fatalf("creating field: %v", err)
-	}
-	_, err = coord.API.CreateField(ctx, index, tqField, pilosa.OptFieldTypeTime("YMDH"))
-	if err != nil {
-		b.Fatalf("creating field: %v", err)
-	}
-	b.ReportAllocs()
-	b.StartTimer()
-	for i := 0; i < b.N; i++ {
-		qcx := m0.API.Txf().NewQcx()
-		defer qcx.Abort()
-		err = coord.API.IngestOperations(ctx, qcx, index, bytes.NewBuffer(data))
-		if err != nil {
-			b.Fatalf("ingest: %v", err)
+		if err := coord.API.ImportRoaringShard(context.Background(), c.Idx(), 8, request); err != nil {
+			t.Fatalf("ingesting: %v", err)
 		}
-		err = qcx.Finish()
-		if err != nil {
-			b.Fatalf("finish: %v", err)
+
+		mustQuery := func(t *testing.T, index, query string) pilosa.QueryResponse {
+			res, err := coord.API.Query(context.Background(), &pilosa.QueryRequest{Index: index, Query: query})
+			if err != nil {
+				t.Fatalf("querying: %v", err)
+			}
+			return res
 		}
-	}
+
+		res := mustQuery(t, c.Idx(), "Row(set=0)")
+		r := res.Results[0].(*pilosa.Row).Columns()
+		if len(r) != 1 || r[0] != pilosa.ShardWidth*8+7 {
+			t.Fatalf("expected row with pilosa.ShardWidth*8+7 set, got %d", r)
+		}
+
+		res = mustQuery(t, c.Idx(), "Row(set=1)")
+		r = res.Results[0].(*pilosa.Row).Columns()
+		if len(r) != 1 || r[0] != pilosa.ShardWidth*8+7 {
+			t.Fatalf("expected row with pilosa.ShardWidth*8+7 set, got %d", r)
+		}
+
+		res = mustQuery(t, c.Idx(), "Row(int==1)")
+		r = res.Results[0].(*pilosa.Row).Columns()
+		if len(r) != 1 || r[0] != pilosa.ShardWidth*8+7 {
+			t.Fatalf("expected row with, pilosa.ShardWidth*8+7 set, got %d", r)
+		}
+
+		request = &pilosa.ImportRoaringShardRequest{
+			Remote: true,
+			Views: []pilosa.RoaringUpdate{
+				{
+					Field: setField,
+					View:  "standard",
+					Clear: setBuf.Bytes(),
+				},
+				{
+					Field: intField,
+					View:  "bsig_" + intField,
+					Clear: intBuf.Bytes(),
+				},
+			},
+		}
+		if err := coord.API.ImportRoaringShard(context.Background(), c.Idx(), 8, request); err != nil {
+			t.Fatalf("ingesting: %v", err)
+		}
+
+		res = mustQuery(t, c.Idx(), "Row(set=0)")
+		r = res.Results[0].(*pilosa.Row).Columns()
+		if len(r) != 0 {
+			t.Fatalf("expected no values after clearing, got: %v", r)
+		}
+
+		res = mustQuery(t, c.Idx(), "Row(set=1)")
+		r = res.Results[0].(*pilosa.Row).Columns()
+		if len(r) != 0 {
+			t.Fatalf("expected no values after clearing, got: %v", r)
+		}
+
+		res = mustQuery(t, c.Idx(), "Row(int==1)")
+		r = res.Results[0].(*pilosa.Row).Columns()
+		if len(r) != 0 {
+			t.Fatalf("expected no values after clearing, got: %v", r)
+		}
+
+	})
 }
 
-// offsetModHasher represents a simple, mod-based hashing offset by 1.
-type offsetModHasher struct{}
-
-func (*offsetModHasher) Hash(key uint64, n int) int {
-	return int(key+1) % n
-}
-
-func (*offsetModHasher) Name() string { return "mod" }
-
 func TestAPI_ClearFlagForImportAndImportValues(t *testing.T) {
-	c := test.MustRunCluster(t, 1,
-		[]server.CommandOption{
-			server.OptCommandServerOptions(
-				pilosa.OptServerNodeID("node0"),
-				pilosa.OptServerClusterHasher(&offsetModHasher{}),
-				pilosa.OptServerOpenTranslateReader(pilosa.GetOpenTranslateReaderFunc(nil)),
-			)},
-	)
+	c := test.MustRunCluster(t, 1)
 	defer c.Close()
 
 	// plan:
@@ -726,7 +628,7 @@ func TestAPI_ClearFlagForImportAndImportValues(t *testing.T) {
 	m0api := m0.API
 
 	ctx := context.Background()
-	index := "i"
+	index := c.Idx()
 	fieldAcct0 := "acct0"
 
 	opts := pilosa.OptFieldTypeInt(-1000, 1000)
@@ -935,7 +837,7 @@ func TestAPI_IDAlloc(t *testing.T) {
 			t.Fatalf("obtaining random bytes: %v", err)
 		}
 		ids3, err := primary.ReserveIDs(key, session, 0, 2)
-		var esync pilosa.ErrIDOffsetDesync
+		var esync pilosa.IDOffsetDesyncError
 		if errors.As(err, &esync) {
 			if esync.Requested != 0 {
 				t.Errorf("incorrect requested offset in error: provided %d but got %d", 0, esync.Requested)
@@ -956,29 +858,6 @@ func TestAPI_IDAlloc(t *testing.T) {
 	})
 }
 
-func TestAPI_SchemaDetailsOff(t *testing.T) {
-	cluster := test.MustRunCluster(t, 2)
-	defer cluster.Close()
-	cmd := cluster.GetNode(0)
-	err := cmd.API.SetAPIOptions(pilosa.OptAPISchemaDetailsOn(false))
-	if err != nil {
-		t.Fatalf("could not toggle schema details to off: %v", err)
-	}
-	schema, err := cmd.API.SchemaDetails(context.Background())
-	if err != nil {
-		t.Fatalf("getting schema: %v", err)
-	}
-
-	for _, i := range schema {
-		for _, f := range i.Fields {
-			if f.Cardinality != nil {
-				t.Fatalf("expected nil cardinality, got: %v", *f.Cardinality)
-			}
-		}
-	}
-
-}
-
 type mutexCheckIndex struct {
 	index     *pilosa.Index
 	indexName string
@@ -993,7 +872,9 @@ type mutexCheckField struct {
 }
 
 func TestAPI_MutexCheck(t *testing.T) {
-	c := test.MustNewCluster(t, 3)
+	// Can't share this one, because it has to get custom option settings and needs
+	// replication.
+	c := test.MustUnsharedCluster(t, 3)
 	for _, c := range c.Nodes {
 		c.Config.Cluster.ReplicaN = 2
 	}
@@ -1015,7 +896,7 @@ func TestAPI_MutexCheck(t *testing.T) {
 
 	ctx := context.Background()
 	for _, keyedIndex := range []bool{false, true} {
-		indexName := fmt.Sprintf("i%t", keyedIndex)
+		indexName := c.Idx(map[bool]string{false: "u", true: "k"}[keyedIndex])
 		index, err := m0.API.CreateIndex(ctx, indexName, pilosa.IndexOptions{Keys: keyedIndex, TrackExistence: true})
 		if err != nil {
 			t.Fatalf("creating index: %v", err)
@@ -1048,7 +929,7 @@ func TestAPI_MutexCheck(t *testing.T) {
 	rowKeysBase := []string{"v0", "v1", "v2", "v3"}
 	colKeysBase := []string{"c0", "c1", "c2", "c3"}
 
-	const nShards = 10
+	const nShards = 9
 
 	// now, try the same thing for each combination of keyed/unkeyed. we
 	// share code between keyed/unkeyed fields, but for indexes, the logic
@@ -1113,7 +994,20 @@ func TestAPI_MutexCheck(t *testing.T) {
 				(4 << shardwidth.Exponent) + 1: true,
 				(5 << shardwidth.Exponent) + 1: true,
 				(8 << shardwidth.Exponent) + 1: true,
-				(9 << shardwidth.Exponent) + 1: true,
+				// So, nShards used to be 10. If you run a complete test,
+				// with go test -race, and you have the sample input for the
+				// unrelated TestImportMutexSampleData configured to use 64K
+				// bit density and 2K rows, everything is fine. If you run a
+				// partial test, everything is fine. If you run a complete
+				// test with -race, but you skip TestImportMutexSampleData,
+				// or reduce either the bit density or the row count, you get
+				// a very strange panic where the go panic handler panics
+				// trying to report what happened so we don't get a valid
+				// stack dump. On Macs. This is as much as I could debug it
+				// after about 6 hours. Since there's no special reason to
+				// think we need all 10 shards, and 9 still tests the
+				// behavior, we're leaving this one a mystery.
+				// (9 << shardwidth.Exponent) + 1: true,
 			}
 
 			results, err := m0.API.MutexCheck(ctx, qcx, indexData.indexName, fieldData.fieldName, true, 0)
@@ -1354,34 +1248,34 @@ func createFieldForTest(index string, field string, coord *test.Command, t *test
 
 func TestVariousApiTranslateCalls(t *testing.T) {
 	for i := 1; i < 8; i += 3 {
-		m := test.MustRunCluster(t, i)
-		defer m.Close()
-		node := m.GetNode(0)
+		c := test.MustRunCluster(t, i)
+		defer c.Close()
+		node := c.GetNode(0)
 		api := node.API
 		// this should never actually get used because we're testing for errors here
 		r := strings.NewReader("")
 		// test index
-		idx, err := api.Holder().CreateIndex("index", pilosa.IndexOptions{})
+		idx, err := api.Holder().CreateIndex(c.Idx(), "", pilosa.IndexOptions{})
 		if err != nil {
 			t.Fatalf("%v: could not create test index", err)
 		}
-		if _, err = idx.CreateFieldIfNotExistsWithOptions("field", &pilosa.FieldOptions{Keys: false}); err != nil {
+		if _, err = idx.CreateFieldIfNotExistsWithOptions("field", "", &pilosa.FieldOptions{Keys: false}); err != nil {
 			t.Fatalf("creating field: %v", err)
 		}
 		t.Run("translateIndexDbOnNilIndex",
 			func(t *testing.T) {
 				err := api.TranslateIndexDB(context.Background(), "nonExistentIndex", 0, r)
-				expected := fmt.Errorf("index %q not found", "nonExistentIndex")
-				if !reflect.DeepEqual(err, expected) {
+				expected := fmt.Sprintf("index %q not found", "nonExistentIndex")
+				if err == nil || err.Error() != expected {
 					t.Fatalf("expected '%#v', got '%#v'", expected, err)
 				}
 			})
 
 		t.Run("translateIndexDbOnNilTranslateStore",
 			func(t *testing.T) {
-				err := api.TranslateIndexDB(context.Background(), "index", 0, r)
-				expected := fmt.Errorf("index %q has no translate store", "index")
-				if !reflect.DeepEqual(err, expected) {
+				err := api.TranslateIndexDB(context.Background(), c.Idx(), 0, r)
+				expected := fmt.Sprintf("index %q has no translate store", c.Idx())
+				if err == nil || err.Error() != expected {
 					t.Fatalf("expected '%#v', got '%#v'", expected, err)
 				}
 			})
@@ -1389,24 +1283,24 @@ func TestVariousApiTranslateCalls(t *testing.T) {
 		t.Run("translateFieldDbOnNilIndex",
 			func(t *testing.T) {
 				err := api.TranslateFieldDB(context.Background(), "nonExistentIndex", "field", r)
-				expected := fmt.Errorf("index %q not found", "nonExistentIndex")
-				if !reflect.DeepEqual(err, expected) {
+				expected := fmt.Sprintf("index %q not found", "nonExistentIndex")
+				if err == nil || err.Error() != expected {
 					t.Fatalf("expected '%#v', got '%#v'", expected, err)
 				}
 			})
 
 		t.Run("translateFieldDbOnNilField",
 			func(t *testing.T) {
-				err := api.TranslateFieldDB(context.Background(), "index", "nonExistentField", r)
-				expected := fmt.Errorf("field %q/%q not found", "index", "nonExistentField")
-				if !reflect.DeepEqual(err, expected) {
+				err := api.TranslateFieldDB(context.Background(), c.Idx(), "nonExistentField", r)
+				expected := fmt.Sprintf("field %q/%q not found", c.Idx(), "nonExistentField")
+				if err == nil || err.Error() != expected {
 					t.Fatalf("expected '%#v', got '%#v'", expected, err)
 				}
 			})
 
 		t.Run("translateFieldDbNilField_keys",
 			func(t *testing.T) {
-				err := api.TranslateFieldDB(context.Background(), "index", "_keys", r)
+				err := api.TranslateFieldDB(context.Background(), c.Idx(), "_keys", r)
 				if err != nil {
 					t.Fatalf("expected 'nil', got '%#v'", err)
 				}
@@ -1416,8 +1310,8 @@ func TestVariousApiTranslateCalls(t *testing.T) {
 		   stores, which is a bug, but one that we will eventually fix. when we do, this
 		   test might come in handy t.Run("translateFieldDbOnNilTranslateStore",
 		   func(t *testing.T) {
-		       err := api.TranslateFieldDB(context.Background(), "index", "field", r)
-		       expected := fmt.Errorf("field %q/%q has no translate store", "index", "field")
+		       err := api.TranslateFieldDB(context.Background(), c.Idx(), "field", r)
+		       expected := fmt.Errorf("field %q/%q has no translate store", c.Idx(), "field")
 		       if !reflect.DeepEqual(err, expected) {
 		           t.Fatalf("expected '%#v', got '%#v'", expected, err)
 		       }
@@ -1426,22 +1320,51 @@ func TestVariousApiTranslateCalls(t *testing.T) {
 	}
 }
 
+func TestAPI_CreateField(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	c := test.MustRunCluster(t, 3)
+	defer c.Close()
+
+	nodes := make([]*test.Command, 3)
+	for i := range nodes {
+		nodes[i] = c.GetNode(i)
+	}
+
+	if _, err := nodes[0].API.CreateIndex(ctx, c.Idx(), pilosa.IndexOptions{}); err != nil {
+		t.Fatal(err)
+	}
+	eg, ctx := errgroup.WithContext(context.Background())
+	for _, n := range nodes {
+		node := n
+		eg.Go(func() error {
+			for i := 0; i < 10; i++ {
+				_, err := node.API.CreateField(ctx, c.Idx(), fmt.Sprintf("f%d", i))
+				if err != nil && !errors.Is(err, pilosa.ErrFieldExists) {
+					return err
+				}
+			}
+			return nil
+		})
+	}
+	err := eg.Wait()
+	if err != nil {
+		if errors.Is(err, pilosa.ErrFieldExists) {
+			t.Fatalf("conflict error: %v", err)
+		}
+		t.Fatalf("unexpected error: %T %v", err, err)
+	}
+}
+
 func TestAPI_RBFDebugInfo(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
-	c := test.MustRunCluster(t, 1,
-		[]server.CommandOption{
-			server.OptCommandServerOptions(
-				pilosa.OptServerNodeID("node0"),
-				pilosa.OptServerClusterHasher(&offsetModHasher{}),
-				pilosa.OptServerOpenTranslateReader(pilosa.GetOpenTranslateReaderFunc(nil)),
-			)},
-	)
+	c := test.MustRunCluster(t, 1)
 	defer c.Close()
 
 	coord := c.GetPrimary()
 
-	if _, err := coord.API.CreateIndex(ctx, "i", pilosa.IndexOptions{}); err != nil {
+	if _, err := coord.API.CreateIndex(ctx, c.Idx(), pilosa.IndexOptions{}); err != nil {
 		t.Fatal(err)
 	} else if infos := coord.API.RBFDebugInfo(); infos == nil {
 		t.Fatal("expected info")
@@ -1460,7 +1383,6 @@ func makeUser(t *testing.T, groups []authn.Group, name, secret string) *authn.Us
 	if err != nil {
 		t.Fatalf("signing string %v", err)
 	}
-	validToken = "Bearer " + validToken
 
 	return &authn.UserInfo{
 		UserID:   "fake" + name,
@@ -1481,23 +1403,11 @@ func TestAuth_MultiNode(t *testing.T) {
     "test": "write"
 admin: "ac97c9e2-346b-42a2-b6da-18bcb61a32fe"`
 	adminUser := makeUser(t, []authn.Group{{GroupID: "ac97c9e2-346b-42a2-b6da-18bcb61a32fe", GroupName: "adminGroup"}}, "admin", "DEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEF")
-	adminCtx := context.WithValue(
-		context.Background(),
-		"userinfo",
-		adminUser,
-	)
+	adminCtx := authn.WithUserInfo(context.Background(), adminUser)
 	readUser := makeUser(t, []authn.Group{{GroupID: "dca35310-ecda-4f23-86cd-876aee55906b", GroupName: "readGroup"}}, "reader", "DEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEF")
-	readCtx := context.WithValue(
-		context.Background(),
-		"userinfo",
-		readUser,
-	)
+	readCtx := authn.WithUserInfo(context.Background(), readUser)
 	writeUser := makeUser(t, []authn.Group{{GroupID: "dca35310-ecda-4f23-86cd-876aee55906f", GroupName: "writeGroup"}}, "writer", "DEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEED")
-	writeCtx := context.WithValue(
-		context.Background(),
-		"userinfo",
-		writeUser,
-	)
+	writeCtx := authn.WithUserInfo(context.Background(), writeUser)
 	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		token, ok := r.Header["Authorization"]
 		if !ok || len(token) == 0 {
@@ -1505,7 +1415,7 @@ admin: "ac97c9e2-346b-42a2-b6da-18bcb61a32fe"`
 			return
 		}
 		g := []authn.Group{}
-		switch token[0] {
+		switch strings.TrimPrefix(token[0], "Bearer ") {
 		case adminUser.Token:
 			g = adminUser.Groups
 		case readUser.Token:
@@ -1572,25 +1482,22 @@ f9Oeos0UUothgiDktdQHxdNEwLjQf7lJJBzV+5OtwswCWA==
 	config.TLS.CertificateKeyPath = writeTestFile(t, "certKey.pem", localhostKey)
 	config.TLS.CertificatePath = writeTestFile(t, "cert.pem", localhostCert)
 
-	c := test.MustRunCluster(t, 3,
+	c := test.MustRunUnsharedCluster(t, 3,
 		[]server.CommandOption{
 			server.OptCommandServerOptions(
 				pilosa.OptServerNodeID("node0"),
-				pilosa.OptServerClusterHasher(&test.ModHasher{}),
 			),
 			server.OptCommandConfig(config),
 		},
 		[]server.CommandOption{
 			server.OptCommandServerOptions(
 				pilosa.OptServerNodeID("node1"),
-				pilosa.OptServerClusterHasher(&test.ModHasher{}),
 			),
 			server.OptCommandConfig(config),
 		},
 		[]server.CommandOption{
 			server.OptCommandServerOptions(
 				pilosa.OptServerNodeID("node2"),
-				pilosa.OptServerClusterHasher(&test.ModHasher{}),
 			),
 			server.OptCommandConfig(config),
 		},
@@ -1600,6 +1507,8 @@ f9Oeos0UUothgiDktdQHxdNEwLjQf7lJJBzV+5OtwswCWA==
 	primaryAPI := c.GetPrimary().API
 
 	// needs internal/cluster/message
+	// Note: This indexName wouldn't be safe on a shared cluster, but we have to use an
+	// unshared cluster to set up the auth config anyway.
 	indexName := "test"
 	_, err := primaryAPI.CreateIndex(adminCtx, indexName, pilosa.IndexOptions{})
 	if err != nil {
@@ -1672,6 +1581,8 @@ func writeTestFile(t *testing.T, filename, content string) string {
 	if err != nil {
 		t.Fatalf("could not write string %v", err)
 	}
-	defer f.Close()
+	if err := f.Close(); err != nil {
+		t.Fatalf("could not close file %v", err)
+	}
 	return fname
 }

apimethod_string.go

@@ -27,27 +27,27 @@ func _() {
 	_ = x[apiIndex-16]
 	_ = x[apiQuery-17]
 	_ = x[apiRecalculateCaches-18]
-	_ = x[apiRemoveNode-19]
-	_ = x[apiResizeAbort-20]
-	_ = x[apiSchema-21]
-	_ = x[apiShardNodes-22]
-	_ = x[apiState-23]
-	_ = x[apiViews-24]
-	_ = x[apiApplySchema-25]
-	_ = x[apiStartTransaction-26]
-	_ = x[apiFinishTransaction-27]
-	_ = x[apiTransactions-28]
-	_ = x[apiGetTransaction-29]
-	_ = x[apiActiveQueries-30]
-	_ = x[apiPastQueries-31]
-	_ = x[apiIDReserve-32]
-	_ = x[apiIDCommit-33]
-	_ = x[apiIDReset-34]
+	_ = x[apiSchema-19]
+	_ = x[apiShardNodes-20]
+	_ = x[apiState-21]
+	_ = x[apiViews-22]
+	_ = x[apiApplySchema-23]
+	_ = x[apiStartTransaction-24]
+	_ = x[apiFinishTransaction-25]
+	_ = x[apiTransactions-26]
+	_ = x[apiGetTransaction-27]
+	_ = x[apiActiveQueries-28]
+	_ = x[apiPastQueries-29]
+	_ = x[apiIDReserve-30]
+	_ = x[apiIDCommit-31]
+	_ = x[apiIDReset-32]
+	_ = x[apiPartitionNodes-33]
+	_ = x[apiMutexCheck-34]
 }
 
-const _apiMethod_name = "apiClusterMessageapiCreateFieldapiCreateIndexapiDeleteFieldapiDeleteAvailableShardapiDeleteIndexapiDeleteViewapiExportCSVapiFragmentBlockDataapiFragmentBlocksapiFragmentDataapiTranslateDataapiFieldTranslateDataapiFieldapiImportapiImportValueapiIndexapiQueryapiRecalculateCachesapiRemoveNodeapiResizeAbortapiSchemaapiShardNodesapiStateapiViewsapiApplySchemaapiStartTransactionapiFinishTransactionapiTransactionsapiGetTransactionapiActiveQueriesapiPastQueriesapiIDReserveapiIDCommitapiIDReset"
+const _apiMethod_name = "apiClusterMessageapiCreateFieldapiCreateIndexapiDeleteFieldapiDeleteAvailableShardapiDeleteIndexapiDeleteViewapiExportCSVapiFragmentBlockDataapiFragmentBlocksapiFragmentDataapiTranslateDataapiFieldTranslateDataapiFieldapiImportapiImportValueapiIndexapiQueryapiRecalculateCachesapiSchemaapiShardNodesapiStateapiViewsapiApplySchemaapiStartTransactionapiFinishTransactionapiTransactionsapiGetTransactionapiActiveQueriesapiPastQueriesapiIDReserveapiIDCommitapiIDResetapiPartitionNodesapiMutexCheck"
 
-var _apiMethod_index = [...]uint16{0, 17, 31, 45, 59, 82, 96, 109, 121, 141, 158, 173, 189, 210, 218, 227, 241, 249, 257, 277, 290, 304, 313, 326, 334, 342, 356, 375, 395, 410, 427, 443, 457, 469, 480, 490}
+var _apiMethod_index = [...]uint16{0, 17, 31, 45, 59, 82, 96, 109, 121, 141, 158, 173, 189, 210, 218, 227, 241, 249, 257, 277, 286, 299, 307, 315, 329, 348, 368, 383, 400, 416, 430, 442, 453, 463, 480, 493}
 
 func (i apiMethod) String() string {
 	if i < 0 || i >= apiMethod(len(_apiMethod_index)-1) {

apply.go

@@ -0,0 +1,686 @@
+// Copyright 2021 Molecula Corp. All rights reserved.
+package pilosa
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+	"sync"
+
+	"github.com/apache/arrow/go/v10/arrow"
+	"github.com/apache/arrow/go/v10/arrow/array"
+	"github.com/apache/arrow/go/v10/arrow/memory"
+	"github.com/featurebasedb/featurebase/v3/pql"
+	"github.com/featurebasedb/featurebase/v3/tracing"
+	"github.com/featurebasedb/featurebase/v3/vprint"
+	"github.com/gomem/gomem/pkg/dataframe"
+	"github.com/pkg/errors"
+
+	ivy "robpike.io/ivy/arrow"
+	config "robpike.io/ivy/config"
+	"robpike.io/ivy/exec"
+	"robpike.io/ivy/parse"
+	"robpike.io/ivy/run"
+	"robpike.io/ivy/scan"
+	"robpike.io/ivy/value"
+)
+
+type (
+	ApplyResult *arrow.Column
+)
+
+func runIvyString(context value.Context, str string) (ok bool, err error) {
+	defer func() {
+		if r := recover(); r != nil {
+			err = r.(value.Error)
+		}
+	}()
+	scanner := scan.New(context, "<args>", strings.NewReader(str))
+	parser := parse.NewParser("<args>", scanner, context)
+	ok = run.Run(parser, context, false)
+	return
+}
+
+// Possibly combine all arrays together then apply some interesting
+// computation at the end?
+func IvyReduce(reduceCode string, opCode string, opt *ExecOptions) (func(ctx context.Context, prev, v interface{}) interface{}, func() (*dataframe.DataFrame, error)) {
+	var accumulator value.Value
+	mu := &sync.Mutex{}
+	concat := value.BinaryOps[opCode]
+	conf := getDefaultConfig()
+	ctxIvy := exec.NewContext(&conf)
+	// concat returned results at coordinating node.
+	reduceFn := func(ctx context.Context, prev, v interface{}) interface{} {
+		if v == nil {
+			return prev
+		}
+		if accumulator == nil {
+			switch val := v.(type) {
+			case *dataframe.DataFrame:
+				col := val.ColumnAt(0)
+				resolver := dataframe.NewChunkResolver(col)
+				accumulator = value.NewArrowVector(col, &conf, &resolver)
+			case value.Value:
+				accumulator = v.(value.Value)
+			default:
+				return errors.New(fmt.Sprintf("ivy reduction failed first unexpected type %T", v))
+			}
+			return nil
+		}
+		switch val := v.(type) {
+		case *dataframe.DataFrame:
+			col := val.ColumnAt(0)
+			resolver := dataframe.NewChunkResolver(col)
+			x := value.NewArrowVector(col, &conf, &resolver)
+			mu.Lock() // i'm being overyerly cautious..need to confirm this can be concurrent
+			accumulator = concat.EvalBinary(ctxIvy, accumulator, x)
+			mu.Unlock()
+		case value.Value:
+			mu.Lock()
+			accumulator = concat.EvalBinary(ctxIvy, accumulator, val)
+			mu.Unlock()
+		default:
+			return errors.New(fmt.Sprintf("ivy reduction failed unexpected type %T", v))
+		}
+
+		return nil
+	}
+	tablerFn := func() (*dataframe.DataFrame, error) {
+		pool := memory.NewGoAllocator() // TODO(twg) 2022/09/01 singledton?
+		if opt.Remote {
+			col := value.ToArrowColumn(accumulator, pool)
+			return dataframe.NewDataFrameFromColumns(pool, []arrow.Column{*col})
+		}
+		// only actually reduce on the initiating node i hate the network
+		// over head but oh well
+		ctxIvy.AssignGlobal("_", accumulator)
+		ok, err := runIvyString(ctxIvy, reduceCode)
+		if err != nil {
+			return nil, err
+		}
+		if ok {
+			v := ctxIvy.Global("_")
+			if v == nil {
+				return nil, errors.New("ivy reduction no result ")
+			}
+			col := value.ToArrowColumn(ctxIvy.Global("_"), pool)
+
+			return dataframe.NewDataFrameFromColumns(pool, []arrow.Column{*col})
+		}
+		return nil, errors.New("ivy reduction failed ")
+	}
+
+	return reduceFn, tablerFn
+}
+
+// executeApply executes a Apply() call.
+func (e *executor) executeApply(ctx context.Context, qcx *Qcx, index string, c *pql.Call, shards []uint64, opt *ExecOptions) (*dataframe.DataFrame, error) {
+	if !e.dataframeEnabled {
+		return nil, errors.New("Dataframe support not enabled")
+	}
+	span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeMax")
+	defer span.Finish()
+
+	if _, err := c.FirstStringArg("_ivy"); err != nil {
+		return nil, errors.Wrap(err, " no ivy program supplied")
+	}
+
+	if len(c.Children) > 1 {
+		return nil, errors.New("Apply() only accepts a single bitmap input filter")
+	}
+
+	// Execute calls in bulk on each remote node and merge.
+	mapFn := func(ctx context.Context, shard uint64, mopt *mapOptions) (_ interface{}, err error) {
+		return e.executeApplyShard(ctx, qcx, index, c, shard)
+	}
+	ivyReduce, ok, err := c.StringArg("_ivyReduce")
+	if err != nil {
+		return nil, err
+	}
+	reduceFn, tablerFn := IvyReduce("_", ",", opt)
+	if ok {
+		reduceFn, tablerFn = IvyReduce(ivyReduce, ",", opt)
+	}
+
+	_, err = e.mapReduce(ctx, index, shards, c, opt, mapFn, reduceFn)
+	if err != nil {
+		return nil, err
+	}
+	return tablerFn()
+}
+
+func getDefaultConfig() config.Config {
+	maxbits := uint(1e9)   // "maximum size of an integer, in bits; 0 means no limit")
+	maxdigits := uint(1e4) // "above this many `digits`, integers print as floating point; 0 disables")
+	maxstack := uint(100000)
+	origin := 1  // "set index origin to `n` (must be 0 or 1)")
+	prompt := "" // flag.String("prompt", "", "command `prompt`")
+	format := ""
+	//      debugFlag := "" // flag.String("debug", "", "comma-separated `names` of debug settings to enable")
+	conf := config.Config{}
+	conf.SetFormat(format)
+	conf.SetMaxBits(maxbits)
+	conf.SetMaxDigits(maxdigits)
+	conf.SetMaxStack(maxstack)
+	conf.SetOrigin(origin)
+	conf.SetPrompt(prompt)
+	conf.SetOutput(io.Discard)
+	conf.SetErrOutput(io.Discard)
+	conf.SetEmbedded(true) // needed to propagate panic
+	return conf
+}
+
+func filterDataframe(resolver dataframe.Resolver, pool memory.Allocator, filter []int64) (*dataframe.IndexResolver, error) {
+	if resolver.NumRows() == 0 {
+		return nil, errors.New("No data")
+	}
+	indexResolver := dataframe.NewIndexResolver(len(filter), uint32(ShardWidth-1))
+	for i, id := range filter {
+		if int(id) >= resolver.NumRows() {
+			continue
+		}
+		c, o := resolver.Resolve(int(id))
+		indexResolver.Set(i, c, o)
+
+	}
+	return indexResolver, nil
+}
+
+func (e *executor) executeApplyShard(ctx context.Context, qcx *Qcx, index string, c *pql.Call, shard uint64) (value.Value, error) {
+	span, _ := tracing.StartSpanFromContext(ctx, "Executor.executeApplyShard")
+	defer span.Finish()
+
+	ivyProgram, ok, err := c.StringArg("_ivy")
+	if err != nil || !ok {
+		return nil, errors.Wrap(err, "finding ivy program")
+	}
+	var filter *Row
+	if len(c.Children) == 1 {
+		row, err := e.executeBitmapCallShard(ctx, qcx, index, c.Children[0], shard)
+		if err != nil {
+			return nil, err
+		}
+		filter = row
+		if !filter.Any() {
+			// no need to actuall run the query for its not operating against any values
+			return value.NewVector([]value.Value{}), nil
+		}
+	}
+	//
+	pool := memory.NewGoAllocator() // TODO(twg) 2022/09/01 singledton?
+
+	ids := filter.ShardColumns() // needs to be shard columns
+	// Fetch index.
+	idx := e.Holder.Index(index)
+	if idx == nil {
+		return nil, newNotFoundError(ErrIndexNotFound, index)
+	}
+
+	fname := idx.GetDataFramePath(shard)
+
+	if !e.dataFrameExists(fname) {
+		return value.NewVector([]value.Value{}), nil
+	}
+
+	table, err := e.getDataTable(ctx, fname, pool)
+	if err != nil {
+		return nil, err
+	}
+	defer table.Release()
+	df, err := dataframe.NewDataFrameFromTable(pool, table)
+	if err != nil {
+		return nil, err
+	}
+	p := dataframe.NewChunkResolver(df.ColumnAt(0))
+	var resolver dataframe.Resolver
+	resolver = &p
+	if filter != nil {
+		if len(ids) == 0 {
+			return value.NewVector([]value.Value{}), nil
+		}
+		resolver, err = filterDataframe(resolver, pool, ids)
+		if err != nil {
+			return nil, err
+		}
+	}
+	conf := getDefaultConfig()
+	context, err := ivy.RunArrow(dataframe.NewTableFacade(df), ivyProgram, conf, resolver)
+	if err != nil {
+		return nil, fmt.Errorf("ivy map error: %w", err)
+	}
+	return context.Global("_"), nil
+}
+
+// ///////////////////////////////////////////////////////
+// all the ingest supporting functions
+// ///////////////////////////////////////////////////////
+
+func NewShardFile(ctx context.Context, name string, mem memory.Allocator, e *executor) (*ShardFile, error) {
+	if !e.dataFrameExists(name) {
+		return &ShardFile{dest: name, executor: e, strings: make(map[key][]string)}, nil
+	}
+	// else read in existing
+	table, err := e.getDataTable(ctx, name, mem)
+	if err != nil {
+		return nil, err
+	}
+	return &ShardFile{table: table, schema: table.Schema(), dest: name, executor: e, strings: make(map[key][]string)}, nil
+}
+
+type NameType struct {
+	Name     string
+	DataType arrow.DataType
+}
+type ChangesetRequest struct {
+	ShardIds     []int64 // only shardwidth bits to provide 0 indexing inside shard file
+	Columns      []interface{}
+	SimpleSchema []NameType
+}
+
+// TODO(twg) 2022/09/30 Needs a refactor
+func cast(v interface{}) arrow.DataType {
+	switch v.(type) {
+	case *arrow.Int64Type:
+		return arrow.PrimitiveTypes.Int64
+	case int64:
+		return arrow.PrimitiveTypes.Int64
+	case *arrow.Float64Type:
+		return arrow.PrimitiveTypes.Float64
+	case float64:
+		return arrow.PrimitiveTypes.Float64
+	case *arrow.StringType:
+		return arrow.BinaryTypes.String
+	default:
+		vprint.VV("%T .... %v", v, v)
+	}
+	return arrow.PrimitiveTypes.Int64
+}
+
+func (cr *ChangesetRequest) ArrowSchema() *arrow.Schema {
+	fields := make([]arrow.Field, len(cr.SimpleSchema))
+	for i := range cr.SimpleSchema {
+		fields[i] = arrow.Field{Name: cr.SimpleSchema[i].Name, Type: cast(cr.SimpleSchema[i].DataType)}
+	}
+	return arrow.NewSchema(fields, nil)
+}
+
+type key struct {
+	col   int
+	chunk int
+}
+
+type ShardFile struct {
+	table      arrow.Table
+	schema     *arrow.Schema
+	beforeRows int64
+	added      int64
+	columns    []interface{}
+	dest       string
+	executor   *executor
+	strings    map[key][]string
+}
+
+func compareSchema(s1, s2 *arrow.Schema) bool {
+	if s1 == nil || s2 == nil {
+		return false
+	}
+	if len(s1.Fields()) != len(s2.Fields()) {
+		return false
+	}
+	for i := 0; i < len(s1.Fields()); i++ {
+		f1 := s1.Field(i)
+		f2 := s2.Field(i)
+		if f1.Name != f2.Name {
+			return false
+		}
+		if f1.Type != f2.Type {
+			return false
+		}
+	}
+	return true
+}
+
+func (sf *ShardFile) EnsureSchema(cs *ChangesetRequest) error {
+	schema := cs.ArrowSchema()
+	if sf.schema == nil {
+		sf.schema = schema
+	} else {
+		if !compareSchema(sf.schema, schema) {
+			vprint.VV("incomeing schema", schema)
+			vprint.VV("existing schema", sf.schema)
+			return errors.New("dataframe schema's don't match")
+		}
+	}
+	sf.columns = make([]interface{}, len(sf.schema.Fields()))
+	return nil
+}
+
+func (sf *ShardFile) buildAppenders(maxid int64) {
+	if sf.table != nil {
+		sf.beforeRows = sf.table.NumRows()
+	}
+	if maxid < sf.beforeRows {
+		// no need to add new rows
+		return
+	}
+	newSize := maxid - sf.beforeRows + 1
+	for i := 0; i < len(sf.schema.Fields()); i++ {
+		switch sf.schema.Field(i).Type {
+		case arrow.PrimitiveTypes.Int64:
+			sf.columns[i] = make([]int64, newSize)
+		case arrow.PrimitiveTypes.Float64:
+			sf.columns[i] = make([]float64, newSize)
+		case arrow.BinaryTypes.String:
+			sf.columns[i] = make([]string, newSize)
+		}
+	}
+	sf.added = newSize
+}
+
+// the row offset must be reset to 0 for the slices being appended
+func (sf *ShardFile) SetIntValue(col int, row int64, val int64) {
+	v := sf.columns[col].([]int64)
+	v[row-sf.beforeRows] = val
+}
+
+func (sf *ShardFile) SetFloatValue(col int, row int64, val float64) {
+	v := sf.columns[col].([]float64)
+	v[row-sf.beforeRows] = val
+}
+
+func (sf *ShardFile) SetStringValue(col int, row int64, val string) {
+	v := sf.columns[col].([]string)
+	v[row-sf.beforeRows] = val
+}
+
+func (sf *ShardFile) Process(cs *ChangesetRequest) error {
+	err := sf.process(cs)
+	if err != nil {
+		return err
+	}
+	rtemp := sf.dest + ".temp"
+	err = sf.Save(rtemp)
+	if err != nil {
+		return err
+	}
+	return os.Rename(rtemp+sf.executor.TableExtension(), sf.dest+sf.executor.TableExtension())
+}
+
+func (sf *ShardFile) LoadBlobs() error {
+	for col := 0; col < len(sf.schema.Fields()); col++ {
+		column := sf.table.Column(col)
+		switch column.DataType() {
+		case arrow.BinaryTypes.String:
+			for i, chunk := range column.Data().Chunks() {
+				stringData := chunk.(*array.String)
+				k := key{col: col, chunk: i}
+				for j := 0; j < stringData.Len(); j++ {
+					v := stringData.Value(j)
+					sf.strings[k] = append(sf.strings[k], v)
+
+				}
+			}
+		}
+	}
+	return nil
+}
+
+func (sf *ShardFile) ReplaceString(col, chunk, l int, s string) {
+	sf.strings[key{col: col, chunk: chunk}][l] = s
+}
+
+func (sf *ShardFile) process(cs *ChangesetRequest) error {
+	offset := 0
+	if sf.table != nil {
+		// need to load blobs prior
+		sf.LoadBlobs()
+		column := sf.table.Column(0)
+		resolver := dataframe.NewChunkResolver(column)
+		for i, rowid := range cs.ShardIds {
+			offset = i
+			if rowid >= sf.table.NumRows() {
+				break
+			}
+			chunk, l := resolver.Resolve(int(rowid))
+			for col := 0; col < len(sf.schema.Fields()); col++ {
+				column := sf.table.Column(col)
+				switch column.DataType() {
+				case arrow.PrimitiveTypes.Int64:
+					v := column.Data().Chunk(chunk).(*array.Int64).Int64Values()
+					v[l] = cs.Columns[col].([]int64)[i]
+				case arrow.PrimitiveTypes.Float64:
+					v := column.Data().Chunk(chunk).(*array.Float64).Float64Values()
+					v[l] = cs.Columns[col].([]float64)[i]
+				case arrow.BinaryTypes.String:
+					// TODO(twg) 2023/01/09 How to update existing?
+					new := cs.Columns[col].([]string)[i]
+					sf.ReplaceString(col, chunk, l, new)
+				default:
+					panic(fmt.Sprintf("Unknown Type %v", column.DataType()))
+				}
+			}
+		}
+	}
+	max := cs.ShardIds[len(cs.ShardIds)-1]
+	sf.buildAppenders(max)
+	// need to check if only replace and no apend
+	if sf.added > 0 {
+		for i, rowid := range cs.ShardIds[offset:] {
+			i += offset
+
+			for col := 0; col < len(sf.schema.Fields()); col++ {
+				switch sf.schema.Field(col).Type {
+				case arrow.PrimitiveTypes.Int64:
+					sf.SetIntValue(col, rowid, cs.Columns[col].([]int64)[i])
+				case arrow.PrimitiveTypes.Float64:
+					sf.SetFloatValue(col, rowid, cs.Columns[col].([]float64)[i])
+				case arrow.BinaryTypes.String:
+					sf.SetStringValue(col, rowid, cs.Columns[col].([]string)[i])
+				default:
+					panic(fmt.Sprintf("2 Unknown Type %v", sf.schema.Field(col).Type))
+				}
+			}
+		}
+	}
+
+	return nil
+}
+
+type twoSlices struct {
+	id_slice    []int
+	lists_slice [][]string
+}
+
+type SortByOther twoSlices
+
+func (sbo SortByOther) Len() int {
+	return len(sbo.id_slice)
+}
+
+func (sbo SortByOther) Swap(i, j int) {
+	sbo.id_slice[i], sbo.id_slice[j] = sbo.id_slice[j], sbo.id_slice[i]
+	sbo.lists_slice[i], sbo.lists_slice[j] = sbo.lists_slice[j], sbo.lists_slice[i]
+}
+
+func (sbo SortByOther) Less(i, j int) bool {
+	return sbo.id_slice[i] < sbo.id_slice[j]
+}
+
+func (sf *ShardFile) buildFromStrings(idx int, mem memory.Allocator) []arrow.Array {
+	ids := make([]int, 0)
+	lists := make([][]string, 0)
+	for k, v := range sf.strings {
+		if k.col == idx { // ugh not ordered :(
+			ids = append(ids, k.chunk)
+			lists = append(lists, v)
+		}
+	}
+	// sort ids/lists
+	parts := twoSlices{id_slice: ids, lists_slice: lists}
+	sort.Sort(SortByOther(parts))
+
+	builder := array.NewStringBuilder(mem)
+	chunks := make([]arrow.Array, 0)
+	for _, v := range parts.lists_slice {
+		builder.AppendValues(v, nil)
+		newChunk := builder.NewArray()
+		chunks = append(chunks, newChunk)
+	}
+	return chunks
+}
+
+func (sf *ShardFile) Save(name string) error {
+	parts := make([]arrow.Array, 0)
+	mem := memory.NewGoAllocator()
+	for col := 0; col < len(sf.schema.Fields()); col++ {
+		chunks := make([]arrow.Array, 0)
+		if sf.table != nil {
+			// we append if there was existing file
+			column := sf.table.Column(col)
+			// if primitive type
+			switch column.DataType() {
+			case arrow.BinaryTypes.String:
+				chunks = sf.buildFromStrings(col, mem)
+			default:
+				chunks = append(chunks, column.Data().Chunks()...)
+			}
+			// else binary type
+		}
+		switch sf.schema.Field(col).Type {
+		case arrow.PrimitiveTypes.Int64:
+			// case *arrow.Int64Type:
+			if sf.added > 0 {
+				ibuild := array.NewInt64Builder(mem)
+				ibuild.AppendValues(sf.columns[col].([]int64), nil) // TODO(twg) 2022/09/28 need to handle null
+				newChunk := ibuild.NewArray()
+				chunks = append(chunks, newChunk)
+			}
+			record, err := array.Concatenate(chunks, mem)
+			if err != nil {
+				return err
+			}
+			parts = append(parts, record)
+		case arrow.PrimitiveTypes.Float64:
+			// case *arrow.Float64Type:
+			if sf.added > 0 {
+				fbuild := array.NewFloat64Builder(mem)
+				fbuild.AppendValues(sf.columns[col].([]float64), nil) // TODO(twg) 2022/09/28 need to handle null
+				newChunk := fbuild.NewArray()
+				chunks = append(chunks, newChunk)
+			}
+			record, err := array.Concatenate(chunks, mem)
+			if err != nil {
+				return err
+			}
+			parts = append(parts, record)
+		case arrow.BinaryTypes.String:
+			if sf.added > 0 {
+				fbuild := array.NewStringBuilder(mem)
+				fbuild.AppendValues(sf.columns[col].([]string), nil) // TODO(twg) 2022/09/28 need to handle null
+				newChunk := fbuild.NewArray()
+				chunks = append(chunks, newChunk)
+			}
+			record, err := array.Concatenate(chunks, mem)
+			if err != nil {
+				return err
+			}
+			parts = append(parts, record)
+		default:
+			vprint.VV("UNKNOWN %T", sf.schema.Field(col).Type)
+		}
+	}
+	rec := array.NewRecord(sf.schema, parts, sf.beforeRows+sf.added)
+	table := array.NewTableFromRecords(sf.schema, []arrow.Record{rec})
+
+	return sf.executor.SaveTable(name, table, mem)
+}
+
+// TODO(twg) 2022/10/03 Not a huge fan of the global variable will look at adding to executor structure
+// when dataframe is fully integrated
+var (
+	dataframeShardLocks map[uint64]*sync.Mutex
+	muWriteDataframe    sync.Mutex
+)
+
+func init() {
+	dataframeShardLocks = make(map[uint64]*sync.Mutex)
+}
+
+func getDataframeWritelock(shard uint64) *sync.Mutex {
+	muWriteDataframe.Lock()
+	defer muWriteDataframe.Unlock()
+	lock, ok := dataframeShardLocks[shard]
+	if ok {
+		return lock
+	}
+	newLock := sync.Mutex{}
+	dataframeShardLocks[shard] = &newLock
+	return &newLock
+}
+
+func (api *API) ApplyDataframeChangeset(ctx context.Context, index string, cs *ChangesetRequest, shard uint64) error {
+	// TODO(twg) 2022/09/29 need to validate api call
+	idx := api.Holder().Index(index)
+
+	// check if dataframe exists
+	fname := idx.GetDataFramePath(shard)
+
+	// only 1 shard writer allowed at at time so wait for it to be available
+	mu := getDataframeWritelock(shard)
+	mu.Lock()
+	defer mu.Unlock()
+	mem := memory.NewGoAllocator()
+	shardFile, err := NewShardFile(ctx, fname, mem, api.server.executor)
+	if err != nil {
+		return err
+	}
+
+	err = shardFile.EnsureSchema(cs)
+	if err != nil {
+		return err
+	}
+
+	return shardFile.Process(cs)
+}
+
+type column struct {
+	Name string
+	Type string
+}
+
+func (api *API) GetDataframeSchema(ctx context.Context, indexName string) (interface{}, error) {
+	idx, err := api.Index(ctx, indexName)
+	if err != nil {
+		return nil, err
+	}
+	base := idx.DataframesPath()
+	dir, _ := os.Open(base)
+	files, _ := dir.Readdir(0)
+	parts := make([]column, 0)
+	mem := memory.NewGoAllocator()
+	for i := range files {
+		file := files[i]
+		name := file.Name()
+		if api.server.executor.IsDataframeFile(name) {
+			// strip off the parquet extenison
+			name = strings.TrimSuffix(name, filepath.Ext(name))
+			// read the parquet file and extract the schema
+			fname := filepath.Join(base, name)
+			table, err := api.server.executor.getDataTable(ctx, fname, mem)
+			if err != nil {
+				return nil, err
+			}
+			for i := 0; i < int(table.NumCols()); i++ {
+				col := table.Column(i)
+				part := column{Name: col.Name(), Type: col.DataType().String()}
+				parts = append(parts, part)
+			}
+			break // only go on first file
+		}
+	}
+	return parts, nil
+}

arrow.go

@@ -0,0 +1,562 @@
+// Copyright 2021 Molecula Corp. All rights reserved.
+package pilosa
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"os"
+	"strings"
+	"sync"
+
+	"github.com/apache/arrow/go/v10/arrow"
+	"github.com/apache/arrow/go/v10/arrow/array"
+	"github.com/apache/arrow/go/v10/arrow/ipc"
+	"github.com/apache/arrow/go/v10/arrow/memory"
+	"github.com/apache/arrow/go/v10/parquet"
+	"github.com/apache/arrow/go/v10/parquet/file"
+	"github.com/apache/arrow/go/v10/parquet/pqarrow"
+	"github.com/featurebasedb/featurebase/v3/pql"
+	"github.com/featurebasedb/featurebase/v3/tracing"
+	"github.com/gomem/gomem/pkg/dataframe"
+	"github.com/pkg/errors"
+)
+
+/*
+The function Arrow provides filtered access to the raw values stored in the dataframe.
+If Arrow is just provided a bitmap filter, such as ConstRow or any Bitmap Operation,
+all the values associated with each column are returned.  This set can be limited with
+the addition of the header parameter
+Example:
+Arrow(ConstRow(columns=[2,4,6]),header=["fval"])
+*/
+
+// executeApply executes a Arrow() call.
+func (e *executor) executeArrow(ctx context.Context, qcx *Qcx, index string, c *pql.Call, shards []uint64, opt *ExecOptions) (arrow.Table, error) {
+	if !e.dataframeEnabled {
+		return nil, errors.New("Dataframe support not enabled")
+	}
+	span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeArrow")
+	defer span.Finish()
+	if len(c.Children) > 1 {
+		return nil, errors.New("Apply() only accepts a single bitmap input filter")
+	}
+	var columnFilter []string
+	if cols, ok := c.Args["header"].([]interface{}); ok {
+		columnFilter = make([]string, 0, len(cols))
+		for _, v := range cols {
+			columnFilter = append(columnFilter, v.(string))
+		}
+	}
+	mapcounter := 0
+	reducecounter := 0
+	pool := memory.NewGoAllocator() // TODO(twg) 2022/09/01 singledton?
+	// Execute calls in bulk on each remote node and merge.
+	mu := &sync.Mutex{}
+	mapFn := func(ctx context.Context, shard uint64, mopt *mapOptions) (_ interface{}, err error) {
+		mu.Lock()
+		mapcounter++
+		mu.Unlock()
+		return e.executeArrowShard(ctx, qcx, index, c, shard, pool, columnFilter)
+	}
+	tables := make([]*BasicTable, 0)
+
+	reduceFn := func(ctx context.Context, prev, v interface{}) interface{} {
+		mu.Lock()
+		reducecounter++
+		mu.Unlock()
+		if v == nil {
+			return prev
+		}
+		switch t := v.(type) {
+		case *BasicTable:
+
+			if t.resolver != nil {
+				mu.Lock()
+				tables = append(tables, t)
+				mu.Unlock()
+			}
+		case arrow.Table:
+			if t.NumRows() > 0 {
+				bt := BasicTableFromArrow(t, pool)
+				mu.Lock()
+				tables = append(tables, bt)
+				mu.Unlock()
+			}
+		}
+		return nil
+	}
+
+	_, err := e.mapReduce(ctx, index, shards, c, opt, mapFn, reduceFn)
+	if err != nil {
+		return nil, err
+	}
+	if len(tables) == 0 {
+		return &BasicTable{name: "empty"}, nil
+	}
+	tbl := Concat(tables[0].Schema(), tables, pool)
+	r := dataframe.NewChunkResolver(tbl.Column(0))
+	return &BasicTable{resolver: &r, table: tbl}, nil
+}
+
+type BasicTable struct {
+	resolver dataframe.Resolver
+	table    arrow.Table
+	filtered bool
+	name     string
+}
+
+func (st *BasicTable) Name() string {
+	return st.name
+}
+
+func (st *BasicTable) Schema() *arrow.Schema {
+	if st.table != nil {
+		return st.table.Schema()
+	}
+	return &arrow.Schema{}
+}
+
+func (st *BasicTable) IsFiltered() bool {
+	return st.filtered
+}
+
+func (st *BasicTable) NumRows() int64 {
+	if st.resolver == nil {
+		return 0
+	}
+	return int64(st.resolver.NumRows())
+}
+
+func (st *BasicTable) NumCols() int64 {
+	if st.table != nil {
+		return st.table.NumCols()
+	}
+	return 0
+}
+
+func (st *BasicTable) Column(i int) *arrow.Column {
+	if st.table != nil {
+		return st.table.Column(i)
+	}
+	return nil
+}
+
+func (st *BasicTable) Retain() {
+	if st.table != nil {
+		st.table.Retain()
+	}
+}
+
+func (st *BasicTable) Release() {
+	if st.table != nil {
+		st.table.Retain()
+	}
+}
+
+func (st *BasicTable) Get(column, row int) interface{} {
+	field := st.Schema().Field(column)
+	c, i := st.resolver.Resolve(row)
+	nullable := field.Nullable
+
+	chunk := st.Column(column).Data().Chunk(c)
+	// TODO(twg) 2023/01/26 potential NULL support?
+	if nullable && chunk.IsNull(i) {
+		return nil
+	}
+	switch field.Type.(type) {
+	case *arrow.BooleanType:
+		return chunk.(*array.Boolean).Value(i)
+	case *arrow.Int8Type:
+		v := chunk.(*array.Int8).Int8Values()
+		return int64(v[i])
+	case *arrow.Int16Type:
+		v := chunk.(*array.Int16).Int16Values()
+		return int64(v[i])
+	case *arrow.Int32Type:
+		v := chunk.(*array.Int32).Int32Values()
+		return int64(v[i])
+	case *arrow.Int64Type:
+		v := chunk.(*array.Int64).Int64Values()
+		return int64(v[i])
+	case *arrow.Uint8Type:
+		v := chunk.(*array.Uint8).Uint8Values()
+		return uint64(v[i])
+	case *arrow.Uint16Type:
+		v := chunk.(*array.Uint16).Uint16Values()
+		return uint64(v[i])
+	case *arrow.Uint32Type:
+		v := chunk.(*array.Uint32).Uint32Values()
+		return uint64(v[i])
+	case *arrow.Uint64Type:
+		v := chunk.(*array.Uint64).Uint64Values()
+		return v[i]
+	case *arrow.Float32Type:
+		v := chunk.(*array.Float32).Float32Values()
+		return float64(v[i])
+	case *arrow.Float64Type:
+		v := chunk.(*array.Float64).Float64Values()
+		return v[i]
+	case *arrow.StringType:
+		return chunk.(*array.String).Value(i)
+	}
+	return 0
+}
+
+func builderFrom(mem memory.Allocator, dt arrow.DataType, size int64) array.Builder {
+	var bldr array.Builder
+	switch dt := dt.(type) {
+	case *arrow.BooleanType:
+		bldr = array.NewBooleanBuilder(mem)
+	case *arrow.Int8Type:
+		bldr = array.NewInt8Builder(mem)
+	case *arrow.Int16Type:
+		bldr = array.NewInt16Builder(mem)
+	case *arrow.Int32Type:
+		bldr = array.NewInt32Builder(mem)
+	case *arrow.Int64Type:
+		bldr = array.NewInt64Builder(mem)
+	case *arrow.Uint8Type:
+		bldr = array.NewUint8Builder(mem)
+	case *arrow.Uint16Type:
+		bldr = array.NewUint16Builder(mem)
+	case *arrow.Uint32Type:
+		bldr = array.NewUint32Builder(mem)
+	case *arrow.Uint64Type:
+		bldr = array.NewUint64Builder(mem)
+	case *arrow.Float32Type:
+		bldr = array.NewFloat32Builder(mem)
+	case *arrow.Float64Type:
+		bldr = array.NewFloat64Builder(mem)
+	case *arrow.StringType:
+		bldr = array.NewStringBuilder(mem)
+	default:
+		panic(fmt.Errorf("builderFrom: invalid Arrow type %v", dt))
+	}
+	bldr.Reserve(int(size))
+	return bldr
+}
+
+func appendData(bldr array.Builder, v interface{}) {
+	switch bldr := bldr.(type) {
+	case *array.BooleanBuilder:
+		bldr.Append(v.(bool))
+	case *array.Int8Builder:
+		bldr.Append(v.(int8))
+	case *array.Int16Builder:
+		bldr.Append(v.(int16))
+	case *array.Int32Builder:
+		bldr.Append(v.(int32))
+	case *array.Int64Builder:
+		bldr.Append(v.(int64))
+	case *array.Uint8Builder:
+		bldr.Append(v.(uint8))
+	case *array.Uint16Builder:
+		bldr.Append(v.(uint16))
+	case *array.Uint32Builder:
+		bldr.Append(v.(uint32))
+	case *array.Uint64Builder:
+		bldr.Append(v.(uint64))
+	case *array.Float32Builder:
+		bldr.Append(v.(float32))
+	case *array.Float64Builder:
+		bldr.Append(v.(float64))
+	case *array.StringBuilder:
+		bldr.Append(v.(string))
+	default:
+		panic(fmt.Errorf("appendData: invalid Arrow builder type %T", bldr))
+	}
+}
+
+func Concat(schema *arrow.Schema, tables []*BasicTable, mem memory.Allocator) arrow.Table {
+	if len(tables) == 1 {
+		if !tables[0].IsFiltered() {
+			return tables[0]
+		}
+	}
+	cols := make([]arrow.Column, len(schema.Fields()))
+
+	defer func(cols []arrow.Column) {
+		for i := range cols {
+			cols[i].Release()
+		}
+	}(cols)
+	sz := 0
+	for i := range tables {
+		sz += int(tables[i].NumRows())
+	}
+	for i := range cols {
+		field := schema.Field(i)
+		arrs := make([]arrow.Array, 0)
+		builder := builderFrom(mem, field.Type, int64(sz))
+		for t := range tables {
+			table := tables[t]
+			if table.IsFiltered() {
+				for row := 0; row < int(table.NumRows()); row++ {
+					v := table.Get(i, row)
+					appendData(builder, v)
+				}
+				arrs = append(arrs, builder.NewArray())
+
+			} else {
+				parts := table.Column(i).Data()
+				arrs = append(arrs, parts.Chunks()...)
+			}
+		}
+		chunk := arrow.NewChunked(field.Type, arrs)
+		cols[i] = *arrow.NewColumn(field, chunk)
+		chunk.Release()
+	}
+	return array.NewTable(schema, cols, -1)
+}
+
+func (st *BasicTable) MarshalJSON() ([]byte, error) {
+	results := make(map[string]interface{})
+	n := 0
+	if st.table != nil {
+		n = int(st.table.NumCols())
+	}
+	for b := 0; b < n; b++ {
+		col := st.table.Column(b)
+		result := make([]interface{}, st.resolver.NumRows())
+		for n := st.resolver.NumRows() - 1; n >= 0; n-- {
+			v := st.Get(b, n)
+			result[n] = v
+
+		}
+		results[col.Name()] = result
+	}
+	return json.Marshal(results)
+}
+
+func BasicTableFromArrow(table arrow.Table, mem memory.Allocator) *BasicTable {
+	col := table.Column(0)
+	r := dataframe.NewChunkResolver(col)
+	return &BasicTable{resolver: &r, table: table}
+}
+
+func filterColumns(filters []string, table arrow.Table) arrow.Table {
+	filters = append(filters, "_ID")
+	schema := table.Schema()
+	// TODO(twg) 2022/11/09 add glob support
+	allFields := schema.Fields()
+	in := func(key string) bool {
+		for _, v := range filters {
+			if v == key {
+				return true
+			}
+		}
+		return false
+	}
+	cols := make([]arrow.Column, 0)
+	fields := make([]arrow.Field, 0)
+	for i := range allFields {
+		field := allFields[i]
+		if in(field.Name) {
+			cols = append(cols, *table.Column(i))
+			fields = append(fields, field)
+		}
+	}
+
+	filterdSchema := arrow.NewSchema(fields, nil) // TODO(twg) 2022/11/09 handle meta:w
+	return array.NewTable(filterdSchema, cols, table.NumRows())
+}
+
+func (e *executor) executeArrowShard(ctx context.Context, qcx *Qcx, index string, c *pql.Call, shard uint64, pool memory.Allocator, columnFilter []string) (*BasicTable, error) {
+	name := fmt.Sprintf("a. %v", shard)
+	span, _ := tracing.StartSpanFromContext(ctx, "Executor.executeArrowShard")
+	defer span.Finish()
+
+	var filter *Row
+	if len(c.Children) == 1 {
+		row, err := e.executeBitmapCallShard(ctx, qcx, index, c.Children[0], shard)
+		if err != nil {
+			return nil, err
+		}
+		filter = row
+		if !filter.Any() {
+			// no need to actuall run the query for its not operating against any values
+			return &BasicTable{name: name}, nil
+		}
+	}
+	//
+	ids := filter.ShardColumns() // needs to be shard columns
+	// Fetch index.
+	idx := e.Holder.Index(index)
+	if idx == nil {
+		return nil, newNotFoundError(ErrIndexNotFound, index)
+	}
+
+	fname := idx.GetDataFramePath(shard)
+
+	if !e.dataFrameExists(fname) {
+		return &BasicTable{name: name}, nil
+	}
+
+	table, err := e.getDataTable(ctx, fname, pool)
+	if err != nil {
+		return nil, errors.Wrap(err, "arrow readTableParquet")
+	}
+	defer table.Release()
+	if len(columnFilter) > 0 {
+		table = filterColumns(columnFilter, table)
+	}
+	df, err := dataframe.NewDataFrameFromTable(pool, table)
+	if err != nil {
+		return nil, errors.Wrap(err, "arrow NewDataFromTable")
+	}
+	p := dataframe.NewChunkResolver(df.ColumnAt(0))
+	var resolver dataframe.Resolver
+	resolver = &p
+	if filter != nil {
+		if len(ids) == 0 {
+			return &BasicTable{name: name}, nil
+		}
+		resolver, err = filterDataframe(resolver, pool, ids)
+		if err != nil {
+			return nil, errors.Wrap(err, "filtering dataframe")
+		}
+	}
+	table.Retain()
+	return &BasicTable{resolver: resolver, table: table, filtered: filter != nil, name: name}, nil
+}
+
+func (e *executor) dataFrameExists(fname string) bool {
+	if e.typeIsParquet() {
+		if _, err := os.Stat(fname + ".parquet"); os.IsNotExist(err) {
+			return false
+		}
+		return true
+	}
+	if _, err := os.Stat(fname + ".arrow"); os.IsNotExist(err) {
+		return false
+	}
+	return true
+}
+
+func (e *executor) getDataTable(ctx context.Context, fname string, mem memory.Allocator) (arrow.Table, error) {
+	if e.typeIsParquet() {
+		table, err := readTableParquetCtx(ctx, fname, mem)
+		return table, err
+	}
+	return readTableArrow(fname, mem)
+}
+
+func (e *executor) typeIsParquet() bool {
+	return e.datafameUseParquet
+}
+
+func (e *executor) IsDataframeFile(name string) bool {
+	if e.typeIsParquet() {
+		return strings.HasSuffix(name, ".parquet")
+	}
+	return strings.HasSuffix(name, ".arrow")
+}
+
+func (e *executor) SaveTable(name string, table arrow.Table, mem memory.Allocator) error {
+	if e.typeIsParquet() {
+		return writeTableParquet(table, name)
+	}
+	return writeTableArrow(table, name, mem)
+}
+
+func (e *executor) TableExtension() string {
+	if e.typeIsParquet() {
+		return ".parquet"
+	}
+	return ".arrow"
+}
+
+func readTableArrow(filename string, mem memory.Allocator) (arrow.Table, error) {
+	r, err := os.Open(filename + ".arrow")
+	if err != nil {
+		return nil, err
+	}
+	rr, err := ipc.NewFileReader(r, ipc.WithAllocator(mem))
+	if err != nil {
+		return nil, err
+	}
+	defer rr.Close()
+	records := make([]arrow.Record, rr.NumRecords())
+	i := 0
+	for {
+		rec, err := rr.Read()
+		if err == io.EOF {
+			break
+		} else if err != nil {
+			return nil, err
+		}
+		records[i] = rec
+		i++
+	}
+	records = records[:i]
+	table := array.NewTableFromRecords(rr.Schema(), records)
+	return table, nil
+}
+
+func readTableParquetCtx(ctx context.Context, filename string, mem memory.Allocator) (arrow.Table, error) {
+	r, err := os.Open(filename + ".parquet")
+	if err != nil {
+		return nil, err
+	}
+	defer r.Close()
+
+	pf, err := file.NewParquetReader(r)
+	if err != nil {
+		return nil, err
+	}
+
+	reader, err := pqarrow.NewFileReader(pf, pqarrow.ArrowReadProperties{}, mem)
+	if err != nil {
+		return nil, err
+	}
+	return reader.ReadTable(ctx)
+}
+
+func writeTableParquet(table arrow.Table, filename string) error {
+	f, err := os.Create(filename + ".parquet")
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+	props := parquet.NewWriterProperties(parquet.WithDictionaryDefault(false))
+	arrProps := pqarrow.DefaultWriterProps()
+	chunkSize := 10 * 1024 * 1024
+	err = pqarrow.WriteTable(table, f, int64(chunkSize), props, arrProps)
+	if err != nil {
+		return err
+	}
+	f.Sync()
+	return nil
+}
+
+func writeTableArrow(table arrow.Table, filename string, mem memory.Allocator) error {
+	f, err := os.Create(filename + ".arrow")
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+	writer, err := ipc.NewFileWriter(f, ipc.WithAllocator(mem), ipc.WithSchema(table.Schema()))
+	if err != nil {
+		panic(err)
+	}
+	chunkSize := int64(0)
+	tr := array.NewTableReader(table, chunkSize)
+	defer tr.Release()
+	n := 0
+	for tr.Next() {
+		arec := tr.Record()
+		err = writer.Write(arec)
+		if err != nil {
+			panic(err)
+		}
+		n++
+	}
+	err = writer.Close()
+	if err != nil {
+		panic(err)
+	}
+	f.Sync()
+	return nil
+}

arrow_test.go

@@ -0,0 +1,53 @@
+// Copyright 2021 Molecula Corp. All rights reserved.
+package pilosa
+
+import (
+	"context"
+	"encoding/hex"
+	"math/rand"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/apache/arrow/go/v10/arrow"
+	"github.com/apache/arrow/go/v10/arrow/array"
+	"github.com/apache/arrow/go/v10/arrow/memory"
+)
+
+func TempFileName(prefix string) string {
+	randBytes := make([]byte, 16)
+	rand.Read(randBytes)
+	return filepath.Join(os.TempDir(), prefix+hex.EncodeToString(randBytes))
+}
+
+func Test_TableParquet(t *testing.T) {
+	// create a arrow table
+	schema := arrow.NewSchema(
+		[]arrow.Field{
+			{Name: "num", Type: arrow.PrimitiveTypes.Float64},
+		},
+		nil, // no metadata
+	)
+	mem := memory.NewGoAllocator()
+	b := array.NewRecordBuilder(mem, schema)
+	defer b.Release()
+	b.Field(0).(*array.Float64Builder).AppendValues([]float64{1.0, 1.5, 2.0}, nil)
+	table := array.NewTableFromRecords(schema, []arrow.Record{b.NewRecord()})
+	defer table.Release()
+	fileName := TempFileName("pq-")
+	// save it as  a parquet file
+	err := writeTableParquet(table, fileName)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.Remove(fileName)
+
+	// read it back in and compare the result
+	got, err := readTableParquetCtx(context.Background(), fileName, mem)
+	if err != nil {
+		t.Fatalf("readTableParquetCtx() error = %v", err)
+	}
+	if got.NumCols() != table.NumCols() {
+		t.Errorf("got:%v expected:%v", got.NumCols(), table.NumCols())
+	}
+}

audit.go

@@ -1,8 +1,9 @@
-// Copyright 2021 Molecula Corp. All rights reserved.
+// Copyright 2022 Molecula Corp. (DBA FeatureBase).
+// SPDX-License-Identifier: Apache-2.0
 package pilosa
 
 import (
-	"github.com/molecula/featurebase/v3/testhook"
+	"github.com/featurebasedb/featurebase/v3/testhook"
 )
 
 var NewAuditor func() testhook.Auditor = NewNopAuditor

audit_internal_test.go

@@ -1,11 +1,12 @@
-// Copyright 2021 Molecula Corp. All rights reserved.
+// Copyright 2022 Molecula Corp. (DBA FeatureBase).
+// SPDX-License-Identifier: Apache-2.0
 package pilosa
 
 import (
 	"fmt"
 	"reflect"
 
-	"github.com/molecula/featurebase/v3/testhook"
+	"github.com/featurebasedb/featurebase/v3/testhook"
 )
 
 // These audit hooks are desireable during testing, but not in

audit_test.go

@@ -1,4 +1,5 @@
-// Copyright 2021 Molecula Corp. All rights reserved.
+// Copyright 2022 Molecula Corp. (DBA FeatureBase).
+// SPDX-License-Identifier: Apache-2.0
 package pilosa_test
 
 import (
@@ -6,8 +7,8 @@ import (
 	"os"
 	"reflect"
 
-	"github.com/molecula/featurebase/v3"
-	"github.com/molecula/featurebase/v3/testhook"
+	"github.com/featurebasedb/featurebase/v3"
+	"github.com/featurebasedb/featurebase/v3/testhook"
 )
 
 // AuditLeaksOn is a global switch to turn on resource

authn/authenticate.go

@@ -1,4 +1,5 @@
-// Copyright 2021 Molecula Corp. All rights reserved.
+// Copyright 2022 Molecula Corp. (DBA FeatureBase).
+// SPDX-License-Identifier: Apache-2.0
 
 // Package authn handles authentication
 package authn
@@ -8,6 +9,7 @@ import (
 	"encoding/hex"
 	"encoding/json"
 	"fmt"
+	"net"
 	"net/http"
 	"net/url"
 	"strconv"
@@ -18,30 +20,45 @@ import (
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/metadata"
 
-	"github.com/molecula/featurebase/v3/logger"
+	"github.com/featurebasedb/featurebase/v3/logger"
 	"github.com/pkg/errors"
 	"golang.org/x/oauth2"
 )
 
+// AuthContextKey is a unique type to prevent collisions when using context.WithValue()
+type AuthContextKey string
+
+const (
+	// AccessCookieName is the name of the cookie that holds the access token.
+	AccessCookieName = "molecula-chip"
+
+	// RefreshCookieName is the name of the cookie that holds the refresh token.
+	RefreshCookieName = "refresh-molecula-chip"
+
+	// RefreshHeaderName is the name of the header that holds the refresh token.
+	RefreshHeaderName = "X-Molecula-Refresh-Token"
+
+	// ContextValueAccessToken is the key used to set AccessTokens in a ctx.
+	ContextValueAccessToken = AuthContextKey("Access")
+
+	// ContextValueRefreshToken is the key used to set RefreshTokens in a ctx.
+	ContextValueRefreshToken = AuthContextKey("Refresh")
+)
+
 // cachedGroups is used to hold groups and when they were last cached
 type cachedGroups struct {
 	cacheTime time.Time
 	groups    []Group
 }
 
-// cacheToken is used to hold tokens and when they were added to the cache
-type cachedToken struct {
-	cacheTime time.Time
-	token     *oauth2.Token
-}
-
 // UserInfo holds the information about the user from the token
 type UserInfo struct {
-	UserID   string    `json:"userid"`
-	UserName string    `json:"username"`
-	Groups   []Group   `json:"groups"`
-	Expiry   time.Time `json:"expiry"`
-	Token    string    `json:"token"`
+	UserID       string    `json:"userid"`
+	UserName     string    `json:"username"`
+	Groups       []Group   `json:"groups"`
+	Expiry       time.Time `json:"expiry"`
+	Token        string    `json:"token"`
+	RefreshToken string    `json:"refreshtoken"`
 }
 
 // Group holds group information for an authenticated user
@@ -52,33 +69,35 @@ type Group struct {
 
 // Groups holds a slice of Group for marshalling from JSON
 type Groups struct {
-	Groups []Group `json:"value"`
+	NextLink string  `json:"@odata.nextLink"`
+	Groups   []Group `json:"value"`
 }
 
 // Auth holds state, configuration, and utilities needed for authentication.
 type Auth struct {
-	logger         logger.Logger
-	cookieName     string
-	secretKey      []byte
-	groupEndpoint  string
-	logoutEndpoint string
-	fbURL          string // fbURL is the domain featurebase is hosted on, used for post logout redirection
-	oAuthConfig    *oauth2.Config
-	cacheTTL       time.Duration           // cacheTTL is used to determine if a cached item should be refreshed or not
-	tokenTTR       time.Duration           // tokenTTR (time to refresh) is used to determine if a token should be refreshed or not
-	tokenCache     map[string]cachedToken  // tokenCache is a map of accessToken -> *oauth2.Token which we can use to refresh the tokens
-	groupsCache    map[string]cachedGroups // groupsCache is a map of accessToken -> group memberships
-	lastCacheClean time.Time               // last cache clean is the time that the cache was last cleaned
+	logger            logger.Logger
+	accessCookieName  string
+	refreshCookieName string
+	secretKey         []byte
+	groupEndpoint     string
+	logoutEndpoint    string
+	fbURL             string // fbURL is the domain featurebase is hosted on, used for post logout redirection
+	oAuthConfig       *oauth2.Config
+	cacheTTL          time.Duration           // cacheTTL is used to determine if a cached item should be refreshed or not
+	groupsCache       map[string]cachedGroups // groupsCache is a map of accessToken -> group memberships
+	lastCacheClean    time.Time               // last cache clean is the time that the cache was last cleaned
+	allowedNetworks   []net.IPNet             // list of allowed networks for ingest
 }
 
 // NewAuth instantiates and returns a new Auth struct
-func NewAuth(logger logger.Logger, url string, scopes []string, authURL, tokenURL, groupEndpoint, logout, clientID, clientSecret, secretKey string) (auth *Auth, err error) {
+func NewAuth(logger logger.Logger, url string, scopes []string, authURL, tokenURL, groupEndpoint, logout, clientID, clientSecret, secretKey string, configuredIPs []string) (auth *Auth, err error) {
 	auth = &Auth{
-		logger:         logger,
-		cookieName:     "molecula-chip",
-		groupEndpoint:  groupEndpoint,
-		logoutEndpoint: logout,
-		fbURL:          url,
+		logger:            logger,
+		accessCookieName:  AccessCookieName,
+		refreshCookieName: RefreshCookieName,
+		groupEndpoint:     groupEndpoint,
+		logoutEndpoint:    logout,
+		fbURL:             url,
 		oAuthConfig: &oauth2.Config{
 			RedirectURL:  fmt.Sprintf("%s/redirect", url),
 			ClientID:     clientID,
@@ -89,84 +108,127 @@ func NewAuth(logger logger.Logger, url string, scopes []string, authURL, tokenUR
 				TokenURL: tokenURL,
 			},
 		},
-		tokenCache:     map[string]cachedToken{},
 		groupsCache:    map[string]cachedGroups{},
 		cacheTTL:       10 * time.Minute,
-		tokenTTR:       7 * time.Minute,
 		lastCacheClean: time.Now(),
 	}
 
 	if auth.secretKey, err = decodeHex(secretKey); err != nil {
 		return nil, errors.Wrap(err, "decoding secret key")
 	}
+
+	// convert IPs and add them to allowed networks
+	err = auth.convertIP(configuredIPs)
+	if err != nil {
+		return nil, err
+	}
+
 	return auth, nil
 }
 
+// CleanOAuthConfig returns a's oauthConfig without the client secret
+func (a Auth) CleanOAuthConfig() oauth2.Config {
+	b := *a.oAuthConfig
+	b.ClientSecret = ""
+	return b
+}
+
 // SecretKey is a convenient function to get the SecretKey from an Auth struct
 func (a Auth) SecretKey() []byte {
 	return a.secretKey
 }
 
-// Authenticate takes in a bearer token `bearer` and returns UserInfo from that token
+// refreshToken refreshes a given access/refresh token pair
+func (a *Auth) refreshToken(access, refresh string) (string, string, error) {
+	resp, err := http.PostForm(a.oAuthConfig.Endpoint.TokenURL,
+		url.Values{
+			"grant_type":    {"refresh_token"},
+			"refresh_token": {refresh},
+			"client_id":     {a.oAuthConfig.ClientID},
+			"client_secret": {a.oAuthConfig.ClientSecret},
+		},
+	)
+	if err != nil {
+		return "", "", errors.Wrap(err, "refreshing token")
+	}
+
+	if resp.StatusCode != http.StatusOK {
+		return "", "", fmt.Errorf("refreshing token: %s", resp.Status)
+	}
+
+	defer resp.Body.Close()
+
+	var t oauth2.Token
+	if err := json.NewDecoder(resp.Body).Decode(&t); err != nil {
+		return "", "", errors.Wrap(err, "decoding refreshed token")
+	}
+
+	// remove the old groups from the groups cache
+	delete(a.groupsCache, access)
+
+	return t.AccessToken, t.RefreshToken, nil
+}
+
+// Authenticate takes in a auth token `access` and returns UserInfo from that token
 // it is caller's responsibility to inform the user that the access token has been refreshed
-func (a *Auth) Authenticate(ctx context.Context, bearer string) (*UserInfo, error) {
+func (a *Auth) Authenticate(access, refresh string) (*UserInfo, error) {
 	// clean up the cache every 30 minutes or so
-	if time.Now().Sub(a.lastCacheClean) >= 30*time.Minute {
+	if time.Since(a.lastCacheClean) >= 30*time.Minute {
 		a.cleanCache()
 	}
 
-	if tkn, ok := a.tokenCache[bearer]; ok && (tkn.token.Expiry.Sub(time.Now()) <= a.tokenTTR || !tkn.token.Valid()) {
-		// refresh the token
-		resp, err := http.PostForm(a.oAuthConfig.Endpoint.TokenURL,
-			url.Values{
-				"grant_type":    {"refresh_token"},
-				"refresh_token": {tkn.token.RefreshToken},
-				"client_id":     {a.oAuthConfig.ClientID},
-				"client_secret": {a.oAuthConfig.ClientSecret},
-			},
-		)
-		if err != nil {
-			return nil, errors.Wrap(err, "refreshing token")
-		}
-		defer resp.Body.Close()
-		var t oauth2.Token
-		if err := json.NewDecoder(resp.Body).Decode(&t); err != nil {
-			return nil, errors.Wrap(err, "decoding refreshed token")
-		}
-
-		// update the cache
-		delete(a.tokenCache, bearer)
-		delete(a.groupsCache, bearer)
-		bearer = t.AccessToken
-		a.tokenCache[bearer] = cachedToken{time.Now(), &t}
+	if len(access) == 0 {
+		return nil, fmt.Errorf("auth token is empty")
 	}
 
 	// NOTE: we are using ParseUnverified here because the IDP validates the
 	// token's signature when we get the user's groups, we just need to make
 	// sure it's not expired and is well-formed
-	token, _, err := new(jwt.Parser).ParseUnverified(bearer, &jwt.MapClaims{})
+	token, _, err := new(jwt.Parser).ParseUnverified(access, &jwt.MapClaims{})
 	// well-formed-ness check
 	if token == nil || token.Claims == nil || err != nil {
-		return nil, fmt.Errorf("parsing bearer token: %v", err)
+		return nil, fmt.Errorf("parsing auth token: %v", err)
 	}
 
 	claims := *token.Claims.(*jwt.MapClaims)
 
 	// expiry check
-	if exp, ok := claims["exp"].(string); ok {
-		if expiry, err := strconv.ParseInt(exp, 10, 64); err != nil || expiry < time.Now().UTC().Unix() {
-			return nil, fmt.Errorf("token is expired")
+	if exp, ok := claims["exp"]; ok {
+		var expiry int64
+		switch v := exp.(type) {
+		case string:
+			expiry, err = strconv.ParseInt(v, 10, 64)
+			if err != nil {
+				return nil, fmt.Errorf("parsing exp string: %v", err)
+			}
+		case float64:
+			expiry = int64(v)
+		case int64:
+			expiry = v
+		}
+
+		if expiry < time.Now().UTC().Unix() {
+			access, refresh, err = a.refreshToken(access, refresh)
+			if err != nil {
+				return nil, fmt.Errorf("token is expired: %w", err)
+			}
 		}
 	}
 
 	userInfo := UserInfo{
-		UserID:   claims["oid"].(string),
-		UserName: claims["name"].(string),
-		Token:    bearer,
-		Groups:   []Group{},
+		Token:        access,
+		RefreshToken: refresh,
+		Groups:       []Group{},
 	}
 
-	if userInfo.Groups, err = a.getGroups(bearer); err != nil {
+	if uid, ok := claims["oid"].(string); ok {
+		userInfo.UserID = uid
+	}
+	if name, ok := claims["name"].(string); ok {
+		userInfo.UserName = name
+	}
+
+	if userInfo.Groups, err = a.getGroups(access); err != nil {
 		return nil, errors.Wrap(err, "getting groups")
 	}
 
@@ -175,18 +237,11 @@ func (a *Auth) Authenticate(ctx context.Context, bearer string) (*UserInfo, erro
 
 // cleanCache removes old items from our cache
 func (a *Auth) cleanCache() {
-	for bearer, tkn := range a.tokenCache {
-		// if it's been more than 24 hours since the token was cached
-		if time.Now().Sub(tkn.cacheTime) >= 24*time.Hour {
-			// remove it from our cache
-			delete(a.tokenCache, bearer)
-		}
-	}
-	for bearer, tkn := range a.groupsCache {
+	for access, tkn := range a.groupsCache {
 		// if it's been more than 24 hours since the groups were cached
-		if time.Now().Sub(tkn.cacheTime) >= 24*time.Hour {
+		if time.Since(tkn.cacheTime) >= 24*time.Hour {
 			// remove it from our cache
-			delete(a.groupsCache, bearer)
+			delete(a.groupsCache, access)
 		}
 	}
 	a.lastCacheClean = time.Now()
@@ -201,14 +256,22 @@ func (a *Auth) Login(w http.ResponseWriter, r *http.Request) {
 // Logout clears out the user's cookie, removes the token from our cache, and
 // redirects user to IdP's logout endpoint
 func (a *Auth) Logout(w http.ResponseWriter, r *http.Request) {
-	// remove the bearer token from a.tokenCache and a.groupsCache
-	if bearer, err := r.Cookie(a.cookieName); err == nil {
-		delete(a.tokenCache, bearer.Value)
-		delete(a.groupsCache, bearer.Value)
+	// remove the access token from a.groupsCache
+	if access, err := r.Cookie(a.accessCookieName); err == nil {
+		delete(a.groupsCache, access.Value)
 	}
 	// clear cookie
 	http.SetCookie(w, &http.Cookie{
-		Name:     a.cookieName,
+		Name:     a.accessCookieName,
+		Value:    "",
+		Path:     "/",
+		Secure:   true,
+		HttpOnly: true,
+		SameSite: http.SameSiteStrictMode,
+		Expires:  time.Unix(0, 0),
+	})
+	http.SetCookie(w, &http.Cookie{
+		Name:     a.refreshCookieName,
 		Value:    "",
 		Path:     "/",
 		Secure:   true,
@@ -230,9 +293,7 @@ func (a *Auth) Redirect(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	a.tokenCache[token.AccessToken] = cachedToken{time.Now(), token}
-
-	a.SetCookie(w, token.AccessToken, token.Expiry)
+	a.SetCookie(w, token.AccessToken, token.RefreshToken, token.Expiry)
 	http.Redirect(w, r, "/", http.StatusTemporaryRedirect)
 }
 
@@ -240,25 +301,39 @@ func (a *Auth) Redirect(w http.ResponseWriter, r *http.Request) {
 func (a *Auth) getGroups(token string) ([]Group, error) {
 	var groups Groups
 
-	g, ok := a.groupsCache[token]
-	if ok && (time.Now().Sub(g.cacheTime) < a.cacheTTL) {
-		return g.groups, nil
+	gc, ok := a.groupsCache[token]
+	if ok && (time.Since(gc.cacheTime) < a.cacheTTL) && len(gc.groups) > 0 {
+		return gc.groups, nil
 	}
 
-	req, err := http.NewRequest("GET", a.groupEndpoint, nil)
-	if err != nil {
-		return groups.Groups, errors.Wrap(err, "creating new request to group endpoint")
+	nextLink := a.groupEndpoint
+	for nextLink != "" {
+		req, err := http.NewRequest("GET", nextLink, nil)
+		if err != nil {
+			return nil, errors.Wrap(err, "creating new request to group endpoint")
+		}
+
+		req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", token))
+		response, err := http.DefaultClient.Do(req)
+		if err != nil {
+			return nil, errors.Wrap(err, "getting group membership info")
+		}
+		if response.StatusCode != http.StatusOK {
+			return nil, fmt.Errorf("getting group membership info: %s", response.Status)
+		}
+
+		var g Groups
+		if err = json.NewDecoder(response.Body).Decode(&g); err != nil {
+			return groups.Groups, errors.Wrap(err, "failed unmarshalling group membership response")
+		}
+
+		response.Body.Close()
+		groups.Groups = append(groups.Groups, g.Groups...)
+		nextLink = g.NextLink
 	}
 
-	req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", token))
-	response, err := http.DefaultClient.Do(req)
-	if err != nil {
-		return groups.Groups, errors.Wrap(err, "getting group membership info")
-	}
-
-	defer response.Body.Close()
-	if err = json.NewDecoder(response.Body).Decode(&groups); err != nil {
-		return groups.Groups, errors.Wrap(err, "failed unmarshalling group membership response")
+	if len(groups.Groups) == 0 {
+		return nil, fmt.Errorf("no groups found")
 	}
 
 	a.groupsCache[token] = cachedGroups{
@@ -268,10 +343,20 @@ func (a *Auth) getGroups(token string) ([]Group, error) {
 	return groups.Groups, nil
 }
 
-func (a *Auth) SetCookie(w http.ResponseWriter, token string, expiry time.Time) error {
+func (a *Auth) SetCookie(w http.ResponseWriter, access, refresh string, expiry time.Time) error {
 	http.SetCookie(w, &http.Cookie{
-		Name:     a.cookieName,
-		Value:    token,
+		Name:     a.refreshCookieName,
+		Value:    refresh,
+		Path:     "/",
+		Secure:   true,
+		HttpOnly: true,
+		SameSite: http.SameSiteStrictMode,
+		Expires:  expiry,
+	})
+
+	http.SetCookie(w, &http.Cookie{
+		Name:     a.accessCookieName,
+		Value:    access,
 		Path:     "/",
 		Secure:   true,
 		HttpOnly: true,
@@ -281,18 +366,25 @@ func (a *Auth) SetCookie(w http.ResponseWriter, token string, expiry time.Time)
 	return nil
 }
 
-func (a *Auth) SetGRPCMetadata(ctx context.Context, md metadata.MD, token string) error {
-	cookies := []string{}
+func (a *Auth) SetGRPCMetadata(ctx context.Context, md metadata.MD, access, refresh string) (context.Context, error) {
+	mCookies := map[string]string{}
 	if c, ok := md["cookie"]; ok {
 		for _, cookie := range c {
-			if strings.HasPrefix(cookie, a.cookieName) {
-				cookie = a.cookieName + "=" + token
-			}
-			cookies = append(cookies, cookie)
+			name, val := parseCookie(cookie)
+			mCookies[name] = val
 		}
 	}
+
+	mCookies[a.accessCookieName] = access
+	mCookies[a.refreshCookieName] = refresh
+
+	cookies := []string{}
+	for name, val := range mCookies {
+		cookies = append(cookies, name+"="+val)
+	}
+
 	md["cookie"] = cookies
-	return grpc.SetHeader(ctx, md)
+	return metadata.NewIncomingContext(ctx, md), grpc.SetHeader(ctx, md)
 }
 
 func decodeHex(hexstr string) ([]byte, error) {
@@ -305,3 +397,49 @@ func decodeHex(hexstr string) ([]byte, error) {
 	}
 	return data, nil
 }
+
+func (a *Auth) convertIP(configuredIPs []string) error {
+	sz := len(configuredIPs)
+	nets := make([]net.IPNet, sz)
+	for i, ip := range configuredIPs {
+		// skip empty strings
+		if ip == "" {
+			sz--
+			continue
+		}
+		// for IPs passed without a subnet, append /32 to only allow 1 IP
+		// this step is needed because ParseCIDR method assumes a CIDR address
+		if !strings.Contains(ip, "/") {
+			ip = ip + "/32"
+		}
+		_, subnet, err := net.ParseCIDR(ip)
+		if err != nil {
+			return errors.Wrapf(err, "parsing CIDR for %v", ip)
+		}
+		nets[i] = *subnet
+	}
+	a.allowedNetworks = nets[:sz]
+	return nil
+}
+
+// if IP is in allowed networks, then return true to grant admin permissions
+func (a *Auth) CheckAllowedNetworks(clientIP string) bool {
+	clientIP = strings.Split(clientIP, ":")[0]
+	convertedIP := net.ParseIP(clientIP)
+	for _, network := range a.allowedNetworks {
+		if network.Contains(convertedIP) {
+			return true
+		}
+	}
+	return false
+}
+
+func parseCookie(cookie string) (name, data string) {
+	vals := strings.Split(cookie, "=")
+	if len(vals) == 0 {
+		vals = []string{"", ""}
+	} else if len(vals) < 2 {
+		vals = append(vals, "")
+	}
+	return vals[0], vals[1]
+}

authn/authenticate_internal_test.go

@@ -6,6 +6,7 @@ import (
 	"encoding/hex"
 	"encoding/json"
 	"fmt"
+	"net"
 	"net/http"
 	"net/http/httptest"
 	"os"
@@ -15,9 +16,8 @@ import (
 	"testing"
 	"time"
 
+	"github.com/featurebasedb/featurebase/v3/logger"
 	"github.com/golang-jwt/jwt"
-	"github.com/molecula/featurebase/v3/logger"
-	"golang.org/x/oauth2"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/metadata"
 )
@@ -33,6 +33,7 @@ func NewTestAuth(t *testing.T) *Auth {
 		LogoutURL        = "https://login.microsoftonline.com/common/oauth2/v2.0/logout"
 		Scopes           = []string{"https://graph.microsoft.com/.default", "offline_access"}
 		Key              = "DEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEF"
+		configuredIPs    = []string{}
 	)
 
 	a, err := NewAuth(
@@ -46,17 +47,98 @@ func NewTestAuth(t *testing.T) *Auth {
 		ClientID,
 		ClientSecret,
 		Key,
+		configuredIPs,
 	)
 	if err != nil {
 		t.Fatalf("building auth object%s", err)
 	}
 	return a
 }
+
+func TestSetGRPCMetadata(t *testing.T) {
+	a := NewTestAuth(t)
+	for name, md := range map[string]metadata.MD{
+		"empty":     {},
+		"something": {"cookie": []string{a.accessCookieName + "=something"}},
+		"somethingElse": {"cookie": []string{
+			a.accessCookieName + "=something",
+			a.refreshCookieName + "=something",
+		}},
+		"otherCookies": {"cookie": []string{a.accessCookieName + "=something", "blah=blah"}},
+	} {
+		t.Run(name, func(t *testing.T) {
+			ogCookies := md["cookie"]
+			ctx := grpc.NewContextWithServerTransportStream(
+				metadata.NewIncomingContext(context.TODO(),
+					md,
+				),
+				NewServerTransportStream(),
+			)
+			md, ok := metadata.FromIncomingContext(ctx)
+			if !ok {
+				t.Fatalf("expected ok, got: %v", ok)
+			}
+			ctx, err := a.SetGRPCMetadata(ctx, md, "accesstoken!", "refreshtoken!")
+			if err != nil {
+				t.Fatalf("expected no errors, got: %v", err)
+			}
+			if err := grpc.SendHeader(ctx, md); err != nil {
+				t.Fatalf("expected no errors, got: %v", err)
+			}
+			md, ok = metadata.FromIncomingContext(ctx)
+			if !ok {
+				t.Fatalf("expected ok, got: %v", ok)
+			}
+			c, ok := md["cookie"]
+			if !ok {
+				t.Fatalf("expected ok, got: %v", ok)
+			}
+			var accessCookie, refreshCookie string
+			for _, cookie := range c {
+				if strings.HasPrefix(cookie, a.accessCookieName) {
+					accessCookie = cookie
+				} else if strings.HasPrefix(cookie, a.refreshCookieName) {
+					refreshCookie = cookie
+				}
+				if refreshCookie != "" && accessCookie != "" {
+					break
+				}
+			}
+
+			exp := a.accessCookieName + "=accesstoken!"
+			if accessCookie != exp {
+				t.Fatalf("expected '%v', got '%v'", exp, accessCookie)
+			}
+			exp = a.refreshCookieName + "=refreshtoken!"
+			if refreshCookie != exp {
+				t.Fatalf("expected '%v', got '%v'", exp, refreshCookie)
+			}
+
+			for _, cookie := range c {
+				if strings.HasPrefix(cookie, a.accessCookieName) || strings.HasPrefix(cookie, a.refreshCookieName) {
+					continue
+				}
+				found := false
+
+				for _, ogCookie := range ogCookies {
+					if cookie == ogCookie {
+						found = true
+						break
+					}
+				}
+				if !found {
+					t.Fatal("SetGRPCMetadata did not maintain the previous cookie list")
+				}
+			}
+		})
+	}
+}
+
 func TestAuth(t *testing.T) {
 	a := NewTestAuth(t)
 	t.Run("SetCookie", func(t *testing.T) {
 		w := httptest.NewRecorder()
-		err := a.SetCookie(w, "a cookie value", time.Now().Add(time.Hour))
+		err := a.SetCookie(w, "access", "refresh", time.Now().Add(time.Hour))
 		if err != nil {
 			t.Fatalf("expected no errors, got: %v", err)
 		}
@@ -69,42 +151,7 @@ func TestAuth(t *testing.T) {
 			t.Fatalf("path=%s, want %s", got, want)
 		}
 	})
-	t.Run("SetGRPCMetadata", func(t *testing.T) {
-		md := metadata.MD{
-			"cookie": []string{a.cookieName + "=something"},
-		}
-		ctx := grpc.NewContextWithServerTransportStream(
-			metadata.NewIncomingContext(context.TODO(),
-				md,
-			),
-			NewServerTransportStream(),
-		)
-		md, ok := metadata.FromIncomingContext(ctx)
-		if !ok {
-			t.Fatalf("expected ok, got: %v", ok)
-		}
-		err := a.SetGRPCMetadata(ctx, md, "this is a token!")
-		if err != nil {
-			t.Fatalf("expected no errors, got: %v", err)
-		}
-		md, ok = metadata.FromIncomingContext(ctx)
-		if !ok {
-			t.Fatalf("expected ok, got: %v", ok)
-		}
-		c, ok := md["cookie"]
-		if !ok {
-			t.Fatalf("expected ok, got: %v", ok)
-		}
-		var cookie string
-		for _, cookie = range c {
-			if strings.HasPrefix(cookie, a.cookieName) {
-				break
-			}
-		}
-		if exp, got := a.cookieName+"=this is a token!", cookie; got != exp {
-			t.Fatalf("expected '%v', got '%v'", exp, got)
-		}
-	})
+
 	t.Run("KeyLength", func(t *testing.T) {
 		_, err := NewAuth(
 			logger.NewStandardLogger(os.Stdout),
@@ -117,6 +164,7 @@ func TestAuth(t *testing.T) {
 			"e9088663-eb08-41d7-8f65-efb5f54bbb71",
 			"DEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEF",
 			"DEADBEEFD",
+			[]string{},
 		)
 		if err == nil || !strings.Contains(err.Error(), "decoding secret key") {
 			t.Fatalf("expected error decoding secret key got: %v", err)
@@ -137,8 +185,9 @@ func TestAuthenticate(t *testing.T) {
 		uname        string
 		exp          int64
 		refresh      bool
-		errOnRefresh bool
+		refreshToken string
 		malformed    bool
+		empty        bool
 		groups       []Group
 		err          error
 	}{
@@ -156,9 +205,13 @@ func TestAuthenticate(t *testing.T) {
 		{
 			name:      "Malformed",
 			malformed: true,
-			err:       fmt.Errorf("parsing bearer token: token contains an invalid number of segments"),
+			err:       fmt.Errorf("parsing auth token: token contains an invalid number of segments"),
+		},
+		{
+			name:  "Empty",
+			empty: true,
+			err:   fmt.Errorf("auth token is empty"),
 		},
-
 		{
 			name:  "ExpiredTokenNoRefresh",
 			uid:   "42",
@@ -170,7 +223,7 @@ func TestAuthenticate(t *testing.T) {
 				},
 			},
 			exp: -17764800,
-			err: fmt.Errorf("token is expired"),
+			err: fmt.Errorf("token is expired: refreshing token: 400 Bad Request"),
 		},
 		{
 			name:  "ExpiredTokenYesRefresh",
@@ -182,8 +235,9 @@ func TestAuthenticate(t *testing.T) {
 					GroupName: "adminGroup",
 				},
 			},
-			refresh: true,
-			exp:     -17764800,
+			refresh:      true,
+			refreshToken: "refreshToken",
+			exp:          -17764800,
 		},
 		{
 			name:  "ExpiredTokenYesRefreshButError",
@@ -196,9 +250,9 @@ func TestAuthenticate(t *testing.T) {
 				},
 			},
 			refresh:      true,
-			errOnRefresh: true,
+			refreshToken: "blah!!",
 			exp:          -17764800,
-			err:          fmt.Errorf("decoding refreshed token: invalid character 'b' looking for beginning of value"),
+			err:          fmt.Errorf("token is expired: refreshing token: 403 Forbidden"),
 		},
 	}
 	for _, test := range cases {
@@ -207,61 +261,58 @@ func TestAuthenticate(t *testing.T) {
 			a := NewTestAuth(t)
 			token := ""
 			var err error
-			if !test.malformed {
+			if !test.malformed && !test.empty {
 				tkn := jwt.New(jwt.SigningMethodHS256)
 				claims := tkn.Claims.(jwt.MapClaims)
 				claims["oid"] = test.uid
 				claims["name"] = test.uname
 				if test.exp != 0 {
-					claims["exp"] = strconv.Itoa(int(test.exp))
+					claims["exp"] = float64(test.exp)
 				}
 				token, err = tkn.SignedString(a.SecretKey())
 				if err != nil {
 					t.Fatalf("unexpected error when signing token %v", err)
 				}
-			} else {
+			} else if !test.empty {
 				token = "asdfasdfasdfasdF"
 			}
 			if len(test.groups) > 0 {
 				a.groupsCache[token] = cachedGroups{time.Now(), test.groups}
 			}
 			if test.refresh {
-				var srv *httptest.Server
-				if !test.errOnRefresh {
-					srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-						tkn := jwt.New(jwt.SigningMethodHS256)
-						claims := tkn.Claims.(jwt.MapClaims)
-						claims["oid"] = test.uid
-						claims["name"] = test.uname
-						expiry := strconv.Itoa(int(time.Now().Add(2 * time.Hour).Unix()))
-						claims["exp"] = expiry
-						fresh, err := tkn.SignedString(a.SecretKey())
-						if err != nil {
-							t.Fatalf("unexpected error when signing token %v", err)
-						}
+				srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+					if err := r.ParseForm(); err != nil {
+						t.Fatalf("unexpected error: %v", err)
+					}
+					refresh := r.Form.Get("refresh_token")
+					if refresh != test.refreshToken {
+						t.Fatalf("refresh token not passed properly, expected %v, got %v", test.refreshToken, refresh)
+						return
+					}
+					if refresh != "refreshToken" {
+						http.Error(w, "bad token", http.StatusForbidden)
+					}
 
-						a.groupsCache[fresh] = cachedGroups{time.Now(), test.groups}
-						fmt.Fprintf(w, `{"access_token": "`+fresh+`",  "refresh_token": "blah",  "token_type": "bearer",  "expires": `+expiry+` }`)
-					}))
-				} else {
-					srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-						http.Error(w, "bad", http.StatusInternalServerError)
-					}))
-				}
+					tkn := jwt.New(jwt.SigningMethodHS256)
+					claims := tkn.Claims.(jwt.MapClaims)
+					claims["oid"] = test.uid
+					claims["name"] = test.uname
+					expiry := float64(time.Now().Add(2 * time.Hour).Unix())
+					claims["exp"] = expiry
+					fresh, err := tkn.SignedString(a.SecretKey())
+					if err != nil {
+						t.Fatalf("unexpected error when signing token %v", err)
+					}
+
+					a.groupsCache[fresh] = cachedGroups{time.Now(), test.groups}
+					fmt.Fprintf(w, `{"access_token": "`+fresh+`",  "refresh_token": "blah",  "token_type": "bearer",  "expires": `+strconv.FormatFloat(expiry, 'f', 0, 64)+` }`)
+				}))
 				defer srv.Close()
 				a.oAuthConfig.Endpoint.TokenURL = srv.URL
-				a.tokenCache[token] = cachedToken{
-					time.Now(),
-					&oauth2.Token{
-						AccessToken:  token,
-						RefreshToken: "blah",
-						Expiry:       time.Unix(test.exp, 0),
-					},
-				}
 			}
 
 			// do the actual testing
-			uinfo, err := a.Authenticate(context.TODO(), token)
+			uinfo, err := a.Authenticate(token, test.refreshToken)
 			// okay this part kind of sucks bc we need to check errors and i
 			// dont want to write a whole new test for things that should have
 			// errors just to avoid this mess. errors.Is doesn't work either
@@ -295,11 +346,9 @@ func TestAuthenticate_CleanCache(t *testing.T) {
 		now := time.Now()
 		a.groupsCache["oldy"] = cachedGroups{now.Add(-24 * time.Hour), []Group{}}
 		a.groupsCache["goldy"] = cachedGroups{now.Add(-4 * time.Hour), []Group{}}
-		a.tokenCache["oldy"] = cachedToken{now.Add(-24 * time.Hour), &oauth2.Token{}}
-		a.tokenCache["goldy"] = cachedToken{now.Add(-4 * time.Hour), &oauth2.Token{}}
 		a.lastCacheClean = now.Add(-45 * time.Minute)
 
-		_, _ = a.Authenticate(context.TODO(), "this doesn't matter")
+		_, _ = a.Authenticate("this doesn't matter", "this doesn't matter?")
 		if a.lastCacheClean.Sub(now) <= time.Nanosecond {
 			t.Fatalf("cache should have been cleaned")
 		}
@@ -309,23 +358,15 @@ func TestAuthenticate_CleanCache(t *testing.T) {
 		if _, ok := a.groupsCache["goldy"]; !ok {
 			t.Errorf("goldy should not have been deleted")
 		}
-		if _, ok := a.tokenCache["oldy"]; ok {
-			t.Errorf("oldy should have been deleted")
-		}
-		if _, ok := a.tokenCache["goldy"]; !ok {
-			t.Errorf("goldy should not have been deleted")
-		}
 	})
 	t.Run("shouldn't clean", func(t *testing.T) {
 		a := NewTestAuth(t)
 		now := time.Now()
 		a.groupsCache["oldy"] = cachedGroups{now.Add(-24 * time.Hour), []Group{}}
 		a.groupsCache["goldy"] = cachedGroups{now.Add(-4 * time.Hour), []Group{}}
-		a.tokenCache["oldy"] = cachedToken{now.Add(-24 * time.Hour), &oauth2.Token{}}
-		a.tokenCache["goldy"] = cachedToken{now.Add(-4 * time.Hour), &oauth2.Token{}}
 		a.lastCacheClean = now
 
-		_, _ = a.Authenticate(context.TODO(), "this doesn't matter")
+		_, _ = a.Authenticate("this doesn't matter", "this doesn't matter?")
 		if a.lastCacheClean.Sub(now) >= time.Nanosecond {
 			t.Fatalf("cache should not have been cleaned")
 		}
@@ -335,14 +376,7 @@ func TestAuthenticate_CleanCache(t *testing.T) {
 		if _, ok := a.groupsCache["goldy"]; !ok {
 			t.Errorf("goldy should not have been deleted")
 		}
-		if _, ok := a.tokenCache["oldy"]; !ok {
-			t.Errorf("oldy should not have been deleted")
-		}
-		if _, ok := a.tokenCache["goldy"]; !ok {
-			t.Errorf("goldy should not have been deleted")
-		}
 	})
-
 }
 
 func TestGetGroups(t *testing.T) {
@@ -352,19 +386,19 @@ func TestGetGroups(t *testing.T) {
 			cacheTime: time.Now(),
 			groups: []Group{
 				{
-					GroupID:   "i feel it in the water",
-					GroupName: "i feel it in the earth",
+					GroupID:   "a han noston ned wilith",
+					GroupName: "I smell it in the air",
 				},
 			},
 		},
 	}
-	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+	srvNext := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		body, err := json.Marshal(
 			Groups{
 				Groups: []Group{
 					{
-						GroupID:   "much that once was is lost",
-						GroupName: "for none now live who remember it",
+						GroupID:   "han mathon ne chae",
+						GroupName: "I feel it in the earth",
 					},
 				},
 			},
@@ -374,6 +408,25 @@ func TestGetGroups(t *testing.T) {
 		}
 		fmt.Fprintf(w, "%s", body)
 	}))
+	defer srvNext.Close()
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		body, err := json.Marshal(
+			Groups{
+				NextLink: srvNext.URL,
+				Groups: []Group{
+					{
+						GroupID:   "han mathon ne nen",
+						GroupName: "i feel it in the water",
+					},
+				},
+			},
+		)
+		if err != nil {
+			t.Fatalf("unexpected error marshalling groups response: %v", err)
+		}
+		fmt.Fprintf(w, "%s", body)
+	}))
+	defer srv.Close()
 	a.groupEndpoint = srv.URL
 
 	for name, test := range map[string]struct {
@@ -384,8 +437,8 @@ func TestGetGroups(t *testing.T) {
 			token: "the world is changed",
 			groups: []Group{
 				{
-					GroupID:   "i feel it in the water",
-					GroupName: "i feel it in the earth",
+					GroupID:   "a han noston ned wilith",
+					GroupName: "I smell it in the air",
 				},
 			},
 		},
@@ -393,8 +446,12 @@ func TestGetGroups(t *testing.T) {
 			token: "i smell it in the air",
 			groups: []Group{
 				{
-					GroupID:   "much that once was is lost",
-					GroupName: "for none now live who remember it",
+					GroupID:   "han mathon ne nen",
+					GroupName: "i feel it in the water",
+				},
+				{
+					GroupID:   "han mathon ne chae",
+					GroupName: "I feel it in the earth",
 				},
 			},
 		},
@@ -405,7 +462,6 @@ func TestGetGroups(t *testing.T) {
 			}
 		})
 	}
-
 }
 
 func TestDecodeHex(t *testing.T) {
@@ -455,7 +511,7 @@ func TestHandlers(t *testing.T) {
 		w := httptest.NewRecorder()
 		req.AddCookie(
 			&http.Cookie{
-				Name:     a.cookieName,
+				Name:     a.accessCookieName,
 				Value:    "test",
 				Path:     "/",
 				Secure:   true,
@@ -464,8 +520,20 @@ func TestHandlers(t *testing.T) {
 				Expires:  time.Unix(3000000, 0),
 			},
 		)
+
+		req.AddCookie(
+			&http.Cookie{
+				Name:     a.refreshCookieName,
+				Value:    "test",
+				Path:     "/",
+				Secure:   true,
+				HttpOnly: true,
+				SameSite: http.SameSiteStrictMode,
+				Expires:  time.Unix(3000000, 0),
+			},
+		)
+
 		a.groupsCache["test"] = cachedGroups{}
-		a.tokenCache["test"] = cachedToken{time.Now(), &oauth2.Token{}}
 		a.Logout(w, req)
 		resp := w.Result()
 		if resp.StatusCode != http.StatusTemporaryRedirect {
@@ -476,7 +544,7 @@ func TestHandlers(t *testing.T) {
 			t.Fatalf("expected %v, got %v", redirect, got.Path)
 		}
 		for _, c := range resp.Cookies() {
-			if c.Name == a.cookieName {
+			if c.Name == a.accessCookieName || c.Name == a.refreshCookieName {
 				if c.Value != "" {
 					t.Fatalf("cookie not set to empty value!")
 				}
@@ -485,15 +553,11 @@ func TestHandlers(t *testing.T) {
 				if want != got {
 					t.Fatalf("expected %v, got %v", want, got)
 				}
-				break
 			}
 		}
 		if _, ok := a.groupsCache["test"]; ok {
 			t.Fatalf("groups not deleted!")
 		}
-		if _, ok := a.tokenCache["test"]; ok {
-			t.Fatalf("token not deleted!")
-		}
 	})
 	t.Run("redirectGood", func(t *testing.T) {
 		req := httptest.NewRequest("GET", "/redirect", nil)
@@ -504,17 +568,12 @@ func TestHandlers(t *testing.T) {
 		claims["name"] = "user name"
 		expiresIn := 2 * time.Hour
 		exp := time.Now().Add(expiresIn)
-		expiry := strconv.Itoa(int(exp.Unix()))
+		expiry := float64(exp.Unix())
 		claims["exp"] = expiry
 		fresh, err := tkn.SignedString(a.SecretKey())
 		if err != nil {
 			t.Fatalf("unexpected error when signing token %v", err)
 		}
-		freshToken := oauth2.Token{
-			AccessToken:  fresh,
-			RefreshToken: "blah",
-			Expiry:       exp,
-		}
 
 		srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 			body := `{"access_token": "` + fresh + `", "refresh_token": "blah", "expires_in": "` + strconv.Itoa(int(expiresIn.Seconds())) + `"}`
@@ -531,15 +590,13 @@ func TestHandlers(t *testing.T) {
 		if got, err := resp.Location(); err != nil || got.String() != "/" {
 			t.Fatalf("expected %v, got %v", "/", got.Path)
 		}
-		cachedToken := a.tokenCache[fresh].token
-		if cachedToken.AccessToken != freshToken.AccessToken {
-			t.Fatalf("expected %v, got %v", freshToken.AccessToken, cachedToken.AccessToken)
-		}
-		if cachedToken.RefreshToken != freshToken.RefreshToken {
-			t.Fatalf("expected %v, got %v", freshToken.RefreshToken, cachedToken.RefreshToken)
-		}
-		if cachedToken.Expiry.Sub(freshToken.Expiry) > time.Second {
-			t.Fatalf("expected %v, got %v", freshToken.Expiry, cachedToken.Expiry)
+		cookies := resp.Cookies()
+		for _, c := range cookies {
+			if c.Name == a.accessCookieName && c.Value != fresh {
+				t.Fatalf("expected %v, got %v", exp, c.Value)
+			} else if c.Name == a.refreshCookieName && c.Value != "blah" {
+				t.Fatalf("expected %v, got %v", "blah", c.Value)
+			}
 		}
 	})
 
@@ -556,7 +613,6 @@ func TestHandlers(t *testing.T) {
 			t.Fatalf("expected BadRequest, got %v", resp.StatusCode)
 		}
 	})
-
 }
 
 // This type is used for mocking ServerTransportStreams in tests
@@ -590,3 +646,115 @@ func (s *ServerTransportStream) SetTrailer(md metadata.MD) error {
 	_ = md
 	return nil
 }
+
+func TestCleanOAuthConfig(t *testing.T) {
+	a := NewTestAuth(t)
+	res := a.CleanOAuthConfig()
+	assertEqual("", res.ClientSecret, t)
+	assertEqual(a.oAuthConfig.ClientID, res.ClientID, t)
+	assertEqual(a.oAuthConfig.RedirectURL, res.RedirectURL, t)
+	assertEqual(a.oAuthConfig.Scopes, res.Scopes, t)
+	assertEqual(a.oAuthConfig.Endpoint, res.Endpoint, t)
+}
+
+func assertEqual(exp, got interface{}, t *testing.T) {
+	if !reflect.DeepEqual(exp, got) {
+		t.Fatalf("expected %v, got %v", exp, got)
+	}
+}
+
+func TestCheckAllowedNetworks(t *testing.T) {
+	tests := []struct {
+		requestIP     string
+		configuredIPs []string
+		isAdmin       bool
+	}{
+		{
+			requestIP:     "10.0.0.1",
+			configuredIPs: []string{"10.0.0.1"},
+			isAdmin:       true,
+		},
+		{
+			requestIP:     "10.0.0.3",
+			configuredIPs: []string{"10.0.0.1", "10.0.0.2"},
+			isAdmin:       false,
+		},
+		{
+			requestIP:     "10.0.0.2",
+			configuredIPs: []string{"10.0.0.1/30"},
+			isAdmin:       true,
+		},
+		// it is possible for the client IP to have a port
+		{
+			requestIP:     "10.0.0.2:22",
+			configuredIPs: []string{"10.0.0.1/30"},
+			isAdmin:       true,
+		},
+		{
+			requestIP:     "10.1.0.3",
+			configuredIPs: []string{"10.0.0.1/32"},
+			isAdmin:       false,
+		},
+		{
+			requestIP:     "10.0.0.254",
+			configuredIPs: []string{"10.0.0.1/24"},
+			isAdmin:       true,
+		},
+	}
+
+	for i, test := range tests {
+		t.Run(fmt.Sprintf("network-%d", i), func(t *testing.T) {
+			a := NewTestAuth(t)
+			if err := a.convertIP(test.configuredIPs); err != nil {
+				t.Fatalf("failed to convert IPs from strings to net.IP: %v", err)
+			}
+			got := a.CheckAllowedNetworks(test.requestIP)
+			if got != test.isAdmin {
+				t.Fatalf("expected %v, got %v", test.isAdmin, got)
+			}
+		})
+	}
+}
+
+func TestConvertIP(t *testing.T) {
+	tests := []struct {
+		configuredIPs []string
+		convertedIPs  []net.IPNet
+	}{
+		{
+			configuredIPs: []string{"10.0.0.1"},
+			convertedIPs: []net.IPNet{
+				{IP: net.ParseIP("10.0.0.1"), Mask: net.CIDRMask(32, 32)},
+			},
+		},
+		{
+			configuredIPs: []string{"10.0.0.1/30"},
+			convertedIPs: []net.IPNet{
+				{IP: net.ParseIP("10.0.0.0"), Mask: net.CIDRMask(30, 32)},
+			},
+		},
+	}
+
+	for i, test := range tests {
+		t.Run(fmt.Sprintf("network-%d", i), func(t *testing.T) {
+			a := NewTestAuth(t)
+			if err := a.convertIP(test.configuredIPs); err != nil {
+				t.Fatalf("failed to convert IPs from strings to net.IP: %v", err)
+			}
+
+			if len(a.allowedNetworks) != len(test.convertedIPs) {
+				t.Fatalf("expected len of %v networks, got %v", len(test.convertedIPs), len(a.allowedNetworks))
+			}
+
+			for i := range a.allowedNetworks {
+				expected, got := test.convertedIPs[i], a.allowedNetworks[i]
+				if got.IP.String() != expected.IP.String() {
+					t.Fatalf("for IP, expected %v, got %v", expected.IP, got.IP)
+				}
+				if got.Mask.String() != expected.Mask.String() {
+					t.Fatalf("for mask, expected %v, got %v", expected.Mask.String(), got.Mask.String())
+				}
+			}
+		})
+	}
+}

authn/context.go

@@ -0,0 +1,54 @@
+// Copyright 2022 Molecula Corp (DBA FeatureBase). All rights reserved.
+package authn
+
+import "context"
+
+// Empty struct to avoid allocations
+type contextKeyAccessToken struct{}
+type contextKeyRefreshToken struct{}
+type contextKeyUserInfo struct{}
+type contextKeyIndexes struct{}
+
+// GetAccessToken gets the access token from a context.
+func GetAccessToken(ctx context.Context) (token string, ok bool) {
+	token, ok = ctx.Value(contextKeyAccessToken{}).(string)
+	return
+}
+
+// WithAccessToken makes a new Context with an access token.
+func WithAccessToken(ctx context.Context, token string) context.Context {
+	return context.WithValue(ctx, contextKeyAccessToken{}, token)
+}
+
+// GetRefreshToken gets the refresh token from a context.
+func GetRefreshToken(ctx context.Context) (token string, ok bool) {
+	token, ok = ctx.Value(contextKeyRefreshToken{}).(string)
+	return
+}
+
+// WithRefreshToken makes a new Context with a refresh token.
+func WithRefreshToken(ctx context.Context, token string) context.Context {
+	return context.WithValue(ctx, contextKeyRefreshToken{}, token)
+}
+
+// GetUserInfo gets the UserInfo from a context.
+func GetUserInfo(ctx context.Context) (userInfo *UserInfo, ok bool) {
+	userInfo, ok = ctx.Value(contextKeyUserInfo{}).(*UserInfo)
+	return
+}
+
+// WithUserInfo makes a new Context with UserInfo.
+func WithUserInfo(ctx context.Context, userInfo *UserInfo) context.Context {
+	return context.WithValue(ctx, contextKeyUserInfo{}, userInfo)
+}
+
+// GetIndexes get the indices from a context.
+func GetIndexes(ctx context.Context) (indexes []string, ok bool) {
+	indexes, ok = ctx.Value(contextKeyIndexes{}).([]string)
+	return
+}
+
+// WithIndexes makes a new Context with a []string containing the indicies.
+func WithIndexes(ctx context.Context, indexes []string) context.Context {
+	return context.WithValue(ctx, contextKeyUserInfo{}, indexes)
+}

authz/authorization.go

@@ -1,25 +1,13 @@
-// Copyright 2017 Pilosa Corp.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
+// Copyright 2022 Molecula Corp. (DBA FeatureBase).
+// SPDX-License-Identifier: Apache-2.0
 
 package authz
 
 import (
 	"fmt"
 	"io"
-	"io/ioutil"
 
-	"github.com/molecula/featurebase/v3/authn"
+	"github.com/featurebasedb/featurebase/v3/authn"
 
 	"gopkg.in/yaml.v2"
 )
@@ -54,7 +42,7 @@ func (p Permission) Satisfies(b Permission) bool {
 }
 
 func (p *GroupPermissions) ReadPermissionsFile(permsFile io.Reader) (err error) {
-	permsData, err := ioutil.ReadAll(permsFile)
+	permsData, err := io.ReadAll(permsFile)
 
 	if err != nil {
 		return fmt.Errorf("reading permissions failed with error: %s", err)

authz/authorization_test.go

@@ -1,16 +1,5 @@
-// Copyright 2017 Pilosa Corp.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
+// Copyright 2022 Molecula Corp. (DBA FeatureBase).
+// SPDX-License-Identifier: Apache-2.0
 package authz_test
 
 import (
@@ -20,8 +9,8 @@ import (
 	"strings"
 	"testing"
 
-	"github.com/molecula/featurebase/v3/authn"
-	"github.com/molecula/featurebase/v3/authz"
+	"github.com/featurebasedb/featurebase/v3/authn"
+	"github.com/featurebasedb/featurebase/v3/authz"
 )
 
 func TestAuth_ReadPermissionsFile(t *testing.T) {

batch/Dockerfile-test

@@ -0,0 +1,11 @@
+ARG GO_VERSION=1.19
+
+FROM golang:${GO_VERSION}
+
+WORKDIR /go/src/github.com/featurebasedb/featurebase/
+
+COPY . .
+
+WORKDIR /go/src/github.com/featurebasedb/featurebase/batch/
+
+CMD ["go","test","-v","-mod=vendor","-tags=odbc,dynamic","./..."]

batch/Dockerfile-wait

@@ -0,0 +1,8 @@
+FROM ubuntu:18.04
+
+RUN ["apt-get", "update", "-y"]
+RUN ["apt-get", "install", "-y", "curl", "netcat"]
+
+ADD wait.sh /wait
+
+ENTRYPOINT ["/wait"]

batch/Makefile

@@ -0,0 +1,44 @@
+GO ?= go
+
+# We allow setting a custom docker-compose "project". Multiple of the
+# same docker-compose environment can exist simultaneously as long as
+# they use different projects (the project name is prepended to
+# container names and such). This is useful in a CI environment where
+# we might be running multiple instances of the tests concurrently.
+PROJECT ?= batch
+DOCKER_COMPOSE = docker-compose -p $(PROJECT)
+
+vendor: ../go.mod
+	$(GO) mod vendor
+
+build-%:
+	$(DOCKER_COMPOSE) build $*
+
+test-all:
+	$(MAKE) startup
+	$(MAKE) test-run
+	$(MAKE) shutdown
+
+start-all: build-wait
+	$(DOCKER_COMPOSE) up -d featurebase
+	$(DOCKER_COMPOSE) run -T wait featurebase curl --silent --fail http://featurebase:10101/status
+
+startup: start-all
+
+shutdown:
+	$(DOCKER_COMPOSE) down -v --remove-orphans
+
+save-%-logs:
+	$(DOCKER_COMPOSE) logs $* > ./testdata/$(PROJECT)_$*_logs.txt
+
+TCMD ?= ./...
+# do "make startup", then e.g. "make test-run-local TCMD='-run=MyFavTest ./kafka'"
+test-run-local: vendor
+	pwd
+	$(DOCKER_COMPOSE) build batch-test
+	$(DOCKER_COMPOSE) run -T batch-test go test -mod=vendor -tags=odbc,dynamic $(TCMD)
+
+TPKG ?= ../...
+test-run: vendor
+	$(DOCKER_COMPOSE) build batch-test
+	$(DOCKER_COMPOSE) run -T batch-test bash -c "set -o pipefail; go test -v -mod=vendor -tags=odbc,dynamic ./... -covermode=atomic -coverpkg=$(TPKG) -coverprofile=/testdata/$(PROJECT)_base_coverage.out"

batch/README.md

@@ -0,0 +1,46 @@
+# batch
+
+The `batch` package provides a standard tool set for batching records in a way
+that is most performant for ingesting those records into FeatureBase. The main
+implementation is `Batch` (which can be initated with the `NewBatch()`
+function). The `NewBatch()` function takes an `Importer` which contains all of
+ the methods required to interact with FeatureBase; these include methods for
+ doing string/id translation as well as for importing shards of data.
+
+ IDK uses the `batch` package internally. Another example where the `batch`
+ package is used in the `sql3` package. When an "INSERT INTO" statement is
+ executed, the SQL engine uses a `Batch` to do key translation and build import
+ batches prior to doing the final import.
+## Integration tests
+
+To run the tests, you will need to install the following dependencies:
+
+1. [Docker](https://docs.docker.com/install/)
+2. [Docker Compose](https://docs.docker.com/compose/install/)
+
+In addition to these dependancies, you will need to be added to the molecula [Gitlab](https://registry.gitlab.com/molecula) account.
+
+First start the test environment. This is a docker-compose environment that includes featurebase.
+
+    make startup
+
+To build and run the integration tests, run:
+
+    make test-run-local
+
+Then to shut down the test environment, run:
+
+    make shutdown
+
+The previous command is equivalent to running the following:
+
+    make startup
+    sleep 30 # wait for services to come up
+    make test-run
+    make shutdown
+
+To run an individual test, you can run the command directly using docker-compose. Note that you must run `docker-compose build batch-test` for docker to run the latest code. Modify the following as needed:
+
+    make startup
+    docker-compose build batch-test
+    docker-compose run batch-test /usr/local/go/bin/go test -count=1 -mod=vendor -run=TestCmdMainOne .

batch/batcher.go

@@ -0,0 +1,20 @@
+package batch
+
+import (
+	"time"
+
+	"github.com/featurebasedb/featurebase/v3/dax"
+)
+
+// Batcher is an interface implemented by anything which can allocate new
+// batches.
+type Batcher interface {
+	NewBatch(cfg Config, tbl *dax.Table, fields []*dax.Field) (RecordBatch, error)
+}
+
+// Config is the configuration options passed to NewBatch for any implementation
+// of the Batcher interface.
+type Config struct {
+	Size         int
+	MaxStaleness time.Duration
+}

batch/convert.go

@@ -0,0 +1,145 @@
+package batch
+
+import (
+	"time"
+
+	featurebase "github.com/featurebasedb/featurebase/v3"
+	"github.com/featurebasedb/featurebase/v3/errors"
+)
+
+var (
+	MinTimestampNano = time.Unix(-1<<32, 0).UTC()       // 1833-11-24T17:31:44Z
+	MaxTimestampNano = time.Unix(1<<32, 0).UTC()        // 2106-02-07T06:28:16Z
+	MinTimestamp     = time.Unix(-62135596799, 0).UTC() // 0001-01-01T00:00:01Z
+	MaxTimestamp     = time.Unix(253402300799, 0).UTC() // 9999-12-31T23:59:59Z
+
+	ErrTimestampOutOfRange = errors.New("", "value provided for timestamp field is out of range")
+)
+
+type TimeUnit string
+
+const (
+	TimeUnitSeconds      = TimeUnit(featurebase.TimeUnitSeconds)
+	TimeUnitMilliseconds = TimeUnit(featurebase.TimeUnitMilliseconds)
+	TimeUnitMicroseconds = TimeUnit(featurebase.TimeUnitMicroseconds)
+	TimeUnitUSeconds     = TimeUnit(featurebase.TimeUnitUSeconds)
+	TimeUnitNanoseconds  = TimeUnit(featurebase.TimeUnitNanoseconds)
+)
+
+// TimestampToInt64 converts the provided timestamp to an int64 as the number of
+// units past the epoch.
+func TimestampToInt64(unit TimeUnit, epoch time.Time, ts time.Time) (int64, error) {
+	var err error
+
+	unit, err = validateTimeUnit(unit)
+	if err != nil {
+		return 0, errors.Wrap(err, "validating time unit")
+	}
+
+	epoch, err = validateEpoch(epoch)
+	if err != nil {
+		return 0, errors.Wrap(err, "validating epoch")
+	}
+
+	// Check if the epoch alone is out-of-range. If so, ingest should halt,
+	// regardless of state of the timestamp out-of-range CLI option.
+	if err := validateTimestamp(unit, epoch); err != nil {
+		return 0, errors.Wrap(err, "validating epoch")
+	}
+
+	epochAsInt64 := timestampToInt(unit, epoch)
+
+	// Check if the timestamp is out-of-range.
+	if err := validateTimestamp(unit, ts); err != nil {
+		return 0, errors.Wrapf(ErrTimestampOutOfRange, "validating timestamp: %s", ts)
+	}
+
+	tsAsInt64 := timestampToInt(unit, ts)
+
+	return tsAsInt64 - epochAsInt64, nil
+}
+
+// validateTimeUnit checks if the time unit is supported. If the provided unit
+// is blank, validateTimeUnit returns the default TimeUnit.
+func validateTimeUnit(unit TimeUnit) (TimeUnit, error) {
+	switch unit {
+	case "":
+		return TimeUnitSeconds, nil
+
+	case TimeUnitSeconds,
+		TimeUnitMilliseconds,
+		TimeUnitMicroseconds,
+		TimeUnitUSeconds,
+		TimeUnitNanoseconds:
+		return unit, nil
+	}
+
+	return "", errors.Errorf("unsupported time unit: %s", unit)
+}
+
+// validateEpoch checks if the epoch is supported. If the provided epoch
+// is "zero", validateEpoch returns the default epoch value.
+func validateEpoch(epoch time.Time) (time.Time, error) {
+	if epoch.IsZero() {
+		return time.Unix(0, 0), nil
+	}
+	return epoch, nil
+}
+
+// validateTimestamp checks if the timestamp is within the range of what FB accepts.
+func validateTimestamp(unit TimeUnit, ts time.Time) error {
+	// Min and Max timestamps that Featurebase accepts
+	var minStamp, maxStamp time.Time
+	switch unit {
+	case TimeUnitNanoseconds:
+		minStamp = MinTimestampNano
+		maxStamp = MaxTimestampNano
+	default:
+		minStamp = MinTimestamp
+		maxStamp = MaxTimestamp
+	}
+
+	if ts.Before(minStamp) || ts.After(maxStamp) {
+		return errors.Errorf("timestamp value (%v) must be within min: %v and max: %v", ts, minStamp, maxStamp)
+	}
+	return nil
+}
+
+// timestampToInt takes a time unit and a time.Time and converts it to an
+// integer value.
+func timestampToInt(unit TimeUnit, ts time.Time) int64 {
+	switch unit {
+	case TimeUnitSeconds:
+		return ts.Unix()
+	case TimeUnitMilliseconds:
+		return ts.UnixMilli()
+	case TimeUnitMicroseconds, TimeUnitUSeconds:
+		return ts.UnixMicro()
+	case TimeUnitNanoseconds:
+		return ts.UnixNano()
+	}
+	return 0
+}
+
+// intToTimestamp takes a timeunit and an integer value and converts it to
+// time.Time.
+func intToTimestamp(unit TimeUnit, val int64) (time.Time, error) {
+	switch unit {
+	case TimeUnitSeconds:
+		return time.Unix(val, 0).UTC(), nil
+	case TimeUnitMilliseconds:
+		return time.UnixMilli(val).UTC(), nil
+	case TimeUnitMicroseconds, TimeUnitUSeconds:
+		return time.UnixMicro(val).UTC(), nil
+	case TimeUnitNanoseconds:
+		return time.Unix(0, val).UTC(), nil
+	default:
+		return time.Time{}, errors.Errorf("Unknown time unit: '%v'", unit)
+	}
+}
+
+// Int64ToTimestamp converts the provided int64 to a timestamp based on the time unit
+// and epoch.
+func Int64ToTimestamp(unit TimeUnit, epoch time.Time, val int64) (time.Time, error) {
+	return intToTimestamp(unit, timestampToInt(unit, epoch)+val)
+}

batch/docker-compose.yml

@@ -0,0 +1,29 @@
+version: '3'
+
+services:
+  featurebase:
+    build:
+      context: ../.
+      dockerfile: ./Dockerfile-clustertests
+    environment:
+      PILOSA_DATA_DIR: /data
+      PILOSA_BIND: 0.0.0.0:10101
+      PILOSA_BIND_GRPC: 0.0.0.0:20101
+      PILOSA_ADVERTISE: featurebase:10101
+    command: /featurebase -test.run=TestRunMain -test.coverprofile=/testdata/batch_coverage.out server
+    volumes:
+      - ./testdata:/testdata
+
+  batch-test:
+    build:
+      context: ../.
+      dockerfile: ./batch/Dockerfile-test
+    volumes:
+      - ./testdata:/testdata
+
+  wait:
+    depends_on:
+      - "featurebase"
+    build:
+      context: .
+      dockerfile: Dockerfile-wait

batch/egpool/egpool.go

@@ -1,4 +1,5 @@
-// Copyright 2021 Molecula Corp. All rights reserved.
+// Copyright 2022 Molecula Corp. (DBA FeatureBase).
+// SPDX-License-Identifier: Apache-2.0
 package egpool
 
 import (
@@ -57,11 +58,11 @@ func (eg *Group) err(err error) {
 	eg.errs = append(eg.errs, err)
 }
 
-type ErrPanic struct {
+type PanicError struct {
 	Value interface{}
 }
 
-func (p ErrPanic) Error() string {
+func (p PanicError) Error() string {
 	return fmt.Sprintf("panic: %v", p.Value)
 }
 
@@ -76,7 +77,7 @@ func (eg *Group) processJobs() {
 	defer func() {
 		if !finished {
 			if p := recover(); p != nil {
-				eg.err(ErrPanic{p})
+				eg.err(PanicError{p})
 			} else {
 				eg.err(ErrGoexit)
 			}

batch/egpool/egpool_test.go

@@ -1,11 +1,12 @@
-// Copyright 2021 Molecula Corp. All rights reserved.
+// Copyright 2022 Molecula Corp. (DBA FeatureBase).
+// SPDX-License-Identifier: Apache-2.0
 package egpool_test
 
 import (
 	"errors"
 	"testing"
 
-	"github.com/molecula/featurebase/v3/client/egpool"
+	"github.com/featurebasedb/featurebase/v3/batch/egpool"
 )
 
 func TestEGPool(t *testing.T) {

batch/error.go

@@ -0,0 +1,8 @@
+package batch
+
+import "github.com/pkg/errors"
+
+// Predefined batch-related errors.
+var (
+	ErrPreconditionFailed = errors.New("Precondition failed")
+)

batch/metrics.go

@@ -0,0 +1,3 @@
+// Copyright 2022 Molecula Corp. (DBA FeatureBase).
+// SPDX-License-Identifier: Apache-2.0
+package batch

batch/testdata/README.md

@@ -0,0 +1,3 @@
+# testdata
+
+This directory is used in CI tests. I think.

batch/wait.sh

@@ -0,0 +1,26 @@
+#!/bin/sh
+
+name=$1
+shift
+
+_start_ts=$(date +%s)
+elapsed=0
+timeout=120
+while :
+do
+    $@ > /dev/null
+    _ret=$?
+    _end_ts=$(date +%s)
+    if [ $_ret -eq 0 ]; then
+        echo "$name is available after $((_end_ts - _start_ts)) seconds."
+        break
+    else
+        echo "Waiting for $name after $((_end_ts - _start_ts)) seconds."
+    fi
+    sleep 1s
+    elapsed=$((elapsed+1))
+    if [ $elapsed -ge $timeout ]; then
+	exit 110
+    fi
+done
+set -ex

broadcast.go

@@ -1,10 +1,11 @@
-// Copyright 2021 Molecula Corp. All rights reserved.
+// Copyright 2022 Molecula Corp. (DBA FeatureBase).
+// SPDX-License-Identifier: Apache-2.0
 package pilosa
 
 import (
 	"fmt"
 
-	"github.com/molecula/featurebase/v3/topology"
+	"github.com/featurebasedb/featurebase/v3/disco"
 	"github.com/pkg/errors"
 )
 
@@ -29,7 +30,7 @@ func (*nopSerializer) Unmarshal([]byte, Message) error { return nil }
 type broadcaster interface {
 	SendSync(Message) error
 	SendAsync(Message) error
-	SendTo(*topology.Node, Message) error
+	SendTo(*disco.Node, Message) error
 }
 
 // Message is the interface implemented by all core pilosa types which can be serialized to messages.
@@ -48,7 +49,7 @@ func (nopBroadcaster) SendSync(Message) error { return nil }
 func (nopBroadcaster) SendAsync(Message) error { return nil }
 
 // SendTo is a no-op implementation of Broadcaster SendTo method.
-func (nopBroadcaster) SendTo(*topology.Node, Message) error { return nil }
+func (nopBroadcaster) SendTo(*disco.Node, Message) error { return nil }
 
 // Broadcast message types.
 const (
@@ -60,16 +61,18 @@ const (
 	messageTypeCreateView
 	messageTypeDeleteView
 	messageTypeClusterStatus
-	messageTypeResizeInstruction
-	messageTypeResizeInstructionComplete
+	messageTypeUNUSED0 // used to be ResizeInstruction
+	messageTypeUNUSED1 // used to be ResizeInstructionComplete
 	messageTypeNodeState
 	messageTypeRecalculateCaches
 	messageTypeLoadSchemaMessage
 	messageTypeNodeEvent
 	messageTypeNodeStatus
 	messageTypeTransaction
-	messageTypeResizeNodeMessage
-	messageTypeResizeAbortMessage
+	messageTypeUNUSED2 // used to be ResizeNodeMessage
+	messageTypeUNUSED3 // used to be ResizeAbortMessage
+	messageTypeUpdateField
+	messageTypeDeleteDataframe
 )
 
 // MarshalInternalMessage serializes the pilosa message and adds pilosa internal
@@ -101,10 +104,6 @@ func getMessage(typ byte) Message {
 		return &DeleteViewMessage{}
 	case messageTypeClusterStatus:
 		return &ClusterStatus{}
-	case messageTypeResizeInstruction:
-		return &ResizeInstruction{}
-	case messageTypeResizeInstructionComplete:
-		return &ResizeInstructionComplete{}
 	case messageTypeNodeState:
 		return &NodeStateMessage{}
 	case messageTypeRecalculateCaches:
@@ -117,10 +116,10 @@ func getMessage(typ byte) Message {
 		return &NodeStatus{}
 	case messageTypeTransaction:
 		return &TransactionMessage{}
-	case messageTypeResizeNodeMessage:
-		return &ResizeNodeMessage{}
-	case messageTypeResizeAbortMessage:
-		return &ResizeAbortMessage{}
+	case messageTypeUpdateField:
+		return &UpdateFieldMessage{}
+	case messageTypeDeleteDataframe:
+		return &DeleteDataframeMessage{}
 	default:
 		panic(fmt.Sprintf("unknown message type %d", typ))
 	}
@@ -144,10 +143,6 @@ func getMessageType(m Message) byte {
 		return messageTypeDeleteView
 	case *ClusterStatus:
 		return messageTypeClusterStatus
-	case *ResizeInstruction:
-		return messageTypeResizeInstruction
-	case *ResizeInstructionComplete:
-		return messageTypeResizeInstructionComplete
 	case *NodeStateMessage:
 		return messageTypeNodeState
 	case *RecalculateCaches:
@@ -160,10 +155,10 @@ func getMessageType(m Message) byte {
 		return messageTypeNodeStatus
 	case *TransactionMessage:
 		return messageTypeTransaction
-	case *ResizeNodeMessage:
-		return messageTypeResizeNodeMessage
-	case *ResizeAbortMessage:
-		return messageTypeResizeAbortMessage
+	case *UpdateFieldMessage:
+		return messageTypeUpdateField
+	case *DeleteDataframeMessage:
+		return messageTypeDeleteDataframe
 	default:
 		panic(fmt.Sprintf("don't have type for message %#v", m))
 	}

bsi.go

@@ -1,20 +1,21 @@
-// Copyright 2021 Molecula Corp. All rights reserved.
+// Copyright 2022 Molecula Corp. (DBA FeatureBase).
+// SPDX-License-Identifier: Apache-2.0
 package pilosa
 
 import (
 	"math/bits"
 
-	"github.com/molecula/featurebase/v3/roaring"
+	"github.com/featurebasedb/featurebase/v3/roaring"
 )
 
-// bsiData contains BSI-structured data.
-type bsiData []*Row
+// BSIData contains BSI-structured data.
+type BSIData []*Row
 
-// pivotDescending loops over nonzero BSI values in descending order.
+// PivotDescending loops over nonzero BSI values in descending order.
 // For each value, the provided function is called with the value and a slice of the associated columns.
 // If limit or offset are not-nil, they will be applied.
 // Applying a limit or offset may modify the pointed-to value.
-func (bsi bsiData) pivotDescending(filter *Row, branch uint64, limit, offset *uint64, fn func(uint64, ...uint64)) {
+func (bsi BSIData) PivotDescending(filter *Row, branch uint64, limit, offset *uint64, fn func(uint64, ...uint64)) {
 	// This "pivot" algorithm works by treating the BSI data as a tree.
 	// Each branch of this tree corresponds to a power-of-2-sized range of BSI values.
 	// Each range is subdivided into 2 ranges of half size, which form lower branches.
@@ -55,8 +56,8 @@ func (bsi bsiData) pivotDescending(filter *Row, branch uint64, limit, offset *ui
 		upperBranch, lowerBranch := branch|(1<<uint(len(bsi)-1)), branch
 		splitBit := bsi[len(bsi)-1]
 		lowerBits := bsi[:len(bsi)-1]
-		lowerBits.pivotDescending(filter.Intersect(splitBit), upperBranch, limit, offset, fn)
-		lowerBits.pivotDescending(filter.Difference(splitBit), lowerBranch, limit, offset, fn)
+		lowerBits.PivotDescending(filter.Intersect(splitBit), upperBranch, limit, offset, fn)
+		lowerBits.PivotDescending(filter.Difference(splitBit), lowerBranch, limit, offset, fn)
 	}
 }
 
@@ -68,7 +69,7 @@ func (bsi bsiData) pivotDescending(filter *Row, branch uint64, limit, offset *ui
 // - TopN on int
 func (bsi bsiData) distribution(filter *Row) bsiData {
 	var dist bsiData
-	bsi.pivotDescending(filter, 0, nil, nil, func(count uint64, values ...uint64) {
+	bsi.PivotDescending(filter, 0, nil, nil, func(count uint64, values ...uint64) {
 		dist.insert(count, uint64(len(values)))
 	})
 	return dist
@@ -77,20 +78,20 @@ func (bsi bsiData) distribution(filter *Row) bsiData {
 
 var placeholderBitmap = roaring.NewBitmap()
 
-// addBSI adds two BSI bitmaps together.
+// AddBSI adds two BSI bitmaps together.
 // It does not handle sign and has no concept of overflow.
-func addBSI(x, y bsiData) bsiData {
+func AddBSI(x, y BSIData) BSIData {
 	// Accumulate row segments.
-	segments := make([][]rowSegment, len(x)+len(y))
+	segments := make([][]RowSegment, len(x)+len(y))
 	xsegs, ysegs := segments[:len(x)], segments[len(x):]
 	for i, r := range x {
-		xsegs[i] = r.segments
+		xsegs[i] = r.Segments
 	}
 	for i, r := range y {
-		ysegs[i] = r.segments
+		ysegs[i] = r.Segments
 	}
 
-	var dst bsiData
+	var dst BSIData
 	var xbitmaps, ybitmaps []*roaring.Bitmap
 	for {
 		// Find the next shard.
@@ -161,7 +162,7 @@ func addBSI(x, y bsiData) bsiData {
 			for len(dst) <= i {
 				dst = append(dst, NewRow())
 			}
-			dst[i].segments = append(dst[i].segments, rowSegment{
+			dst[i].Segments = append(dst[i].Segments, RowSegment{
 				shard:    next,
 				writable: true,
 				data:     b,
@@ -272,10 +273,10 @@ func (b *bsiBuilder) Insert(col, val uint64) {
 
 // Build BSI data.
 // This resets the builder.
-func (b *bsiBuilder) Build() bsiData {
+func (b *bsiBuilder) Build() BSIData {
 	builders := *b
 	*b = builders[:0]
-	rows := make(bsiData, len(builders))
+	rows := make(BSIData, len(builders))
 	for i := range builders {
 		rows[i] = builders[i].Build()
 	}

bsi_test.go

@@ -1,4 +1,5 @@
-// Copyright 2021 Molecula Corp. All rights reserved.
+// Copyright 2022 Molecula Corp. (DBA FeatureBase).
+// SPDX-License-Identifier: Apache-2.0
 package pilosa
 
 import (
@@ -68,11 +69,11 @@ func TestBSIAdd(t *testing.T) {
 				builderB.Insert(uint64(id), vb)
 			}
 			dataA, dataB := builderA.Build(), builderB.Build()
-			dataC := addBSI(dataA, dataB)
+			dataC := AddBSI(dataA, dataB)
 
 			// build results from added bsiData; results[i] should hold a[i]+b[i]
 			results := make([]uint64, len(a))
-			dataC.pivotDescending(NewRow().Union(dataC...), 0, nil, nil, func(count uint64, ids ...uint64) {
+			dataC.PivotDescending(NewRow().Union(dataC...), 0, nil, nil, func(count uint64, ids ...uint64) {
 				for _, id := range ids {
 					results[idToIndex[int(id)]] = count
 				}
@@ -140,10 +141,10 @@ func TestBSIAddCases(t *testing.T) {
 			}
 
 			dataA, dataB := builderA.Build(), builderB.Build()
-			dataC := addBSI(dataA, dataB)
+			dataC := AddBSI(dataA, dataB)
 			// maps id to count
 			results := make(map[uint64]uint64)
-			dataC.pivotDescending(NewRow().Union(dataC...), 0, nil, nil, func(count uint64, ids ...uint64) {
+			dataC.PivotDescending(NewRow().Union(dataC...), 0, nil, nil, func(count uint64, ids ...uint64) {
 				for _, id := range ids {
 					results[id] = count
 				}

buffer/filebuffer.go

@@ -0,0 +1,110 @@
+package buffer
+
+import (
+	"bytes"
+	"io"
+	"io/ioutil"
+	"os"
+	"sync"
+)
+
+// NewFileBuffer returns a file buffer which will use an in-memory buffer, until `max` bytes have been written, at which point it will write the contents of memory to a file, and continue writing future data to the file.
+// The file will be written to `temp` directory. The buffer fulfills the io.Reader and io.Writer interface
+func NewFileBuffer(max int, temp string) *FileBuffer {
+	return &FileBuffer{max: max, tempDir: temp}
+}
+
+type FileBuffer struct {
+	max     int
+	buf     bytes.Buffer
+	file    *os.File
+	tempDir string
+	reading bool
+	files   []*os.File
+	mu      sync.Mutex
+}
+
+func (fb *FileBuffer) Write(p []byte) (n int, err error) {
+	if fb.reading {
+		panic("cannot write after read")
+	}
+	if fb.file != nil {
+		return fb.file.Write(p)
+	}
+	n, err = fb.buf.Write(p)
+	if err != nil {
+		return
+	}
+	if fb.buf.Len() > fb.max {
+		fb.file, err = ioutil.TempFile(fb.tempDir, "filebuffer-")
+		if err != nil {
+			return
+		}
+		_, err = io.Copy(fb.file, &fb.buf)
+		fb.buf.Reset()
+	}
+	return
+}
+
+func (fb *FileBuffer) Len() (int64, error) {
+	if fb.file == nil {
+		return int64(fb.buf.Len()), nil
+	}
+	fi, err := fb.file.Stat()
+	if err != nil {
+		return 0, err
+	}
+
+	return fi.Size(), nil
+}
+
+func (fb *FileBuffer) Read(p []byte) (n int, err error) {
+	if fb.file != nil {
+		if !fb.reading {
+			fb.reading = true
+			_, err = fb.file.Seek(0, 0)
+			if err != nil {
+				return
+			}
+		}
+		return fb.file.Read(p)
+	}
+	fb.reading = true
+	return fb.buf.Read(p)
+}
+
+func (fb *FileBuffer) Close() error {
+	if fb.file != nil {
+		name := fb.file.Name()
+		if err := fb.file.Close(); err != nil {
+			return err
+		}
+		for _, f := range fb.files {
+			f.Close()
+		}
+		fb.files = fb.files[:0]
+		fb.file = nil
+		return os.Remove(name)
+	}
+	return nil
+}
+
+func (fb *FileBuffer) Reset() error {
+	fb.mu.Lock()
+	defer fb.mu.Unlock()
+	fb.reading = false
+	fb.buf.Reset()
+	return fb.Close()
+}
+
+func (fb *FileBuffer) NewReader() (io.Reader, error) {
+	fb.mu.Lock()
+	defer fb.mu.Unlock()
+	fb.reading = true
+	if fb.file == nil {
+		return bytes.NewReader(fb.buf.Bytes()), nil
+	}
+	f, err := os.OpenFile(fb.file.Name(), os.O_RDONLY, 0)
+	fb.files = append(fb.files, f)
+	return f, err
+}

bufferpool/bufferpool.go

@@ -0,0 +1,262 @@
+package bufferpool
+
+import (
+	"errors"
+	"fmt"
+	"sync"
+)
+
+// FrameID is the type for frame id
+type FrameID int
+
+// PageID is the type for page id
+type PageID int
+
+var pageSyncPool = sync.Pool{
+	New: func() any {
+		pg := new(Page)
+		pg.id = PageID(INVALID_PAGE)
+		pg.isDirty = false
+		pg.pinCount = 0
+		return pg
+	},
+}
+
+// BufferPool represents a buffer pool of pages
+type BufferPool struct {
+	// the underlying storage
+	diskManager DiskManager
+	// the actual pages in the buffer pool
+	pages []*Page
+	// the replacer that will elect replacements when buffer pool is full
+	replacer *ClockReplacer
+	// the list of free frames
+	freeList []FrameID
+	// the map of frames to page ids to frame ids
+	// frame ids are the offset into pages
+	// if you ask the pool for page 673, this will know at
+	// what offset in pages page 673 will exist
+	pageTable map[PageID]FrameID
+}
+
+// TODO(pok) implement a lazy writer
+//  * if free list is 'low' then
+//		* increase size of cache if there is physical memory available
+//		* write out old pages and boot them from the cache to increase free list
+
+// TODO(pok) implement a checkpoint that scans the pool and writes out dirty pages every
+// minute or so
+
+// NewBufferPool returns a buffer pool
+func NewBufferPool(maxSize int, diskManager DiskManager) *BufferPool {
+	freeList := make([]FrameID, 0)
+	pages := make([]*Page, maxSize)
+	for i := 0; i < maxSize; i++ {
+		frameNumber := FrameID(i)
+		freeList = append(freeList, frameNumber)
+	}
+	clockReplacer := NewClockReplacer(maxSize)
+	return &BufferPool{
+		diskManager: diskManager,
+		pages:       pages,
+		replacer:    clockReplacer,
+		freeList:    freeList,
+		pageTable:   make(map[PageID]FrameID),
+	}
+}
+
+// Dumps all the pages in the buffer pool
+func (b *BufferPool) Dump() {
+	fmt.Println()
+	fmt.Printf("------------------------------------------------------------------------------------------\n")
+	fmt.Printf("BUFFER POOL\n")
+	for _, p := range b.pages {
+		if p != nil {
+			p.Dump("")
+		}
+	}
+	fmt.Printf("------------------------------------------------------------------------------------------\n")
+	fmt.Println()
+}
+
+// FetchPage fetches the requested page from the buffer pool.
+func (b *BufferPool) FetchPage(pageID PageID) (*Page, error) {
+	// if it is in buffer pool already then just return it
+	if frameID, ok := b.pageTable[pageID]; ok {
+		page := b.pages[frameID]
+		page.pinCount++
+		b.replacer.Pin(frameID)
+		return page, nil
+	}
+
+	// not in the buffer pool so try the free list or
+	// the replacer will vote a page off the island
+	frameID, isFromFreeList, err := b.getFrameID()
+	if err != nil {
+		return nil, err
+	}
+
+	if !isFromFreeList {
+		// if it didn't come from the freelist then
+		// remove page from current frame, writing it out if dirty
+		currentPage := b.pages[frameID]
+		if currentPage != nil {
+			if currentPage.isDirty {
+				b.diskManager.WritePage(currentPage)
+			}
+
+			delete(b.pageTable, currentPage.id)
+		}
+	}
+
+	// if we got to here, sorry, have to do an I/O
+	page, err := b.diskManager.ReadPage(pageID)
+	if err != nil {
+		return nil, err
+	}
+	page.pinCount = 1
+	b.pageTable[pageID] = frameID
+	pageSyncPool.Put(b.pages[frameID])
+	b.pages[frameID] = page
+	b.replacer.Pin(frameID)
+
+	return page, nil
+}
+
+// UnpinPage unpins the target page from the buffer pool
+func (b *BufferPool) UnpinPage(pageID PageID) error {
+	if frameID, ok := b.pageTable[pageID]; ok {
+		page := b.pages[frameID]
+		page.DecPinCount()
+
+		if page.pinCount <= 0 {
+			b.replacer.Unpin(frameID)
+		}
+		return nil
+	}
+
+	return errors.New("could not find page")
+}
+
+// FlushPage Flushes the target page to disk
+func (b *BufferPool) FlushPage(pageID PageID) bool {
+	if frameID, ok := b.pageTable[pageID]; ok {
+		page := b.pages[frameID]
+		page.DecPinCount()
+
+		b.diskManager.WritePage(page)
+		page.isDirty = false
+
+		return true
+	}
+	return false
+}
+
+// NewPage allocates a new page in the buffer pool with the disk manager help
+func (b *BufferPool) NewPage() (*Page, error) {
+	// get a free frame
+	frameID, isFromFreeList, err := b.getFrameID()
+	if err != nil {
+		return nil, err
+	}
+
+	if !isFromFreeList {
+		// remove page from current frame
+		currentPage := b.pages[frameID]
+		if currentPage != nil {
+			if currentPage.isDirty {
+				b.diskManager.WritePage(currentPage)
+			}
+
+			delete(b.pageTable, currentPage.id)
+		}
+	}
+
+	// allocates new page
+	pageID, err := b.diskManager.AllocatePage()
+	if err != nil {
+		return nil, err
+	}
+	page := &Page{pageID, 1, false, [PAGE_SIZE]byte{}}
+	page.WritePageNumber(int32(pageID))
+	page.WriteFreeSpaceOffset(int16(PAGE_SIZE))
+	page.WriteNextPointer(int32(INVALID_PAGE))
+	page.WritePrevPointer(int32(INVALID_PAGE))
+
+	// update the frame table
+	b.pageTable[pageID] = frameID
+	pageSyncPool.Put(b.pages[frameID])
+	b.pages[frameID] = page
+
+	return page, nil
+}
+
+// ScratchPage returns a page outside the buffer pool - do not use if you intend the page
+// to be in the buffer pool (use NewPage() for that)
+// ScratchPage is intended to be used in cases where you need the Page primitives
+// and will copy the scratch page back over a real page later
+func (b *BufferPool) ScratchPage() *Page {
+	page := &Page{
+		id:       PageID(INVALID_PAGE),
+		pinCount: 0,
+		isDirty:  false,
+		data:     [PAGE_SIZE]byte{},
+	}
+	page.WritePageNumber(int32(INVALID_PAGE))
+	page.WriteFreeSpaceOffset(int16(PAGE_SIZE))
+	page.WriteNextPointer(int32(INVALID_PAGE))
+	page.WritePrevPointer(int32(INVALID_PAGE))
+	return page
+}
+
+// DeletePage deletes a page from the buffer pool
+func (b *BufferPool) DeletePage(pageID PageID) error {
+	var frameID FrameID
+	var ok bool
+	if frameID, ok = b.pageTable[pageID]; !ok {
+		return nil
+	}
+
+	page := b.pages[frameID]
+
+	if page.pinCount > 0 {
+		return errors.New("pin count greater than 0")
+	}
+	delete(b.pageTable, page.id)
+	b.replacer.Pin(frameID)
+	b.diskManager.DeallocatePage(pageID)
+
+	b.freeList = append(b.freeList, frameID)
+
+	return nil
+}
+
+// FlushAllpages flushes all the pages in the buffer pool to disk
+// Yeah, never call this unless you know what you are doing
+func (b *BufferPool) FlushAllpages() {
+	for pageID := range b.pageTable {
+		b.FlushPage(pageID)
+	}
+}
+
+func (b *BufferPool) getFrameID() (FrameID, bool, error) {
+	if len(b.freeList) > 0 {
+		frameID, newFreeList := b.freeList[0], b.freeList[1:]
+		b.freeList = newFreeList
+		return frameID, true, nil
+	}
+
+	victim, err := b.replacer.Victim()
+	return victim, false, err
+}
+
+// OnDiskSize exposes the on disk size of the backing store
+// behind this buffer pool
+func (b *BufferPool) OnDiskSize() int64 {
+	return b.diskManager.FileSize()
+}
+
+// Close closes the buffer pool
+func (b *BufferPool) Close() {
+	b.diskManager.Close()
+}

bufferpool/circularlist.go

@@ -0,0 +1,93 @@
+package bufferpool
+
+import (
+	"errors"
+)
+
+type circularListNode struct {
+	key   interface{}
+	value interface{}
+	next  *circularListNode
+	prev  *circularListNode
+}
+
+type circularList struct {
+	head     *circularListNode
+	tail     *circularListNode
+	size     int
+	capacity int
+}
+
+func newCircularList(maxSize int) *circularList {
+	return &circularList{nil, nil, 0, maxSize}
+}
+
+func (c *circularList) find(key interface{}) *circularListNode {
+	ptr := c.head
+	for i := 0; i < c.size; i++ {
+		if ptr.key == key {
+			return ptr
+		}
+		ptr = ptr.next
+	}
+	return nil
+}
+
+func (c *circularList) hasKey(key interface{}) bool {
+	return c.find(key) != nil
+}
+
+func (c *circularList) insert(key interface{}, value interface{}) error {
+	if c.size == c.capacity {
+		return errors.New("list is full")
+	}
+	newNode := &circularListNode{key, value, nil, nil}
+	if c.size == 0 {
+		newNode.next = newNode
+		newNode.prev = newNode
+		c.head = newNode
+		c.tail = newNode
+		c.size++
+		return nil
+	}
+
+	node := c.find(key)
+	if node != nil {
+		node.value = value
+		return nil
+	}
+
+	newNode.next = c.head
+	newNode.prev = c.tail
+	c.tail.next = newNode
+	if c.head == c.tail {
+		c.head.next = newNode
+	}
+	c.tail = newNode
+	c.head.prev = c.tail
+
+	c.size++
+	return nil
+}
+
+func (c *circularList) remove(key interface{}) {
+	node := c.find(key)
+	if node == nil {
+		return
+	}
+	if c.size == 1 {
+		c.head = nil
+		c.tail = nil
+		c.size--
+		return
+	}
+	if node == c.head {
+		c.head = c.head.next
+	}
+	if node == c.tail {
+		c.tail = c.tail.prev
+	}
+	node.next.prev = node.prev
+	node.prev.next = node.next
+	c.size--
+}

bufferpool/clockreplacer.go

@@ -0,0 +1,64 @@
+package bufferpool
+
+import "errors"
+
+// ClockReplacer implements a clock replacer algorithm
+type ClockReplacer struct {
+	cList     *circularList
+	clockHand **circularListNode
+}
+
+// NewClockReplacer instantiates a new clock replacer
+func NewClockReplacer(poolSize int) *ClockReplacer {
+	cList := newCircularList(poolSize)
+	return &ClockReplacer{cList, &cList.head}
+}
+
+// Victim removes the victim frame as defined by the replacement policy
+func (c *ClockReplacer) Victim() (FrameID, error) {
+	if c.cList.size == 0 {
+		return FrameID(INVALID_PAGE), errors.New("no victims available")
+	}
+	var victimFrameID FrameID
+	currentNode := (*c.clockHand)
+	for {
+
+		if currentNode.value.(bool) {
+			currentNode.value = false
+			c.clockHand = &currentNode.next
+		} else {
+			frameID := currentNode.key.(FrameID)
+			victimFrameID = frameID
+			c.clockHand = &currentNode.next
+			c.cList.remove(currentNode.key)
+			return victimFrameID, nil
+		}
+	}
+}
+
+// Unpin unpins a frame, indicating that it can now be victimized
+func (c *ClockReplacer) Unpin(id FrameID) {
+	if !c.cList.hasKey(id) {
+		c.cList.insert(id, true)
+		if c.cList.size == 1 {
+			c.clockHand = &c.cList.head
+		}
+	}
+}
+
+// Pin pins a frame, indicating that it should not be victimized until it is unpinned
+func (c *ClockReplacer) Pin(id FrameID) {
+	node := c.cList.find(id)
+	if node == nil {
+		return
+	}
+	if (*c.clockHand) == node {
+		c.clockHand = &(*c.clockHand).next
+	}
+	c.cList.remove(id)
+}
+
+// Size returns the size of the clock
+func (c *ClockReplacer) Size() int {
+	return c.cList.size
+}

bufferpool/diskmanager.go

@@ -0,0 +1,21 @@
+package bufferpool
+
+// DiskManager is responsible for interacting with disk
+type DiskManager interface {
+	// reads a page from the disk
+	ReadPage(PageID) (*Page, error)
+	// writes a page to the disk
+	WritePage(*Page) error
+
+	// allocates a page
+	AllocatePage() (PageID, error)
+
+	// deallocates a page
+	DeallocatePage(PageID) error
+
+	// returns on disk file size
+	FileSize() int64
+
+	// closes and does any clean up
+	Close()
+}

bufferpool/inmemdiskmanager.go

@@ -0,0 +1,162 @@
+package bufferpool
+
+import (
+	"errors"
+	"fmt"
+	"os"
+
+	uuid "github.com/satori/go.uuid"
+)
+
+// InMemDiskSpillingDiskManager is a memory implementation for a DiskManager interface
+// that can spill to disk when a threshold is reached
+type InMemDiskSpillingDiskManager struct {
+	// tracks the number of pages
+	numPages int
+
+	onDiskPages int
+
+	// tracks the number of pages we can consume before spilling
+	thresholdPages int
+	hasSpilled     *struct{}
+	fd             *os.File
+
+	// the data buffer
+	data []byte
+}
+
+// NewInMemDiskSpillingDiskManager returns a in-memory version of disk manager
+func NewInMemDiskSpillingDiskManager(thresholdPages int) *InMemDiskSpillingDiskManager {
+	dm := &InMemDiskSpillingDiskManager{
+		numPages:       0,
+		thresholdPages: thresholdPages,
+		data:           make([]byte, 0),
+	}
+	return dm
+}
+
+// ReadPage reads a page from pages
+func (d *InMemDiskSpillingDiskManager) ReadPage(pageID PageID) (*Page, error) {
+	// check we're not asking for page out of range
+	if pageID < 0 || int(pageID) >= d.numPages {
+		return nil, errors.New("page not found")
+	}
+	// check that the offset is within range
+	offset := int(pageID) * PAGE_SIZE
+
+	var page = pageSyncPool.Get().(*Page)
+	// we have to do this stupid check because if -cpuprofile is set for go test, this
+	// the previous line return a weird nil-ish thing...
+	if page == (*Page)(nil) {
+		page = pageSyncPool.New().(*Page)
+	}
+	page.id = pageID
+
+	// do the read
+	if d.hasSpilled == nil {
+		if offset+PAGE_SIZE > len(d.data) {
+			return nil, errors.New("offset out of range")
+		}
+		b := copy(page.data[:], d.data[offset:offset+PAGE_SIZE])
+		fmt.Printf("bytes read: %d", b)
+	} else {
+		var err error
+		if offset+PAGE_SIZE > d.numPages*PAGE_SIZE {
+			return nil, errors.New("offset out of range")
+		}
+		_, err = d.fd.ReadAt(page.data[:], int64(offset))
+		if err != nil {
+			return nil, err
+		}
+	}
+	return page, nil
+}
+
+// WritePage writes a page in memory to pages
+func (d *InMemDiskSpillingDiskManager) WritePage(page *Page) error {
+	// make sure the offset is sensible
+	offset := int(page.ID()) * PAGE_SIZE
+	// do the write
+	if d.hasSpilled == nil {
+		if offset+PAGE_SIZE > len(d.data) {
+			return errors.New("offset out of range")
+		}
+		copy(d.data[offset:], page.data[:])
+	} else {
+		var err error
+		if offset+PAGE_SIZE > d.numPages*PAGE_SIZE {
+			return errors.New("offset out of range")
+		}
+		_, err = d.fd.WriteAt(page.data[:], int64(offset))
+		if err != nil {
+			return err
+		}
+		// err = d.fd.Sync()
+		// if err != nil {
+		// 	return err
+		// }
+	}
+	return nil
+}
+
+// AllocatePage allocates a page and returns the page number
+func (d *InMemDiskSpillingDiskManager) AllocatePage() (PageID, error) {
+	d.numPages = d.numPages + 1
+	pageID := PageID(d.numPages - 1)
+
+	if d.hasSpilled == nil {
+		// we have not spilled (yet), so make storage bigger
+		newData := make([]byte, PAGE_SIZE)
+		d.data = append(d.data, newData...)
+
+		// check to see if we need to spill
+		if d.numPages > d.thresholdPages {
+			fileUUID, err := uuid.NewV4()
+			if err != nil {
+				return PageID(INVALID_PAGE), err
+			}
+			// TODO(pok) we should try to tell the OS not to cache this file
+			d.fd, err = os.CreateTemp("", fmt.Sprintf("fb-ehash-%s", fileUUID.String()))
+			if err != nil {
+				return PageID(INVALID_PAGE), err
+			}
+			_, err = d.fd.WriteAt(d.data, 0)
+			if err != nil {
+				return PageID(INVALID_PAGE), err
+			}
+			d.data = []byte{}
+			d.hasSpilled = &struct{}{}
+		}
+	} else {
+		if d.numPages >= d.onDiskPages {
+			// grow the file by a chunk - 512 pages
+			d.onDiskPages += 512
+			var err error
+			size := int64(d.onDiskPages * PAGE_SIZE)
+			_, err = d.fd.WriteAt([]byte{0}, size-1)
+			if err != nil {
+				return PageID(INVALID_PAGE), err
+			}
+		}
+	}
+
+	return pageID, nil
+}
+
+// DeallocatePage removes page from disk
+func (d *InMemDiskSpillingDiskManager) DeallocatePage(pageID PageID) error {
+	// nothing to do right now
+	return nil
+}
+
+func (d *InMemDiskSpillingDiskManager) FileSize() int64 {
+	return int64(len(d.data))
+}
+
+func (d *InMemDiskSpillingDiskManager) Close() {
+	// close and delete the file if we spilled
+	if d.fd != nil {
+		_ = d.fd.Close()
+		os.Remove(d.fd.Name())
+	}
+}

bufferpool/page.go

@@ -0,0 +1,371 @@
+package bufferpool
+
+import (
+	"encoding/binary"
+	"errors"
+	"fmt"
+)
+
+const PAGE_SIZE int = 8192
+
+const INVALID_PAGE int = -1
+
+const PAGE_TYPE_BTREE_INTERNAL = 10
+const PAGE_TYPE_BTREE_LEAF = 11
+const PAGE_TYPE_HASH_TABLE = 12
+
+// PAGE
+// page size 8192 bytes
+// byte aligned, big endian
+
+// |====================================================|
+// | offset | length        |                           |
+// |----------------------------------------------------|
+// | header                                             |
+// |====================================================|
+// | 0      | 4             |  pageNumber (int32)       |
+// | 4      | 2             |  pageType (int16)         |
+// | 6      | 2             |  slotCount (int16)        |
+// | 8      | 2             |  localDepth (int16)       |
+// | 10     | 2             |  freeSpaceOffset (int16)  |
+// | 12     | 4             |  prevPointer (int32)      |
+// | 16     | 4             |  nextPointer (int32)      |
+// |====================================================|
+// | <start of slot array 1..slotCount>                 |
+// |----------------------------------------------------|
+// | 20     | slotcount     |  slot entry is 2 int16    |
+// |        |  * slotwidth  |   values (payloadOffset,  |
+// |        |  * #slots     |   payloadLength)          |
+// |----------------------------------------------------|
+// | <free space>                                       |
+// |----------------------------------------------------|
+// | <payload starting at freeSpaceOffset>              |
+// |  payload entries are keylength (int16), key bytes, |
+// |  payload length (int32), payload bytes             |
+// |====================================================|
+
+const PAGE_NUMBER_OFFSET = 0        // offset 0, length 4, end 4
+const PAGE_TYPE_OFFSET = 4          // offset 4, length 2, end 6
+const PAGE_SLOT_COUNT_OFFSET = 6    // offset 6, length 2, end 8
+const PAGE_LOCAL_DEPTH_OFFSET = 8   // offset 8, length 2, end 10
+const PAGE_FREE_SPACE_OFFSET = 10   // offset 10, length 2, end 12
+const PAGE_PREV_POINTER_OFFSET = 12 // offset 12, length 4, end 16
+const PAGE_NEXT_POINTER_OFFSET = 16 // offset 16, length 4, end 20
+const PAGE_SLOTS_START_OFFSET = 20  // offset 20
+
+// PAGE_SLOT_LENGTH is the size of the page slot key/value.
+//
+// key offset int16    	//offset 0, length 2, end 2
+// value offset int16	    //offset 2, length 2, end 4
+const PAGE_SLOT_LENGTH = 4
+
+// Page represents a page on disk
+type Page struct {
+	id       PageID
+	pinCount int
+	isDirty  bool
+	data     [PAGE_SIZE]byte
+}
+
+type PageSlot struct {
+	KeyOffset   int16
+	ValueOffset int16
+}
+
+func (s *PageSlot) KeyBytes(page *Page) []byte {
+	offset := s.KeyOffset
+	keyLen := int16(binary.BigEndian.Uint16(page.data[offset:]))
+	offset += 2
+	result := make([]byte, keyLen)
+	copy(result, page.data[offset:offset+keyLen])
+	return result
+}
+
+func (s *PageSlot) KeyAsInt(page *Page) int32 {
+	return int32(binary.BigEndian.Uint32(page.data[s.KeyOffset+2:]))
+}
+
+func (s *PageSlot) ValueBytes(page *Page) []byte {
+	offset := s.ValueOffset
+	valueLen := int32(binary.BigEndian.Uint32(page.data[offset:]))
+	offset += 4
+	result := make([]byte, valueLen)
+	copy(result, page.data[offset:int32(offset)+valueLen])
+	return result
+}
+
+func (s *PageSlot) ValueAsPagePointer(page *Page) int32 {
+	return int32(binary.BigEndian.Uint32(page.data[s.ValueOffset+4:]))
+}
+
+type PageChunk struct {
+	KeyLength int16
+	KeyBytes  []byte
+	// TODO(pok) ValueBytes can be up to int32 long
+	// this requires an overflow page mechanism, that is not implemented
+	// yet, so be aware of this when storing stuff...
+	ValueLength int32
+	ValueBytes  []byte
+}
+
+func (pc *PageChunk) Length() int {
+	return 2 + len(pc.KeyBytes) + 4 + len(pc.ValueBytes)
+}
+
+func (pc *PageChunk) ComputeKeyOffset(pageOffset int) int {
+	return pageOffset
+}
+
+func (pc *PageChunk) ComputeValueOffset(pageOffset int) int {
+	return pageOffset + 2 + len(pc.KeyBytes)
+}
+
+func (p *Page) WritePageNumber(pageNumber int32) {
+	p.id = PageID(pageNumber)
+	binary.BigEndian.PutUint32(p.data[PAGE_NUMBER_OFFSET:], uint32(pageNumber))
+	p.isDirty = true
+}
+
+func (p *Page) ReadPageNumber() int {
+	return int(binary.BigEndian.Uint32(p.data[PAGE_NUMBER_OFFSET:]))
+}
+
+func (p *Page) WritePageType(pageType int16) {
+	binary.BigEndian.PutUint16(p.data[PAGE_TYPE_OFFSET:], uint16(pageType))
+	p.isDirty = true
+}
+
+func (p *Page) ReadPageType() int16 {
+	return int16(binary.BigEndian.Uint16(p.data[PAGE_TYPE_OFFSET:]))
+}
+
+func (p *Page) WriteSlotCount(slotCount int16) {
+	binary.BigEndian.PutUint16(p.data[PAGE_SLOT_COUNT_OFFSET:], uint16(slotCount))
+	p.isDirty = true
+}
+
+func (p *Page) ReadSlotCount() int16 {
+	return int16(binary.BigEndian.Uint16(p.data[PAGE_SLOT_COUNT_OFFSET:]))
+}
+
+func (p *Page) WriteLocalDepth(localDepth int16) {
+	binary.BigEndian.PutUint16(p.data[PAGE_LOCAL_DEPTH_OFFSET:], uint16(localDepth))
+	p.isDirty = true
+}
+
+func (p *Page) ReadLocalDepth() int16 {
+	return int16(binary.BigEndian.Uint16(p.data[PAGE_LOCAL_DEPTH_OFFSET:]))
+}
+
+func (p *Page) ReadSlot(slot int16) PageSlot {
+	offset := PAGE_SLOTS_START_OFFSET + PAGE_SLOT_LENGTH*slot
+	keyOffset := int16(binary.BigEndian.Uint16(p.data[offset:]))
+	offset += 2
+	valueOffset := int16(binary.BigEndian.Uint16(p.data[offset:]))
+	return PageSlot{
+		KeyOffset:   keyOffset,
+		ValueOffset: valueOffset,
+	}
+}
+
+func (p *Page) WriteSlot(slot int16, value PageSlot) {
+	offset := PAGE_SLOTS_START_OFFSET + PAGE_SLOT_LENGTH*slot
+	binary.BigEndian.PutUint16(p.data[offset:], uint16(value.KeyOffset))
+	offset += 2
+	binary.BigEndian.PutUint16(p.data[offset:], uint16(value.ValueOffset))
+}
+
+func (p *Page) WriteFreeSpaceOffset(offset int16) {
+	binary.BigEndian.PutUint16(p.data[PAGE_FREE_SPACE_OFFSET:], uint16(offset))
+	p.isDirty = true
+}
+
+func (p *Page) ReadFreeSpaceOffset() int16 {
+	return int16(binary.BigEndian.Uint16(p.data[PAGE_FREE_SPACE_OFFSET:]))
+}
+
+func (p *Page) WritePrevPointer(prevPointer int32) {
+	binary.BigEndian.PutUint32(p.data[PAGE_PREV_POINTER_OFFSET:], uint32(prevPointer))
+	p.isDirty = true
+}
+
+func (p *Page) ReadPrevPointer() int {
+	return int(binary.BigEndian.Uint32(p.data[PAGE_PREV_POINTER_OFFSET:]))
+}
+
+func (p *Page) WriteNextPointer(nextPointer int32) {
+	binary.BigEndian.PutUint32(p.data[PAGE_NEXT_POINTER_OFFSET:], uint32(nextPointer))
+	p.isDirty = true
+}
+
+func (p *Page) ReadNextPointer() int {
+	return int(binary.BigEndian.Uint32(p.data[PAGE_NEXT_POINTER_OFFSET:]))
+}
+
+func (p *Page) WriteChunk(offset int16, chunk PageChunk) {
+	binary.BigEndian.PutUint16(p.data[offset:], uint16(chunk.KeyLength))
+	offset += 2
+	copy(p.data[offset:], chunk.KeyBytes)
+	offset += int16(len(chunk.KeyBytes))
+	binary.BigEndian.PutUint32(p.data[offset:], uint32(chunk.ValueLength))
+	offset += 4
+	copy(p.data[offset:], chunk.ValueBytes)
+	p.isDirty = true
+}
+
+func (p *Page) ReadChunk(offset int16) PageChunk {
+	keyLen := int16(binary.BigEndian.Uint16(p.data[offset:]))
+	offset += 2
+	keyBytes := make([]byte, keyLen)
+	copy(keyBytes, p.data[offset:offset+keyLen])
+	offset += keyLen
+	valueLen := int32(binary.BigEndian.Uint32(p.data[offset:]))
+	offset += 4
+	valueBytes := make([]byte, valueLen)
+	copy(valueBytes, p.data[offset:int32(offset)+valueLen])
+	return PageChunk{
+		KeyLength:   keyLen,
+		KeyBytes:    keyBytes,
+		ValueLength: valueLen,
+		ValueBytes:  valueBytes,
+	}
+}
+
+func (p *Page) FreeSpace() int16 {
+	freeSpaceOffset := p.ReadFreeSpaceOffset()
+	freespace := freeSpaceOffset - (p.ReadSlotCount()*PAGE_SLOT_LENGTH + PAGE_SLOT_LENGTH + PAGE_SLOTS_START_OFFSET)
+	return freespace
+}
+
+func (p *Page) WriteKeyValueInSlot(slotNumber int16, key []byte, value []byte) error {
+	freeSpaceOffset := p.ReadFreeSpaceOffset()
+
+	// build a chunk
+	chunk := PageChunk{
+		KeyLength:   int16(len(key)),
+		KeyBytes:    key,
+		ValueLength: int32(len(value)),
+		ValueBytes:  value,
+	}
+
+	// compute the new free space offset
+	freeSpaceOffset -= int16(chunk.Length())
+
+	// check we won't blow free space on page
+	slotCount := p.ReadSlotCount()
+	slotEndOffset := slotCount*PAGE_SLOT_LENGTH + PAGE_SLOT_LENGTH + PAGE_SLOTS_START_OFFSET
+
+	// DEBUG!!
+	//fmt.Printf("freeSpaceOffset: %d, slotCount: %d, slotCount*4 + 4 + 20: %d, freeSpace: %d\n", freeSpaceOffset, slotCount, slotEndOffset, freeSpaceOffset-slotEndOffset)
+
+	if freeSpaceOffset-slotEndOffset <= 0 {
+		return errors.New("page is full")
+	}
+
+	keyOffset := chunk.ComputeKeyOffset(int(freeSpaceOffset))
+	valueOffset := chunk.ComputeValueOffset(int(freeSpaceOffset))
+
+	p.WriteChunk(freeSpaceOffset, chunk)
+
+	// update the free space offset
+	p.WriteFreeSpaceOffset(int16(freeSpaceOffset))
+
+	// make a slot
+	slot := PageSlot{
+		KeyOffset:   int16(keyOffset),
+		ValueOffset: int16(valueOffset),
+	}
+	// write the slot
+	p.WriteSlot(slotNumber, slot)
+
+	return nil
+}
+
+func (p *Page) WritePage(page *Page) {
+	// copy everything but pageNumber & pageType
+	offset := PAGE_SLOT_COUNT_OFFSET
+	copy(page.data[offset:], p.data[offset:offset+PAGE_SIZE-offset])
+}
+
+func (p *Page) PinCount() int {
+	return p.pinCount
+}
+
+func (p *Page) ID() PageID {
+	return p.id
+}
+
+func (p *Page) DecPinCount() {
+	if p.pinCount > 0 {
+		p.pinCount--
+	}
+}
+
+type PageSlotIterator struct {
+	page      *Page
+	slotCount int16
+	cursor    int16
+}
+
+func NewPageSlotIterator(page *Page, fromSlot int16) *PageSlotIterator {
+	i := &PageSlotIterator{
+		page:      page,
+		slotCount: page.ReadSlotCount(),
+		cursor:    fromSlot,
+	}
+	return i
+}
+
+func (i *PageSlotIterator) Next() *PageSlot {
+	if i.cursor < i.slotCount {
+		s := i.page.ReadSlot(i.cursor)
+		i.cursor++
+		return &s
+	}
+	return nil
+}
+
+func (i *PageSlotIterator) Cursor() int16 {
+	return i.cursor
+}
+
+func (pg *Page) Dump(label string) {
+	indent := 0
+	if len(label) > 0 {
+		fmt.Printf("%s%s:\n", fmt.Sprintf("%*s", indent, ""), label)
+		indent += 4
+	}
+	pageType := pg.ReadPageType()
+	fmt.Printf("%sPAGE(%d) pageType: %d slotCount: %d, prevPtr: %d, nextPtr: %d\n", fmt.Sprintf("%*s", indent, ""), pg.ID(), pageType, pg.ReadSlotCount(), pg.ReadPrevPointer(), pg.ReadNextPointer())
+	fmt.Printf("%sKEYS: -->\n", fmt.Sprintf("%*s", indent, ""))
+	indent += 4
+
+	// get the keys off the page
+	keys := make([]int, 0)
+	pointers := make([]int, 0)
+	iter := NewPageSlotIterator(pg, 0)
+	for {
+		ps := iter.Next()
+		if ps == nil {
+			break
+		}
+		keys = append(keys, int(ps.KeyAsInt(pg)))
+		if pageType == /*nodeTypeInternal*/ 10 {
+			pointers = append(pointers, int(ps.ValueAsPagePointer(pg)))
+		}
+	}
+
+	if pageType == /*nodeTypeLeaf*/ 11 {
+		for _, key := range keys {
+			fmt.Printf("%s(%d)\n", fmt.Sprintf("%*s", indent, ""), key)
+		}
+	} else {
+		for idx, key := range keys {
+			ptr := pointers[idx]
+			fmt.Printf("%s(%d, %d)\n", fmt.Sprintf("%*s", indent, ""), key, ptr)
+		}
+		ptr := pg.ReadNextPointer()
+		fmt.Printf("%s(-->, %d)\n", fmt.Sprintf("%*s", indent, ""), ptr)
+	}
+
+}

cache.go

@@ -1,4 +1,5 @@
-// Copyright 2021 Molecula Corp. All rights reserved.
+// Copyright 2022 Molecula Corp. (DBA FeatureBase).
+// SPDX-License-Identifier: Apache-2.0
 package pilosa
 
 import (
@@ -10,9 +11,8 @@ import (
 	"sync"
 	"time"
 
-	"github.com/molecula/featurebase/v3/lru"
-	pb "github.com/molecula/featurebase/v3/proto"
-	"github.com/molecula/featurebase/v3/stats"
+	"github.com/featurebasedb/featurebase/v3/lru"
+	pb "github.com/featurebasedb/featurebase/v3/proto"
 	"github.com/pkg/errors"
 )
 
@@ -40,23 +40,24 @@ type cache interface {
 	// Returns an ordered list of the top ranked bitmaps.
 	Top() []bitmapPair
 
-	// SetStats defines the stats client used in the cache.
-	SetStats(s stats.StatsClient)
+	// Clear removes everything from the cache. If possible it should leave allocated structures in place to be reused.
+	Clear()
 }
 
 // lruCache represents a least recently used Cache implementation.
 type lruCache struct {
 	cache  *lru.Cache
 	counts map[uint64]uint64
-	stats  stats.StatsClient
+	// maxEntries is saved to support Clear which recreates the cache.
+	maxEntries uint32
 }
 
 // newLRUCache returns a new instance of LRUCache.
 func newLRUCache(maxEntries uint32) *lruCache {
 	c := &lruCache{
-		cache:  lru.New(int(maxEntries)),
-		counts: make(map[uint64]uint64),
-		stats:  stats.NopStatsClient,
+		cache:      lru.New(int(maxEntries)),
+		counts:     make(map[uint64]uint64),
+		maxEntries: maxEntries,
 	}
 	c.cache.OnEvicted = c.onEvicted
 	return c
@@ -113,9 +114,11 @@ func (c *lruCache) Top() []bitmapPair {
 	return a
 }
 
-// SetStats defines the stats client used in the cache.
-func (c *lruCache) SetStats(s stats.StatsClient) {
-	c.stats = s
+func (c *lruCache) Clear() {
+	for k := range c.counts {
+		delete(c.counts, k)
+	}
+	c.cache = lru.New(int(c.maxEntries))
 }
 
 func (c *lruCache) onEvicted(key lru.Key, _ interface{}) { delete(c.counts, key.(uint64)) }
@@ -125,6 +128,7 @@ var _ cache = &lruCache{}
 
 // rankCache represents a cache with sorted entries.
 type rankCache struct {
+	// TODO why does this have a lock and lruCache doesn't?
 	mu           sync.Mutex
 	entries      map[uint64]uint64
 	rankings     bitmapPairs // cached, ordered list
@@ -143,8 +147,6 @@ type rankCache struct {
 
 	// thresholdValue is the value of the last item in the cache
 	thresholdValue uint64
-
-	stats stats.StatsClient
 }
 
 // NewRankCache returns a new instance of RankCache.
@@ -153,10 +155,24 @@ func NewRankCache(maxEntries uint32) *rankCache {
 		maxEntries:      maxEntries,
 		thresholdBuffer: int(thresholdFactor * float64(maxEntries)),
 		entries:         make(map[uint64]uint64),
-		stats:           stats.NopStatsClient,
 	}
 }
 
+func (c *rankCache) Clear() {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	for k := range c.entries {
+		delete(c.entries, k)
+	}
+	c.rankings = c.rankings[:0]
+	c.rankingsRead = false
+	c.dirty = false
+
+	c.updateN = 0
+	c.updateTime = time.Time{}
+	c.thresholdValue = 0
+}
+
 // Add adds a count to the cache.
 func (c *rankCache) Add(id uint64, n uint64) {
 	c.mu.Lock()
@@ -199,7 +215,7 @@ func (c *rankCache) BulkAdd(id uint64, n uint64) {
 	// as this can take up an upbounded amount of memory. This is especially
 	// true when restoring shards as all rows will be added.
 	if len(c.entries) > int(2*c.maxEntries) {
-		c.stats.Count(MetricRecalculateCache, 1, 1.0)
+		CounterRecalculateCache.Inc()
 		c.recalculate()
 	}
 }
@@ -244,7 +260,7 @@ func (c *rankCache) Invalidate() {
 func (c *rankCache) Recalculate() {
 	c.mu.Lock()
 	defer c.mu.Unlock()
-	c.stats.Count(MetricRecalculateCache, 1, 1.0)
+	CounterRecalculateCache.Inc()
 	c.recalculate()
 }
 
@@ -256,12 +272,12 @@ func (c *rankCache) invalidate() {
 		// This is somewhat necessary for now since recalculation is not cheap.
 		// The cache will remain flagged as dirty and will be recalculated if Top is called.
 		// This may cause unexpected memory growth, so record it in metrics for debugging purposes.
-		c.stats.Count(MetricInvalidateCacheSkipped, 1, 1.0)
+		CounterInvalidateCacheSkipped.Inc()
 		// Ensure that we're marked as dirty even if we weren't otherwise.
 		c.dirty = true
 		return
 	}
-	c.stats.Count(MetricInvalidateCache, 1, 1.0)
+	CounterInvalidateCache.Inc()
 	c.recalculate()
 }
 
@@ -287,7 +303,7 @@ func (c *rankCache) recalculate() {
 
 	// Store the count of the item at the threshold index.
 	length := len(c.rankings)
-	c.stats.Gauge(MetricRankCacheLength, float64(length), 1.0)
+	GaugeRankCacheLength.Set(float64(length))
 
 	var removeItems []bitmapPair // cached, ordered list
 	if length > int(c.maxEntries) {
@@ -303,7 +319,7 @@ func (c *rankCache) recalculate() {
 
 	// If size is larger than the threshold then trim it.
 	if len(c.entries) > c.thresholdBuffer {
-		c.stats.Count(MetricCacheThresholdReached, 1, 1.0)
+		CounterCacheThresholdReached.Inc()
 		for _, pair := range removeItems {
 			delete(c.entries, pair.ID)
 		}
@@ -313,11 +329,6 @@ func (c *rankCache) recalculate() {
 	c.dirty = false
 }
 
-// SetStats defines the stats client used in the cache.
-func (c *rankCache) SetStats(s stats.StatsClient) {
-	c.stats = s
-}
-
 // Top returns an ordered list of pairs.
 func (c *rankCache) Top() []bitmapPair {
 	c.mu.Lock()
@@ -325,7 +336,7 @@ func (c *rankCache) Top() []bitmapPair {
 
 	if c.dirty {
 		// The cache is dirty, so we need to recalculate it to get a consistent view.
-		c.stats.Count(MetricReadDirtyCache, 1, 1.0)
+		CounterReadDirtyCache.Inc()
 		c.recalculate()
 	}
 
@@ -576,24 +587,21 @@ func (p uint64Slice) Len() int           { return len(p) }
 func (p uint64Slice) Less(i, j int) bool { return p[i] < p[j] }
 
 // nopCache represents a no-op Cache implementation.
-type nopCache struct {
-	stats stats.StatsClient
-}
+type nopCache struct{}
 
 // Ensure NopCache implements Cache.
-var globalNopCache cache = nopCache{
-	stats: stats.NopStatsClient,
-}
+var globalNopCache cache = nopCache{}
 
 func (c nopCache) Add(uint64, uint64)     {}
 func (c nopCache) BulkAdd(uint64, uint64) {}
 func (c nopCache) Get(uint64) uint64      { return 0 }
 func (c nopCache) IDs() []uint64          { return []uint64{} }
 
-func (c nopCache) Invalidate()                {}
-func (c nopCache) Len() int                   { return 0 }
-func (c nopCache) Recalculate()               {}
-func (c nopCache) SetStats(stats.StatsClient) {}
+func (c nopCache) Invalidate()  {}
+func (c nopCache) Len() int     { return 0 }
+func (c nopCache) Recalculate() {}
+
+func (c nopCache) Clear() {}
 
 func (c nopCache) Top() []bitmapPair {
 	return []bitmapPair{}

cache_test.go

@@ -1,11 +1,12 @@
-// Copyright 2021 Molecula Corp. All rights reserved.
+// Copyright 2022 Molecula Corp. (DBA FeatureBase).
+// SPDX-License-Identifier: Apache-2.0
 package pilosa_test
 
 import (
 	"reflect"
 	"testing"
 
-	"github.com/molecula/featurebase/v3"
+	pilosa "github.com/featurebasedb/featurebase/v3"
 )
 
 // Ensure cache stays constrained to its configured size.
@@ -61,7 +62,7 @@ func TestCache_Rank_Dirty(t *testing.T) {
 		cache.Add(v.ID, v.Count)
 	}
 
-	var got []pair
+	var got []pair //nolint:prealloc
 	for _, p := range cache.Top() {
 		got = append(got, pair(p))
 	}

catcher.go

@@ -1,10 +1,11 @@
-// Copyright 2021 Molecula Corp. All rights reserved.
+// Copyright 2022 Molecula Corp. (DBA FeatureBase).
+// SPDX-License-Identifier: Apache-2.0
 package pilosa
 
 import (
-	"github.com/molecula/featurebase/v3/roaring"
-	txkey "github.com/molecula/featurebase/v3/short_txkey"
-	"github.com/molecula/featurebase/v3/vprint"
+	"github.com/featurebasedb/featurebase/v3/roaring"
+	txkey "github.com/featurebasedb/featurebase/v3/short_txkey"
+	"github.com/featurebasedb/featurebase/v3/vprint"
 )
 
 // catcher is useful to report error locations with a
@@ -26,10 +27,6 @@ func init() {
 
 var _ Tx = (*catcherTx)(nil)
 
-func (c *catcherTx) NewTxIterator(index, field, view string, shard uint64) *roaring.Iterator {
-	return c.b.NewTxIterator(index, field, view, shard)
-}
-
 func (c *catcherTx) ImportRoaringBits(index, field, view string, shard uint64, rit roaring.RoaringIterator, clear bool, log bool, rowSize uint64) (changed int, rowSet map[uint64]int, err error) {
 	defer func() {
 		if r := recover(); r != nil {
@@ -127,6 +124,17 @@ func (c *catcherTx) Remove(index, field, view string, shard uint64, a ...uint64)
 	return c.b.Remove(index, field, view, shard, a...)
 }
 
+func (c *catcherTx) Removed(index, field, view string, shard uint64, a ...uint64) (changed []uint64, err error) {
+
+	defer func() {
+		if r := recover(); r != nil {
+			vprint.AlwaysPrintf("see Removed() PanicOn '%v' at '%v'", r, vprint.Stack())
+			vprint.PanicOn(r)
+		}
+	}()
+	return c.b.Removed(index, field, view, shard, a...)
+}
+
 func (c *catcherTx) Contains(index, field, view string, shard uint64, key uint64) (exists bool, err error) {
 
 	defer func() {
@@ -149,28 +157,6 @@ func (c *catcherTx) ContainerIterator(index, field, view string, shard uint64, f
 	return c.b.ContainerIterator(index, field, view, shard, firstRoaringContainerKey)
 }
 
-func (c *catcherTx) ForEach(index, field, view string, shard uint64, fn func(i uint64) error) error {
-
-	defer func() {
-		if r := recover(); r != nil {
-			vprint.AlwaysPrintf("see ForEach() PanicOn '%v' at '%v'", r, vprint.Stack())
-			vprint.PanicOn(r)
-		}
-	}()
-	return c.b.ForEach(index, field, view, shard, fn)
-}
-
-func (c *catcherTx) ForEachRange(index, field, view string, shard uint64, start, end uint64, fn func(uint64) error) error {
-
-	defer func() {
-		if r := recover(); r != nil {
-			vprint.AlwaysPrintf("see ForEachRange() PanicOn '%v' at '%v'", r, vprint.Stack())
-			vprint.PanicOn(r)
-		}
-	}()
-	return c.b.ForEachRange(index, field, view, shard, start, end, fn)
-}
-
 func (c *catcherTx) Count(index, field, view string, shard uint64) (uint64, error) {
 
 	defer func() {
@@ -234,10 +220,14 @@ func (c *catcherTx) ApplyFilter(index, field, view string, shard uint64, ckey ui
 	return GenericApplyFilter(c, index, field, view, shard, ckey, filter)
 }
 
+func (c *catcherTx) ApplyRewriter(index, field, view string, shard uint64, ckey uint64, filter roaring.BitmapRewriter) (err error) {
+	return c.b.ApplyRewriter(index, field, view, shard, ckey, filter)
+}
+
 func (c *catcherTx) GetSortedFieldViewList(idx *Index, shard uint64) (fvs []txkey.FieldView, err error) {
 	return c.b.GetSortedFieldViewList(idx, shard)
 }
 
-func (tx *catcherTx) GetFieldSizeBytes(index, field string) (uint64, error) {
+func (c *catcherTx) GetFieldSizeBytes(index, field string) (uint64, error) {
 	return 0, nil
 }

cli/Makefile

@@ -0,0 +1,15 @@
+.PHONY: test testv test-integration testv-integration
+
+GO=go
+
+test:
+	$(GO) test ./... -short
+
+testv:
+	$(GO) test -v ./... -short
+
+test-integration:
+	$(GO) test . -count 1 -timeout 20m -run TestCLIIntegration/$(RUN)
+
+testv-integration:
+	$(GO) test -v . -count 1 -timeout 20m -run TestCLIIntegration/$(RUN)

cli/batch/inserter.go

@@ -0,0 +1,8 @@
+package batch
+
+// Inserter can be implemented by anything which can handle a SQL statement
+// representing a write operation. An example is `BULK INSERT`. The Insert()
+// method on this interface does not return any results other than an error.
+type Inserter interface {
+	Insert(sql string) error
+}

cli/batch/sql.go

@@ -0,0 +1,215 @@
+package batch
+
+import (
+	"encoding/json"
+	"fmt"
+	"strings"
+	"time"
+
+	fbbatch "github.com/featurebasedb/featurebase/v3/batch"
+	"github.com/featurebasedb/featurebase/v3/dax"
+	"github.com/featurebasedb/featurebase/v3/errors"
+	"github.com/featurebasedb/featurebase/v3/pql"
+)
+
+// Ensure type implements interface.
+var _ fbbatch.Batcher = (*sqlBatcher)(nil)
+
+type sqlBatcher struct {
+	inserter Inserter
+	fields   []*dax.Field
+}
+
+func NewSQLBatcher(i Inserter, flds []*dax.Field) *sqlBatcher {
+	return &sqlBatcher{
+		inserter: i,
+		fields:   flds,
+	}
+}
+
+func (b *sqlBatcher) NewBatch(cfg fbbatch.Config, tbl *dax.Table, flds []*dax.Field) (fbbatch.RecordBatch, error) {
+	fields := flds
+	if b.fields != nil {
+		fields = b.fields
+	}
+	return &sqlBatch{
+		table:        tbl,
+		fields:       fields,
+		size:         cfg.Size,
+		maxStaleness: cfg.MaxStaleness,
+		ids:          make([]interface{}, 0, cfg.Size),
+		rows:         make([][]interface{}, 0, cfg.Size),
+		inserter:     b.inserter,
+	}, nil
+}
+
+// Ensure type implements interface.
+var _ fbbatch.RecordBatch = (*sqlBatch)(nil)
+
+type sqlBatch struct {
+	table  *dax.Table
+	fields []*dax.Field
+	size   int
+
+	ids  []interface{}
+	rows [][]interface{}
+
+	// staleTime tracks the time the first record of the batch was inserted
+	// plus the maxStaleness, in order to raise ErrBatchNowStale if the
+	// maxStaleness has elapsed
+	staleTime    time.Time
+	maxStaleness time.Duration
+
+	// inserter handles SQL INSERT statements generated for each batch.
+	inserter Inserter
+}
+
+func (b *sqlBatch) Add(rec fbbatch.Row) error {
+	// Clear rec.Values and rec.Clears upon return.
+	defer func() {
+		for i := range rec.Values {
+			rec.Values[i] = nil
+		}
+		for k := range rec.Clears {
+			delete(rec.Clears, k)
+		}
+	}()
+
+	if len(b.ids) == cap(b.ids) {
+		return fbbatch.ErrBatchAlreadyFull
+	}
+	if len(rec.Values) != len(b.fields) {
+		return errors.Errorf("record needs to match up with batch fields, got %d fields and %d record", len(b.fields), len(rec.Values))
+	}
+
+	// Append the ID to b.ids.
+	b.ids = append(b.ids, rec.ID)
+
+	// Convert decimal fields (which come in as int64, along with the scale in
+	// field) to pql.Decimal.
+	for i, fld := range b.fields {
+		switch b.fields[i].Type {
+		case dax.BaseTypeDecimal:
+			if val, ok := rec.Values[i].(int64); ok {
+				rec.Values[i] = pql.NewDecimal(val, fld.Options.Scale)
+			}
+		case dax.BaseTypeTimestamp:
+			if val, ok := rec.Values[i].(int64); ok {
+				ts := time.Unix(val, 0)
+				rec.Values[i] = ts.Format(time.RFC3339)
+			}
+		}
+	}
+
+	// Append the record to b.rows.
+	vals := make([]interface{}, 0, len(rec.Values))
+	vals = append(vals, rec.Values...)
+	b.rows = append(b.rows, vals)
+
+	// Check for batch full or stale.
+	if len(b.ids) == cap(b.ids) {
+		return fbbatch.ErrBatchNowFull
+	}
+	if b.maxStaleness != time.Duration(0) { // set maxStaleness to 0 to disable staleness checking
+		if len(b.ids) == 1 {
+			b.staleTime = time.Now().Add(b.maxStaleness)
+		} else if time.Now().After(b.staleTime) {
+			return fbbatch.ErrBatchNowStale
+		}
+	}
+	return nil
+}
+
+func (b *sqlBatch) Import() error {
+	if len(b.rows) == 0 {
+		return nil
+	}
+
+	// Construct the BULK INSERT statement based on the table and fields.
+	sql, err := buildBulkInsert(b.table, b.fields, b.ids, b.rows)
+	if err != nil {
+		return errors.Wrap(err, "building bulk insert statement")
+	}
+
+	// Reset batch data.
+	b.reset()
+
+	// Submit the SQL statement.
+	return b.inserter.Insert(sql)
+}
+
+func (b *sqlBatch) reset() {
+	b.ids = b.ids[:0]
+	b.rows = b.rows[:0]
+}
+
+func (b *sqlBatch) Len() int {
+	return len(b.rows)
+}
+
+func (b *sqlBatch) Flush() error {
+	return nil
+}
+
+func buildBulkInsert(tbl *dax.Table, fields []*dax.Field, ids []interface{}, rows [][]interface{}) (string, error) {
+	// Validation.
+	if tbl.Name == "" {
+		return "", errors.New(errors.ErrUncoded, "table name is required")
+	} else if len(fields) == 0 {
+		return "", errors.New(errors.ErrUncoded, "at least one field is required")
+	}
+
+	var sb strings.Builder
+
+	sb.WriteString(`BULK INSERT INTO `)
+	sb.WriteString(string(tbl.Name))
+	sb.WriteString(` (_id,`)
+
+	flds := make([]string, 0, len(fields))
+	maps := make([]string, 0, len(fields))
+	for i := range fields {
+		flds = append(flds, string(fields[i].Name))
+		maps = append(maps, fmt.Sprintf("'$.col_%d' %s", i, fields[i].FullType()))
+	}
+	// Fields
+	sb.WriteString(strings.Join(flds, ","))
+
+	// MAP
+	keyType := dax.BaseTypeID
+	if tbl.StringKeys() {
+		keyType = dax.BaseTypeString
+	}
+	sb.WriteString(`) MAP ('$._id' `)
+	sb.WriteString(keyType)
+	sb.WriteString(`,`)
+	sb.WriteString(strings.Join(maps, ","))
+	sb.WriteString(`) FROM x'`)
+
+	// Row values.
+
+	// m is a map representing a single row to be marshalled and added to the
+	// bulk insert as one line in the NDJSON payload. We re-use the map for each
+	// row.
+	m := make(map[string]interface{})
+	for i := range rows {
+		// Write the ID value.
+		m[string(dax.PrimaryKeyFieldName)] = ids[i]
+		// Write the rest of the data values.
+		for col := range rows[i] {
+			m[fmt.Sprintf("col_%d", col)] = rows[i][col]
+		}
+
+		// Marshal the map to json and add to the sql statement.
+		if j, err := json.Marshal(m); err != nil {
+			return "", errors.Wrap(err, "marshalling row to json")
+		} else {
+			sb.Write(j)
+			sb.WriteString("\n")
+		}
+	}
+
+	// WITH
+	sb.WriteString(fmt.Sprintf(`' WITH BATCHSIZE %d FORMAT 'NDJSON' INPUT 'STREAM'`, len(rows)))
+
+	return sb.String(), nil
+}

cli/batch/sql_test.go

@@ -0,0 +1,47 @@
+package batch
+
+import (
+	"testing"
+
+	"github.com/featurebasedb/featurebase/v3/dax"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestBatchSQL(t *testing.T) {
+	tbl := &dax.Table{
+		Name: "foo",
+	}
+	fields := []*dax.Field{
+		{
+			Name: "name",
+			Type: dax.BaseTypeString,
+		},
+		{
+			Name: "age",
+			Type: dax.BaseTypeInt,
+		},
+	}
+	ids := []interface{}{
+		0, 1, 2,
+	}
+	rows := [][]interface{}{
+		{
+			[]interface{}{"Alice", int64(11)},
+		},
+		{
+			[]interface{}{"Bob", int64(22)},
+		},
+		{
+			[]interface{}{"Carl,Comma", int64(33)},
+		},
+	}
+
+	s, err := buildBulkInsert(tbl, fields, ids, rows)
+	assert.NoError(t, err)
+
+	exp := `BULK INSERT INTO foo (_id,name,age) MAP ('$._id' id,'$.col_0' string,'$.col_1' int) FROM x'{"_id":0,"col_0":["Alice",11]}
+{"_id":1,"col_0":["Bob",22]}
+{"_id":2,"col_0":["Carl,Comma",33]}
+' WITH BATCHSIZE 3 FORMAT 'NDJSON' INPUT 'STREAM'`
+	assert.Equal(t, exp, s)
+}

cli/buffer.go

@@ -0,0 +1,91 @@
+package cli
+
+import (
+	"io"
+	"strings"
+
+	"github.com/featurebasedb/featurebase/v3/errors"
+)
+
+// buffer is a query buffer for SQL statements. Note that this is not a query
+// buffer as you would find on a database server (buffering query results).
+// Rather, this buffers the working SQL statement. The buffer has two
+// components: the buffer of query parts making up the working, incomplete SQL
+// statement, and the last completed SQL statement submitted to the Queryer.
+type buffer struct {
+	parts     []queryPart
+	lastQuery query
+
+	hasBatchFile bool
+}
+
+func newBuffer() *buffer {
+	return &buffer{}
+}
+
+// addPart adds the given queryPart to the buffer. If the part is of type
+// `partTerminator` (which is generally singified in the CLI by a ";"), the
+// buffer will finalize the query and return it. In all other cases, the
+// returned query is nil.
+func (b *buffer) addPart(part queryPart) (query, error) {
+	// Check for part type compatibility. For example, multiple batchFile parts
+	// are not allowed in the same query.
+	switch part.(type) {
+	case *partBatchFile:
+		if b.hasBatchFile {
+			return nil, errors.Errorf("multiple batch files in one query is not supported")
+		}
+		b.hasBatchFile = true
+	case *partTerminator:
+		return b.finalize(), nil
+	}
+
+	b.parts = append(b.parts, part)
+	return nil, nil
+}
+
+// finalize copies the contents (queryParts) of buffer to lastQuery and then
+// resets the buffer. It returns the query that was finalized.
+func (b *buffer) finalize() query {
+	q := make(query, len(b.parts))
+	copy(q, b.parts)
+	b.lastQuery = q
+	b.reset()
+	return q
+
+}
+
+// print returns the contents of the buffer as a string. This is generally used
+// to visually inspect the state of the buffer (for example, when a user issues
+// a `\p` meta-command in the CLI).
+func (b *buffer) print() string {
+	if len(b.parts) > 0 {
+		return query(b.parts).String()
+	} else if b.lastQuery != nil {
+		return b.lastQuery.String() + ";"
+	}
+	return "Query buffer is empty."
+}
+
+// reset clears the buffer. It returns a message which may optionally be used to
+// display to a user.
+func (b *buffer) reset() string {
+	b.parts = b.parts[:0]
+	b.hasBatchFile = false
+	return "Query buffer reset (cleared)."
+}
+
+func (b *buffer) Reader() io.Reader {
+	if len(b.parts) > 0 {
+		return query(b.parts).Reader()
+	} else if b.lastQuery != nil {
+		r := b.lastQuery.Reader()
+		// TODO(tlt): terminating the query here results in a line feed just
+		// before the semi-colon (for example, when you print out the query
+		// buffer using `\w [FILE]`). The removal and re-introduction of line
+		// feeds is kind of a mess.
+		term := strings.NewReader(";")
+		return io.MultiReader(r, term)
+	}
+	return strings.NewReader("")
+}

cli/cli.go

@@ -0,0 +1,816 @@
+// Package cli contains a FeatureBase command line interface.
+package cli
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"github.com/chzyer/readline"
+	featurebase "github.com/featurebasedb/featurebase/v3"
+	"github.com/featurebasedb/featurebase/v3/cli/batch"
+	"github.com/featurebasedb/featurebase/v3/cli/fbcloud"
+	"github.com/featurebasedb/featurebase/v3/errors"
+	"github.com/featurebasedb/featurebase/v3/logger"
+)
+
+const (
+	defaultHost     string = "localhost"
+	defaultClientID string = "6i2gs7mu215ab23cnvmshdoq6t" // production Cognito client ID
+	defaultRegion   string = "us-east-2"
+	terminationChar string = ";"
+	nullValue       string = "NULL"
+)
+
+var (
+	Stdin  io.ReadCloser = os.Stdin
+	Stdout io.Writer     = os.Stdout
+	Stderr io.Writer     = os.Stderr
+)
+
+var splash string = fmt.Sprintf(`FeatureBase CLI (%s)
+Type "\q" to quit.
+`, featurebase.Version)
+
+// Ensure type implments interfaces.
+var _ printer = (*Command)(nil)
+var _ batch.Inserter = (*Command)(nil)
+
+type Command struct {
+	host string
+	port string
+
+	splitter   *splitter
+	buffer     *buffer
+	workingDir *workingDir
+
+	organizationID string
+	database       string
+	databaseID     string
+	databaseName   string
+
+	Queryer Queryer `json:"-"`
+
+	stdin  io.ReadCloser `json:"-"`
+	stdout io.Writer     `json:"-"`
+	stderr io.Writer     `json:"-"`
+
+	// output is where actual results are written. This might point to stdout,
+	// or to a file, based on the current configuration.
+	output       io.Writer `json:"-"`
+	writeOptions *writeOptions
+
+	Config *Config `json:"config"`
+
+	historyPath string
+
+	// Commands contains optional commands provided via one or more `-c` (or
+	// `--command`) flags. If this is non-empty, the cli will run in
+	// non-interactive mode; i.e. it will quit after the command is complete.
+	Commands []string `json:"commands"`
+
+	// Files contains optional files provided via one or more `-f` (or `--file`)
+	// flags. If this is non-empty, the cli will run in non-interactive mode;
+	// i.e. it will quit after the command is complete.
+	Files []string `json:"files"`
+
+	// variables holds the variables created with the \set meta-command.
+	variables map[string]string
+
+	// nonInteractiveMode is set to true when fbsql is running in
+	// non-ineracative mode. And example of this is when the user has provided a
+	// `-c` flag in the command line.
+	nonInteractiveMode bool
+
+	// quit gets closed when Run should stop listening for input.
+	quit chan struct{}
+}
+
+func NewCommand(logdest logger.Logger) *Command {
+	variables := make(map[string]string)
+
+	return &Command{
+		Config: &Config{
+			Host: defaultHost,
+			Port: "",
+
+			OrganizationID: "",
+			Database:       "",
+
+			CloudAuth: CloudAuthConfig{
+				ClientID: defaultClientID,
+				Region:   defaultRegion,
+				Email:    "",
+				Password: "",
+			},
+
+			HistoryPath: "",
+
+			CSV: false,
+		},
+
+		buffer:     newBuffer(),
+		splitter:   newSplitter(newReplacer(variables)),
+		workingDir: newWorkingDir(),
+
+		stdin:  Stdin,
+		stdout: Stdout,
+		stderr: Stderr,
+
+		output:       Stdout,
+		writeOptions: defaultWriteOptions(),
+
+		variables: variables,
+
+		quit: make(chan struct{}),
+	}
+}
+
+// SetStdin sets stdin. This is useful for initial configuration in tests.
+func (cmd *Command) SetStdin(rc io.ReadCloser) {
+	cmd.stdin = rc
+}
+
+// SetStdout sets both stdout and output to the value provided. This is useful
+// for initial configuration in tests.
+func (cmd *Command) SetStdout(w io.Writer) {
+	cmd.stdout = w
+	cmd.output = w
+}
+
+// SetStderr sets stderr. This is useful for initial configuration in tests.
+func (cmd *Command) SetStderr(w io.Writer) {
+	cmd.stderr = w
+}
+
+// Run is the main entry-point to the CLI.
+func (cmd *Command) Run(ctx context.Context) error {
+	if err := cmd.run(ctx); err != nil {
+		cmd.Errorf(err.Error() + "\n")
+		return err
+	}
+	return nil
+}
+
+// run is effectively wrapped by the Run() method, but it's split out this way
+// so that run() can simply return errors, rather than worrying about how errors
+// should be printed; printing errors returned by run() is left up to the Run()
+// method.
+func (cmd *Command) run(ctx context.Context) error {
+	if err := cmd.setupConfig(); err != nil {
+		return errors.Wrap(err, "setting up config")
+	}
+
+	// Check to see if Command needs to run in non-interactive mode.
+	if len(cmd.Commands) > 0 ||
+		len(cmd.Files) > 0 ||
+		cmd.Config.KafkaConfig != "" ||
+		cmd.Config.CSV {
+		cmd.nonInteractiveMode = true
+	}
+
+	// Print the splash message.
+	if !cmd.nonInteractiveMode {
+		cmd.Printf(splash)
+	}
+
+	if err := cmd.setupClient(); err != nil {
+		return errors.Wrap(err, "setting up client")
+	}
+
+	// Print the connection info.
+	if !cmd.nonInteractiveMode {
+		cmd.printConnInfo()
+	}
+
+	if err := cmd.connectToDatabase(cmd.database); err != nil {
+		cmd.Errorf(errors.Wrap(err, "connecting to database").Error() + "\n")
+		// We intentionally do not return err here.
+	}
+
+	// Run in non-interactive mode based on flags and configuration.
+	// This includes either handling `-c` and/or `-f` flags, or handling a
+	// `--kafka-config` flag.
+	if len(cmd.Commands) > 0 || len(cmd.Files) > 0 {
+		// Run Commands.
+		for _, line := range cmd.Commands {
+			if err := cmd.handleLine(line); err != nil {
+				return errors.Wrapf(err, "handling line: %s", line)
+			}
+		}
+
+		// Run Files.
+		for _, fname := range cmd.Files {
+			if _, err := executeFile(cmd, fname); err != nil {
+				return errors.Wrapf(err, "executing file: %s", fname)
+			}
+		}
+
+		return nil
+	} else if cmd.Config.KafkaConfig != "" {
+		runner, err := cmd.newKafkaRunner(cmd.Config.KafkaConfig)
+		if err != nil {
+			return errors.Wrap(err, "getting new kafka runner")
+		}
+		if err := runner.Main.Run(); err != nil {
+			return errors.Wrap(err, "running kafka")
+		}
+		return nil
+	}
+
+	// From this point on, we should be in interactive mode.
+
+	// Set up history for saving user input.
+	cmd.setupHistory()
+
+	rl, err := readline.NewEx(&readline.Config{
+		Prompt:                 cmd.prompt(false),
+		HistoryFile:            cmd.historyPath,
+		HistoryLimit:           100000,
+		DisableAutoSaveHistory: true,
+
+		Stdin:  cmd.stdin,
+		Stdout: cmd.stdout,
+		Stderr: cmd.stderr,
+	})
+	if err != nil {
+		return errors.Wrap(err, "getting readline")
+	}
+	defer rl.Close()
+
+	// inMidCommand indicates whether a partial command has been received and
+	// we're still waiting for a termination character.
+	var inMidCommand bool
+
+	for {
+		rl.SetPrompt(cmd.prompt(inMidCommand))
+
+		// Read user provided input.
+		line, err := rl.Readline()
+		if err == readline.ErrInterrupt {
+			inMidCommand = false
+			cmd.buffer.reset()
+			continue
+		} else if err != nil {
+			return errors.Wrap(err, "reading line")
+		}
+
+		// We append a line feed at the end of each line because at this point
+		// we have effectively stripped any intentional line feeds (since we are
+		// reading a line at a time), and we don't want to do that. An example
+		// of an intentional line feed is in a BULK INSERT CSV STREAM like this
+		// example:
+		//
+		// bulk replace
+		// 	 into foo (_id, age)
+		// 	 map (0 id, 1 int)
+		// from
+		// 	 x'3,33
+		// 	 4,44
+		// 	 5,55'
+		// with
+		// 	 format 'CSV'
+		// 	 input 'STREAM';
+		//
+		// We want to preserve the line feeds that are contained in the x''
+		// block; those are intentional as they demarc records within the csv.
+		qps, mcs, err := cmd.splitter.split(line + "\n")
+		if err != nil {
+			cmd.Errorf("error splitting line: %s\n", err)
+			continue
+		}
+
+		// Save line in the history.
+		if err := rl.SaveHistory(line); err != nil {
+			cmd.Errorf("Couldn't save history: %v\n", err)
+		}
+
+		// This is wrapped in an anonymous function so we can capture any
+		// errors, ignore the rest of the line, and return back to a prompt.
+		if err := func() error {
+			for i := range qps {
+				if qry, err := cmd.buffer.addPart(qps[i]); err != nil {
+					return errors.Wrap(err, "adding part to buffer")
+				} else if qry != nil {
+					if err := cmd.executeAndWriteQuery(qry); err != nil {
+						return errors.Wrap(err, "executing query")
+					}
+					// In addition to saving each line in the history, we also
+					// save each successful query.
+					if err := rl.SaveHistory(qry.String() + ";"); err != nil {
+						cmd.Errorf("Couldn't save query in history: %v\n", err)
+					}
+					inMidCommand = false
+				} else {
+					inMidCommand = true
+				}
+			}
+			return nil
+		}(); err != nil {
+			cmd.Errorf(err.Error() + "\n")
+			inMidCommand = false
+			continue
+		}
+
+		// This is wrapped in an anonymous function so we can capture any
+		// errors, ignore the rest of the line, and return back to a prompt.
+		if err := func() error {
+			for i := range mcs {
+				action, err := mcs[i].execute(cmd)
+				if err != nil {
+					return errors.Wrap(err, "executing meta command")
+				}
+				switch action {
+				case actionQuit:
+					close(cmd.quit)
+					return nil
+				case actionReset:
+					inMidCommand = false
+				}
+			}
+			return nil
+		}(); err != nil {
+			cmd.Errorf(err.Error() + "\n")
+			inMidCommand = false
+			continue
+		}
+
+		select {
+		case <-cmd.quit:
+			if err := cmd.close(); err != nil {
+				cmd.Errorf("closing: %s\n", err)
+			}
+			return nil
+		default:
+			// pass
+		}
+	}
+}
+
+// prompt constructs the prompt that the user sees based on the currently
+// connected database and whether the user is in the middle of a sql statement.
+func (cmd *Command) prompt(mid bool) string {
+	db := "fbsql" // default prompt when a database is not set.
+	if cmd.databaseName != "" {
+		db = cmd.databaseName
+	}
+
+	if mid {
+		return strings.Repeat(" ", len(db)) + "-# "
+	}
+	return db + "=# "
+}
+
+// close is called upon quitting. It should close any remaining open file
+// handles used by the CLICommand.
+func (cmd *Command) close() error {
+	return cmd.closeOutput()
+}
+
+// setupConfig sets up private struct members based on values provided via the
+// configuration flags.
+func (cmd *Command) setupConfig() error {
+	if cmd.Config == nil {
+		return nil
+	}
+
+	cmd.host = cmd.Config.Host
+	cmd.port = cmd.Config.Port
+
+	cmd.organizationID = cmd.Config.OrganizationID
+	cmd.database = cmd.Config.Database
+
+	cmd.historyPath = cmd.Config.HistoryPath
+
+	// Apply any pset flag arguments.
+	for _, pset := range cmd.Config.PSets {
+		if err := cmd.applyPSet(pset); err != nil {
+			return errors.Wrapf(err, "applying pset: %s", pset)
+		}
+	}
+
+	// If running with the `--csv` flag, configure things to ensure the output
+	// is correct (i.e. that it's just the csv).
+	if cmd.Config.CSV {
+		cmd.writeOptions.format = formatCSV
+	}
+
+	return nil
+}
+
+// applyPSet takes a pset string of the form `arg` or `arg=val` and applies it
+// as if the user had run `\pset arg val`. The only difference is that applying
+// pset here suppresses any output to stdout.
+func (cmd *Command) applyPSet(pset string) error {
+	// We expect arg to be one of the folowing formats:
+	// arg
+	// arg=val
+	args := strings.SplitN(pset, "=", 2)
+
+	// This is kind of hacky, but until we re-think the metaCommand interface to
+	// take a printer interface somewhere (so we can pass in the nopPrinter
+	// here), we're just going to discard stdout for the duration of this apply,
+	// and then set stdout back to its previous writer after the apply.
+	hold := cmd.stdout
+	cmd.stdout = io.Discard
+	defer func() {
+		cmd.stdout = hold
+	}()
+
+	_, err := newMetaPSet(args).execute(cmd)
+	return err
+}
+
+func (cmd *Command) executeAndWriteQuery(qry query) error {
+	queryResponse, err := cmd.executeQuery(qry)
+	if err != nil {
+		if errors.Is(err, ErrOrganizationRequired) {
+			// Print an error message and return nil, effectively aborting any
+			// further writes for this query.
+			cmd.Errorf("Organization required. Use \\org to set an organization.\n")
+			return nil
+		}
+		return errors.Wrap(err, "making query")
+	}
+	if err := writeOutput(queryResponse, cmd.writeOptions, cmd.output, cmd.stdout, cmd.stderr); err != nil {
+		return errors.Wrap(err, "writing out response")
+	}
+
+	return nil
+}
+
+func (cmd *Command) executeQuery(qry query) (*featurebase.WireQueryResponse, error) {
+	wqr, err := cmd.Queryer.Query(cmd.organizationID, cmd.databaseID, qry.Reader())
+	if err != nil {
+		return nil, errors.Wrap(err, "executing query")
+	}
+
+	// If we're running in non-interactive mode, we need to check the error that
+	// comes back in the WireQueryResponse. If there's an error, we want to
+	// return it now (rather than just printing it later) so that we immediately
+	// stop any further execution of commands.
+	if cmd.nonInteractiveMode && wqr.Error != "" {
+		return nil, errors.Errorf(wqr.Error)
+	}
+
+	return wqr, nil
+}
+
+// printer is an interface which encapsulates the methods used to print output
+// to the various io.Writers.
+type printer interface {
+	Printf(format string, a ...any)
+	Outputf(format string, a ...any)
+	Errorf(format string, a ...any)
+}
+
+type nopPrinter struct{}
+
+func newNopPrinter() *nopPrinter {
+	return &nopPrinter{}
+}
+
+func (n *nopPrinter) Printf(format string, a ...any)  {}
+func (n *nopPrinter) Outputf(format string, a ...any) {}
+func (n *nopPrinter) Errorf(format string, a ...any)  {}
+
+// Printf is a helper method which sends the given payload to stdout.
+func (cmd *Command) Printf(format string, a ...any) {
+	out := fmt.Sprintf(format, a...)
+	cmd.stdout.Write([]byte(out))
+}
+
+// Outputf is a helper method which sends the given payload to output.
+func (cmd *Command) Outputf(format string, a ...any) {
+	out := fmt.Sprintf(format, a...)
+	cmd.output.Write([]byte(out))
+}
+
+// Errorf is a helper method which sends the given payload to stderr.
+func (cmd *Command) Errorf(format string, a ...any) {
+	out := fmt.Sprintf(format, a...)
+	cmd.stderr.Write([]byte(out))
+}
+
+func (cmd *Command) setupHistory() {
+	// If HistoryPath has already been configured (i.e. with a command flag),
+	// don't bother setting up the default in the home directory.
+	if cmd.historyPath != "" {
+		return
+	}
+
+	historyPath := ""
+	if home, err := os.UserHomeDir(); err != nil {
+		cmd.Errorf("Error getting home directory, command history persistence will be disabled: %v\n", err)
+	} else {
+		historyDir := filepath.Join(home, ".featurebase")
+		err := os.MkdirAll(historyDir, 0o750)
+		if err != nil {
+			cmd.Errorf("Creating directory for history: %v\n", err)
+		} else {
+			historyPath = filepath.Join(historyDir, "fbsql_history")
+		}
+	}
+	cmd.historyPath = historyPath
+}
+
+// printConnInfo displays the currently set host.
+// TODO(tlt): extend this to be the output of the /conninfo meta-command.
+func (cmd *Command) printConnInfo() {
+	cmd.Printf("Host: %s\n", hostPort(cmd.host, cmd.port))
+}
+
+func (cmd *Command) connectToDatabase(dbName string) error {
+	var p printer = cmd
+	if cmd.nonInteractiveMode {
+		p = newNopPrinter()
+	}
+
+	// Providing a blank ("") or hyphen ("-") dbName is the equivalent of
+	// disconnecting from the current database. We support the hyphen option
+	// because calling the `\c` meta-command without an argument is how you
+	// print the current connection.
+	switch dbName {
+	case "-", "":
+		cmd.databaseID = ""
+		cmd.databaseName = ""
+		p.Printf(cmd.connectionMessage())
+		return nil
+	}
+
+	// Look up dbID based on dbName.
+	wqr, err := cmd.executeQuery(newRawQuery("SHOW DATABASES"))
+	if err != nil {
+		return errors.Wrap(err, "executing query")
+	}
+
+	for _, db := range wqr.Data {
+		// 0: _id
+		// 1: name
+		if db[1] == dbName {
+			cmd.databaseName = dbName
+			cmd.databaseID = db[0].(string)
+			p.Printf(cmd.connectionMessage())
+			return nil
+		}
+	}
+	return errors.Errorf("invalid database: %s", dbName)
+}
+
+func (cmd *Command) orgMessage() string {
+	if cmd.organizationID == "" {
+		return "You have not set an organization.\n"
+	}
+	return fmt.Sprintf("You have set organization \"%s\".\n", cmd.organizationID)
+}
+
+func (cmd *Command) connectionMessage() string {
+	if cmd.databaseName == "" {
+		return "You are not connected to a database.\n"
+	}
+	return fmt.Sprintf("You are now connected to database \"%s\" (%s).\n", cmd.databaseName, cmd.databaseID)
+}
+
+func (cmd *Command) setupClient() error {
+	// If the Queryer has already been set (in tests for example), don't bother
+	// trying to detect it.
+	if cmd.Queryer != nil {
+		return nil
+	}
+
+	var p printer = cmd
+	if cmd.nonInteractiveMode {
+		p = newNopPrinter()
+	}
+
+	if strings.TrimSpace(cmd.host) == "" {
+		return errors.Errorf("no host provided\n")
+	}
+
+	if !strings.HasPrefix(cmd.host, "http") {
+		cmd.host = "http://" + cmd.host
+	}
+
+	typ, err := cmd.detectFBType()
+	if err != nil {
+		return errors.Wrap(err, "detecting FeatureBase deployment type")
+	}
+
+	switch typ {
+	case featurebaseTypeOnPremClassic:
+		p.Printf("Detected on-prem, classic deployment.\n")
+		cmd.Queryer = &standardQueryer{
+			Host: cmd.host,
+			Port: cmd.port,
+		}
+	case featurebaseTypeOnPremServerless:
+		p.Printf("Detected on-prem, serverless deployment.\n")
+		cmd.Queryer = &serverlessQueryer{
+			Host: cmd.host,
+			Port: cmd.port,
+		}
+	case featurebaseTypeCloud:
+		p.Printf("Detected cloud deployment.\n")
+
+		cmd.Queryer = &fbcloud.Queryer{
+			Host: hostPort(cmd.host, cmd.port),
+
+			ClientID: cmd.Config.CloudAuth.ClientID,
+			Region:   cmd.Config.CloudAuth.Region,
+			Email:    cmd.Config.CloudAuth.Email,
+			Password: cmd.Config.CloudAuth.Password,
+		}
+	case featurebaseTypeUnknown:
+		p.Printf("Could not detect deployment\n")
+		// cmd.Queryer = &nopQueryer{}
+		// Instead of using a no-op queryer when the type can't be detected, we
+		// default to using a cloud queryer.
+		cmd.Queryer = &fbcloud.Queryer{
+			Host: hostPort(cmd.host, cmd.port),
+
+			ClientID: cmd.Config.CloudAuth.ClientID,
+			Region:   cmd.Config.CloudAuth.Region,
+			Email:    cmd.Config.CloudAuth.Email,
+			Password: cmd.Config.CloudAuth.Password,
+		}
+	default:
+		return errors.Errorf("unknown type: %s", typ)
+	}
+	return nil
+}
+
+type featurebaseType string
+
+const (
+	featurebaseTypeUnknown          featurebaseType = "unknown"            // unknown
+	featurebaseTypeOnPremClassic    featurebaseType = "on-prem-standard"   // on-prem, classic
+	featurebaseTypeOnPremServerless featurebaseType = "on-prem-serverless" // on-prem, serverless
+	featurebaseTypeCloud            featurebaseType = "cloud"              // cloud, (both classic and serverless)?
+)
+
+func hostPort(host, port string) string {
+	if port == "" {
+		return host
+	}
+	return host + ":" + port
+}
+
+// detectFBType determines if we're talking to standalone FeatureBase
+// or FeatureBase Cloud
+func (cmd *Command) detectFBType() (featurebaseType, error) {
+	type trial struct {
+		port   string
+		health string
+		typ    featurebaseType
+	}
+
+	// trials is populated with the url/endpoints to try in order to detect if a
+	// process is running there which can support the cli requests.
+	trials := []trial{}
+
+	var clientTimeout time.Duration
+	if cmd.port != "" {
+		clientTimeout = 100 * time.Millisecond
+		trials = append(trials,
+			// on-prem, serverless
+			trial{
+				port:   cmd.port,
+				health: "/queryer/health",
+				typ:    featurebaseTypeOnPremServerless,
+			},
+			// on-prem, classic
+			trial{
+				port:   cmd.port,
+				health: "/status",
+				typ:    featurebaseTypeOnPremClassic,
+			},
+		)
+	} else if strings.HasPrefix(cmd.host, "https") {
+		// https suggesting we might be connecting to a cloud host
+		clientTimeout = 1 * time.Second
+		trials = append(trials,
+			// cloud
+			trial{
+				port:   "",
+				health: "/health",
+				typ:    featurebaseTypeCloud,
+			},
+		)
+	} else {
+		// Try default ports just in case.
+		clientTimeout = 100 * time.Millisecond
+		trials = append(trials,
+			// on-prem, serverless
+			trial{
+				port:   "8080",
+				health: "/queryer/health",
+				typ:    featurebaseTypeOnPremServerless,
+			},
+			// on-prem, classic
+			trial{
+				port:   "10101",
+				health: "/status",
+				typ:    featurebaseTypeOnPremClassic,
+			},
+		)
+	}
+
+	client := http.Client{
+		Timeout: clientTimeout,
+	}
+	for _, trial := range trials {
+		url := hostPort(cmd.host, trial.port) + trial.health
+		if resp, err := client.Get(url); err != nil {
+			continue
+		} else if resp.StatusCode/100 == 2 {
+			cmd.port = trial.port
+			return trial.typ, nil
+		}
+	}
+
+	return featurebaseTypeUnknown, nil
+}
+
+func (cmd *Command) closeOutput() error {
+	if cmd.output == nil {
+		return nil
+	}
+
+	if closer, ok := cmd.output.(io.Closer); ok {
+		return closer.Close()
+	}
+
+	return nil
+}
+
+func (cmd *Command) handleLine(line string) error {
+	// For single-line command handling, we handle either a meta-command, or
+	// query parts, but not both. The logic is that any line which begins with
+	// "\" will be handled as a meta-command, otherwise it will be handled as a
+	// query.
+	if len(line) == 0 {
+		return nil
+	} else if line[0] == byte('\\') {
+		return cmd.handleLineAsMetaCommand(line)
+	} else {
+		return cmd.handleLineAsQueryParts(line)
+	}
+}
+
+func (cmd *Command) handleLineAsMetaCommand(line string) error {
+	_, mcs, err := cmd.splitter.split(line)
+	if err != nil {
+		return errors.Wrapf(err, "splitting line")
+	}
+
+	for i := range mcs {
+		_, err := mcs[i].execute(cmd)
+		if err != nil {
+			return errors.Wrap(err, "executing meta command")
+		}
+	}
+
+	return nil
+}
+
+func (cmd *Command) handleLineAsQueryParts(line string) error {
+	qps, mcs, err := cmd.splitter.split(line)
+	if err != nil {
+		return errors.Wrapf(err, "splitting line")
+	} else if len(mcs) > 0 {
+		return errors.Errorf("--command does not support meta-commands")
+	}
+
+	// Add a termintor part to the end of []queryPart. We do this because the
+	// command is coming in from the --command flag, it may not end with a
+	// semi-colon, but we still want to execute it.
+	if len(qps) > 0 {
+		if _, ok := qps[len(qps)-1].(*partTerminator); !ok {
+			qps = append(qps, newPartTerminator())
+		}
+	}
+
+	for i := range qps {
+		if qry, err := cmd.buffer.addPart(qps[i]); err != nil {
+			return errors.Wrap(err, "adding part to buffer")
+		} else if qry != nil {
+			if err := cmd.executeAndWriteQuery(qry); err != nil {
+				return errors.Wrap(err, "executing query")
+			}
+		}
+	}
+	return nil
+}
+
+func (cmd *Command) Insert(sql string) error {
+	wqr, err := cmd.executeQuery(newRawQuery(sql))
+	if wqr.Error != "" {
+		return errors.Errorf(wqr.Error)
+	}
+	return err
+}

cli/cli_integration_test.go

@@ -0,0 +1,257 @@
+package cli_test
+
+import (
+	"bufio"
+	"context"
+	"fmt"
+	"os"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/featurebasedb/featurebase/v3/cli"
+	"github.com/featurebasedb/featurebase/v3/dax/server/test"
+	"github.com/featurebasedb/featurebase/v3/logger"
+	"github.com/stretchr/testify/require"
+)
+
+func TestCLIIntegration(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test")
+	}
+
+	ctx := context.Background()
+
+	t.Run("Stubbed Framework", func(t *testing.T) {
+		mc := test.MustRunManagedCommand(t)
+		defer mc.Close()
+
+		addr := mc.Address()
+
+		capture := newCapture(t)
+
+		comparer := newComparer(t)
+		comparer.run()
+
+		fbsql := cli.NewCommand(logger.StderrLogger)
+		fbsql.SetStdin(capture)
+		fbsql.SetStdout(comparer)
+		fbsql.SetStderr(comparer)
+
+		fbsql.Config = &cli.Config{
+			Host: addr.Host(),
+			Port: fmt.Sprintf("%d", addr.Port()),
+		}
+
+		// Run fbsql in a goroutine so we can continue to send it commands
+		// below.
+		didQuit := make(chan struct{})
+		go func() {
+			require.NoError(t, fbsql.Run(ctx))
+			close(didQuit)
+		}()
+
+		// testFiles reference files located in the cli/testdata directory. All
+		// tests should be placed there; other than adding another test file to
+		// this list, you probably shouldn't be editing this file unless you are
+		// trying to modify the way the test framework itself works.
+		testFiles := []string{
+			"setup",
+			"database",
+			"table",
+			// the tests below may be dependent on the previous tests, which do
+			// setup and some shared database and table creation.
+			"query_buffer",
+			// meta commands
+			"meta_bang",
+			"meta_cd",
+			"meta_echo",
+			"meta_describe",
+			"meta_file",
+			"meta_pset_border",
+			"meta_pset_expanded",
+			"meta_pset_format_csv",
+			"meta_pset_tuples_only",
+			"meta_include",
+			"meta_output",
+			"meta_set",
+			"meta_timing",
+			"meta_write",
+		}
+
+		for _, testFile := range testFiles {
+			t.Run(testFile, func(t *testing.T) {
+				f, err := os.Open("testdata/" + testFile)
+				require.NoError(t, err)
+
+				scanner := bufio.NewScanner(f)
+				var lineNo int
+				for scanner.Scan() {
+					line := scanner.Text()
+					lineNo++
+
+					// Empty lines and comments (//) are ignored.
+					if line == "" {
+						continue
+					} else if strings.HasPrefix(line, "//") {
+						continue
+					}
+
+					parts := strings.SplitN(line, ":", 2)
+
+					switch parts[0] {
+					case "SEND":
+						v := ""
+						if len(parts) == 2 {
+							v = parts[1]
+						}
+						capture.sendLine(v)
+					case "EXPECT":
+						v := ""
+						if len(parts) == 2 {
+							v = parts[1]
+						}
+						comparer.expectLine(v, testFile, lineNo)
+					case "EXPECTCOMP":
+						if len(parts) == 2 {
+							comps := strings.SplitN(parts[1], ":", 2)
+							v := ""
+							if len(comps) == 2 {
+								v = comps[1]
+							}
+							comparer.expectLineComp(comparator(comps[0]), v, testFile, lineNo)
+						} else {
+							t.Errorf("unexpected line: %s[%d]:%s", testFile, lineNo, line)
+						}
+					default:
+						t.Errorf("unexpected line: %s[%d]:%s", testFile, lineNo, line)
+					}
+				}
+				require.NoError(t, scanner.Err())
+			})
+		}
+
+		// End with quit to ensure that fbsql closes without error.
+		capture.sendLine(`\q`)
+
+		// Ensure fbsql quits cleanly.
+		select {
+		case <-didQuit:
+		case <-time.After(time.Second):
+			t.Fatalf("expected fbsql to quit")
+		}
+	})
+}
+
+// compare is used to compare fbsql output written to its Stdout with expected
+// lines.
+type comparer struct {
+	t       *testing.T
+	out     chan byte
+	outline chan []byte
+	exp     chan []byte
+}
+
+func newComparer(t *testing.T) *comparer {
+	return &comparer{
+		t:       t,
+		out:     make(chan byte, 1024),
+		outline: make(chan []byte, 128),
+		exp:     make(chan []byte, 1024),
+	}
+}
+
+func (c *comparer) run() {
+	// Read bytes off output, and for every line (designated by a line feed "\n"),
+	// push the line onto the outline channel.
+	go func() {
+		var line []byte
+		for {
+			b := <-c.out
+			if b == byte('\n') {
+				c.outline <- line
+				line = []byte{}
+				continue
+			}
+			line = append(line, b)
+		}
+	}()
+}
+
+type comparator string
+
+const (
+	compEquals     = "Equals"
+	compHasPrefix  = "HasPrefix"
+	compWithFormat = "WithFormat"
+)
+
+// expectLine is a convenience method which calls expectLineComp with the compEq
+// comparator and the given line.
+func (c *comparer) expectLine(line string, fileName string, lineNo int) {
+	c.expectLineComp(compEquals, line, fileName, lineNo)
+}
+
+// expectLineComp reads the next line from the outline channel and compares it
+// with the given `line`. A comparator can be provided to inform how the lines
+// should be compared (for example, the compHasPrefix comparator will just
+// compare the beginning part of the outline).
+func (c *comparer) expectLineComp(comp comparator, line string, fileName string, lineNo int) {
+	var outline []byte
+	select {
+	case outline = <-c.outline:
+	case <-time.After(10 * time.Second):
+		// TODO(tlt): this is 10 seconds to account for the fb_views creation on
+		// a local mac. This should really be something like 2 seconds. Put this
+		// back to 2 once fb_views issue is addressed.
+		c.t.Fatalf("expected output line %s[%d]: >%s<", fileName, lineNo, line)
+	}
+
+	// msg is included in any require which fails.
+	msg := []interface{}{"exp: %s[%d], got: >%s<", fileName, lineNo, outline}
+
+	switch comp {
+	case compEquals:
+		require.Equal(c.t, []byte(line), outline, msg...)
+	case compHasPrefix:
+		require.True(c.t, strings.HasPrefix(string(outline), line), msg...)
+	case compWithFormat:
+		require.True(c.t, compareByteSlices(outline, []byte(line)), msg...)
+	default:
+		c.t.Fatalf("invalid comparator: %s", comp)
+	}
+}
+
+func (c *comparer) Write(b []byte) (n int, err error) {
+	for i := range b {
+		c.out <- b[i]
+	}
+	return len(b), err
+}
+
+// compareByteSlices compares a byte slice s with another byte slice format and
+// returns true if they are the same. It will accept underscore as a
+// single-character wildcard anywhere in slice format.
+func compareByteSlices(s, format []byte) bool {
+	// Replace some helpers in format before comparing.
+	f := string(format)
+	f = strings.ReplaceAll(f, `{uuid}`, `________-____-____-____-____________`)
+	f = strings.ReplaceAll(f, `{timestamp}`, `____-__-__T__:__:__Z`)
+	format = []byte(f)
+
+	if len(s) != len(format) {
+		return false
+	}
+
+	for i := range s {
+		if format[i] == '_' {
+			continue
+		}
+		if s[i] != format[i] {
+			// log.Printf("DEBUG: characters differ: (%d): '%v' != '%v'", i, s[i], format[i])
+			return false
+		}
+	}
+
+	return true
+}

cli/cli_test.go

@@ -0,0 +1,199 @@
+package cli_test
+
+import (
+	"context"
+	"io"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	featurebase "github.com/featurebasedb/featurebase/v3"
+	"github.com/featurebasedb/featurebase/v3/cli"
+	"github.com/featurebasedb/featurebase/v3/logger"
+	"github.com/pkg/errors"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestCLI(t *testing.T) {
+	t.Run("Input", func(t *testing.T) {
+		ctx := context.Background()
+
+		capture := newCapture(t)
+
+		cli := cli.NewCommand(logger.StderrLogger)
+		cli.SetStdin(capture)
+		cli.SetStdout(capture)
+		cli.Queryer = capture
+
+		go func() {
+			assert.NoError(t, cli.Run(ctx))
+		}()
+
+		none := []string{}
+
+		// One statement, one line.
+		capture.Assert("one;", []string{"one\n"})
+
+		// One statement, multiple lines.
+		capture.Assert("one", none)
+		capture.Assert(" two ", none)
+		capture.Assert("three;", []string{"one\ntwo\nthree\n"})
+
+		// Multiple statements, one line.
+		capture.Assert("foo; bar;", []string{"foo\n", "bar\n"})
+
+		// Multiple statements, multiple lines.
+		capture.Assert("a1", none)
+		capture.Assert("a2; b1", []string{"a1\na2\n"})
+		capture.Assert("b2;", []string{"b1\nb2\n"})
+
+		// Blank lines.
+		capture.Assert("one", none)
+		capture.Assert("", none)
+		capture.Assert("three;", []string{"one\nthree\n"})
+
+		// Just a semi-colon.
+		capture.Assert(";", []string{""})
+
+		// Multi-line with just a semi-colon.
+		capture.Assert("one", none)
+		capture.Assert(";", []string{"one\n"})
+
+		// Ensure a clean exit with no errors.
+		assert.NoError(t, capture.Exit())
+	})
+}
+
+////////////////////////////////////////////////////////
+
+// Ensure type implementes interface.
+var _ io.ReadCloser = (*capture)(nil)
+var _ io.Writer = (*capture)(nil)
+var _ cli.Queryer = (*capture)(nil)
+
+// capture implements the various CLI interfaces in order to capture test input
+// and submit it as though that input were being read from the command line. It
+// also captures calls made to the Queryer.Query method and ensures the sql they
+// contain is expected.
+type capture struct {
+	t *testing.T
+
+	// ch is a channel of strings (one line at a time) of CLI input.
+	ch chan string
+
+	mu   sync.RWMutex
+	sqls []string
+
+	// queryDone will receive an event any time the Query method is called and
+	// has completed. This is to tell the Assert method that it's safe to
+	// compare the sqls slice.
+	queryDone chan struct{}
+
+	asserting chan struct{}
+
+	err error
+}
+
+func newCapture(t *testing.T) *capture {
+	return &capture{
+		t:         t,
+		ch:        make(chan string),
+		sqls:      make([]string, 0),
+		queryDone: make(chan struct{}),
+	}
+}
+
+func (c *capture) Exit() error {
+	c.sendLine(`\q`)
+	c.mu.RLock()
+	defer c.mu.RUnlock()
+	return c.err
+}
+
+func (c *capture) Assert(in string, out []string) {
+	c.asserting = make(chan struct{})
+
+	c.sendLine(in)
+
+	// Wait for the CLI command to complete processing the input and send the
+	// sql to Query() by blocking on the queryDone channel. Because Query gets
+	// called for every sql statement in the input, an input resulting in
+	// multiple sql statements needs to wait for all expected queries to
+	// complete. A timeout is included to this so it doesn't deadlock in the
+	// case where Query is expected to be called, but isn't; after the timeout,
+	// the test should fail completely. In summary: we wait on queryDone the
+	// number of sql statements we expect. If we receive fewer than expected,
+	// the timeout will occur. If we receive more than expected, the Query()
+	// method will effectively deadlock, reach its own timout, then write to
+	// capture.err, which will be reported upon Exit().
+	for range out {
+		select {
+		case <-c.queryDone:
+		case <-time.After(2 * time.Second):
+			c.t.Fatalf("expected Query() to be called")
+		}
+	}
+
+	close(c.asserting)
+
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	assert.Equal(c.t, out, c.sqls)
+
+	// Reset the slice.
+	c.sqls = c.sqls[:0]
+
+}
+
+// sendLine sends the given string as a line input to the CLI command. It
+// appends a line feed to the end of string in order to mimic the user hitting
+// the return key.
+func (c *capture) sendLine(s string) {
+	// Add a line feed before putting s on the channel in order to mimic the
+	// user hitting the return key.
+	c.ch <- s + "\n"
+}
+
+// Read is read by the CLI in place of user input. It effectively sends lines of
+// input to the CLI, getting each line to be sent off the channel.
+func (c *capture) Read(b []byte) (n int, err error) {
+	s := <-c.ch
+	return strings.NewReader(s).Read(b)
+}
+
+func (c *capture) Close() error {
+	close(c.ch)
+	return nil
+}
+
+// Write is called with anything written to output. This would included results
+// from calling Query() under normal, non-testing conditions, as well as other
+// informational text sent to output, such as the splash message.
+func (c *capture) Write(b []byte) (n int, err error) {
+	return 0, nil
+}
+
+// Query is called by the CLI command once a full SQL statement is received
+// (signified by the terminator: `;`).
+func (c *capture) Query(org string, db string, sql io.Reader) (*featurebase.WireQueryResponse, error) {
+	tmpBuf := new(strings.Builder)
+	_, err := io.Copy(tmpBuf, sql)
+	if err != nil {
+		return nil, err
+	}
+
+	c.mu.Lock()
+	c.sqls = append(c.sqls, tmpBuf.String())
+	c.mu.Unlock()
+
+	select {
+	case c.queryDone <- struct{}{}:
+	case <-c.asserting:
+		c.mu.Lock()
+		c.err = errors.Errorf("unexpected query: %s", sql)
+		c.mu.Unlock()
+	}
+	return &featurebase.WireQueryResponse{}, nil
+}

cli/config.go

@@ -0,0 +1,31 @@
+package cli
+
+// Config represents the configuration for the command.
+type Config struct {
+	Host string `json:"host"`
+	Port string `json:"port"`
+
+	OrganizationID string `json:"org-id"`
+	Database       string `json:"db"`
+
+	// CloudAuth
+	CloudAuth CloudAuthConfig `json:"cloud-auth"`
+
+	// Kafka
+	KafkaConfig string `json:"kafka-config"`
+
+	HistoryPath string `json:"history-path"`
+
+	// CSV (Comma-Separated Values) table output mode.
+	CSV bool `json:"csv"`
+
+	// PSet takes one or more pset arguments of the form: `--pset=VAR[=ARG]`.
+	PSets []string `json:"pset"`
+}
+
+type CloudAuthConfig struct {
+	ClientID string `json:"client-id"`
+	Region   string `json:"region"`
+	Email    string `json:"email"`
+	Password string `json:"password"`
+}

cli/errors.go

@@ -0,0 +1,16 @@
+package cli
+
+import (
+	"github.com/featurebasedb/featurebase/v3/errors"
+)
+
+const (
+	ErrOrganizationRequired errors.Code = "OrganizationRequired"
+)
+
+func NewErrOrganizationRequired() error {
+	return errors.New(
+		ErrOrganizationRequired,
+		"organization required",
+	)
+}

cli/fbcloud/auth.go

@@ -0,0 +1,79 @@
+package fbcloud
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+
+	"github.com/pkg/errors"
+)
+
+const (
+	authFlow           = "USER_PASSWORD_AUTH"
+	cognitoURLTemplate = "https://cognito-idp.%s.amazonaws.com"
+)
+
+type cognitoParameters struct {
+	Email    string `json:"USERNAME"`
+	Password string `json:"PASSWORD"`
+}
+
+type cognitoAuthRequest struct {
+	AuthParameters cognitoParameters `json:"AuthParameters"`
+	AuthFlow       string            `json:"AuthFlow"`
+	AppClientID    string            `json:"ClientId"`
+}
+
+type cognitoAuthResult struct {
+	IDToken string `json:"IdToken"`
+}
+
+type cognitoAuthResponse struct {
+	Result cognitoAuthResult `json:"AuthenticationResult"`
+}
+
+func authenticate(clientID, region, email, password string) (string, error) {
+	authPayload := cognitoAuthRequest{
+		AuthParameters: cognitoParameters{
+			Email:    email,
+			Password: password,
+		},
+		AuthFlow:    authFlow,
+		AppClientID: clientID,
+	}
+
+	data, err := json.Marshal(authPayload)
+	if err != nil {
+		return "", errors.Wrap(err, "marshaling json")
+	}
+
+	url := fmt.Sprintf(cognitoURLTemplate, region)
+	req, err := http.NewRequest(http.MethodPost, url, bytes.NewBuffer(data))
+	if err != nil {
+		return "", errors.Wrap(err, "creating authentication request object")
+	}
+	req.Header.Add("Content-Type", "application/x-amz-json-1.1")
+	req.Header.Add("X-Amz-Target", "AWSCognitoIdentityProviderService.InitiateAuth")
+
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return "", errors.Wrap(err, "making request")
+	}
+	defer resp.Body.Close()
+
+	fullbod, err := io.ReadAll(resp.Body)
+
+	if resp.StatusCode != http.StatusOK || err != nil {
+		return "", errors.Errorf("HTTP status code=%d from Cognito authentication response. reading body: %v, body: '%s'", resp.StatusCode, err, fullbod)
+	}
+
+	var auth cognitoAuthResponse
+	err = json.Unmarshal(fullbod, &auth)
+	if err != nil {
+		return "", errors.Wrap(err, "decoding cognito auth response")
+	}
+
+	return auth.Result.IDToken, nil
+}

cli/fbcloud/client.go

@@ -0,0 +1,131 @@
+package fbcloud
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"strings"
+	"time"
+
+	featurebase "github.com/featurebasedb/featurebase/v3"
+	"github.com/pkg/errors"
+)
+
+// TokenRefreshTimeout is currently hardcoded to be just under the
+// Cognito token timeout for cloud which is 15 minutes (I think I
+// heard that somewhere anyway). It seems to work.
+const TokenRefreshTimeout = time.Minute * 13
+
+type Queryer struct {
+	Host string
+
+	ClientID string
+	Region   string
+	Email    string
+	Password string
+
+	token       string
+	lastRefresh time.Time
+}
+
+func (cq *Queryer) tokenRefresh() error {
+	token, err := authenticate(cq.ClientID, cq.Region, cq.Email, cq.Password)
+	if err != nil {
+		return errors.Wrap(err, "getting token")
+	}
+	cq.token = token
+	cq.lastRefresh = time.Now()
+	return nil
+}
+
+// Query issues a SQL query formatted for the FeatureBase cloud query endpoint.
+func (cq *Queryer) Query(org string, db string, sql io.Reader) (*featurebase.WireQueryResponse, error) {
+	if time.Since(cq.lastRefresh) > TokenRefreshTimeout {
+		if err := cq.tokenRefresh(); err != nil {
+			return nil, errors.Wrap(err, "refreshing token")
+		}
+	}
+	url := fmt.Sprintf("%s/databases/%s/sql", cq.Host, db)
+	if db == "" {
+		url = fmt.Sprintf("%s/sql", cq.Host)
+	}
+
+	client := &http.Client{
+		Timeout: time.Second * 30,
+	}
+	req, err := http.NewRequest(http.MethodPost, url, sql)
+	if err != nil {
+		return nil, errors.Wrap(err, "creating new post request")
+	}
+	req.Header.Add("Content-Type", "text/plain")
+	req.Header.Add("Authorization", cq.token)
+
+	var resp *http.Response
+	if resp, err = client.Do(req); err != nil {
+		return nil, errors.Wrap(err, "executing post request")
+	}
+
+	fullbod, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return nil, errors.Wrap(err, "reading cloud response")
+	}
+	if resp.StatusCode/100 != 2 {
+		return nil, errors.Errorf("unexpected status: %s, full body: '%s'", resp.Status, fullbod)
+	}
+
+	var sqlResponse featurebase.WireQueryResponse
+	if err := json.Unmarshal(fullbod, &sqlResponse); err != nil {
+		return nil, errors.Wrapf(err, "decoding cloud response, body:\n%s", fullbod)
+	}
+	return &sqlResponse, nil
+}
+
+// HTTPRequest can make an arbitrary http request to the host and
+// tries to json unmarshal the response body into v if v is
+// non-nil. This is handy for hitting cloud endpoints other than the
+// query endpoint which is handled by Query. I don't think this is
+// currently used, but I'd like to keep it around for debugging.
+func (cq *Queryer) HTTPRequest(method, path, body string, v interface{}) ([]byte, error) {
+	if time.Since(cq.lastRefresh) > TokenRefreshTimeout {
+		if err := cq.tokenRefresh(); err != nil {
+			return nil, errors.Wrap(err, "refreshing token")
+		}
+	}
+	var bod io.Reader
+	if body == "" {
+		bod = nil
+	} else {
+		bod = strings.NewReader(body)
+	}
+	req, err := http.NewRequest(method, fmt.Sprintf("%s%s", cq.Host, path), bod)
+	if err != nil {
+		return nil, errors.Errorf("creating request: %v", err)
+	}
+	// fmt.Printf("%+v\n", req)
+
+	req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", cq.token))
+	if bod != nil {
+		req.Header.Add("Content-Type", "application/json")
+	}
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return nil, errors.Errorf("doing request: %v", err)
+	}
+	bodbytes, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return nil, errors.Errorf("reading response body: %v", err)
+	}
+	if resp.StatusCode/100 != 2 {
+		return nil, errors.Errorf("bad status: %s. body: '%s'", resp.Status, bodbytes)
+	}
+
+	if v != nil {
+		err = json.Unmarshal(bodbytes, v)
+		if err != nil {
+			return nil, errors.Errorf("unmarshaling: %v", err)
+		}
+	}
+
+	return bodbytes, nil
+}

cli/kafka.go

@@ -0,0 +1,70 @@
+package cli
+
+import (
+	"fmt"
+
+	"github.com/featurebasedb/featurebase/v3/cli/batch"
+	"github.com/featurebasedb/featurebase/v3/cli/kafka"
+	"github.com/featurebasedb/featurebase/v3/errors"
+	"github.com/spf13/viper"
+)
+
+func (cmd *Command) newKafkaRunner(cfgFile string) (*kafka.Runner, error) {
+	// Read the kafka config file.
+	v := viper.New()
+	v.SetConfigFile(cfgFile)
+	v.SetConfigType("toml")
+	err := v.ReadInConfig()
+	if err != nil {
+		return nil, fmt.Errorf("error reading configuration file '%s': %v", cfgFile, err)
+	}
+
+	cfg := kafka.Config{}
+	if err := v.Unmarshal(&cfg); err != nil {
+		return nil, errors.Wrap(err, "unmarshalling config")
+	}
+
+	if err := kafka.ValidateConfig(cfg); err != nil {
+		return nil, errors.Wrap(err, "validating config")
+	}
+
+	// Create a new config with defaults.
+
+	// Look up fields based on table provided in the config.
+	wqr, err := cmd.executeQuery(newRawQuery("SHOW COLUMNS FROM " + cfg.Table))
+	if err != nil {
+		return nil, errors.Wrap(err, "executing query")
+	}
+
+	scr, err := wqr.ShowColumnsResponse()
+	if err != nil {
+		return nil, errors.Wrap(err, "getting show columns from wire query response")
+	}
+
+	// If no fields were provided in the config, use the fields defined on the
+	// table and assume a 1-to-1 mapping of source to destination.
+	if len(cfg.Fields) == 0 {
+		cfg.Fields = kafka.FieldsToConfig(scr.Fields)
+	} else {
+		cfg.Fields, err = kafka.CheckFieldCompatibility(cfg.Fields, scr)
+		if err != nil {
+			return nil, errors.Wrap(err, "validating config fields")
+		}
+	}
+
+	idkCfg, err := kafka.ConvertConfig(cfg)
+	if err != nil {
+		return nil, errors.Wrap(err, "cleaning config")
+	}
+
+	flds, err := kafka.ConfigToFields(cfg)
+	if err != nil {
+		return nil, errors.Wrap(err, "getting fields from config")
+	}
+
+	return kafka.NewRunner(
+		idkCfg,
+		batch.NewSQLBatcher(cmd, flds),
+		cmd.stderr,
+	), nil
+}

cli/kafka/config.go

@@ -0,0 +1,251 @@
+package kafka
+
+import (
+	"fmt"
+	"time"
+
+	featurebase "github.com/featurebasedb/featurebase/v3"
+	"github.com/featurebasedb/featurebase/v3/dax"
+	"github.com/featurebasedb/featurebase/v3/idk"
+	"github.com/pkg/errors"
+)
+
+// Config is the user-facing configuration for kafka support in the CLI. This is
+// unmarshalled from the the toml config file supplied by the user.
+type Config struct {
+	Hosts  []string `mapstructure:"hosts" help:"Kafka hosts."`
+	Group  string   `mapstructure:"group" help:"Kafka group."`
+	Topics []string `mapstructure:"topics" help:"Kafka topics to read from."`
+
+	BatchSize         int           `mapstructure:"batch-size" help:"Batch size."`
+	BatchMaxStaleness time.Duration `mapstructure:"batch-max-staleness" help:"Maximum length of time that the oldest record in a batch can exist before flushing the batch. Note that this can potentially stack with timeouts waiting for the source."`
+	Timeout           time.Duration `mapstructure:"timeout" help:"Time to wait for more records from Kafka before flushing a batch. 0 to disable."`
+
+	Table  string  `mapstructure:"table" help:"Destination table name."`
+	Fields []Field `mapstructure:"fields"`
+}
+
+// Field is a user-facing configuration field.
+type Field struct {
+	Name       string   `mapstructure:"name"`
+	SourceType string   `mapstructure:"source-type"`
+	SourcePath []string `mapstructure:"source-path"`
+	PrimaryKey bool     `mapstructure:"primary-key"`
+}
+
+// ConfigForIDK represents Config converted to values suitable for IDK. In
+// particular, the idk.RawField is used in parsing the schema in IDK.
+type ConfigForIDK struct {
+	Hosts  []string
+	Group  string
+	Topics []string
+
+	BatchSize         int
+	BatchMaxStaleness time.Duration
+	Timeout           time.Duration
+
+	Table   string
+	IDField string
+	Fields  []idk.RawField
+}
+
+// ValidateConfig validates the config is usable.
+func ValidateConfig(c Config) error {
+	if c.Table == "" {
+		return errors.Errorf("table is required")
+	} else if len(c.Topics) == 0 {
+		return errors.Errorf("at least one topic is required")
+	} else if len(c.Fields) > 0 {
+		// We only need to do these checks if any fields are specified at all.
+		// If no fields are specified, that's ok because then we default to
+		// using fields based off the existing table.
+		if len(c.Fields) < 2 {
+			return errors.Errorf("at least two fields are required (one should be a primary key)")
+		} else {
+			var found int
+			for i := range c.Fields {
+				if c.Fields[i].PrimaryKey {
+					found++
+				}
+				if c.Fields[i].Name == "" {
+					return errors.Errorf("a name attribute (which isn't equal to \"\") should exist for all fields")
+				}
+			}
+			if found != 1 {
+				return errors.Errorf("exactly one primary key field is required")
+			}
+		}
+	}
+	return nil
+}
+
+// ConvertConfig converts a Config to one that suitable for IDK.
+func ConvertConfig(c Config) (ConfigForIDK, error) {
+	// Set a default kafka host in case one isn't provided.
+	hosts := []string{"localhost:9092"}
+	if len(c.Hosts) > 0 {
+		hosts = c.Hosts
+	}
+
+	// Copy all the shared members from Config to ConfigForIDK.
+	out := ConfigForIDK{
+		Hosts:             hosts,
+		Group:             c.Group,
+		Topics:            c.Topics,
+		BatchSize:         c.BatchSize,
+		BatchMaxStaleness: c.BatchMaxStaleness,
+		Timeout:           c.Timeout,
+		Table:             c.Table,
+	}
+
+	if len(c.Fields) == 0 {
+		return out, errors.New("fields cannot be empty")
+	}
+
+	// rawFields wil be the same as c.Fields, but possibly enhanced.
+	rawFields := make([]idk.RawField, 0, len(c.Fields))
+
+	var foundPK bool
+	for _, fld := range c.Fields {
+		if fld.PrimaryKey {
+			out.IDField = fld.Name
+			foundPK = true
+		}
+
+		typ, quals, err := dax.SplitFieldType(fld.SourceType)
+		if err != nil {
+			return out, errors.Wrap(err, "getting base type")
+		}
+
+		rawFld := idk.RawField{
+			Name: fld.Name,
+			Type: string(typ),
+			Path: fld.SourcePath,
+		}
+		// If a SourcePath wasn't provided, default to using the field name.
+		if len(rawFld.Path) == 0 {
+			rawFld.Path = []string{fld.Name}
+		}
+
+		switch typ {
+		case dax.BaseTypeInt:
+			// We don't have to handle min/max because we don't create the table.
+		case dax.BaseTypeDecimal:
+			if len(quals) != 1 {
+				return out, errors.Errorf("expected decimal scale")
+			}
+			rawFld.Config = []byte(fmt.Sprintf(`{"scale":%d}`, quals[0]))
+		case dax.BaseTypeID:
+			rawFld.Config = []byte("{\"mutex\":true}")
+		case dax.BaseTypeIDSet:
+			rawFld.Type = "ids"
+		case dax.BaseTypeString:
+			rawFld.Config = []byte("{\"mutex\":true}")
+		case dax.BaseTypeStringSet:
+			rawFld.Type = "strings"
+		case dax.BaseTypeTimestamp:
+			// No timestamp options are handled for now.
+		}
+
+		rawFields = append(rawFields, rawFld)
+	}
+	if !foundPK {
+		return out, errors.New("primary-key not found in fields")
+	}
+
+	out.Fields = rawFields
+
+	return out, nil
+}
+
+// ConfigToFields returns a list of *dax.Field based on the IDField and Fields
+// in the Config.
+func ConfigToFields(c Config) ([]*dax.Field, error) {
+	// We don't know if a primary key will be found, so we can't set the
+	// capacity to `len(c.Fields)-1`.
+	out := make([]*dax.Field, 0, len(c.Fields))
+
+	for _, fld := range c.Fields {
+		if fld.PrimaryKey {
+			continue
+		}
+		typ, quals, err := dax.SplitFieldType(fld.SourceType)
+		if err != nil {
+			return nil, errors.Wrap(err, "splitting field type")
+		}
+		dfld := &dax.Field{
+			Name: dax.FieldName(fld.Name),
+			Type: typ,
+		}
+		switch typ {
+		case dax.BaseTypeDecimal:
+			if len(quals) != 1 {
+				return nil, errors.Errorf("expected decimal scale")
+			}
+			scale, ok := quals[0].(int64)
+			if !ok {
+				return nil, errors.Errorf("invalid decimal scale: %v", quals[0])
+			}
+			dfld.Options.Scale = scale
+		}
+		out = append(out, dfld)
+	}
+
+	return out, nil
+}
+
+// FieldsToConfig returns a Config.Fields based on a list of *dax.Field.
+func FieldsToConfig(flds []*dax.Field) []Field {
+	out := make([]Field, 0, len(flds))
+	for _, fld := range flds {
+		out = append(out, Field{
+			Name:       string(fld.Name),
+			SourceType: fld.FullType(),
+			PrimaryKey: fld.IsPrimaryKey(),
+		})
+	}
+	return out
+}
+
+// CheckFieldCompatibility ensures that the fields provided in the kafka config
+// are compatible with the fields in the existing table. It returns a copy of
+// the kafka config fields with empty values defaulted to the table field
+// configuration.
+func CheckFieldCompatibility(cflds []Field, scr *featurebase.ShowColumnsResponse) ([]Field, error) {
+	out := make([]Field, len(cflds))
+	for i, cfld := range cflds {
+		out[i] = cfld
+		cfldName := dax.FieldName(cfld.Name)
+
+		// Primary key field.
+		if cfld.PrimaryKey {
+			f := scr.Field(dax.PrimaryKeyFieldName)
+			if f == nil {
+				return nil, dax.NewErrFieldDoesNotExist(dax.PrimaryKeyFieldName) // It should be impossible to hit this.
+			}
+			if out[i].SourceType == "" {
+				if f.StringKeys() {
+					out[i].SourceType = dax.BaseTypeString
+				} else {
+					out[i].SourceType = dax.BaseTypeID
+				}
+			}
+			continue
+		}
+
+		// Non primary key fields.
+		if cfldName == dax.PrimaryKeyFieldName {
+			return nil, errors.Errorf("field named '%s' must be a primary key", dax.PrimaryKeyFieldName)
+		}
+
+		f := scr.Field(cfldName)
+		if f == nil {
+			return nil, dax.NewErrFieldDoesNotExist(cfldName)
+		}
+
+		if out[i].SourceType == "" {
+			out[i].SourceType = f.FullType()
+		}
+	}
+	return out, nil
+}

cli/kafka/runner.go

@@ -0,0 +1,67 @@
+package kafka
+
+import (
+	"io"
+	"time"
+
+	fbbatch "github.com/featurebasedb/featurebase/v3/batch"
+	"github.com/featurebasedb/featurebase/v3/errors"
+	"github.com/featurebasedb/featurebase/v3/idk"
+	"github.com/featurebasedb/featurebase/v3/idk/kafka_static"
+	"github.com/featurebasedb/featurebase/v3/logger"
+)
+
+// Runner is a CLI-specific kafka consumer. It's similar to
+// idk.kafka_static.Main in that it embeds idk.Main and contains additional
+// functionality specific to its use case.
+type Runner struct {
+	idk.Main   `flag:"!embed"`
+	KafkaHosts []string       `help:"Comma separated list of host:port pairs for Kafka."`
+	Group      string         `help:"Kafka group."`
+	Topics     []string       `help:"Kafka topics to read from."`
+	Timeout    time.Duration  `help:"Time to wait for more records from Kafka before flushing a batch. 0 to disable."`
+	Header     []idk.RawField `help:"Header configuration."`
+}
+
+func NewRunner(cfg ConfigForIDK, batcher fbbatch.Batcher, logWriter io.Writer) *Runner {
+	idkMain := idk.NewMain()
+	idkMain.IDField = cfg.IDField
+	idkMain.Index = cfg.Table
+	idkMain.Batcher = batcher
+	idkMain.BatchSize = cfg.BatchSize
+	idkMain.BatchMaxStaleness = cfg.BatchMaxStaleness
+	idkMain.SetBasic()
+	idkMain.SetLog(logger.NewStandardLogger(logWriter))
+
+	kr := &Runner{
+		Main:       *idkMain,
+		KafkaHosts: cfg.Hosts,
+		Group:      cfg.Group,
+		Topics:     cfg.Topics,
+		Header:     cfg.Fields,
+		Timeout:    cfg.Timeout,
+	}
+	kr.OffsetMode = true
+	kr.Main.Namespace = "cli_kafka_runner"
+	kr.Main.Pprof = "" // don't initialize pprof until we actually use it in tests
+	kr.NewSource = func() (idk.Source, error) {
+		source := kafka_static.NewSource()
+		source.Hosts = kr.KafkaHosts
+		source.Group = kr.Group
+		source.Topics = kr.Topics
+		source.Log = kr.Main.Log()
+		// source.TLS = m.KafkaTLS
+		source.Timeout = kr.Timeout
+		// source.SkipOld = m.SkipOld
+		source.HeaderFields = kr.Header
+		// source.S3Region = m.S3Region
+		// source.AllowMissingFields = m.AllowMissingFields
+
+		err := source.Open()
+		if err != nil {
+			return nil, errors.Wrap(err, "opening source")
+		}
+		return source, nil
+	}
+	return kr
+}

cli/parts.go

@@ -0,0 +1,136 @@
+package cli
+
+import (
+	"fmt"
+	"io"
+	"os"
+	"strings"
+)
+
+// query is a collection of queryParts which, when applied together, make up an
+// executable SQL query.
+type query []queryPart
+
+func (q query) String() string {
+	var sb strings.Builder
+	for i := range q {
+		sb.WriteString(q[i].String())
+		if i < len(q)-1 {
+			sb.WriteRune('\n')
+		}
+	}
+	return sb.String()
+}
+
+// Reader returns the query as an io.Reader so that it can be passed to, for
+// example, http.Post().
+func (q query) Reader() io.Reader {
+	readers := make([]io.Reader, 0, len(q))
+	for i := range q {
+		readers = append(readers, q[i].Reader())
+	}
+	return io.MultiReader(readers...)
+}
+
+// queryPart is an interface representing anything which can use to build up a
+// query.
+type queryPart interface {
+	fmt.Stringer
+	Reader() io.Reader
+}
+
+func newRawQuery(s string) query {
+	return []queryPart{
+		newPartRaw(s),
+	}
+}
+
+// ////////////////////////////////////////////////////////////////////////////
+// raw
+// ////////////////////////////////////////////////////////////////////////////
+
+// Ensure type implements interface.
+var _ queryPart = (*partRaw)(nil)
+
+type partRaw struct {
+	raw string
+}
+
+func newPartRaw(s string) *partRaw {
+	return &partRaw{
+		raw: s,
+	}
+}
+
+func (p *partRaw) Reader() io.Reader {
+	return strings.NewReader(p.raw + "\n")
+}
+
+func (p *partRaw) String() string {
+	return p.raw
+}
+
+// ////////////////////////////////////////////////////////////////////////////
+// file
+// ////////////////////////////////////////////////////////////////////////////
+
+// Ensure type implements interface.
+var _ queryPart = (*partFile)(nil)
+
+type partFile struct {
+	file *os.File
+}
+
+func newPartFile(f *os.File) *partFile {
+	return &partFile{
+		file: f,
+	}
+}
+
+func (p *partFile) Reader() io.Reader {
+	return p.file
+}
+
+func (p *partFile) String() string {
+	return fmt.Sprintf("[file: %s]", p.file.Name())
+}
+
+// ////////////////////////////////////////////////////////////////////////////
+// batch file
+// ////////////////////////////////////////////////////////////////////////////
+
+// Ensure type implements interface.
+var _ queryPart = (*partBatchFile)(nil)
+
+type partBatchFile struct {
+	file *os.File
+}
+
+func (p *partBatchFile) Reader() io.Reader {
+	return p.file
+}
+
+func (p *partBatchFile) String() string {
+	return p.file.Name()
+}
+
+// ////////////////////////////////////////////////////////////////////////////
+// terminator (i.e. ";")
+// ////////////////////////////////////////////////////////////////////////////
+
+// Ensure type implements interface.
+var _ queryPart = (*partTerminator)(nil)
+
+type partTerminator struct{}
+
+func newPartTerminator() *partTerminator {
+	return &partTerminator{}
+}
+
+func (p *partTerminator) Reader() io.Reader {
+	return nil
+}
+
+func (p *partTerminator) String() string {
+	return terminationChar
+}

cli/queryer.go

@@ -0,0 +1,108 @@
+package cli
+
+import (
+	"fmt"
+	"io"
+	"net/http"
+	"time"
+
+	featurebase "github.com/featurebasedb/featurebase/v3"
+	"github.com/pkg/errors"
+)
+
+type Queryer interface {
+	Query(org string, db string, sql io.Reader) (*featurebase.WireQueryResponse, error)
+}
+
+// Ensure type implements interface.
+var _ Queryer = (*nopQueryer)(nil)
+
+type nopQueryer struct{}
+
+func (qryr *nopQueryer) Query(org string, db string, sql io.Reader) (*featurebase.WireQueryResponse, error) {
+	return nil, errors.Errorf("no-op queryer")
+}
+
+// Ensure type implements interface.
+var _ Queryer = (*standardQueryer)(nil)
+
+// standardQueryer supports a standard featurebase deployment hitting the /sql
+// endpoint with a payload containing only the sql statement.
+type standardQueryer struct {
+	Host string
+	Port string
+}
+
+func (qryr *standardQueryer) Query(org string, db string, sql io.Reader) (*featurebase.WireQueryResponse, error) {
+	url := fmt.Sprintf("%s/sql", hostPort(qryr.Host, qryr.Port))
+
+	resp, err := http.Post(url, "application/json", sql)
+	if err != nil {
+		return nil, errors.Wrapf(err, "posting query")
+	}
+
+	fullbod, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return nil, errors.Wrap(err, "reading response")
+	}
+
+	sqlResponse := &featurebase.WireQueryResponse{}
+	// TODO(tlt): switch this back once all responses are typed
+	// TODO(twg) 2023/03/01 using json.Number to decode large ints so care must be made
+	// if err := json.Unmarshal(fullbod, sqlResponse); err != nil {
+	if err := sqlResponse.UnmarshalJSONTyped(fullbod, true); err != nil {
+		return nil, errors.Wrapf(err, "unmarshaling query response, body:\n'%s'\n", fullbod)
+	}
+
+	return sqlResponse, nil
+}
+
+// Ensure type implements interface.
+var _ Queryer = (*serverlessQueryer)(nil)
+
+// serverlessQueryer is similar to the standardQueryer except that it hits a
+// different endpoint, and its payload is database-aware.
+type serverlessQueryer struct {
+	Host string
+	Port string
+}
+
+func (qryr *serverlessQueryer) Query(org string, db string, sql io.Reader) (*featurebase.WireQueryResponse, error) {
+	if org == "" {
+		return nil, NewErrOrganizationRequired()
+	}
+
+	url := fmt.Sprintf("%s/queryer/databases/%s/sql", hostPort(qryr.Host, qryr.Port), db)
+	if db == "" {
+		url = fmt.Sprintf("%s/queryer/sql", hostPort(qryr.Host, qryr.Port))
+	}
+
+	client := &http.Client{
+		Timeout: time.Second * 30,
+	}
+
+	req, err := http.NewRequest(http.MethodPost, url, sql)
+	if err != nil {
+		return nil, errors.Wrap(err, "creating new post request")
+	}
+	req.Header.Add("Content-Type", "text/plain")
+	req.Header.Add("OrganizationID", org)
+
+	var resp *http.Response
+	if resp, err = client.Do(req); err != nil {
+		return nil, errors.Wrap(err, "executing post request")
+	}
+
+	fullbod, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return nil, errors.Wrap(err, "reading response")
+	}
+	sqlResponse := &featurebase.WireQueryResponse{}
+	// TODO(tlt): switch this back once all responses are typed
+	// if err := json.Unmarshal(fullbod, sqlResponse); err != nil {
+	if err := sqlResponse.UnmarshalJSONTyped(fullbod, true); err != nil {
+		return nil, errors.Wrapf(err, "unmarshaling query response, body:\n'%s'\n", fullbod)
+	}
+
+	return sqlResponse, nil
+}

cli/replacer.go

@@ -0,0 +1,109 @@
+package cli
+
+import (
+	"strings"
+
+	"github.com/benhoyt/goawk/lexer"
+)
+
+// replacer can replace parts of a string based on some rules and the provided
+// map[string]string. For example, the Command can replace strings with values
+// in its `variables` map.
+type replacer struct {
+	m map[string]string
+}
+
+func newReplacer(m map[string]string) *replacer {
+	return &replacer{
+		m: m,
+	}
+}
+
+// replace replaces all instances of the string pattern `:key` with the value at
+// m[key]. For example we want something like this:
+//
+// GIVEN: `start :one,:'two', :"three" ::four ::`
+//
+// with map
+//
+//	map[string]string{
+//		"one":   "repl1",
+//		"three": "repl3",
+//	}
+//
+// WANT: `start repl1,:'two', "repl3" ::four ::`
+func (r *replacer) replace(s string) string {
+	// If no variables have been added to the map, there's no need to parse the
+	// string for variable replacement.
+	if len(r.m) == 0 {
+		return s
+	}
+
+	line := []byte(s)
+	lex := lexer.NewLexer(line)
+
+	// finger contains the index into line at the start of non-variable text
+	// that we want to include, as-is in the output.
+	var finger int
+
+	// sb builds the string which will be the final output.
+	var sb strings.Builder
+	for {
+		pos, tok, _ := lex.Scan()
+
+		switch tok {
+		case lexer.COLON:
+			// last is the last normal character position before the colon.
+			last := pos.Column - 1
+
+			// Get the next byte to see if the colon value is quoted, and if so,
+			// whether its has single or double quotes.
+			b := lex.PeekByte()
+
+			// padding is the amount of padding we have to consider around the
+			// variable name. If the variable is not quoted, it doesn't require
+			// any padding. But if it has quotes, it needs 2 characters of
+			// paddings to accomodate the quotes.
+			padding := 0
+
+			// quote holds the character to use to quote the final, replaced
+			// output value. Because the lexer doesn't tell us how a certain
+			// `string` token was quoted, we need to keep track of that here so
+			// we can put them back.
+			quote := ""
+			switch b {
+			case byte('\''): // single quote
+				quote = `'`
+				padding = 2
+			case byte('"'): // double quote
+				quote = `"`
+				padding = 2
+			}
+
+			pos, tok, key := lex.Scan()
+			switch tok {
+			case lexer.NAME, lexer.STRING:
+				// Write the normal text up to the variable replacement
+				// position.
+				sb.Write(line[finger:last])
+
+				if v, ok := r.m[key]; ok {
+					// Write replaced variable with the quotes it had.
+					sb.WriteString(quote + v + quote)
+				} else {
+					// Since the variable was not found in the map, just write
+					// back what was already there.
+					sb.WriteString(":" + quote + key + quote)
+				}
+
+				// Reset finger to point to the next position after the
+				// variable.
+				finger = pos.Column + len(key) + padding - 1
+			}
+		case lexer.EOF:
+			// Write the remainder of the string and return.
+			sb.Write(line[finger:])
+			return sb.String()
+		}
+	}
+}

cli/replacer_test.go

@@ -0,0 +1,109 @@
+package cli
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestReplacer(t *testing.T) {
+	t.Run("general replace function", func(t *testing.T) {
+
+		m := map[string]string{
+			"v1": "newVone",
+			"v2": "newVtwo",
+		}
+
+		tests := []struct {
+			s   string
+			m   map[string]string
+			exp string
+		}{
+			{
+				// no variables present
+				s:   "foo",
+				m:   m,
+				exp: "foo",
+			},
+			{
+				// variable prefix, but not in map
+				s:   ":foo",
+				m:   m,
+				exp: ":foo",
+			},
+			{
+				// variable name match, but missing prefix
+				s:   "v1",
+				m:   m,
+				exp: "v1",
+			},
+			{
+				// variable name match
+				s:   ":v1",
+				m:   m,
+				exp: "newVone",
+			},
+			{
+				// two variables, the same, no space
+				s:   ":v1:v1",
+				m:   m,
+				exp: "newVonenewVone",
+			},
+			{
+				// two variables, different, no space
+				s:   ":v1:v2",
+				m:   m,
+				exp: "newVonenewVtwo",
+			},
+			{
+				// two variables, different, spaces
+				s:   ":v1  :v2",
+				m:   m,
+				exp: "newVone  newVtwo",
+			},
+			{
+				// one variable, one non-variable, no space
+				s:   ":v1:foo",
+				m:   m,
+				exp: "newVone:foo",
+			},
+			{
+				// one non-variable, one variable, no space
+				s:   "foo:v1",
+				m:   m,
+				exp: "foonewVone",
+			},
+			{
+				// two variables, different, comma
+				s:   ":v1, :v2",
+				m:   m,
+				exp: "newVone, newVtwo",
+			},
+			{
+				// single quotes
+				s:   ":'v1'",
+				m:   m,
+				exp: "'newVone'",
+			},
+			{
+				// double quotes
+				s:   `:"v2"`,
+				m:   m,
+				exp: `"newVtwo"`,
+			},
+			{
+				// more quotes
+				s:   `start :v1,:'two', :"v2" ::four :: `,
+				m:   m,
+				exp: `start newVone,:'two', "newVtwo" ::four :: `,
+			},
+		}
+		for i, test := range tests {
+			t.Run(fmt.Sprintf("test-%d", i), func(t *testing.T) {
+				replacer := newReplacer(test.m)
+				assert.Equal(t, test.exp, replacer.replace(test.s))
+			})
+		}
+	})
+}

cli/splitter.go

@@ -0,0 +1,133 @@
+package cli
+
+import (
+	"strings"
+
+	"github.com/pkg/errors"
+)
+
+// splitter is a line splitter which splits a line into queryParts and
+// metaCommands. It may not be necessary to have this be a separate struct since
+// it contains no members and just has the one `split()` method, but here we
+// are.
+type splitter struct {
+	replacer *replacer
+}
+
+func newSplitter(r *replacer) *splitter {
+	return &splitter{
+		replacer: r,
+	}
+}
+
+// split splits the given line into queryParts and metaCommands.
+// If a metaCommand is found, everything after that is considered either arguments to that
+// metaCommand, or additional metaCommands. In other words, queryParts can not follow
+// metaCommands in the same line.
+//
+// A line can contain any of the following patterns:
+// 1- [queryParts...]: "select * from tbl; select"
+// 2- [metaCommands...]: "\! pwd \q"
+// 3- [queryParts...][metaCommands...]: "select * from \i file.sql"
+func (s *splitter) split(line string) ([]queryPart, []metaCommand, error) {
+	// Look for a comment line.
+	if strings.HasPrefix(line, "--") {
+		return nil, nil, nil
+	}
+
+	// Look for a meta command.
+	parts := strings.SplitN(line, `\`, 2)
+
+	switch len(parts) {
+	case 1:
+		// slice of queryParts (pattern 1)
+		if qps, err := s.splitQueryParts(strings.TrimSpace(parts[0])); err != nil {
+			return nil, nil, errors.Wrap(err, "splitting query parts")
+		} else {
+			return qps, nil, nil
+		}
+	case 2:
+		// slice of parts + slice of meta commands (pattern 3)
+		// or
+		// slice of meta commands (pattern 2)
+		qps, err := s.splitQueryParts(strings.TrimSpace(parts[0]))
+		if err != nil {
+			return nil, nil, errors.Wrap(err, "splitting query parts")
+		}
+
+		mcs, err := s.splitMetaCommands(strings.TrimSpace(parts[1]))
+		if err != nil {
+			return nil, nil, errors.Wrap(err, "splitting meta commands")
+		}
+
+		return qps, mcs, nil
+	}
+
+	return nil, nil, nil
+}
+
+func (s *splitter) splitQueryParts(line string) ([]queryPart, error) {
+	if line == "" {
+		return nil, nil
+	}
+
+	// Look for a termination character;
+	parts := strings.Split(line, terminationChar)
+
+	// Do variable replacement.
+	for i := range parts {
+		parts[i] = s.replacer.replace(parts[i])
+	}
+
+	if len(parts) == 1 {
+		part0 := strings.TrimSpace(parts[0])
+		return []queryPart{
+			newPartRaw(part0),
+		}, nil
+	}
+
+	qps := make([]queryPart, 0)
+	for i := range parts {
+		part := strings.TrimSpace(parts[i])
+		if part == "" {
+			// If the line starts with a ";", treat it as a terminator for a
+			// previous line.
+			if i == 0 {
+				qps = append(qps, &partTerminator{})
+			}
+			continue
+		}
+		qps = append(qps, newPartRaw(part))
+		if i < len(parts)-1 {
+			qps = append(qps, &partTerminator{})
+		}
+	}
+
+	return qps, nil
+}
+
+func (s *splitter) splitMetaCommands(in string) ([]metaCommand, error) {
+	parts := strings.Split(in, `\`)
+	if len(parts) == 1 {
+		mc, err := splitMetaCommand(parts[0], s.replacer)
+		if err != nil {
+			return nil, errors.Wrapf(err, "splitting meta command: %s", parts[0])
+		}
+		return []metaCommand{mc}, nil
+	}
+
+	mcs := make([]metaCommand, 0)
+	for i := range parts {
+		part := strings.TrimSpace(parts[i])
+		if part == "" {
+			continue
+		}
+		mc, err := splitMetaCommand(part, s.replacer)
+		if err != nil {
+			return nil, errors.Wrapf(err, "splitting meta command: %s", part)
+		}
+		mcs = append(mcs, mc)
+	}
+
+	return mcs, nil
+}

cli/splitter_test.go

@@ -0,0 +1,146 @@
+package cli
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestSplitter(t *testing.T) {
+	s := newSplitter(newReplacer(nil))
+	t.Run("Split", func(t *testing.T) {
+		tests := []struct {
+			line            string
+			expQueryParts   []queryPart
+			expMetaCommands []metaCommand
+			expError        string
+		}{
+			{
+				line: `foo`,
+				expQueryParts: []queryPart{
+					newPartRaw("foo"),
+				},
+			},
+			{
+				line: `foo;`,
+				expQueryParts: []queryPart{
+					newPartRaw("foo"),
+					newPartTerminator(),
+				},
+			},
+			{
+				line: `foo; `,
+				expQueryParts: []queryPart{
+					newPartRaw("foo"),
+					newPartTerminator(),
+				},
+			},
+			{
+				line: `foo; ; `,
+				expQueryParts: []queryPart{
+					newPartRaw("foo"),
+					newPartTerminator(),
+				},
+			},
+			{
+				line: `foo; bar`,
+				expQueryParts: []queryPart{
+					newPartRaw("foo"),
+					newPartTerminator(),
+					newPartRaw("bar"),
+				},
+			},
+			{
+				line: `foo; bar;`,
+				expQueryParts: []queryPart{
+					newPartRaw("foo"),
+					newPartTerminator(),
+					newPartRaw("bar"),
+					newPartTerminator(),
+				},
+			},
+			{
+				line: `\q`,
+				expMetaCommands: []metaCommand{
+					&metaQuit{},
+				},
+			},
+			{
+				line: ` \p`,
+				expMetaCommands: []metaCommand{
+					&metaPrint{},
+				},
+			},
+			{
+				line: `\q \p`,
+				expMetaCommands: []metaCommand{
+					&metaQuit{},
+					&metaPrint{},
+				},
+			},
+			{
+				line: `\q \p arg1 arg2`,
+				expMetaCommands: []metaCommand{
+					&metaQuit{},
+					&metaPrint{},
+				},
+			},
+			{
+				line: `\set`,
+				expMetaCommands: []metaCommand{
+					&metaSet{
+						args: []string{},
+					},
+				},
+			},
+			{
+				line: `\set arg1 arg2`,
+				expMetaCommands: []metaCommand{
+					&metaSet{
+						args: []string{"arg1", "arg2"},
+					},
+				},
+			},
+			{
+				line: `\set 'arg1' 'arg2'`,
+				expMetaCommands: []metaCommand{
+					&metaSet{
+						args: []string{"arg1", "arg2"},
+					},
+				},
+			},
+			{
+				line: `\set 'arg1' '"arg2"'`,
+				expMetaCommands: []metaCommand{
+					&metaSet{
+						args: []string{"arg1", "\"arg2\""},
+					},
+				},
+			},
+			{
+				line:     `\`,
+				expError: "unsupported meta-command:",
+			},
+			{
+				line:     `\xyzxyz`,
+				expError: "unsupported meta-command:",
+			},
+		}
+		for i, tt := range tests {
+			t.Run(fmt.Sprintf("test-%d-%s", i, tt.line), func(t *testing.T) {
+				qps, mcs, err := s.split(tt.line)
+				if tt.expError != "" {
+					if assert.Error(t, err) {
+						assert.Contains(t, err.Error(), tt.expError)
+					}
+					return
+				}
+
+				assert.NoError(t, err)
+				assert.ElementsMatch(t, tt.expQueryParts, qps)
+				assert.ElementsMatch(t, tt.expMetaCommands, mcs)
+			})
+		}
+	})
+}

cli/testdata/database

@@ -0,0 +1,53 @@
+// Show databases now that we have set org.
+SEND:SHOW DATABASES;
+EXPECT:+-----+------+-------+------------+------------+------------+-------+-------------+
+EXPECT:| _id | name | owner | updated_by | created_at | updated_at | units | description |
+EXPECT:+-----+------+-------+------------+------------+------------+-------+-------------+
+EXPECT:+-----+------+-------+------------+------------+------------+-------+-------------+
+EXPECT:
+
+// Create db1.
+SEND:CREATE DATABASE db1 WITH UNITS 1;
+EXPECT:
+
+// List databases via SHOW DATABASES.
+SEND:SHOW DATABASES;
+EXPECT:+--------------------------------------+------+-------+------------+----------------------+----------------------+-------+-------------+
+EXPECT:| _id                                  | name | owner | updated_by | created_at           | updated_at           | units | description |
+EXPECT:+--------------------------------------+------+-------+------------+----------------------+----------------------+-------+-------------+
+EXPECTCOMP:WithFormat:| {uuid} | db1  |       |            | {timestamp} | {timestamp} |     1 |             |
+EXPECT:+--------------------------------------+------+-------+------------+----------------------+----------------------+-------+-------------+
+EXPECT:
+
+// List databases via SHOW DATABASES.
+SEND:\l
+EXPECT:+--------------------------------------+------+-------+------------+----------------------+----------------------+-------+-------------+
+EXPECT:| _id                                  | name | owner | updated_by | created_at           | updated_at           | units | description |
+EXPECT:+--------------------------------------+------+-------+------------+----------------------+----------------------+-------+-------------+
+EXPECTCOMP:WithFormat:| {uuid} | db1  |       |            | {timestamp} | {timestamp} |     1 |             |
+EXPECT:+--------------------------------------+------+-------+------------+----------------------+----------------------+-------+-------------+
+EXPECT:
+
+// Check database connection.
+SEND:\c
+EXPECT:You are not connected to a database.
+
+// Try connecting to an invalid database.
+SEND:\c invalid
+EXPECT:executing meta command: invalid database: invalid
+
+// Try connecting with too many arguments.
+SEND:\c db1 extra
+EXPECT:executing meta command: meta command 'connect' takes zero or one argument
+
+// Connect to a database.
+SEND:\c db1
+EXPECTCOMP:WithFormat:You are now connected to database "db1" ({uuid}).
+
+// Disconnect from the current database.
+SEND:\c -
+EXPECT:You are not connected to a database.
+
+// Connect to a database again.
+SEND:\c db1
+EXPECTCOMP:WithFormat:You are now connected to database "db1" ({uuid}).

cli/testdata/famous.csv

@@ -0,0 +1,10 @@
+
+"Id", "Name", "Short description", "Gender", "Country", "Occupation", "Birth year", "Death year", "Manner of death", "Age of death"
+1, "George Washington", "1st president of the United States (1732–1799)", "Male", "United States of America; Kingdom of Great Britain", "Politician", "1732", "1799", "natural causes", "67"
+2, "Douglas Adams", "English writer and humorist", "Male", "United Kingdom", "Artist", "1952", "2001", "natural causes", "49"
+3, "Abraham Lincoln", "16th president of the United States (1809-1865)", "Male", "United States of America", "Politician", "1809", "1865", "homicide", "56"
+4, "Wolfgang Amadeus Mozart", "Austrian composer of the Classical period", "Male", "Archduchy of Austria; Archbishopric of Salzburg", "Artist", "1756", "1791", "0", "35"
+5, "Ludwig van Beethoven", "German classical and romantic composer", "Male", "Holy Roman Empire; Austrian Empire", "Artist", "1770", "1827", "0", "57"
+6, "Jean-François Champollion", "French classical scholar", "Male", "Kingdom of France; First French Empire", "Egyptologist", "1790", "1832", "natural causes", "42"
+7, "Paul Morand", "French writer", "Male", "France", "Artist", "1888", "1976", "0", "88"
+8, "Claude Monet", "French impressionist painter (1840-1926)", "Male", "France", "Artist", "1840", "1926", "natural causes", "86"

cli/testdata/meta_bang

@@ -0,0 +1,8 @@
+SEND:\! echo 'foo'
+EXPECT:foo
+
+SEND:\! echo "foo"
+EXPECT:"foo"
+
+SEND:\!
+EXPECT:executing meta command: meta command '!' requires at least one argument

cli/testdata/meta_cd

@@ -0,0 +1,17 @@
+// Make a directory so we can test \cd'ing into it.
+SEND:\! mkdir cli-test-dir
+SEND:\cd cli-test-dir
+SEND:\cd ..
+SEND:\! rmdir cli-test-dir
+
+// TODO(tlt): before we do this, we should implement the ability to execute
+// commands in a \set like:
+// \set homedir `pwd`
+// then we can store what directory we're in so we can move back to it 
+// at the end of the test
+// Switch to home directory.
+// SEND:\cd
+
+// Expect error on extra argument to \cd.
+SEND:\cd dir extra
+EXPECT:executing meta command: meta command 'cd' takes zero or one argument

cli/testdata/meta_describe

@@ -0,0 +1,33 @@
+// TODO(tlt): we can't run this test until we get the system tables under control (i.e. sorted). Currently, fb_views is in a map with users, so the following can fail 50% of the time.
+// Show tables for database by calling describe with no args.
+// SEND:\d
+// EXPECT:+-------------------------+-------------------------+-------+------------+----------------------+----------------------+-------+------------+------------------------+
+// EXPECT:| _id                     | name                    | owner | updated_by | created_at           | updated_at           | keys  | space_used | description            |
+// EXPECT:+-------------------------+-------------------------+-------+------------+----------------------+----------------------+-------+------------+------------------------+
+// EXPECTCOMP:WithFormat:| fb_veiws                | fb_views                |       |            | {timestamp} | {timestamp} | true  |          0 | system table for views |
+// EXPECTCOMP:WithFormat:| users                   | users                   |       |            | {timestamp} | {timestamp} | false |          0 |                        |
+// EXPECTCOMP:WithFormat:| fb_____________________ | fb_____________________ |       |            | {timestamp} | {timestamp} | false |          0 |                        |
+// EXPECTCOMP:WithFormat:| fb_____________________ | fb_____________________ |       |            | {timestamp} | {timestamp} | false |          0 |                        |
+// EXPECTCOMP:WithFormat:| fb_____________________ | fb_____________________ |       |            | {timestamp} | {timestamp} | false |          0 |                        |
+// EXPECTCOMP:WithFormat:| fb_____________________ | fb_____________________ |       |            | {timestamp} | {timestamp} | false |          0 |                        |
+// EXPECTCOMP:WithFormat:| fb_____________________ | fb_____________________ |       |            | {timestamp} | {timestamp} | false |          0 |                        |
+// EXPECT:+-------------------------+-------------------------+-------+------------+----------------------+----------------------+-------+------------+------------------------+
+// EXPECT:
+
+// Show columns for table.
+SEND:\d users
+EXPECT:+------+------+--------+----------------------+-------+------------+------------+-------+----------------------+---------------------+----------+-------+-------------+-----+
+EXPECT:| _id  | name | type   | created_at           | keys  | cache_type | cache_size | scale |                  min |                 max | timeunit | epoch | timequantum | ttl |
+EXPECT:+------+------+--------+----------------------+-------+------------+------------+-------+----------------------+---------------------+----------+-------+-------------+-----+
+EXPECTCOMP:WithFormat:| _id  | _id  | id     | {timestamp} | false |            |          0 |     0 |                    0 |                   0 |          |     0 |             | 0s  |
+EXPECTCOMP:WithFormat:| name | name | string | {timestamp} | true  | ranked     |      50000 |     0 |                    0 |                   0 |          |     0 |             | 0s  |
+EXPECTCOMP:WithFormat:| age  | age  | int    | {timestamp} | false |            |          0 |     0 | -9223372036854775808 | 9223372036854775807 |          |     0 |             | 0s  |
+EXPECT:+------+------+--------+----------------------+-------+------------+------------+-------+----------------------+---------------------+----------+-------+-------------+-----+
+EXPECT:
+
+// Show columns for an invalid table.
+SEND:\d invalid
+EXPECT:Error: compiling plan: [1:19] table 'invalid' not found
+
+SEND:\d users extra
+EXPECT:executing meta command: meta command 'describe' takes zero or one argument

cli/testdata/meta_echo

@@ -0,0 +1,6 @@
+SEND:\echo
+EXPECT:
+
+// Simple \echo.
+SEND:\echo foo bar
+EXPECT:foo bar

cli/testdata/meta_file

@@ -0,0 +1,70 @@
+// Create a table.
+SEND:CREATE TABLE famous (
+SEND:    _id ID,
+SEND:    name STRING,
+SEND:    description STRING,
+SEND:    gender STRING,
+SEND:    country STRING,
+SEND:    occupation STRING,
+SEND:    birth_year INT min -32767 max 32767,
+SEND:    death_year INT min -32767 max 32767,
+SEND:    death_manner STRING,
+SEND:    birth_age INT min -32767 max 32767
+SEND:);
+EXPECT:
+
+// Open bulk insert.
+SEND:BULK INSERT
+SEND:INTO famous (_id, name, description, gender, country, occupation,
+SEND:    birth_year, death_year, death_manner, birth_age )
+SEND:MAP(0 INT,
+SEND:1 STRING,
+SEND:2 STRING,
+SEND:3 STRING,
+SEND:4 STRING,
+SEND:5 STRING,
+SEND:6 INT,
+SEND:7 INT,
+SEND:8 STRING,
+SEND:9 INT )
+SEND:FROM
+SEND:    x'
+
+// Call \file
+SEND:\file testdata/famous.csv
+
+// Close bulk insert.
+SEND:'
+SEND:WITH
+SEND:    BATCHSIZE 100000
+SEND:    FORMAT 'CSV'
+SEND:    INPUT 'STREAM'
+SEND:    HEADER_ROW;
+EXPECT:
+
+// Query table to ensure we have data.
+SEND:SELECT * FROM famous;
+EXPECT:+-----+---------------------------+-------------------------------------------------+--------+----------------------------------------------------+--------------+------------+------------+----------------+-----------+
+EXPECT:| _id | name                      | description                                     | gender | country                                            | occupation   | birth_year | death_year | death_manner   | birth_age |
+EXPECT:+-----+---------------------------+-------------------------------------------------+--------+----------------------------------------------------+--------------+------------+------------+----------------+-----------+
+EXPECT:|   1 | George Washington         | 1st president of the United States (1732–1799)  | Male   | United States of America; Kingdom of Great Britain | Politician   |       1732 |       1799 | natural causes |        67 |
+EXPECT:|   2 | Douglas Adams             | English writer and humorist                     | Male   | United Kingdom                                     | Artist       |       1952 |       2001 | natural causes |        49 |
+EXPECT:|   3 | Abraham Lincoln           | 16th president of the United States (1809-1865) | Male   | United States of America                           | Politician   |       1809 |       1865 | homicide       |        56 |
+EXPECT:|   4 | Wolfgang Amadeus Mozart   | Austrian composer of the Classical period       | Male   | Archduchy of Austria; Archbishopric of Salzburg    | Artist       |       1756 |       1791 | 0              |        35 |
+EXPECT:|   5 | Ludwig van Beethoven      | German classical and romantic composer          | Male   | Holy Roman Empire; Austrian Empire                 | Artist       |       1770 |       1827 | 0              |        57 |
+EXPECT:|   6 | Jean-François Champollion | French classical scholar                        | Male   | Kingdom of France; First French Empire             | Egyptologist |       1790 |       1832 | natural causes |        42 |
+EXPECT:|   7 | Paul Morand               | French writer                                   | Male   | France                                             | Artist       |       1888 |       1976 | 0              |        88 |
+EXPECT:|   8 | Claude Monet              | French impressionist painter (1840-1926)        | Male   | France                                             | Artist       |       1840 |       1926 | natural causes |        86 |
+EXPECT:+-----+---------------------------+-------------------------------------------------+--------+----------------------------------------------------+--------------+------------+------------+----------------+-----------+
+EXPECT:
+
+// TODO(tlt): dropping the table seems to cause problems.
+// Drop the table.
+//SEND:DROP TABLE famous;
+
+// Ensure that invalid aruments (none or too many) return an error.
+SEND:\file
+EXPECT:executing meta command: meta command 'file' requires exactly one argument
+SEND:\file filename extra
+EXPECT:executing meta command: meta command 'file' requires exactly one argument
+