Update README.md

changed links to the community help repo

.github/workflows/ci.yml

@@ -36,7 +36,7 @@ jobs:
         with:
           # Optional: version of golangci-lint to use in form of v1.2 or v1.2.3 or `latest` to use the latest version
           # version: v1.29
-          args: --timeout=5m
+          args: --timeout=8m
 
   validate:
     name: Code Checks

.gitignore

@@ -83,3 +83,5 @@ staticcheck.conf
 dax/dax-data
 
 coverage-from-docker
+*.client_id.txt
+

.gitlab/.gitlab-ci.yml

@@ -12,10 +12,22 @@ include:
   - template: Security/License-Scanning.gitlab-ci.yml
   - template: Security/Dependency-Scanning.gitlab-ci.yml
 
+default:
+  retry:
+    max: 2 # This is confusing but this means "3 runs at max".
+    when:
+      - unknown_failure
+      - api_failure
+      - runner_system_failure
+      - job_execution_timeout
+      - stuck_or_timeout_failure
+
 variables:
   GOVERSION: "1.19.3"
   GOFUTURE: "latest"
   CI_IMAGE: "${CI_REGISTRY_IMAGE}/ci-builder:0.0.1"
+  CI_PRE_CLONE_SCRIPT: |
+    set -x
 
 stages:
   - ci_image_build
@@ -157,19 +169,52 @@ build featurebase:
     - go install github.com/rakyll/statik@v0.1.7
     - $GOPATH/bin/statik -src=lattice
     - export SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct)
-    - GOOS="linux" GOARCH="amd64" make build FLAGS="-o featurebase_linux_amd64"
-    - GOOS="linux" GOARCH="arm64" make build FLAGS="-o featurebase_linux_arm64"
-    - GOOS="darwin" GOARCH="amd64" make build FLAGS="-o featurebase_darwin_amd64"
-    - GOOS="darwin" GOARCH="arm64" make build FLAGS="-o featurebase_darwin_arm64"
+    - |
+      for goos in "darwin" "linux"; do
+        for goarch in "amd64" "arm64"; do
+          GOOS="${goos}" GOARCH="${goarch}" make build FLAGS="-o featurebase_${goos}_${goarch}"
+        done
+      done
   artifacts:
     paths:
-      - featurebase_linux_amd64
-      - featurebase_linux_arm64
-      - featurebase_darwin_amd64
-      - featurebase_darwin_arm64
+      - featurebase_*
   needs:
     - job: build lattice
 
+build fbsql amd64:
+  stage: test
+  variables:
+    BUILD_NAME: build_${CI_COMMIT_SHA}_${CI_CONCURRENT_ID}
+  tags:
+    - shell
+  rules:
+    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
+  script:
+    - export SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct)
+    - date
+    - GOOS="linux" GOARCH="amd64" make docker-build-fbsql BUILD_CGO=1
+    - GOOS="darwin" GOARCH="amd64" make docker-build-fbsql
+  artifacts:
+    paths:
+      - ./build/fbsql_*
+
+build fbsql arm64:
+  stage: test
+  variables:
+    BUILD_NAME: build_${CI_COMMIT_SHA}_${CI_CONCURRENT_ID}
+  tags:
+    - shell-arm64
+  rules:
+    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
+  script:
+    - export SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct)
+    - date
+    - GOOS="linux" GOARCH="arm64" make docker-build-fbsql BUILD_CGO=1
+    - GOOS="darwin" GOARCH="arm64" make docker-build-fbsql
+  artifacts:
+    paths:
+      - ./build/fbsql_*
+
 build amd container fb:
   stage: test
   tags:
@@ -210,13 +255,27 @@ run go tests race:
   stage: nonblocking # don't let this job block any other jobs because it takes much longer than the other tests.
   image: golang:$GOVERSION
   extends: .go-cache
+  variables:
+    FEATUREBASE_CONTROLLER_CONFIG_SQLDB_DATABASE: run_go_tests_race
+    POSTGRES_DB: run_go_tests_race
+    FEATUREBASE_CONTROLLER_CONFIG_SQLDB_USER: postgres
+    POSTGRES_USER: postgres
+    FEATUREBASE_CONTROLLER_CONFIG_SQLDB_PASSWORD: $POSTGRES_PASSWORD
+    POSTGRES_PASSWORD: $POSTGRES_PASSWORD
+    FEATUREBASE_CONTROLLER_CONFIG_SQLDB_HOST: postgres
+  services:
+    - postgres:14.7
   rules:
     - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
   needs: ["smoke build"] # we do block on smoke build though bc it's pretty dumb to test stuff if it doesn't build
   script:
     - echo "Running featurebase race tests..."
     - PKG_LIST=$(go list ./... | grep -Ev 'internal/clustertests|simulacraData|batch|idk|v3/dax/test/dax' | paste -s -d, -)
-    - RAMDISK=/mnt/ramdisk go test -race -v -timeout=10m ${PKG_LIST//,/ }
+    - export TMPDIR=/mnt/ramdisk/test-$CI_JOB_ID
+    - mkdir -p $TMPDIR
+    - go test -race -v -timeout=10m ${PKG_LIST//,/ }
+  after_script:
+    - rm -rf /mnt/ramdisk/test-$CI_JOB_ID
   tags:
     - docker
 
@@ -227,12 +286,26 @@ run go tests:
   stage: test
   image: golang:$GOVERSION
   extends: .go-cache
+  variables:
+    FEATUREBASE_CONTROLLER_CONFIG_SQLDB_DATABASE: run_go_tests
+    POSTGRES_DB: run_go_tests
+    FEATUREBASE_CONTROLLER_CONFIG_SQLDB_USER: postgres
+    POSTGRES_USER: postgres
+    FEATUREBASE_CONTROLLER_CONFIG_SQLDB_PASSWORD: $POSTGRES_PASSWORD
+    POSTGRES_PASSWORD: $POSTGRES_PASSWORD
+    FEATUREBASE_CONTROLLER_CONFIG_SQLDB_HOST: postgres
+  services:
+    - postgres:14.7
   rules:
     - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
   script:
     - echo "Running featurebase unit tests..."
     - PKG_LIST=$(go list ./... | grep -Ev 'internal/clustertests|simulacraData|batch|idk|v3/dax/test/dax' | paste -s -d, -)
-    - RAMDISK=/mnt/ramdisk go test -tags=shardwidth22 -timeout=10m -coverprofile=coverage.out -covermode=atomic -coverpkg=${PKG_LIST} ${PKG_LIST//,/ }
+    - export TMPDIR=/mnt/ramdisk/test-$CI_JOB_ID
+    - mkdir -p $TMPDIR
+    - go test -tags=shardwidth22 -timeout=10m -coverprofile=coverage.out -covermode=atomic -coverpkg=${PKG_LIST} ${PKG_LIST//,/ }
+  after_script:
+    - rm -rf /mnt/ramdisk/test-$CI_JOB_ID
   artifacts:
     paths:
       - coverage.out
@@ -242,42 +315,45 @@ run go tests:
 run go tests dax/test/dax:
   stage: test
   image: golang:$GOVERSION
+  extends: .go-cache
   tags:
-    - aws
+    - docker
+  variables:
+    FEATUREBASE_CONTROLLER_CONFIG_SQLDB_DATABASE: run_go_tests_dax
+    POSTGRES_DB: run_go_tests_dax
+    FEATUREBASE_CONTROLLER_CONFIG_SQLDB_USER: postgres
+    POSTGRES_USER: postgres
+    FEATUREBASE_CONTROLLER_CONFIG_SQLDB_PASSWORD: $POSTGRES_PASSWORD
+    POSTGRES_PASSWORD: $POSTGRES_PASSWORD
+    FEATUREBASE_CONTROLLER_CONFIG_SQLDB_HOST: postgres
+  services:
+    - postgres:14.7
   rules:
     - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
   script:
     - echo "Building FB and Datagen docker images for DAX tests"
     - PKG_LIST=$(go list ./... | grep -Ev 'internal/clustertests|simulacraData' | paste -s -d, -)
+    - export TMPDIR=/mnt/ramdisk/test-$CI_JOB_ID
+    - mkdir -p $TMPDIR
     - go test -coverprofile=coverage-dax-integration.out -covermode=atomic -coverpkg=${PKG_LIST} -timeout=20m ./dax/test/dax
+  after_script:
+    - rm -rf /mnt/ramdisk/test-$CI_JOB_ID
   artifacts:
     paths:
       - coverage-dax-integration.out
 
-.idk_changed:
-  rules:
-    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
-      changes:
-        compare_to: refs/heads/master
-        paths:
-          - idk/**
-          - client/**
-          - batch/**
-
 # idk tests
 run go tests idk race:
   variables:
     PROJECT: race_${CI_CONCURRENT_ID}
   stage: nonblocking
   retry: 1
-  rules:
-    - !reference [.idk_changed, rules]
   script:
     - echo "Running test-all-race"
     - cd ./idk/
     - echo $PROJECT
     - echo $CI_COMMIT_REF_SLUG
-    - BRANCH_NAME=${CI_COMMIT_REF_SLUG} make test-all-race
+    - BRANCH_NAME=${CI_COMMIT_REF_SLUG} IDK_FEATUREBASE_TAG=${CI_COMMIT_TAG} IDK_FEATUREBASE_HASH=${CI_COMMIT_SHA} make test-all-race
   after_script:
     - cd ./idk/
     - make save-pilosa-logs
@@ -297,14 +373,12 @@ run go tests idk shard transactional:
     PROJECT: shardttrans_${CI_CONCURRENT_ID}
   stage: nonblocking
   retry: 1
-  rules:
-    - !reference [.idk_changed, rules]
   script:
     - echo "Running shard transactional tests"
     - cd ./idk/
     - echo $PROJECT
     - echo $CI_COMMIT_REF_SLUG
-    - BRANCH_NAME=${CI_COMMIT_REF_SLUG} make test-all
+    - BRANCH_NAME=${CI_COMMIT_REF_SLUG} IDK_FEATUREBASE_TAG=${CI_COMMIT_TAG} IDK_FEATUREBASE_HASH=${CI_COMMIT_SHA} make test-all
   after_script:
     - cd ./idk/
     - make save-pilosa-logs
@@ -330,13 +404,11 @@ run go tests idk 533:
     - cd ./idk/
     - echo $PROJECT
     - echo $CI_COMMIT_REF_SLUG
-    - CONFLUENT_VERSION=5.3.3 BRANCH_NAME=${CI_COMMIT_REF_SLUG} make test-all
+    - CONFLUENT_VERSION=5.3.3 BRANCH_NAME=${CI_COMMIT_REF_SLUG} IDK_FEATUREBASE_TAG=${CI_COMMIT_TAG} IDK_FEATUREBASE_HASH=${CI_COMMIT_SHA} make test-all
   after_script:
     - cd ./idk/
     - make save-pilosa-logs
     - make shutdown
-  rules:
-    - !reference [.idk_changed, rules]
   tags:
     - shell
     - aws
@@ -356,13 +428,11 @@ run go tests idk sasl:
     - cd ./idk/
     - echo $PROJECT
     - echo $CI_COMMIT_REF_SLUG
-    - BRANCH_NAME=${CI_COMMIT_REF_SLUG} make test-all-kafka-sasl
+    - BRANCH_NAME=${CI_COMMIT_REF_SLUG} IDK_FEATUREBASE_TAG=${CI_COMMIT_TAG} IDK_FEATUREBASE_HASH=${CI_COMMIT_SHA} make test-all-kafka-sasl
   after_script:
     - cd ./idk/
     - make save-pilosa-logs
     - make shutdown
-  rules:
-    - !reference [.idk_changed, rules]
   tags:
     - shell
     - aws
@@ -422,7 +492,8 @@ upload_artifacts_to_nexus:
   rules:
     - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
   script:
-    - find . -maxdepth 1 -name '*.rpm' -exec curl -v --user "$NEXUS_YUM_CREDS" --upload-file {} https://nexus.molecula.com/repository/molecula-yum/release/ \;
+    - curl -v --user "$NEXUS_YUM_CREDS" --upload-file *.arm64.rpm https://nexus.molecula.com/repository/molecula-yum/release/
+    - curl -v --user "$NEXUS_YUM_CREDS" --upload-file *.amd64.rpm https://nexus.molecula.com/repository/molecula-yum/release/
   dependencies:
     - package for linux amd64
     - package for linux arm64
@@ -624,16 +695,19 @@ s3 dump:
     - aws configure set aws_secret_access_key $AWS_FBCI_SECRET_ACCESS_KEY
     - aws configure set region "us-east-2"
     - aws configure set aws_profile $PROFILE
-    - aws s3 cp featurebase_linux_amd64 s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/${CI_COMMIT_SHORT_SHA}/featurebase_linux_amd64
-    - aws s3 cp featurebase_linux_amd64 s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/_latest/featurebase_linux_amd64
-    - aws s3 cp featurebase_linux_arm64 s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/${CI_COMMIT_SHORT_SHA}/featurebase_linux_arm64
-    - aws s3 cp featurebase_linux_arm64 s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/_latest/featurebase_linux_arm64
-    - aws s3 cp featurebase_darwin_amd64 s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/${CI_COMMIT_SHORT_SHA}/featurebase_darwin_amd64
-    - aws s3 cp featurebase_darwin_amd64 s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/_latest/featurebase_darwin_amd64
-    - aws s3 cp featurebase_darwin_arm64 s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/${CI_COMMIT_SHORT_SHA}/featurebase_darwin_arm64
-    - aws s3 cp featurebase_darwin_arm64 s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/_latest/featurebase_darwin_arm64
+    - |
+      for goos in "darwin" "linux"; do
+        for goarch in "amd64" "arm64"; do
+          aws s3 cp featurebase_${goos}_${goarch} s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/${CI_COMMIT_SHORT_SHA}/featurebase_${goos}_${goarch}
+          aws s3 cp featurebase_${goos}_${goarch} s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/_latest/featurebase_${goos}_${goarch}
+          aws s3 cp ./build/fbsql_${goos}_${goarch} s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/${CI_COMMIT_SHORT_SHA}/fbsql_${goos}_${goarch}
+          aws s3 cp ./build/fbsql_${goos}_${goarch} s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/_latest/fbsql_${goos}_${goarch}
+        done
+      done
   needs:
     - job: build featurebase
+    - job: build fbsql amd64
+    - job: build fbsql arm64
 
 s3 dump tag:
   stage: post build
@@ -659,6 +733,7 @@ s3 dump tag:
           echo "Directory ${dir}"
           mkdir $dir
           mv featurebase_${goos}_${goarch} ${dir}/featurebase
+          mv ./build/fbsql_${goos}_${goarch} ${dir}/fbsql
           cp NOTICE install/featurebase.conf install/featurebase.*.service ${dir}/
           tar cvzf ${dir}.tar.gz ${dir}
           aws s3 cp ${dir} s3://${LOCATION}/${CI_COMMIT_TAG}/${dir}/ --recursive
@@ -668,3 +743,5 @@ s3 dump tag:
 
   needs:
     - job: build featurebase
+    - job: build fbsql amd64
+    - job: build fbsql arm64

.golangci.yml

@@ -10,9 +10,26 @@ run:
     - pql/pql.peg.go
 linters:
   enable:
+    # Recommended to be enabled by default (https://golangci-lint.run).
+    # - errcheck (lots to fix)
+    - gosimple
     - govet
-    - gofmt
+    - ineffassign
     - staticcheck
+    - typecheck
+    # - unused (about 20 to fix)
+
+    # Additional linters we choose to enable.
+    # - bodyclose (lots to fix, but we should)
+    - errchkjson
+    - errname
+    - gofmt
+    # - misspell (lots to fix, but we should)
+    - prealloc
+    # - predeclared (20 to fix)
+    # - stylecheck (quite a lot to fix, but we should definitely work on this)
+    - stylecheck
+    # - unconvert (not at all critical, but makes for cleaner code)
   enable-all: false
   disable-all: true
 
@@ -50,6 +67,12 @@ linters-settings:
       - shadow
     disable-all: false
 
+  stylecheck:
+    # ST1000: at least one file in a package should have a package comment
+    # ST1003: golang naming standards
+    # ST1016: methods on the same type should have the same receiver name
+    # ST1020: comment on exported function
+    checks: ["all", "-ST1000", "-ST1003", "-ST1016", "-ST1020"]
 
 issues:
   exclude-use-default: false

Dockerfile-dax

@@ -16,7 +16,7 @@ RUN make build FLAGS="-o build/featurebase" ${MAKE_FLAGS}
 ### FeatureBase runner ###
 ##########################
 
-FROM alpine:3.13.2 as runner
+FROM golang:alpine as runner
 
 LABEL maintainer "dev@featurebase.com"
 

Dockerfile-fbsql

@@ -0,0 +1,48 @@
+ARG GO_VERSION=1.19
+
+FROM golang:1.19-buster as builder
+
+WORKDIR /
+RUN apt-get update -y -qq && apt-get install -y -qq \
+  build-essential \
+  git \
+  musl-tools \
+  netcat \
+  unixodbc \
+  unixodbc-dev \
+  && rm -rf /var/lib/apt/lists/*
+RUN ["git", "clone", "https://github.com/edenhill/librdkafka.git"]
+WORKDIR /librdkafka
+RUN ./configure --prefix /usr && \
+  make && \
+  make install
+
+WORKDIR /featurebase
+
+COPY . .
+
+ARG MAKE_FLAGS
+ARG GO_BUILD_FLAGS
+ARG SOURCE_DATE_EPOCH
+
+WORKDIR /featurebase/
+
+ENV SOURCE_DATE_EPOCH=${SOURCE_DATE_EPOCH}
+RUN make build-fbsql GO_BUILD_FLAGS="-mod=vendor ${GO_BUILD_FLAGS}" ${MAKE_FLAGS}
+
+FROM ubuntu:20.04 as runner
+
+RUN apt-get update -y -qq && apt-get install -y -qq \
+  ca-certificates \
+  musl-tools \
+  netcat \
+  unixodbc-dev \
+  && rm -rf /var/lib/apt/lists/*
+
+COPY --from=builder /featurebase/fbsql /usr/local/bin/
+
+# Verify that the linker can find everything.
+FROM runner as linkcheck
+RUN if [ -e /usr/local/bin/fbsql ] ; then ldd /usr/local/bin/fbsql; fi
+
+FROM runner

Makefile

@@ -1,4 +1,4 @@
-.PHONY: build clean build-lattice cover cover-viz default docker docker-build docker-tag-push generate generate-protoc generate-pql generate-statik generate-stringer install install-protoc-gen-gofast install-protoc install-statik install-peg test docker-login
+.PHONY: build clean build-lattice cover cover-viz default docker docker-build docker-build-fbsql docker-tag-push generate generate-protoc generate-pql generate-statik generate-stringer install install-protoc-gen-gofast install-protoc install-statik install-peg test docker-login
 
 SHELL := /bin/bash
 VERSION := $(shell git describe --tags 2> /dev/null || echo unknown)
@@ -19,11 +19,15 @@ SHARD_WIDTH = 20
 COMMIT := $(shell git describe --exact-match >/dev/null 2>&1 || git rev-parse --short HEAD)
 LDFLAGS="-X github.com/featurebasedb/featurebase/v3.Version=$(VERSION) -X github.com/featurebasedb/featurebase/v3.BuildTime=$(BUILD_TIME) -X github.com/featurebasedb/featurebase/v3.Variant=$(VARIANT) -X github.com/featurebasedb/featurebase/v3.Commit=$(COMMIT) -X github.com/featurebasedb/featurebase/v3.TrialDeadline=$(TRIAL_DEADLINE)"
 GO_VERSION=1.19
+GO_BUILD_FLAGS=
 DOCKER_BUILD= # set to 1 to use `docker-build` instead of `build` when creating a release
 BUILD_TAGS += 
 TEST_TAGS = roaringparanoia
 TEST_TIMEOUT=10m
 RACE_TEST_TIMEOUT=10m
+# size in GB to use for ramdisk, ?= so you can override it with env
+# 4GB is not enough for `make test`, 8GB usually is.
+RAMDISK_SIZE ?= 8
 
 export GO111MODULE=on
 export GOPRIVATE=github.com/molecula
@@ -47,11 +51,11 @@ version:
 
 # We build a list of packages that omits the IDK and batch packages because
 # those packages require fancy environment setup.
-GOPACKAGES := $(shell $(GO) list ./... | grep -v "/idk" | grep -v "/batch")
+GOPACKAGES := $(shell $(GO) list ./... | grep -v "/v3/idk" | grep -v "/v3/batch")
 
 # Run test suite
 test:
-	$(GO) test $(GOPACKAGES) -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) -v -timeout $(TEST_TIMEOUT)
+	$(GO) test $(GOPACKAGES) -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) -v -timeout $(TEST_TIMEOUT) -count=1
 
 # Run test suite with race flag
 test-race:
@@ -77,13 +81,18 @@ testvsub:
 			echo; echo "999 done testing subpkg $$pkg"; \
 		done
 
-# make a 2GB RAMDisk. Speed up tests by running them with RAMDISK=/mnt/ramdisk
+# make a $(RAMDISK_SIZE)GB RAMDisk. Speed up tests by running
+# them with TMPDIR=/mnt/ramdisk.
 ramdisk-linux:
-	mount -o size=2G -t tmpfs none /mnt/ramdisk
+	mount -o size=$(RAMDISK__SIZE)G -t tmpfs none /mnt/ramdisk
 
-# make a 2GB RAMDisk. Speed up tests by running them with RAMDISK=/Volumes/RAMDisk
+# make a $(RAMDISK_SIZE)GB RAMDisk. Speed up tests by running
+# them with TMPDIR=/Volumes/RAMDisk. This is more important on
+# OS X than it is on Linux, because there's performance issues
+# with fsync on OS X that can make the SSD slow down to moving-platters
+# drive speeds. Oops.
 ramdisk-osx:
-	diskutil erasevolume HFS+ 'RAMDisk' `hdiutil attach -nobrowse -nomount ram://4194304`
+	diskutil erasevolume HFS+ 'RAMDisk' $$(hdiutil attach -nobrowse -nomount ram://$$(expr 2097152 \* $(RAMDISK_SIZE)))
 
 detach-ramdisk-osx:
 	hdiutil detach /Volumes/RAMDisk
@@ -108,12 +117,13 @@ cover:
 cover-viz: cover
 	$(GO) tool cover -html=build/coverage.out
 
-# Compile Pilosa
+# Build featurebase
 build:
 	$(GO) build -tags='$(BUILD_TAGS)' -ldflags $(LDFLAGS) $(FLAGS) ./cmd/featurebase
 
 package:
-	GOOS=$(GOOS) GOARCH=$(GOARCH) FLAGS="-o featurebase" $(MAKE) build
+	GOOS=$(GOOS) GOARCH=$(GOARCH) $(MAKE) build
+	GOOS=$(GOOS) GOARCH=$(GOARCH) $(MAKE) build-fbsql
 	GOARCH=$(GOARCH) VERSION=$(VERSION) nfpm package --packager deb --target featurebase.$(VERSION).$(GOARCH).deb
 	GOARCH=$(GOARCH) VERSION=$(VERSION) nfpm package --packager rpm --target featurebase.$(VERSION).$(GOARCH).rpm
 
@@ -144,7 +154,7 @@ authclustertests: vendor
 	CLUSTERTESTS_FB_ARGS=$(AUTH_ARGS) $(DOCKER_COMPOSE) -f internal/clustertests/docker-compose.yml down
 
 # Install FeatureBase and IDK
-install: install-featurebase install-idk
+install: install-featurebase install-idk install-fbsql
 
 install-featurebase:
 	$(GO) install -tags='$(BUILD_TAGS)' -ldflags $(LDFLAGS) $(FLAGS) ./cmd/featurebase
@@ -152,6 +162,9 @@ install-featurebase:
 install-idk:
 	$(MAKE) -C ./idk install
 
+install-fbsql:
+	CGO_ENABLED=1 $(GO) install ./cmd/fbsql
+
 # Build the lattice assets
 build-lattice:
 	docker build -t lattice:build ./lattice
@@ -220,6 +233,13 @@ docker-image-featurebase: vendor
 		--file Dockerfile-dax \
 		--tag dax/featurebase .
 
+docker-image-featurebase-linux-amd64: vendor
+	docker build \
+	    --build-arg GO_VERSION=$(GO_VERSION) \
+	    --platform linux/amd64 \
+	    --file Dockerfile-dax \
+	    --tag dax/featurebase .
+
 docker-image-featurebase-test: vendor
 	docker build \
 	    --build-arg GO_VERSION=$(GO_VERSION) \
@@ -246,7 +266,12 @@ docker-image-featurebase-quick: build-for-quick
 docker-image-datagen: vendor
 	docker build --tag dax/datagen --file Dockerfile-datagen .
 
+get-account-id:
+	$(eval AWS_ACCOUNTID := $(shell aws sts get-caller-identity --output=json | jq -r .Account))
+
+
 ecr-push-featurebase: docker-login
+	echo "Pushing to account $(AWS_ACCOUNTID), profile $(AWS_PROFILE)"
 	docker tag dax/featurebase:latest $(AWS_ACCOUNTID).dkr.ecr.us-east-2.amazonaws.com/dax/featurebase:latest
 	docker push $(AWS_ACCOUNTID).dkr.ecr.us-east-2.amazonaws.com/dax/featurebase:latest
 
@@ -254,7 +279,7 @@ ecr-push-datagen: docker-login
 	docker tag dax/datagen:latest $(AWS_ACCOUNTID).dkr.ecr.us-east-2.amazonaws.com/dax/datagen:latest
 	docker push $(AWS_ACCOUNTID).dkr.ecr.us-east-2.amazonaws.com/dax/datagen:latest
 
-docker-login:
+docker-login: get-account-id
 	aws ecr get-login-password --region us-east-2 | docker login --username AWS --password-stdin $(AWS_ACCOUNTID).dkr.ecr.us-east-2.amazonaws.com
 
 # Create docker image (alias)
@@ -328,3 +353,53 @@ test-external-lookup:
 
 bnf:
 	ebnf2railroad --no-overview-diagram  --no-optimizations ./sql3/sql3.ebnf
+
+#################################
+# fbsql builds in docker
+#################################
+
+# This allows multiple concurrent builds to happen in CI without
+# creating container name conflicts and such. (different BUILD_NAMEs
+# are passed in from gitlab-ci.yml)
+BUILD_NAME ?= fbsql-build
+
+LDFLAGS_STATIC="-linkmode external -extldflags \"-static\" -X 'github.com/featurebasedb/featurebase/v3/fbsql.Version=$(VERSION)' -X 'github.com/featurebasedb/featurebase/v3/fbsql.BuildTime=$(BUILD_TIME)' "
+
+UNAME_P := $(shell uname -p)
+BUILD_CGO ?= 0
+
+# Build fbsql
+build-fbsql:
+	@echo GOOS=$(GOOS) GOARCH=$(GOARCH) uname -p=$(UNAME_P) build_cgo=$(BUILD_CGO)
+ifeq ($(BUILD_CGO), 0)
+	make build-fbsql-non-cgo
+endif
+ifeq ($(BUILD_CGO), 1)
+	make build-fbsql-cgo
+endif
+
+build-fbsql-non-cgo:
+	CGO_ENABLED=0 $(GO) build -ldflags $(LDFLAGS) $(GO_BUILD_FLAGS) -o fbsql ./cmd/fbsql
+
+build-fbsql-cgo:
+ifeq ($(GOARCH), arm64)
+	CGO_ENABLED=1 $(GO) build -tags dynamic $(GO_BUILD_FLAGS) -o fbsql ./cmd/fbsql
+endif
+ifeq ($(GOARCH), amd64)
+	CC=/usr/bin/musl-gcc CGO_ENABLED=1 $(GO) build -tags "musl static" -ldflags $(LDFLAGS_STATIC) $(GO_BUILD_FLAGS) -o fbsql ./cmd/fbsql
+endif
+
+docker-build-fbsql: vendor
+	DOCKER_BUILDKIT=0 docker build \
+		--file Dockerfile-fbsql \
+	    --build-arg GO_VERSION=$(GO_VERSION) \
+	    --build-arg MAKE_FLAGS="GOOS=$(GOOS) GOARCH=$(GOARCH) BUILD_CGO=$(BUILD_CGO)" \
+	    --build-arg GO_BUILD_FLAGS=$(GO_BUILD_FLAGS) \
+		--build-arg SOURCE_DATE_EPOCH=$(SOURCE_DATE_EPOCH) \
+	    --target builder \
+	    --tag fbsql:$(BUILD_NAME) .
+	mkdir -p build
+	docker create --name $(BUILD_NAME) fbsql:$(BUILD_NAME)
+	docker cp $(BUILD_NAME):/featurebase/fbsql ./build/fbsql_$(GOOS)_$(GOARCH)
+	docker rm $(BUILD_NAME)
+

OPENSOURCE.md

@@ -0,0 +1,45 @@
+## User Contribution Guidelines for FeatureBase
+
+Thank you for your interest in contributing to FeatureBase! We appreciate your support in making this open-source project even better. Here are some guidelines to help you get started with contributing to FeatureBase:
+
+1. Familiarize Yourself with the Project:
+   - Visit the FeatureBase website at www.featurebase.com to understand the project's goals, capabilities, and features.
+   - Read the documentation available on the website, including the installation guide, configuration options, and data modeling concepts.
+   - Explore the codebase by cloning the repository and reviewing the source code.
+
+2. Join the Community:
+   - Visit the FeatureBase community page at https://www.featurebase.com/community to learn more about the project's community and how to get involved.
+   - Join the Discord server at https://discord.gg/FBn2vEp7Na to chat with other contributors and users, ask questions, and share your ideas.
+
+3. Set Up Your Development Environment:
+   - Ensure you have Go installed on your machine. Make sure your shell's search path includes the go/bin directory.
+   - Clone the FeatureBase repository or download it as a zip file from the repository's page.
+   - Follow the "Build FeatureBase Server from source" instructions in the README file to compile the server binary and the ingester binaries.
+
+4. Choose a Contribution Area:
+   - Identify the area you'd like to contribute to, such as bug fixes, new features, performance improvements, documentation updates, or community support.
+   - Check the issue tracker on the repository or the FeatureBase community for open issues or feature requests that align with your interests and skills. Alternatively, propose your own idea by creating a new issue.
+
+5. Create a New Branch:
+   - Before making any changes, create a new branch in the repository's Git repository. This branch will contain your contributions.
+   - Give your branch a descriptive name that reflects the nature of your contribution.
+
+6. Make Your Changes:
+   - Follow the coding style and conventions used in the existing codebase.
+   - Write clear and concise commit messages for each logical change.
+   - If you're introducing new features or modifying existing behavior, make sure to update the documentation to reflect the changes.
+
+7. Test Your Changes:
+   - Run the existing test suite to ensure that your modifications do not introduce any regressions.
+   - If applicable, write additional tests to cover the changes you made.
+   - Document any new testing procedures required for your contribution.
+
+8. Submitting Your Contribution:
+   - Push your branch to the main repository or create a fork and submit a pull request to the main repository.
+   - Provide a detailed description of your changes, including the problem you solved and the approach you took.
+   - Be responsive to any feedback or suggestions provided by the project maintainers or other contributors.
+   - Once your contribution is approved, it will be reviewed and merged into the main codebase.
+
+Please note that by contributing to FeatureBase, you agree that your contributions will be licensed under the Apache 2.0 license, which governs the project.
+
+Thank you for considering contributing to FeatureBase! Your contributions are valuable and help improve the project for everyone.

README.md

@@ -1,4 +1,10 @@
-# FeatureBase
+# FeatureBase Community
+
+FeatureBase Community is now archived and no longer maintained.
+
+* [FeatureBase Community Help](https://github.com/FeatureBaseDB/FB-community-help)
+
+
 
 ## Pilosa is now FeatureBase
 
@@ -10,6 +16,8 @@ For more information about FeatureBase, please visit [www.featurebase.com][HomeP
 
 ## Getting Started
 
+* [Learn how to install FeatureBase Community](https://github.com/FeatureBaseDB/FB-community-help/blob/main/docs/community/com-getstart/com-getstart-home.md)
+
 ### Build FeatureBase Server from source
 
 0. Install go. Ensure that your shell's search path includes the go/bin directory.
@@ -19,42 +27,20 @@ For more information about FeatureBase, please visit [www.featurebase.com][HomeP
 4. Run `featurebase server --handler.allowed-origins=http://localhost:3000` to run FeatureBase server with default settings (learn more about configuring FeatureBase at the link below). The `--handler.allowed-origins` parameter allows the standalone web UI to talk to the server; this can be omitted if the web UI is not needed.
 5. Run `curl localhost:10101/status` to verify the server is running and accessible.
 
-### Ingest Data and Query
-
-1. Run 
-```
-molecula-consumer-csv \
-    --index repository \
-    --header "language__ID_F,project_id__ID_F" \
-    --id-field project_id \
-    --batch-size 1000 \
-    --files example.csv
-```
-
-This will ingest the `example.csv` file into a FeatureBase table called `repository`. If the table does not exist, it will be automatically created. Learn more about [ingesting data into FeatureBase][Ingest]
-
-2. Query your data. 
-```
-curl localhost:10101/index/repository/query \
-     -X POST \
-     -d 'Row(example=5)'
-```
-Learn about supported [SQL][SQL], native [Pilosa Query Language (PQL)][PQL].
-
 ### Data Model
 
 Because FeatureBase is built on bitmaps, there is bit of a learning curve to grasp how your data is represented. 
-[Learn about Data Modeling][DataModel].
 
-### More Information
+* [Learn about Data Modeling](https://github.com/FeatureBaseDB/FB-community-help/blob/main/docs/concepts/concepts-home.md)
 
-[Installation][Install]
 
-[Configuration][Config]
+### Ingest Data and Query
+
+* [Learn how to ingest data from multiple data sources](https://github.com/FeatureBaseDB/FB-community-help/blob/main/docs/community/com-ingest/com-ingest-manage.md)
 
 ## Community
 
-You can email us at community@featurebase.com or learn more about contributing at [https://www.featurebase.com/community][Community].
+You can email us at community@featurebase.com and [learn more about contributing](https://github.com/FeatureBaseDB/featurebase/blob/master/OPENSOURCE.md).
 
 Chat with us: [https://discord.gg/FBn2vEp7Na][Discord]
 
@@ -73,13 +59,14 @@ A lot has changed since the days of Pilosa. This list highlights some new capabi
 
 FeatureBase is licensed under the [Apache License, Version 2.0][License]
 
-[Community]: http://www.featurebase.com/community?utm_campaign=Open%20Source&utm_source=GitHub
-[Config]: https://docs.featurebase.com/community/community-setup/featurebase-configuration?utm_campaign=Open%20Source&utm_source=GitHub
-[DataModel]: https://docs.featurebase.com/concepts/data-modeling-overview?utm_campaign=Open%20Source&utm_source=GitHub
+[Community]: https://github.com/FeatureBaseDB/FB-community-help/tree/main
+[Install]:https://github.com/FeatureBaseDB/FB-community-help/blob/main/docs/community/com-getstart/com-getstart-home.md
+[Config]: https://github.com/FeatureBaseDB/FB-community-help/tree/main/docs/community/com-config
+[DataModel]: https://github.com/FeatureBaseDB/FB-community-help/blob/main/docs/concepts/concepts-home.md
 [Discord]: https://discord.gg/FBn2vEp7Na
 [HomePage]: http://featurebase.com?utm_campaign=Open%20Source&utm_source=GitHub
-[Ingest]: https://docs.featurebase.com/community/community-data-ingestion/ingesters?utm_campaign=Open%20Source&utm_source=GitHub
-[Install]: https://docs.featurebase.com/community/community-setup/community-install-config?utm_campaign=Open%20Source&utm_source=GitHub
+[Ingest]: https://github.com/FeatureBaseDB/FB-community-help/blob/main/docs/community/com-ingest/com-ingest-manage.md
+ 
 [License]: http://www.apache.org/licenses/LICENSE-2.0
-[PQL]: https://docs.featurebase.com/pql-guide/pql-introduction?utm_campaign=Open%20Source&utm_source=GitHub
-[SQL]: https://docs.featurebase.com/sql-preview/sql-preview?utm_campaign=Open%20Source&utm_source=GitHub
+[PQL]: https://docs.featurebase.com/docs/pql-guide/pql-home/?utm_campaign=Open%20Source&utm_source=GitHub
+[SQL]: https://docs.featurebase.com/docs/sql-guide/sql-guide-home/?utm_campaign=Open%20Source&utm_source=GitHub

api.go

@@ -367,8 +367,8 @@ func (api *API) DeleteIndex(ctx context.Context, indexName string) error {
 }
 
 // CreateField makes the named field in the named index with the given options.
-// This method currently only takes a single functional option, but that may be
-// changed in the future to support multiple options.
+//
+// The resulting field will always have TrackExistence set.
 func (api *API) CreateField(ctx context.Context, indexName string, fieldName string, opts ...FieldOption) (*Field, error) {
 	span, _ := tracing.StartSpanFromContext(ctx, "API.CreateField")
 	defer span.Finish()
@@ -381,6 +381,11 @@ func (api *API) CreateField(ctx context.Context, indexName string, fieldName str
 	// authN/Z info
 	requestUserID, _ := fbcontext.UserID(ctx) // requestUserID is "" if not in ctx
 
+	// newFieldOptions is also used in the path through the index creating
+	// a field from an update from DAX, so it can't assume it can always
+	// override this. But we're the call path for creating new fields, and
+	// new fields should always have TrackExistence on.
+	opts = append(opts, OptFieldTrackExistence())
 	// Apply and validate functional options.
 	fo, err := newFieldOptions(opts...)
 	if err != nil {
@@ -494,16 +499,9 @@ func importWorker(importWork chan importJob) {
 	for j := range importWork {
 		err := func() (err0 error) {
 			for viewName, viewData := range j.req.Views {
-				// The logic here corresponds to the logic in fragment.cleanViewName().
-				// Unfortunately, the logic in that method is not completely exclusive
-				// (i.e. an "other" view named with format YYYYMMDD would be handled
-				// incorrectly). One way to address this would be to change the logic
-				// overall so there weren't conflicts. For now, we just
-				// rely on the field type to inform the intended view name.
-				if viewName == "" {
-					viewName = viewStandard
-				} else if j.field.Type() == FieldTypeTime {
-					viewName = fmt.Sprintf("%s_%s", viewStandard, viewName)
+				viewName, err0 = j.field.cleanupViewName(viewName)
+				if err0 != nil {
+					return err0
 				}
 				if len(viewData) == 0 {
 					return fmt.Errorf("no data to import for view: %s", viewName)
@@ -1316,7 +1314,6 @@ type ImportOptions struct {
 	Clear          bool
 	IgnoreKeyCheck bool
 	Presorted      bool
-	fullySorted    bool // format-aware sorting, internal use only please.
 	suppressLog    bool
 
 	// test Tx atomicity if > 0
@@ -1523,7 +1520,6 @@ func (api *API) ImportWithTx(ctx context.Context, qcx *Qcx, req *ImportRequest,
 		return errors.Wrap(err, "validating api method")
 	}
 
-	api.server.logger.Debugf("ImportWithTx: %v %v %v", req.Index, req.Field, req.Shard)
 	idx, field, err := api.indexField(req.Index, req.Field, req.Shard)
 	if err != nil {
 		return errors.Wrap(err, "getting index and field")
@@ -1642,6 +1638,12 @@ func (api *API) ImportWithTx(ctx context.Context, qcx *Qcx, req *ImportRequest,
 // across many fields in a single shard. It can both set and clear
 // bits and updates caches/bitDepth as appropriate, although only the
 // bitmap parts happen truly transactionally.
+//
+// This function does not attempt to do existence tracking, because
+// it can't; there's no way to distinguish empty sets from not setting
+// bits. As a result, users of this endpoint are responsible for
+// providing corrected existence views for fields with existence
+// tracking. Our batch API does that.
 func (api *API) ImportRoaringShard(ctx context.Context, indexName string, shard uint64, req *ImportRoaringShardRequest) error {
 	index, err := api.Index(ctx, indexName)
 	if err != nil {
@@ -1672,7 +1674,7 @@ func (api *API) ImportRoaringShard(ctx context.Context, indexName string, shard
 		}
 
 		fieldType := field.Options().Type
-		if err1 = cleanupView(fieldType, &viewUpdate); err1 != nil {
+		if viewUpdate.View, err1 = field.cleanupViewName(viewUpdate.View); err1 != nil {
 			return err1
 		}
 
@@ -1764,27 +1766,6 @@ func (api *API) ImportRoaringShard(ctx context.Context, indexName string, shard
 	return nil
 }
 
-func cleanupView(fieldType string, viewUpdate *RoaringUpdate) error {
-	// TODO wouldn't hurt to have consolidated logic somewhere for validating view names.
-	switch fieldType {
-	case FieldTypeSet, FieldTypeTime:
-		if viewUpdate.View == "" {
-			viewUpdate.View = "standard"
-		}
-		// add 'standard_' if we just have a time... this is how IDK works by default
-		if fieldType == FieldTypeTime && !strings.HasPrefix(viewUpdate.View, viewStandard) {
-			viewUpdate.View = fmt.Sprintf("%s_%s", viewStandard, viewUpdate.View)
-		}
-	case FieldTypeInt, FieldTypeDecimal, FieldTypeTimestamp:
-		if viewUpdate.View == "" {
-			viewUpdate.View = "bsig_" + viewUpdate.Field
-		} else if viewUpdate.View != "bsig_"+viewUpdate.Field {
-			return NewBadRequestError(errors.Errorf("invalid view name (%s) for field %s of type %s", viewUpdate.View, viewUpdate.Field, fieldType))
-		}
-	}
-	return nil
-}
-
 // ImportValue is a wrapper around the common code in ImportValueWithTx, which
 // currently just translates req.Clear into a clear ImportOption.
 func (api *API) ImportValue(ctx context.Context, qcx *Qcx, req *ImportValueRequest, opts ...ImportOption) error {
@@ -2038,21 +2019,20 @@ func (api *API) ImportValueWithTx(ctx context.Context, qcx *Qcx, req *ImportValu
 	return nil
 }
 
-func importExistenceColumns(qcx *Qcx, index *Index, columnIDs []uint64, shard uint64) error {
+func importExistenceColumns(qcx *Qcx, index *Index, columnIDs []uint64, shard uint64) (err0 error) {
 	ef := index.existenceField()
 	if ef == nil {
 		return nil
 	}
-
-	existenceRowIDs := make([]uint64, len(columnIDs))
-	// If we don't gratuitously hand-duplicate things in field.Import,
-	// the fact that fragment.bulkImport rewrites its row and column
-	// lists can burn us if we don't make a copy before doing the
-	// existence field write.
-	columnCopy := make([]uint64, len(columnIDs))
-	copy(columnCopy, columnIDs)
-	options := ImportOptions{}
-	return ef.Import(qcx, existenceRowIDs, columnCopy, nil, shard, &options)
+	tx, finisher, err := qcx.GetTx(Txo{Write: true, Index: index, Shard: shard})
+	if err != nil {
+		return err
+	}
+	defer finisher(&err0)
+	// markExistingInView is simpler/faster than Import, but unusually, we use the
+	// standard view of the existence field, instead of the existence view of
+	// a specific field, when doing the index-wide update.
+	return ef.markExistingInView(tx, columnIDs, viewStandard, shard)
 }
 
 // ShardDistribution returns an object representing the distribution of shards
@@ -3070,9 +3050,9 @@ func (api *API) Directive(ctx context.Context, d *dax.Directive) error {
 }
 
 // DirectiveApplied returns true if the computer's current Directive has been
-// applied and is ready to be queried. This it temporary (primarily for tests)
-// and needs to be refactored as we improve the logic around mds-to-computer
-// communication.
+// applied and is ready to be queried. This is temporary (primarily for tests)
+// and needs to be refactored as we improve the logic around
+// controller-to-computer communication.
 func (api *API) DirectiveApplied(ctx context.Context) (bool, error) {
 	return api.holder.DirectiveApplied(), nil
 }
@@ -3085,7 +3065,7 @@ func (api *API) SnapshotShardData(ctx context.Context, req *dax.SnapshotShardDat
 	}
 	// TODO(jaffee) confirm this node is actually responsible for the given
 	// shard? Not sure we need to given that this request comes from
-	// MDS, but might be a belt&suspenders situation.
+	// the Controller, but might be a belt&suspenders situation.
 
 	qtid := req.TableKey.QualifiedTableID()
 
@@ -3316,6 +3296,14 @@ func shardInShards(i dax.ShardNum, s dax.ShardNums) bool {
 }
 
 type SchemaAPI interface {
+	CreateDatabase(context.Context, *dax.Database) error
+	DropDatabase(context.Context, dax.DatabaseID) error
+
+	DatabaseByName(ctx context.Context, dbname dax.DatabaseName) (*dax.Database, error)
+	DatabaseByID(ctx context.Context, dbid dax.DatabaseID) (*dax.Database, error)
+	SetDatabaseOption(ctx context.Context, dbid dax.DatabaseID, option string, value string) error
+	Databases(context.Context, ...dax.DatabaseID) ([]*dax.Database, error)
+
 	TableByName(ctx context.Context, tname dax.TableName) (*dax.Table, error)
 	TableByID(ctx context.Context, tid dax.TableID) (*dax.Table, error)
 	Tables(ctx context.Context) ([]*dax.Table, error)
@@ -3327,8 +3315,49 @@ type SchemaAPI interface {
 	DeleteField(ctx context.Context, tname dax.TableName, fname dax.FieldName) error
 }
 
+// Ensure type implements interface.
+var _ SchemaAPI = (*NopSchemaAPI)(nil)
+
+// NopSchemaAPI is a no-op implementation of the SchemaAPI.
+type NopSchemaAPI struct{}
+
+func (n *NopSchemaAPI) ClusterName() string {
+	return ""
+}
+func (n *NopSchemaAPI) CreateDatabase(context.Context, *dax.Database) error { return nil }
+func (n *NopSchemaAPI) DropDatabase(context.Context, dax.DatabaseID) error  { return nil }
+func (n *NopSchemaAPI) DatabaseByName(ctx context.Context, dbname dax.DatabaseName) (*dax.Database, error) {
+	return nil, nil
+}
+func (n *NopSchemaAPI) DatabaseByID(ctx context.Context, dbid dax.DatabaseID) (*dax.Database, error) {
+	return nil, nil
+}
+func (n *NopSchemaAPI) SetDatabaseOption(ctx context.Context, dbid dax.DatabaseID, option string, value string) error {
+	return nil
+}
+func (n *NopSchemaAPI) Databases(context.Context, ...dax.DatabaseID) ([]*dax.Database, error) {
+	return nil, nil
+}
+func (n *NopSchemaAPI) TableByName(ctx context.Context, tname dax.TableName) (*dax.Table, error) {
+	return nil, nil
+}
+func (n *NopSchemaAPI) TableByID(ctx context.Context, tid dax.TableID) (*dax.Table, error) {
+	return nil, nil
+}
+func (n *NopSchemaAPI) Tables(ctx context.Context) ([]*dax.Table, error) { return nil, nil }
+
+func (n *NopSchemaAPI) CreateTable(ctx context.Context, tbl *dax.Table) error { return nil }
+func (n *NopSchemaAPI) CreateField(ctx context.Context, tname dax.TableName, fld *dax.Field) error {
+	return nil
+}
+func (n *NopSchemaAPI) DeleteTable(ctx context.Context, tname dax.TableName) error { return nil }
+func (n *NopSchemaAPI) DeleteField(ctx context.Context, tname dax.TableName, fname dax.FieldName) error {
+	return nil
+}
+
 type ClusterNode struct {
 	ID        string
+	Type      string
 	State     string
 	URI       string
 	GRPCURI   string

api_directive.go

@@ -35,6 +35,22 @@ func (api *API) ApplyDirective(ctx context.Context, d *dax.Directive) error {
 	// Handle the operations based on the directive method.
 	switch d.Method {
 	case dax.DirectiveMethodDiff:
+		// In order to prevent adding too much code specific to handling a diff
+		// directive (e.g. adding something like an `enactDirectiveDiff()`
+		// method), we are instead going to build a full Directive based on the
+		// diff, and then proceed normally as if we had received a full
+		// Directive. We do that by copying the previous Directive and then
+		// applying the diffs to the copy.
+		newD := previousDirective.Copy()
+
+		// Apply the diffs from the incoming Directive to the new, copied
+		// Directive.
+		newD.ApplyDiff(d)
+
+		// Now proceed with the new diff as if we had received it as a full diff.
+		d = newD
+
+	case dax.DirectiveMethodFull:
 		// pass: normal operation
 
 	case dax.DirectiveMethodReset:
@@ -324,6 +340,17 @@ func (api *API) pushJobsTableKeys(ctx context.Context, jobs chan<- directiveJobT
 	// Get the diff between from/to directive.partitions.
 	partComp := newPartitionsComparer(fromD.TranslatePartitionsMap(), toPartitionsMap)
 
+	// Remove any partitions which are no longer assigned to this worker.
+	// TODO(tlt): currently, this is just removing the file lock on the
+	// resource; it's not actually removing the resource from the local
+	// computer. We should do that.
+	for tkey, partitions := range partComp.removed() {
+		qtid := tkey.QualifiedTableID()
+		for _, partition := range partitions {
+			api.serverlessStorage.RemoveTableKeyResource(qtid, partition)
+		}
+	}
+
 	// Loop over the partition map and load from Writelogger.
 	for tkey, partitions := range partComp.added() {
 		// Get index in order to find the translate stores (by partition) for
@@ -411,6 +438,17 @@ func (api *API) pushJobsFieldKeys(ctx context.Context, jobs chan<- directiveJobT
 	// Get the diff between from/to directive.fields.
 	fieldComp := newFieldsComparer(fromD.TranslateFieldsMap(), toD.TranslateFieldsMap())
 
+	// Remove any field keys which are no longer assigned to this worker.
+	// TODO(tlt): currently, this is just removing the file lock on the
+	// resource; it's not actually removing the resource from the local
+	// computer. We should do that.
+	for tkey, fields := range fieldComp.removed() {
+		qtid := tkey.QualifiedTableID()
+		for _, field := range fields {
+			api.serverlessStorage.RemoveFieldKeyResource(qtid, field)
+		}
+	}
+
 	// Loop over the field map and load from Writelogger.
 	for tkey, fields := range fieldComp.added() {
 		for _, field := range fields {
@@ -495,6 +533,18 @@ func (api *API) pushJobsShards(ctx context.Context, jobs chan<- directiveJobType
 	// Get the diff between from/to directive shards.
 	shardComp := newShardsComparer(fromD.ComputeShardsMap(), shardMap)
 
+	// Remove any shards which are no longer assigned to this worker.
+	// TODO(tlt): currently, this is just removing the file lock on the
+	// resource; it's not actually removing the resource from the local
+	// computer. We should do that.
+	for tkey, shards := range shardComp.removed() {
+		qtid := tkey.QualifiedTableID()
+		for _, shard := range shards {
+			partition := dax.PartitionNum(disco.ShardToShardPartition(string(tkey), uint64(shard), disco.DefaultPartitionN))
+			api.serverlessStorage.RemoveShardResource(qtid, partition, shard)
+		}
+	}
+
 	// Loop over the shard map and load from Writelogger.
 	for tkey, shards := range shardComp.added() {
 		for _, shard := range shards {
@@ -930,7 +980,7 @@ func createField(idx *Index, fld *dax.Field) error {
 		return errors.Wrapf(err, "creating field options from field: %s", fld.Name)
 	}
 
-	if _, err := idx.CreateField(string(fld.Name), "", opts...); err != nil {
+	if _, err := idx.createNullableField(string(fld.Name), "", opts...); err != nil {
 		return errors.Wrapf(err, "creating field on index: %s", fld.Name)
 	}
 	return nil

api_directive_test.go

@@ -30,7 +30,7 @@ func TestAPI_Directive(t *testing.T) {
 		// Empty directive (and empty holder).
 		{
 			d := &dax.Directive{
-				Method:  dax.DirectiveMethodDiff,
+				Method:  dax.DirectiveMethodFull,
 				Version: 1,
 			}
 			err := api.ApplyDirective(ctx, d)
@@ -41,7 +41,7 @@ func TestAPI_Directive(t *testing.T) {
 		// Add a new table.
 		{
 			d := &dax.Directive{
-				Method: dax.DirectiveMethodDiff,
+				Method: dax.DirectiveMethodFull,
 				Tables: []*dax.QualifiedTable{
 					tbl1,
 				},
@@ -55,7 +55,7 @@ func TestAPI_Directive(t *testing.T) {
 		// Add a new table, and keep the existing table.
 		{
 			d := &dax.Directive{
-				Method: dax.DirectiveMethodDiff,
+				Method: dax.DirectiveMethodFull,
 				Tables: []*dax.QualifiedTable{
 					tbl1,
 					tbl2,
@@ -70,7 +70,7 @@ func TestAPI_Directive(t *testing.T) {
 		// Add a new table and remove one of the existing tables.
 		{
 			d := &dax.Directive{
-				Method: dax.DirectiveMethodDiff,
+				Method: dax.DirectiveMethodFull,
 				Tables: []*dax.QualifiedTable{
 					tbl2,
 					tbl3,

api_test.go

@@ -837,7 +837,7 @@ func TestAPI_IDAlloc(t *testing.T) {
 			t.Fatalf("obtaining random bytes: %v", err)
 		}
 		ids3, err := primary.ReserveIDs(key, session, 0, 2)
-		var esync pilosa.ErrIDOffsetDesync
+		var esync pilosa.IDOffsetDesyncError
 		if errors.As(err, &esync) {
 			if esync.Requested != 0 {
 				t.Errorf("incorrect requested offset in error: provided %d but got %d", 0, esync.Requested)

arrow.go

@@ -60,7 +60,7 @@ func (e *executor) executeArrow(ctx context.Context, qcx *Qcx, index string, c *
 		mu.Unlock()
 		return e.executeArrowShard(ctx, qcx, index, c, shard, pool, columnFilter)
 	}
-	tables := make([]*basicTable, 0)
+	tables := make([]*BasicTable, 0)
 
 	reduceFn := func(ctx context.Context, prev, v interface{}) interface{} {
 		mu.Lock()
@@ -70,7 +70,7 @@ func (e *executor) executeArrow(ctx context.Context, qcx *Qcx, index string, c *
 			return prev
 		}
 		switch t := v.(type) {
-		case *basicTable:
+		case *BasicTable:
 
 			if t.resolver != nil {
 				mu.Lock()
@@ -93,104 +93,108 @@ func (e *executor) executeArrow(ctx context.Context, qcx *Qcx, index string, c *
 		return nil, err
 	}
 	if len(tables) == 0 {
-		return &basicTable{name: "empty"}, nil
+		return &BasicTable{name: "empty"}, nil
 	}
 	tbl := Concat(tables[0].Schema(), tables, pool)
 	r := dataframe.NewChunkResolver(tbl.Column(0))
-	return &basicTable{resolver: &r, table: tbl}, nil
+	return &BasicTable{resolver: &r, table: tbl}, nil
 }
 
-type basicTable struct {
+type BasicTable struct {
 	resolver dataframe.Resolver
 	table    arrow.Table
 	filtered bool
 	name     string
 }
 
-func (st *basicTable) Name() string {
+func (st *BasicTable) Name() string {
 	return st.name
 }
 
-func (st *basicTable) Schema() *arrow.Schema {
+func (st *BasicTable) Schema() *arrow.Schema {
 	if st.table != nil {
 		return st.table.Schema()
 	}
 	return &arrow.Schema{}
 }
 
-func (st *basicTable) IsFiltered() bool {
+func (st *BasicTable) IsFiltered() bool {
 	return st.filtered
 }
 
-func (st *basicTable) NumRows() int64 {
+func (st *BasicTable) NumRows() int64 {
 	if st.resolver == nil {
 		return 0
 	}
 	return int64(st.resolver.NumRows())
 }
 
-func (st *basicTable) NumCols() int64 {
+func (st *BasicTable) NumCols() int64 {
 	if st.table != nil {
 		return st.table.NumCols()
 	}
 	return 0
 }
 
-func (st *basicTable) Column(i int) *arrow.Column {
+func (st *BasicTable) Column(i int) *arrow.Column {
 	if st.table != nil {
 		return st.table.Column(i)
 	}
 	return nil
 }
 
-func (st *basicTable) Retain() {
+func (st *BasicTable) Retain() {
 	if st.table != nil {
 		st.table.Retain()
 	}
 }
 
-func (st *basicTable) Release() {
+func (st *BasicTable) Release() {
 	if st.table != nil {
 		st.table.Retain()
 	}
 }
 
-func (st *basicTable) Get(column, row int) interface{} {
+func (st *BasicTable) Get(column, row int) interface{} {
 	field := st.Schema().Field(column)
 	c, i := st.resolver.Resolve(row)
+	nullable := field.Nullable
 
 	chunk := st.Column(column).Data().Chunk(c)
+	// TODO(twg) 2023/01/26 potential NULL support?
+	if nullable && chunk.IsNull(i) {
+		return nil
+	}
 	switch field.Type.(type) {
-	// case *arrow.BooleanType:
-	//	v := chunk.(*array.Boolean).BooleanValues()
-	//	return v[i]
+	case *arrow.BooleanType:
+		return chunk.(*array.Boolean).Value(i)
 	case *arrow.Int8Type:
 		v := chunk.(*array.Int8).Int8Values()
-		return v[i]
+		return int64(v[i])
 	case *arrow.Int16Type:
 		v := chunk.(*array.Int16).Int16Values()
-		return v[i]
+		return int64(v[i])
 	case *arrow.Int32Type:
 		v := chunk.(*array.Int32).Int32Values()
-		return v[i]
+		return int64(v[i])
 	case *arrow.Int64Type:
 		v := chunk.(*array.Int64).Int64Values()
-		return v[i]
+		return int64(v[i])
 	case *arrow.Uint8Type:
 		v := chunk.(*array.Uint8).Uint8Values()
-		return v[i]
+		return uint64(v[i])
 	case *arrow.Uint16Type:
 		v := chunk.(*array.Uint16).Uint16Values()
-		return v[i]
+		return uint64(v[i])
 	case *arrow.Uint32Type:
 		v := chunk.(*array.Uint32).Uint32Values()
-		return v[i]
+		return uint64(v[i])
 	case *arrow.Uint64Type:
 		v := chunk.(*array.Uint64).Uint64Values()
 		return v[i]
 	case *arrow.Float32Type:
 		v := chunk.(*array.Float32).Float32Values()
-		return v[i]
+		return float64(v[i])
 	case *arrow.Float64Type:
 		v := chunk.(*array.Float64).Float64Values()
 		return v[i]
@@ -265,7 +269,7 @@ func appendData(bldr array.Builder, v interface{}) {
 	}
 }
 
-func Concat(schema *arrow.Schema, tables []*basicTable, mem memory.Allocator) arrow.Table {
+func Concat(schema *arrow.Schema, tables []*BasicTable, mem memory.Allocator) arrow.Table {
 	if len(tables) == 1 {
 		if !tables[0].IsFiltered() {
 			return tables[0]
@@ -307,7 +311,7 @@ func Concat(schema *arrow.Schema, tables []*basicTable, mem memory.Allocator) ar
 	return array.NewTable(schema, cols, -1)
 }
 
-func (st *basicTable) MarshalJSON() ([]byte, error) {
+func (st *BasicTable) MarshalJSON() ([]byte, error) {
 	results := make(map[string]interface{})
 	n := 0
 	if st.table != nil {
@@ -326,10 +330,10 @@ func (st *basicTable) MarshalJSON() ([]byte, error) {
 	return json.Marshal(results)
 }
 
-func BasicTableFromArrow(table arrow.Table, mem memory.Allocator) *basicTable {
+func BasicTableFromArrow(table arrow.Table, mem memory.Allocator) *BasicTable {
 	col := table.Column(0)
 	r := dataframe.NewChunkResolver(col)
-	return &basicTable{resolver: &r, table: table}
+	return &BasicTable{resolver: &r, table: table}
 }
 
 func filterColumns(filters []string, table arrow.Table) arrow.Table {
@@ -359,7 +363,7 @@ func filterColumns(filters []string, table arrow.Table) arrow.Table {
 	return array.NewTable(filterdSchema, cols, table.NumRows())
 }
 
-func (e *executor) executeArrowShard(ctx context.Context, qcx *Qcx, index string, c *pql.Call, shard uint64, pool memory.Allocator, columnFilter []string) (*basicTable, error) {
+func (e *executor) executeArrowShard(ctx context.Context, qcx *Qcx, index string, c *pql.Call, shard uint64, pool memory.Allocator, columnFilter []string) (*BasicTable, error) {
 	name := fmt.Sprintf("a. %v", shard)
 	span, _ := tracing.StartSpanFromContext(ctx, "Executor.executeArrowShard")
 	defer span.Finish()
@@ -373,7 +377,7 @@ func (e *executor) executeArrowShard(ctx context.Context, qcx *Qcx, index string
 		filter = row
 		if !filter.Any() {
 			// no need to actuall run the query for its not operating against any values
-			return &basicTable{name: name}, nil
+			return &BasicTable{name: name}, nil
 		}
 	}
 	//
@@ -387,7 +391,7 @@ func (e *executor) executeArrowShard(ctx context.Context, qcx *Qcx, index string
 	fname := idx.GetDataFramePath(shard)
 
 	if !e.dataFrameExists(fname) {
-		return &basicTable{name: name}, nil
+		return &BasicTable{name: name}, nil
 	}
 
 	table, err := e.getDataTable(ctx, fname, pool)
@@ -407,7 +411,7 @@ func (e *executor) executeArrowShard(ctx context.Context, qcx *Qcx, index string
 	resolver = &p
 	if filter != nil {
 		if len(ids) == 0 {
-			return &basicTable{name: name}, nil
+			return &BasicTable{name: name}, nil
 		}
 		resolver, err = filterDataframe(resolver, pool, ids)
 		if err != nil {
@@ -415,7 +419,7 @@ func (e *executor) executeArrowShard(ctx context.Context, qcx *Qcx, index string
 		}
 	}
 	table.Retain()
-	return &basicTable{resolver: resolver, table: table, filtered: filter != nil, name: name}, nil
+	return &BasicTable{resolver: resolver, table: table, filtered: filter != nil, name: name}, nil
 }
 
 func (e *executor) dataFrameExists(fname string) bool {
@@ -474,7 +478,7 @@ func readTableArrow(filename string, mem memory.Allocator) (arrow.Table, error)
 		return nil, err
 	}
 	defer rr.Close()
-	records := make([]arrow.Record, rr.NumRecords(), rr.NumRecords())
+	records := make([]arrow.Record, rr.NumRecords())
 	i := 0
 	for {
 		rec, err := rr.Read()

batch/batch.go

@@ -23,6 +23,7 @@ import (
 const (
 	DefaultKeyTranslateBatchSize = 100000
 	existenceFieldName           = "_exists"
+	existenceViewName            = "existence" // this should match top level featurebase viewExistence
 )
 
 // TODO if using column translation, column ids might get way out of
@@ -573,7 +574,11 @@ func (b *Batch) Add(rec Row) error {
 		case int64:
 			b.values[field.Name] = append(b.values[field.Name], val)
 		case []string:
-			if len(val) == 0 {
+			// note that a length of 0 can be valid, and represents an
+			// empty set. an empty set counts as a non-NULL value for
+			// SQL purposes -- it means the existence view bit should
+			// get set.
+			if val == nil {
 				continue
 			}
 			rowIDSets, ok := b.rowIDSets[field.Name]
@@ -608,7 +613,11 @@ func (b *Batch) Add(rec Row) error {
 			}
 			b.rowIDSets[field.Name] = append(rowIDSets, rowIDs)
 		case []uint64:
-			if len(val) == 0 {
+			// note that a length of 0 can be valid, and represents an
+			// empty set. an empty set counts as a non-NULL value for
+			// SQL purposes -- it means the existence view bit should
+			// get set.
+			if val == nil {
 				continue
 			}
 			rowIDSets, ok := b.rowIDSets[field.Name]
@@ -663,6 +672,9 @@ func (b *Batch) Add(rec Row) error {
 
 	for i, uval := range rec.Clears {
 		field := b.header[i]
+		if field.Options.Type == featurebase.FieldTypeMutex && uval != nil {
+			return errors.Errorf("individual-bit clears not allowed on mutex fields; use nil to clear a mutex")
+		}
 		if _, ok := b.clearRowIDs[i]; !ok {
 			b.clearRowIDs[i] = make(map[int]uint64)
 		}
@@ -1245,7 +1257,7 @@ func (b *Batch) doImport(frags, clearFrags fragments) error {
 			}
 
 			ferr := b.importer.ImportRoaringBitmap(ctx, b.tbl.ID, fld, shard, viewMap, false)
-			b.log.Debugf("imp-roar    field: %s, shard:%d, views:%d %v", field, shard, len(clearViewMap), time.Since(starty))
+			b.log.Debugf("imp-roar    field: %s, shard:%d, views:%d %v", field, shard, len(viewMap), time.Since(starty))
 			return errors.Wrapf(ferr, "importing data for %s", field)
 		})
 	}
@@ -1343,6 +1355,7 @@ func (b *Batch) makeFragments(frags, clearFrags fragments) (fragments, fragments
 		curShard := ^uint64(0) // impossible sentinel value for shard.
 		var curBM *roaring.Bitmap
 		var clearBM *roaring.Bitmap
+		var existCurBM *roaring.Bitmap
 		for j := range b.ids {
 			col := b.ids[j]
 			row := nilSentinel
@@ -1355,8 +1368,12 @@ func (b *Batch) makeFragments(frags, clearFrags fragments) (fragments, fragments
 
 			if col/shardWidth != curShard {
 				curShard = col / shardWidth
+				// the API treats "" as standard
 				curBM = frags.GetOrCreate(curShard, field.Name, "")
 				clearBM = clearFrags.GetOrCreate(curShard, field.Name, "")
+				if opts.ActuallyTrackingExistence() {
+					existCurBM = frags.GetOrCreate(curShard, field.Name, existenceViewName)
+				}
 			}
 			if row != nilSentinel {
 				// TODO this is super ugly, but we want to avoid setting
@@ -1366,6 +1383,9 @@ func (b *Batch) makeFragments(frags, clearFrags fragments) (fragments, fragments
 				// the NoStandardView case would be great.
 				if !(opts.Type == featurebase.FieldTypeTime && opts.NoStandardView) {
 					curBM.DirectAdd(row*shardWidth + (col % shardWidth))
+					if opts.ActuallyTrackingExistence() {
+						existCurBM.DirectAdd(col % shardWidth)
+					}
 				}
 				if opts.Type == featurebase.FieldTypeTime {
 					views, err := b.times[j].views(opts.TimeQuantum)
@@ -1386,6 +1406,16 @@ func (b *Batch) makeFragments(frags, clearFrags fragments) (fragments, fragments
 				// we want to make sure that at this point, the "set"
 				// fragments don't contain the bit that we're clearing
 				curBM.DirectRemoveN(clearRow*shardWidth + (col % shardWidth))
+				// Because this is RowIDs, not RowIDSets, there's only one
+				// bit. We should not be setting the existence bit based on
+				// this value, if we're actually clearing it. This doesn't
+				// mean we will clear an existing existence bit, though.
+				// The case where we would clear an existence bit is the
+				// case where someone specified row[mutexField].Clears = nil,
+				// which is far from here.
+				if opts.ActuallyTrackingExistence() {
+					existCurBM.DirectRemoveN(col % shardWidth)
+				}
 			}
 		}
 	}
@@ -1404,14 +1434,23 @@ func (b *Batch) makeFragments(frags, clearFrags fragments) (fragments, fragments
 		opts := field.Options
 		curShard := ^uint64(0) // impossible sentinel value for shard.
 		var curBM *roaring.Bitmap
+		var existCurBM *roaring.Bitmap
 		for j := range b.ids {
 			col, rowIDs := b.ids[j], rowIDSets[j]
-			if len(rowIDs) == 0 {
-				continue
-			}
 			if col/shardWidth != curShard {
 				curShard = col / shardWidth
 				curBM = frags.GetOrCreate(curShard, fname, "")
+				if opts.ActuallyTrackingExistence() {
+					existCurBM = frags.GetOrCreate(curShard, fname, existenceViewName)
+				}
+			}
+			if len(rowIDs) == 0 {
+				// you can validly specify an empty set, which is not the same as a null,
+				// but which still ought to set the existence bit if we're tracking that.
+				if opts.ActuallyTrackingExistence() && rowIDs != nil {
+					existCurBM.DirectAdd(col % shardWidth)
+				}
+				continue
 			}
 			// TODO this is super ugly, but we want to avoid setting
 			// bits on the standard view in the specific case when
@@ -1422,6 +1461,9 @@ func (b *Batch) makeFragments(frags, clearFrags fragments) (fragments, fragments
 				for _, row := range rowIDs {
 					curBM.DirectAdd(row*shardWidth + (col % shardWidth))
 				}
+				if opts.ActuallyTrackingExistence() {
+					existCurBM.DirectAdd(col % shardWidth)
+				}
 			}
 			if opts.Type == featurebase.FieldTypeTime {
 				views, err := b.times[j].views(opts.TimeQuantum)
@@ -1549,6 +1591,11 @@ func (b *Batch) makeSingleValFragments(frags, clearFrags fragments) (fragments,
 		shard := ids[0] / shardWidth
 		bitmap := frags.GetOrCreate(shard, field.Name, "standard")
 		clearBM := clearFrags.GetOrCreate(shard, field.Name, "standard")
+		var existBM, existClearBM *roaring.Bitmap
+		if field.Options.ActuallyTrackingExistence() {
+			existBM = frags.GetOrCreate(shard, field.Name, existenceViewName)
+			existClearBM = clearFrags.GetOrCreate(shard, field.Name, existenceViewName)
+		}
 		for i, id := range ids {
 			if i+1 < len(ids) {
 				// we only want the last value set for each id
@@ -1561,6 +1608,10 @@ func (b *Batch) makeSingleValFragments(frags, clearFrags fragments) (fragments,
 				shard = id / shardWidth
 				bitmap = frags.GetOrCreate(shard, field.Name, "standard")
 				clearBM = clearFrags.GetOrCreate(shard, field.Name, "standard")
+				if field.Options.ActuallyTrackingExistence() {
+					existBM = frags.GetOrCreate(shard, field.Name, existenceViewName)
+					existClearBM = clearFrags.GetOrCreate(shard, field.Name, existenceViewName)
+				}
 			}
 			fragmentColumn := id % shardWidth
 			clearBM.Add(fragmentColumn) // Will use this to clear columns.
@@ -1568,6 +1619,11 @@ func (b *Batch) makeSingleValFragments(frags, clearFrags fragments) (fragments,
 				// clearSentinel is used for deletion
 				// so this value should only be added if its not clearSentinel
 				bitmap.Add(row*shardWidth + fragmentColumn)
+				if field.Options.ActuallyTrackingExistence() {
+					existBM.Add(fragmentColumn)
+				}
+			} else if field.Options.ActuallyTrackingExistence() {
+				existClearBM.Add(fragmentColumn)
 			}
 		}
 	}
@@ -1596,6 +1652,11 @@ func (b *Batch) makeSingleValFragments(frags, clearFrags fragments) (fragments,
 			fragmentColumn := recID % shardWidth
 
 			clearBM.Add(fragmentColumn)
+			if field.Options.ActuallyTrackingExistence() {
+				existClearBM := clearFrags.GetOrCreate(shard, field.Name, existenceViewName)
+
+				existClearBM.Add(fragmentColumn)
+			}
 		}
 	}
 
@@ -1618,6 +1679,10 @@ func (b *Batch) makeSingleValFragments(frags, clearFrags fragments) (fragments,
 
 			fragmentColumn := recID % shardWidth
 			clearBM.Add(fragmentColumn)
+			if field.Options.ActuallyTrackingExistence() {
+				exist := frags.GetOrCreate(shard, field.Name, existenceViewName)
+				exist.Add(fragmentColumn)
+			}
 
 			if boolVal {
 				bitmap.Add(trueRowOffset + fragmentColumn)

batch/batch_test.go

@@ -103,6 +103,12 @@ func testStringSliceCombos(t *testing.T, importer featurebase.Importer, sapi fea
 		Index: idx.Name,
 		Query: "TopN(a1, n=10)",
 	})
+	if resp.Err != nil {
+		t.Fatalf("unexpected error from TopN query: %v", resp.Err)
+	}
+	if len(resp.Results) < 1 {
+		t.Fatalf("expected non-empty result set, got empty results")
+	}
 	pairsField, ok := resp.Results[0].(*featurebase.PairsField)
 	assert.True(t, ok, "wrong return type: %T", resp.Results[0])
 
@@ -508,10 +514,11 @@ func testStringSliceEmptyAndNil(t *testing.T, importer featurebase.Importer, sap
 			{
 				Name: "strslice",
 				Options: featurebase.FieldOptions{
-					Type:      featurebase.FieldTypeSet,
-					Keys:      true,
-					CacheType: featurebase.CacheTypeRanked,
-					CacheSize: 100,
+					Type:           featurebase.FieldTypeSet,
+					Keys:           true,
+					CacheType:      featurebase.CacheTypeRanked,
+					CacheSize:      100,
+					TrackExistence: true,
 				},
 			},
 		},
@@ -611,6 +618,14 @@ func testStringSliceEmptyAndNil(t *testing.T, importer featurebase.Importer, sap
 			pql: "Row(strslice='z')",
 			exp: []uint64{2},
 		},
+		{
+			pql: "Row(strslice==null)",
+			exp: []uint64{1},
+		},
+		{
+			pql: "Row(strslice!=null)",
+			exp: []uint64{0, 2, 3, 4},
+		},
 	}
 	for i, test := range tests {
 		t.Run(fmt.Sprintf("test-%d", i), func(t *testing.T) {
@@ -2045,7 +2060,7 @@ func mutexClearRegression(t *testing.T, importer featurebase.Importer, sapi feat
 	}
 
 	col := uint64(0)
-	row := uint64(1)
+	row := uint64(0)
 	for i := uint64(0); i <= 21; i++ {
 		col = (i%2+1)*featurebase.ShardWidth + i%5
 		row = i % 3
@@ -2126,7 +2141,7 @@ func mutexNilClearID(t *testing.T, importer featurebase.Importer, sapi featureba
 	}
 
 	col := uint64(0)
-	row := uint64(1)
+	row := uint64(0)
 	// populate mutex with some data
 	for i := uint64(0); i < 11; i++ {
 		col = (i%2+1)*featurebase.ShardWidth + i%5
@@ -2341,3 +2356,58 @@ func testImportBatchBools(t *testing.T, importer featurebase.Importer, sapi feat
 	assert.True(t, ok, "wrong return type: %T", resp.Results[0])
 	assert.Equal(t, uint64(2), count)
 }
+
+func TestConvert(t *testing.T) {
+	t.Run("timestampToInt", func(t *testing.T) {
+		tests := []struct {
+			unit TimeUnit
+			ts   string
+			exp  int64
+		}{
+			{unit: "s", ts: "2022-01-01T00:00:00Z", exp: 1640995200},
+			{unit: "ms", ts: "2022-01-01T00:00:00Z", exp: 1640995200000},
+			{unit: "us", ts: "2022-01-01T00:00:00Z", exp: 1640995200000000},
+			{unit: "ns", ts: "2022-01-01T00:00:00Z", exp: 1640995200000000000},
+			{unit: "x", ts: "2022-01-01T00:00:00Z", exp: 0},
+		}
+		for i, test := range tests {
+			t.Run(fmt.Sprintf("test-%d", i), func(t *testing.T) {
+				ts, err := time.Parse(time.RFC3339, test.ts)
+				assert.NoError(t, err)
+				v := timestampToInt(test.unit, ts)
+				assert.Equal(t, test.exp, v)
+			})
+		}
+	})
+
+	t.Run("Int64ToTimestamp", func(t *testing.T) {
+		tests := []struct {
+			unit  TimeUnit
+			epoch string
+			val   int64
+			exp   time.Time
+		}{
+			{
+				unit:  "ms",
+				epoch: "2022-01-01T00:00:00Z",
+				val:   0,
+				exp:   time.Date(2022, 1, 1, 0, 0, 0, 0, time.UTC),
+			},
+			{
+				unit:  "s",
+				epoch: "2022-01-01T00:00:00Z",
+				val:   86400,
+				exp:   time.Date(2022, 1, 2, 0, 0, 0, 0, time.UTC),
+			},
+		}
+		for i, test := range tests {
+			t.Run(fmt.Sprintf("test-%d", i), func(t *testing.T) {
+				epoch, err := time.Parse(time.RFC3339, test.epoch)
+				assert.NoError(t, err)
+				ts, err := Int64ToTimestamp(test.unit, epoch, test.val)
+				assert.NoError(t, err)
+				assert.Equal(t, test.exp, ts)
+			})
+		}
+	})
+}

batch/batcher.go

@@ -0,0 +1,20 @@
+package batch
+
+import (
+	"time"
+
+	"github.com/featurebasedb/featurebase/v3/dax"
+)
+
+// Batcher is an interface implemented by anything which can allocate new
+// batches.
+type Batcher interface {
+	NewBatch(cfg Config, tbl *dax.Table, fields []*dax.Field) (RecordBatch, error)
+}
+
+// Config is the configuration options passed to NewBatch for any implementation
+// of the Batcher interface.
+type Config struct {
+	Size         int
+	MaxStaleness time.Duration
+}

batch/egpool/egpool.go

@@ -58,11 +58,11 @@ func (eg *Group) err(err error) {
 	eg.errs = append(eg.errs, err)
 }
 
-type ErrPanic struct {
+type PanicError struct {
 	Value interface{}
 }
 
-func (p ErrPanic) Error() string {
+func (p PanicError) Error() string {
 	return fmt.Sprintf("panic: %v", p.Value)
 }
 
@@ -77,7 +77,7 @@ func (eg *Group) processJobs() {
 	defer func() {
 		if !finished {
 			if p := recover(); p != nil {
-				eg.err(ErrPanic{p})
+				eg.err(PanicError{p})
 			} else {
 				eg.err(ErrGoexit)
 			}

buffer/filebuffer.go

@@ -0,0 +1,110 @@
+package buffer
+
+import (
+	"bytes"
+	"io"
+	"io/ioutil"
+	"os"
+	"sync"
+)
+
+// NewFileBuffer returns a file buffer which will use an in-memory buffer, until `max` bytes have been written, at which point it will write the contents of memory to a file, and continue writing future data to the file.
+// The file will be written to `temp` directory. The buffer fulfills the io.Reader and io.Writer interface
+func NewFileBuffer(max int, temp string) *FileBuffer {
+	return &FileBuffer{max: max, tempDir: temp}
+}
+
+type FileBuffer struct {
+	max     int
+	buf     bytes.Buffer
+	file    *os.File
+	tempDir string
+	reading bool
+	files   []*os.File
+	mu      sync.Mutex
+}
+
+func (fb *FileBuffer) Write(p []byte) (n int, err error) {
+	if fb.reading {
+		panic("cannot write after read")
+	}
+	if fb.file != nil {
+		return fb.file.Write(p)
+	}
+	n, err = fb.buf.Write(p)
+	if err != nil {
+		return
+	}
+	if fb.buf.Len() > fb.max {
+		fb.file, err = ioutil.TempFile(fb.tempDir, "filebuffer-")
+		if err != nil {
+			return
+		}
+		_, err = io.Copy(fb.file, &fb.buf)
+		fb.buf.Reset()
+	}
+	return
+}
+
+func (fb *FileBuffer) Len() (int64, error) {
+	if fb.file == nil {
+		return int64(fb.buf.Len()), nil
+	}
+	fi, err := fb.file.Stat()
+	if err != nil {
+		return 0, err
+	}
+
+	return fi.Size(), nil
+}
+
+func (fb *FileBuffer) Read(p []byte) (n int, err error) {
+	if fb.file != nil {
+		if !fb.reading {
+			fb.reading = true
+			_, err = fb.file.Seek(0, 0)
+			if err != nil {
+				return
+			}
+		}
+		return fb.file.Read(p)
+	}
+	fb.reading = true
+	return fb.buf.Read(p)
+}
+
+func (fb *FileBuffer) Close() error {
+	if fb.file != nil {
+		name := fb.file.Name()
+		if err := fb.file.Close(); err != nil {
+			return err
+		}
+		for _, f := range fb.files {
+			f.Close()
+		}
+		fb.files = fb.files[:0]
+		fb.file = nil
+		return os.Remove(name)
+	}
+	return nil
+}
+
+func (fb *FileBuffer) Reset() error {
+	fb.mu.Lock()
+	defer fb.mu.Unlock()
+	fb.reading = false
+	fb.buf.Reset()
+	return fb.Close()
+}
+
+func (fb *FileBuffer) NewReader() (io.Reader, error) {
+	fb.mu.Lock()
+	defer fb.mu.Unlock()
+	fb.reading = true
+	if fb.file == nil {
+		return bytes.NewReader(fb.buf.Bytes()), nil
+	}
+	f, err := os.OpenFile(fb.file.Name(), os.O_RDONLY, 0)
+	fb.files = append(fb.files, f)
+	return f, err
+}

bufferpool/page.go

@@ -53,10 +53,10 @@ const PAGE_PREV_POINTER_OFFSET = 12 // offset 12, length 4, end 16
 const PAGE_NEXT_POINTER_OFFSET = 16 // offset 16, length 4, end 20
 const PAGE_SLOTS_START_OFFSET = 20  // offset 20
 
-// page slots
+// PAGE_SLOT_LENGTH is the size of the page slot key/value.
 //
-//	key offset int16    	//offset 0, length 2, end 2
-//	value offset int16	    //offset 2, length 2, end 4
+// key offset int16    	//offset 0, length 2, end 2
+// value offset int16	    //offset 2, length 2, end 4
 const PAGE_SLOT_LENGTH = 4
 
 // Page represents a page on disk

cache_test.go

@@ -6,7 +6,7 @@ import (
 	"reflect"
 	"testing"
 
-	"github.com/featurebasedb/featurebase/v3"
+	pilosa "github.com/featurebasedb/featurebase/v3"
 )
 
 // Ensure cache stays constrained to its configured size.
@@ -62,7 +62,7 @@ func TestCache_Rank_Dirty(t *testing.T) {
 		cache.Add(v.ID, v.Count)
 	}
 
-	var got []pair
+	var got []pair //nolint:prealloc
 	for _, p := range cache.Top() {
 		got = append(got, pair(p))
 	}

catcher.go

@@ -124,6 +124,17 @@ func (c *catcherTx) Remove(index, field, view string, shard uint64, a ...uint64)
 	return c.b.Remove(index, field, view, shard, a...)
 }
 
+func (c *catcherTx) Removed(index, field, view string, shard uint64, a ...uint64) (changed []uint64, err error) {
+
+	defer func() {
+		if r := recover(); r != nil {
+			vprint.AlwaysPrintf("see Removed() PanicOn '%v' at '%v'", r, vprint.Stack())
+			vprint.PanicOn(r)
+		}
+	}()
+	return c.b.Removed(index, field, view, shard, a...)
+}
+
 func (c *catcherTx) Contains(index, field, view string, shard uint64, key uint64) (exists bool, err error) {
 
 	defer func() {

cli/Makefile

@@ -0,0 +1,15 @@
+.PHONY: test testv test-integration testv-integration
+
+GO=go
+
+test:
+	$(GO) test ./... -short
+
+testv:
+	$(GO) test -v ./... -short
+
+test-integration:
+	$(GO) test . -count 1 -timeout 20m -run TestCLIIntegration/$(RUN)
+
+testv-integration:
+	$(GO) test -v . -count 1 -timeout 20m -run TestCLIIntegration/$(RUN)

cli/batch/inserter.go

@@ -0,0 +1,8 @@
+package batch
+
+// Inserter can be implemented by anything which can handle a SQL statement
+// representing a write operation. An example is `BULK INSERT`. The Insert()
+// method on this interface does not return any results other than an error.
+type Inserter interface {
+	Insert(sql string) error
+}

cli/batch/sql.go

@@ -0,0 +1,215 @@
+package batch
+
+import (
+	"encoding/json"
+	"fmt"
+	"strings"
+	"time"
+
+	fbbatch "github.com/featurebasedb/featurebase/v3/batch"
+	"github.com/featurebasedb/featurebase/v3/dax"
+	"github.com/featurebasedb/featurebase/v3/errors"
+	"github.com/featurebasedb/featurebase/v3/pql"
+)
+
+// Ensure type implements interface.
+var _ fbbatch.Batcher = (*sqlBatcher)(nil)
+
+type sqlBatcher struct {
+	inserter Inserter
+	fields   []*dax.Field
+}
+
+func NewSQLBatcher(i Inserter, flds []*dax.Field) *sqlBatcher {
+	return &sqlBatcher{
+		inserter: i,
+		fields:   flds,
+	}
+}
+
+func (b *sqlBatcher) NewBatch(cfg fbbatch.Config, tbl *dax.Table, flds []*dax.Field) (fbbatch.RecordBatch, error) {
+	fields := flds
+	if b.fields != nil {
+		fields = b.fields
+	}
+	return &sqlBatch{
+		table:        tbl,
+		fields:       fields,
+		size:         cfg.Size,
+		maxStaleness: cfg.MaxStaleness,
+		ids:          make([]interface{}, 0, cfg.Size),
+		rows:         make([][]interface{}, 0, cfg.Size),
+		inserter:     b.inserter,
+	}, nil
+}
+
+// Ensure type implements interface.
+var _ fbbatch.RecordBatch = (*sqlBatch)(nil)
+
+type sqlBatch struct {
+	table  *dax.Table
+	fields []*dax.Field
+	size   int
+
+	ids  []interface{}
+	rows [][]interface{}
+
+	// staleTime tracks the time the first record of the batch was inserted
+	// plus the maxStaleness, in order to raise ErrBatchNowStale if the
+	// maxStaleness has elapsed
+	staleTime    time.Time
+	maxStaleness time.Duration
+
+	// inserter handles SQL INSERT statements generated for each batch.
+	inserter Inserter
+}
+
+func (b *sqlBatch) Add(rec fbbatch.Row) error {
+	// Clear rec.Values and rec.Clears upon return.
+	defer func() {
+		for i := range rec.Values {
+			rec.Values[i] = nil
+		}
+		for k := range rec.Clears {
+			delete(rec.Clears, k)
+		}
+	}()
+
+	if len(b.ids) == cap(b.ids) {
+		return fbbatch.ErrBatchAlreadyFull
+	}
+	if len(rec.Values) != len(b.fields) {
+		return errors.Errorf("record needs to match up with batch fields, got %d fields and %d record", len(b.fields), len(rec.Values))
+	}
+
+	// Append the ID to b.ids.
+	b.ids = append(b.ids, rec.ID)
+
+	// Convert decimal fields (which come in as int64, along with the scale in
+	// field) to pql.Decimal.
+	for i, fld := range b.fields {
+		switch b.fields[i].Type {
+		case dax.BaseTypeDecimal:
+			if val, ok := rec.Values[i].(int64); ok {
+				rec.Values[i] = pql.NewDecimal(val, fld.Options.Scale)
+			}
+		case dax.BaseTypeTimestamp:
+			if val, ok := rec.Values[i].(int64); ok {
+				ts := time.Unix(val, 0)
+				rec.Values[i] = ts.Format(time.RFC3339)
+			}
+		}
+	}
+
+	// Append the record to b.rows.
+	vals := make([]interface{}, 0, len(rec.Values))
+	vals = append(vals, rec.Values...)
+	b.rows = append(b.rows, vals)
+
+	// Check for batch full or stale.
+	if len(b.ids) == cap(b.ids) {
+		return fbbatch.ErrBatchNowFull
+	}
+	if b.maxStaleness != time.Duration(0) { // set maxStaleness to 0 to disable staleness checking
+		if len(b.ids) == 1 {
+			b.staleTime = time.Now().Add(b.maxStaleness)
+		} else if time.Now().After(b.staleTime) {
+			return fbbatch.ErrBatchNowStale
+		}
+	}
+	return nil
+}
+
+func (b *sqlBatch) Import() error {
+	if len(b.rows) == 0 {
+		return nil
+	}
+
+	// Construct the BULK INSERT statement based on the table and fields.
+	sql, err := buildBulkInsert(b.table, b.fields, b.ids, b.rows)
+	if err != nil {
+		return errors.Wrap(err, "building bulk insert statement")
+	}
+
+	// Reset batch data.
+	b.reset()
+
+	// Submit the SQL statement.
+	return b.inserter.Insert(sql)
+}
+
+func (b *sqlBatch) reset() {
+	b.ids = b.ids[:0]
+	b.rows = b.rows[:0]
+}
+
+func (b *sqlBatch) Len() int {
+	return len(b.rows)
+}
+
+func (b *sqlBatch) Flush() error {
+	return nil
+}
+
+func buildBulkInsert(tbl *dax.Table, fields []*dax.Field, ids []interface{}, rows [][]interface{}) (string, error) {
+	// Validation.
+	if tbl.Name == "" {
+		return "", errors.New(errors.ErrUncoded, "table name is required")
+	} else if len(fields) == 0 {
+		return "", errors.New(errors.ErrUncoded, "at least one field is required")
+	}
+
+	var sb strings.Builder
+
+	sb.WriteString(`BULK INSERT INTO `)
+	sb.WriteString(string(tbl.Name))
+	sb.WriteString(` (_id,`)
+
+	flds := make([]string, 0, len(fields))
+	maps := make([]string, 0, len(fields))
+	for i := range fields {
+		flds = append(flds, string(fields[i].Name))
+		maps = append(maps, fmt.Sprintf("'$.col_%d' %s", i, fields[i].FullType()))
+	}
+	// Fields
+	sb.WriteString(strings.Join(flds, ","))
+
+	// MAP
+	keyType := dax.BaseTypeID
+	if tbl.StringKeys() {
+		keyType = dax.BaseTypeString
+	}
+	sb.WriteString(`) MAP ('$._id' `)
+	sb.WriteString(keyType)
+	sb.WriteString(`,`)
+	sb.WriteString(strings.Join(maps, ","))
+	sb.WriteString(`) FROM x'`)
+
+	// Row values.
+
+	// m is a map representing a single row to be marshalled and added to the
+	// bulk insert as one line in the NDJSON payload. We re-use the map for each
+	// row.
+	m := make(map[string]interface{})
+	for i := range rows {
+		// Write the ID value.
+		m[string(dax.PrimaryKeyFieldName)] = ids[i]
+		// Write the rest of the data values.
+		for col := range rows[i] {
+			m[fmt.Sprintf("col_%d", col)] = rows[i][col]
+		}
+
+		// Marshal the map to json and add to the sql statement.
+		if j, err := json.Marshal(m); err != nil {
+			return "", errors.Wrap(err, "marshalling row to json")
+		} else {
+			sb.Write(j)
+			sb.WriteString("\n")
+		}
+	}
+
+	// WITH
+	sb.WriteString(fmt.Sprintf(`' WITH BATCHSIZE %d FORMAT 'NDJSON' INPUT 'STREAM'`, len(rows)))
+
+	return sb.String(), nil
+}

cli/batch/sql_test.go

@@ -0,0 +1,47 @@
+package batch
+
+import (
+	"testing"
+
+	"github.com/featurebasedb/featurebase/v3/dax"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestBatchSQL(t *testing.T) {
+	tbl := &dax.Table{
+		Name: "foo",
+	}
+	fields := []*dax.Field{
+		{
+			Name: "name",
+			Type: dax.BaseTypeString,
+		},
+		{
+			Name: "age",
+			Type: dax.BaseTypeInt,
+		},
+	}
+	ids := []interface{}{
+		0, 1, 2,
+	}
+	rows := [][]interface{}{
+		{
+			[]interface{}{"Alice", int64(11)},
+		},
+		{
+			[]interface{}{"Bob", int64(22)},
+		},
+		{
+			[]interface{}{"Carl,Comma", int64(33)},
+		},
+	}
+
+	s, err := buildBulkInsert(tbl, fields, ids, rows)
+	assert.NoError(t, err)
+
+	exp := `BULK INSERT INTO foo (_id,name,age) MAP ('$._id' id,'$.col_0' string,'$.col_1' int) FROM x'{"_id":0,"col_0":["Alice",11]}
+{"_id":1,"col_0":["Bob",22]}
+{"_id":2,"col_0":["Carl,Comma",33]}
+' WITH BATCHSIZE 3 FORMAT 'NDJSON' INPUT 'STREAM'`
+	assert.Equal(t, exp, s)
+}

cli/buffer.go

@@ -0,0 +1,91 @@
+package cli
+
+import (
+	"io"
+	"strings"
+
+	"github.com/featurebasedb/featurebase/v3/errors"
+)
+
+// buffer is a query buffer for SQL statements. Note that this is not a query
+// buffer as you would find on a database server (buffering query results).
+// Rather, this buffers the working SQL statement. The buffer has two
+// components: the buffer of query parts making up the working, incomplete SQL
+// statement, and the last completed SQL statement submitted to the Queryer.
+type buffer struct {
+	parts     []queryPart
+	lastQuery query
+
+	hasBatchFile bool
+}
+
+func newBuffer() *buffer {
+	return &buffer{}
+}
+
+// addPart adds the given queryPart to the buffer. If the part is of type
+// `partTerminator` (which is generally singified in the CLI by a ";"), the
+// buffer will finalize the query and return it. In all other cases, the
+// returned query is nil.
+func (b *buffer) addPart(part queryPart) (query, error) {
+	// Check for part type compatibility. For example, multiple batchFile parts
+	// are not allowed in the same query.
+	switch part.(type) {
+	case *partBatchFile:
+		if b.hasBatchFile {
+			return nil, errors.Errorf("multiple batch files in one query is not supported")
+		}
+		b.hasBatchFile = true
+	case *partTerminator:
+		return b.finalize(), nil
+	}
+
+	b.parts = append(b.parts, part)
+	return nil, nil
+}
+
+// finalize copies the contents (queryParts) of buffer to lastQuery and then
+// resets the buffer. It returns the query that was finalized.
+func (b *buffer) finalize() query {
+	q := make(query, len(b.parts))
+	copy(q, b.parts)
+	b.lastQuery = q
+	b.reset()
+	return q
+
+}
+
+// print returns the contents of the buffer as a string. This is generally used
+// to visually inspect the state of the buffer (for example, when a user issues
+// a `\p` meta-command in the CLI).
+func (b *buffer) print() string {
+	if len(b.parts) > 0 {
+		return query(b.parts).String()
+	} else if b.lastQuery != nil {
+		return b.lastQuery.String() + ";"
+	}
+	return "Query buffer is empty."
+}
+
+// reset clears the buffer. It returns a message which may optionally be used to
+// display to a user.
+func (b *buffer) reset() string {
+	b.parts = b.parts[:0]
+	b.hasBatchFile = false
+	return "Query buffer reset (cleared)."
+}
+
+func (b *buffer) Reader() io.Reader {
+	if len(b.parts) > 0 {
+		return query(b.parts).Reader()
+	} else if b.lastQuery != nil {
+		r := b.lastQuery.Reader()
+		// TODO(tlt): terminating the query here results in a line feed just
+		// before the semi-colon (for example, when you print out the query
+		// buffer using `\w [FILE]`). The removal and re-introduction of line
+		// feeds is kind of a mess.
+		term := strings.NewReader(";")
+		return io.MultiReader(r, term)
+	}
+	return strings.NewReader("")
+}

cli/cli_integration_test.go

@@ -0,0 +1,257 @@
+package cli_test
+
+import (
+	"bufio"
+	"context"
+	"fmt"
+	"os"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/featurebasedb/featurebase/v3/cli"
+	"github.com/featurebasedb/featurebase/v3/dax/server/test"
+	"github.com/featurebasedb/featurebase/v3/logger"
+	"github.com/stretchr/testify/require"
+)
+
+func TestCLIIntegration(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test")
+	}
+
+	ctx := context.Background()
+
+	t.Run("Stubbed Framework", func(t *testing.T) {
+		mc := test.MustRunManagedCommand(t)
+		defer mc.Close()
+
+		addr := mc.Address()
+
+		capture := newCapture(t)
+
+		comparer := newComparer(t)
+		comparer.run()
+
+		fbsql := cli.NewCommand(logger.StderrLogger)
+		fbsql.SetStdin(capture)
+		fbsql.SetStdout(comparer)
+		fbsql.SetStderr(comparer)
+
+		fbsql.Config = &cli.Config{
+			Host: addr.Host(),
+			Port: fmt.Sprintf("%d", addr.Port()),
+		}
+
+		// Run fbsql in a goroutine so we can continue to send it commands
+		// below.
+		didQuit := make(chan struct{})
+		go func() {
+			require.NoError(t, fbsql.Run(ctx))
+			close(didQuit)
+		}()
+
+		// testFiles reference files located in the cli/testdata directory. All
+		// tests should be placed there; other than adding another test file to
+		// this list, you probably shouldn't be editing this file unless you are
+		// trying to modify the way the test framework itself works.
+		testFiles := []string{
+			"setup",
+			"database",
+			"table",
+			// the tests below may be dependent on the previous tests, which do
+			// setup and some shared database and table creation.
+			"query_buffer",
+			// meta commands
+			"meta_bang",
+			"meta_cd",
+			"meta_echo",
+			"meta_describe",
+			"meta_file",
+			"meta_pset_border",
+			"meta_pset_expanded",
+			"meta_pset_format_csv",
+			"meta_pset_tuples_only",
+			"meta_include",
+			"meta_output",
+			"meta_set",
+			"meta_timing",
+			"meta_write",
+		}
+
+		for _, testFile := range testFiles {
+			t.Run(testFile, func(t *testing.T) {
+				f, err := os.Open("testdata/" + testFile)
+				require.NoError(t, err)
+
+				scanner := bufio.NewScanner(f)
+				var lineNo int
+				for scanner.Scan() {
+					line := scanner.Text()
+					lineNo++
+
+					// Empty lines and comments (//) are ignored.
+					if line == "" {
+						continue
+					} else if strings.HasPrefix(line, "//") {
+						continue
+					}
+
+					parts := strings.SplitN(line, ":", 2)
+
+					switch parts[0] {
+					case "SEND":
+						v := ""
+						if len(parts) == 2 {
+							v = parts[1]
+						}
+						capture.sendLine(v)
+					case "EXPECT":
+						v := ""
+						if len(parts) == 2 {
+							v = parts[1]
+						}
+						comparer.expectLine(v, testFile, lineNo)
+					case "EXPECTCOMP":
+						if len(parts) == 2 {
+							comps := strings.SplitN(parts[1], ":", 2)
+							v := ""
+							if len(comps) == 2 {
+								v = comps[1]
+							}
+							comparer.expectLineComp(comparator(comps[0]), v, testFile, lineNo)
+						} else {
+							t.Errorf("unexpected line: %s[%d]:%s", testFile, lineNo, line)
+						}
+					default:
+						t.Errorf("unexpected line: %s[%d]:%s", testFile, lineNo, line)
+					}
+				}
+				require.NoError(t, scanner.Err())
+			})
+		}
+
+		// End with quit to ensure that fbsql closes without error.
+		capture.sendLine(`\q`)
+
+		// Ensure fbsql quits cleanly.
+		select {
+		case <-didQuit:
+		case <-time.After(time.Second):
+			t.Fatalf("expected fbsql to quit")
+		}
+	})
+}
+
+// compare is used to compare fbsql output written to its Stdout with expected
+// lines.
+type comparer struct {
+	t       *testing.T
+	out     chan byte
+	outline chan []byte
+	exp     chan []byte
+}
+
+func newComparer(t *testing.T) *comparer {
+	return &comparer{
+		t:       t,
+		out:     make(chan byte, 1024),
+		outline: make(chan []byte, 128),
+		exp:     make(chan []byte, 1024),
+	}
+}
+
+func (c *comparer) run() {
+	// Read bytes off output, and for every line (designated by a line feed "\n"),
+	// push the line onto the outline channel.
+	go func() {
+		var line []byte
+		for {
+			b := <-c.out
+			if b == byte('\n') {
+				c.outline <- line
+				line = []byte{}
+				continue
+			}
+			line = append(line, b)
+		}
+	}()
+}
+
+type comparator string
+
+const (
+	compEquals     = "Equals"
+	compHasPrefix  = "HasPrefix"
+	compWithFormat = "WithFormat"
+)
+
+// expectLine is a convenience method which calls expectLineComp with the compEq
+// comparator and the given line.
+func (c *comparer) expectLine(line string, fileName string, lineNo int) {
+	c.expectLineComp(compEquals, line, fileName, lineNo)
+}
+
+// expectLineComp reads the next line from the outline channel and compares it
+// with the given `line`. A comparator can be provided to inform how the lines
+// should be compared (for example, the compHasPrefix comparator will just
+// compare the beginning part of the outline).
+func (c *comparer) expectLineComp(comp comparator, line string, fileName string, lineNo int) {
+	var outline []byte
+	select {
+	case outline = <-c.outline:
+	case <-time.After(10 * time.Second):
+		// TODO(tlt): this is 10 seconds to account for the fb_views creation on
+		// a local mac. This should really be something like 2 seconds. Put this
+		// back to 2 once fb_views issue is addressed.
+		c.t.Fatalf("expected output line %s[%d]: >%s<", fileName, lineNo, line)
+	}
+
+	// msg is included in any require which fails.
+	msg := []interface{}{"exp: %s[%d], got: >%s<", fileName, lineNo, outline}
+
+	switch comp {
+	case compEquals:
+		require.Equal(c.t, []byte(line), outline, msg...)
+	case compHasPrefix:
+		require.True(c.t, strings.HasPrefix(string(outline), line), msg...)
+	case compWithFormat:
+		require.True(c.t, compareByteSlices(outline, []byte(line)), msg...)
+	default:
+		c.t.Fatalf("invalid comparator: %s", comp)
+	}
+}
+
+func (c *comparer) Write(b []byte) (n int, err error) {
+	for i := range b {
+		c.out <- b[i]
+	}
+	return len(b), err
+}
+
+// compareByteSlices compares a byte slice s with another byte slice format and
+// returns true if they are the same. It will accept underscore as a
+// single-character wildcard anywhere in slice format.
+func compareByteSlices(s, format []byte) bool {
+	// Replace some helpers in format before comparing.
+	f := string(format)
+	f = strings.ReplaceAll(f, `{uuid}`, `________-____-____-____-____________`)
+	f = strings.ReplaceAll(f, `{timestamp}`, `____-__-__T__:__:__Z`)
+	format = []byte(f)
+
+	if len(s) != len(format) {
+		return false
+	}
+
+	for i := range s {
+		if format[i] == '_' {
+			continue
+		}
+		if s[i] != format[i] {
+			// log.Printf("DEBUG: characters differ: (%d): '%v' != '%v'", i, s[i], format[i])
+			return false
+		}
+	}
+
+	return true
+}

cli/cli_test.go

@@ -21,9 +21,9 @@ func TestCLI(t *testing.T) {
 
 		capture := newCapture(t)
 
-		cli := cli.NewCLICommand(logger.StderrLogger)
-		cli.Stdin = capture
-		cli.Stdout = capture
+		cli := cli.NewCommand(logger.StderrLogger)
+		cli.SetStdin(capture)
+		cli.SetStdout(capture)
 		cli.Queryer = capture
 
 		go func() {
@@ -33,38 +33,32 @@ func TestCLI(t *testing.T) {
 		none := []string{}
 
 		// One statement, one line.
-		capture.Assert("one;", []string{`one`})
+		capture.Assert("one;", []string{"one\n"})
 
 		// One statement, multiple lines.
 		capture.Assert("one", none)
 		capture.Assert(" two ", none)
-		capture.Assert("three;", []string{`one
- two 
-three`})
+		capture.Assert("three;", []string{"one\ntwo\nthree\n"})
 
 		// Multiple statements, one line.
-		capture.Assert("foo; bar;", []string{`foo`, `bar`})
+		capture.Assert("foo; bar;", []string{"foo\n", "bar\n"})
 
 		// Multiple statements, multiple lines.
 		capture.Assert("a1", none)
-		capture.Assert("a2; b1", []string{`a1
-a2`})
-		capture.Assert("b2;", []string{`b1
-b2`})
+		capture.Assert("a2; b1", []string{"a1\na2\n"})
+		capture.Assert("b2;", []string{"b1\nb2\n"})
 
 		// Blank lines.
 		capture.Assert("one", none)
 		capture.Assert("", none)
-		capture.Assert("three;", []string{`one
-
-three`})
+		capture.Assert("three;", []string{"one\nthree\n"})
 
 		// Just a semi-colon.
-		capture.Assert(";", none)
+		capture.Assert(";", []string{""})
 
 		// Multi-line with just a semi-colon.
 		capture.Assert("one", none)
-		capture.Assert(";", []string{`one`})
+		capture.Assert(";", []string{"one\n"})
 
 		// Ensure a clean exit with no errors.
 		assert.NoError(t, capture.Exit())
@@ -80,7 +74,7 @@ var _ cli.Queryer = (*capture)(nil)
 
 // capture implements the various CLI interfaces in order to capture test input
 // and submit it as though that input were being read from the command line. It
-// also captures calls made to the Queryer.Query method and ensures the sql the
+// also captures calls made to the Queryer.Query method and ensures the sql they
 // contain is expected.
 type capture struct {
 	t *testing.T
@@ -111,7 +105,7 @@ func newCapture(t *testing.T) *capture {
 }
 
 func (c *capture) Exit() error {
-	c.sendLine("exit")
+	c.sendLine(`\q`)
 	c.mu.RLock()
 	defer c.mu.RUnlock()
 	return c.err
@@ -183,9 +177,15 @@ func (c *capture) Write(b []byte) (n int, err error) {
 
 // Query is called by the CLI command once a full SQL statement is received
 // (signified by the terminator: `;`).
-func (c *capture) Query(org, db, sql string) (*featurebase.WireQueryResponse, error) {
+func (c *capture) Query(org string, db string, sql io.Reader) (*featurebase.WireQueryResponse, error) {
+	tmpBuf := new(strings.Builder)
+	_, err := io.Copy(tmpBuf, sql)
+	if err != nil {
+		return nil, err
+	}
+
 	c.mu.Lock()
-	c.sqls = append(c.sqls, sql)
+	c.sqls = append(c.sqls, tmpBuf.String())
 	c.mu.Unlock()
 
 	select {

cli/config.go

@@ -0,0 +1,31 @@
+package cli
+
+// Config represents the configuration for the command.
+type Config struct {
+	Host string `json:"host"`
+	Port string `json:"port"`
+
+	OrganizationID string `json:"org-id"`
+	Database       string `json:"db"`
+
+	// CloudAuth
+	CloudAuth CloudAuthConfig `json:"cloud-auth"`
+
+	// Kafka
+	KafkaConfig string `json:"kafka-config"`
+
+	HistoryPath string `json:"history-path"`
+
+	// CSV (Comma-Separated Values) table output mode.
+	CSV bool `json:"csv"`
+
+	// PSet takes one or more pset arguments of the form: `--pset=VAR[=ARG]`.
+	PSets []string `json:"pset"`
+}
+
+type CloudAuthConfig struct {
+	ClientID string `json:"client-id"`
+	Region   string `json:"region"`
+	Email    string `json:"email"`
+	Password string `json:"password"`
+}

cli/errors.go

@@ -0,0 +1,16 @@
+package cli
+
+import (
+	"github.com/featurebasedb/featurebase/v3/errors"
+)
+
+const (
+	ErrOrganizationRequired errors.Code = "OrganizationRequired"
+)
+
+func NewErrOrganizationRequired() error {
+	return errors.New(
+		ErrOrganizationRequired,
+		"organization required",
+	)
+}

cli/fbcloud/client.go

@@ -1,7 +1,6 @@
 package fbcloud
 
 import (
-	"bytes"
 	"encoding/json"
 	"fmt"
 	"io"
@@ -40,37 +39,26 @@ func (cq *Queryer) tokenRefresh() error {
 	return nil
 }
 
-type tokenizedSQL struct {
-	Language  string `json:"language"`
-	Statement string `json:"statement"`
-}
-
 // Query issues a SQL query formatted for the FeatureBase cloud query endpoint.
-func (cq *Queryer) Query(org, db, sql string) (*featurebase.WireQueryResponse, error) {
+func (cq *Queryer) Query(org string, db string, sql io.Reader) (*featurebase.WireQueryResponse, error) {
 	if time.Since(cq.lastRefresh) > TokenRefreshTimeout {
 		if err := cq.tokenRefresh(); err != nil {
 			return nil, errors.Wrap(err, "refreshing token")
 		}
 	}
-	url := fmt.Sprintf("%s/v2/databases/%s/query/sql", cq.Host, db)
-
-	sqlReq := &tokenizedSQL{
-		Language:  "sql",
-		Statement: sql,
-	}
-	var buf bytes.Buffer
-	if err := json.NewEncoder(&buf).Encode(sqlReq); err != nil {
-		return nil, errors.Wrapf(err, "encoding sql request: %s", sql)
+	url := fmt.Sprintf("%s/databases/%s/sql", cq.Host, db)
+	if db == "" {
+		url = fmt.Sprintf("%s/sql", cq.Host)
 	}
 
 	client := &http.Client{
 		Timeout: time.Second * 30,
 	}
-	req, err := http.NewRequest(http.MethodPost, url, &buf)
+	req, err := http.NewRequest(http.MethodPost, url, sql)
 	if err != nil {
 		return nil, errors.Wrap(err, "creating new post request")
 	}
-	req.Header.Add("Content-Type", "application/json")
+	req.Header.Add("Content-Type", "text/plain")
 	req.Header.Add("Authorization", cq.token)
 
 	var resp *http.Response
@@ -141,7 +129,3 @@ func (cq *Queryer) HTTPRequest(method, path, body string, v interface{}) ([]byte
 
 	return bodbytes, nil
 }
-
-type cloudResponse struct {
-	Results featurebase.WireQueryResponse `json:"results"`
-}

cli/kafka.go

@@ -0,0 +1,70 @@
+package cli
+
+import (
+	"fmt"
+
+	"github.com/featurebasedb/featurebase/v3/cli/batch"
+	"github.com/featurebasedb/featurebase/v3/cli/kafka"
+	"github.com/featurebasedb/featurebase/v3/errors"
+	"github.com/spf13/viper"
+)
+
+func (cmd *Command) newKafkaRunner(cfgFile string) (*kafka.Runner, error) {
+	// Read the kafka config file.
+	v := viper.New()
+	v.SetConfigFile(cfgFile)
+	v.SetConfigType("toml")
+	err := v.ReadInConfig()
+	if err != nil {
+		return nil, fmt.Errorf("error reading configuration file '%s': %v", cfgFile, err)
+	}
+
+	cfg := kafka.Config{}
+	if err := v.Unmarshal(&cfg); err != nil {
+		return nil, errors.Wrap(err, "unmarshalling config")
+	}
+
+	if err := kafka.ValidateConfig(cfg); err != nil {
+		return nil, errors.Wrap(err, "validating config")
+	}
+
+	// Create a new config with defaults.
+
+	// Look up fields based on table provided in the config.
+	wqr, err := cmd.executeQuery(newRawQuery("SHOW COLUMNS FROM " + cfg.Table))
+	if err != nil {
+		return nil, errors.Wrap(err, "executing query")
+	}
+
+	scr, err := wqr.ShowColumnsResponse()
+	if err != nil {
+		return nil, errors.Wrap(err, "getting show columns from wire query response")
+	}
+
+	// If no fields were provided in the config, use the fields defined on the
+	// table and assume a 1-to-1 mapping of source to destination.
+	if len(cfg.Fields) == 0 {
+		cfg.Fields = kafka.FieldsToConfig(scr.Fields)
+	} else {
+		cfg.Fields, err = kafka.CheckFieldCompatibility(cfg.Fields, scr)
+		if err != nil {
+			return nil, errors.Wrap(err, "validating config fields")
+		}
+	}
+
+	idkCfg, err := kafka.ConvertConfig(cfg)
+	if err != nil {
+		return nil, errors.Wrap(err, "cleaning config")
+	}
+
+	flds, err := kafka.ConfigToFields(cfg)
+	if err != nil {
+		return nil, errors.Wrap(err, "getting fields from config")
+	}
+
+	return kafka.NewRunner(
+		idkCfg,
+		batch.NewSQLBatcher(cmd, flds),
+		cmd.stderr,
+	), nil
+}

cli/kafka/config.go

@@ -0,0 +1,251 @@
+package kafka
+
+import (
+	"fmt"
+	"time"
+
+	featurebase "github.com/featurebasedb/featurebase/v3"
+	"github.com/featurebasedb/featurebase/v3/dax"
+	"github.com/featurebasedb/featurebase/v3/idk"
+	"github.com/pkg/errors"
+)
+
+// Config is the user-facing configuration for kafka support in the CLI. This is
+// unmarshalled from the the toml config file supplied by the user.
+type Config struct {
+	Hosts  []string `mapstructure:"hosts" help:"Kafka hosts."`
+	Group  string   `mapstructure:"group" help:"Kafka group."`
+	Topics []string `mapstructure:"topics" help:"Kafka topics to read from."`
+
+	BatchSize         int           `mapstructure:"batch-size" help:"Batch size."`
+	BatchMaxStaleness time.Duration `mapstructure:"batch-max-staleness" help:"Maximum length of time that the oldest record in a batch can exist before flushing the batch. Note that this can potentially stack with timeouts waiting for the source."`
+	Timeout           time.Duration `mapstructure:"timeout" help:"Time to wait for more records from Kafka before flushing a batch. 0 to disable."`
+
+	Table  string  `mapstructure:"table" help:"Destination table name."`
+	Fields []Field `mapstructure:"fields"`
+}
+
+// Field is a user-facing configuration field.
+type Field struct {
+	Name       string   `mapstructure:"name"`
+	SourceType string   `mapstructure:"source-type"`
+	SourcePath []string `mapstructure:"source-path"`
+	PrimaryKey bool     `mapstructure:"primary-key"`
+}
+
+// ConfigForIDK represents Config converted to values suitable for IDK. In
+// particular, the idk.RawField is used in parsing the schema in IDK.
+type ConfigForIDK struct {
+	Hosts  []string
+	Group  string
+	Topics []string
+
+	BatchSize         int
+	BatchMaxStaleness time.Duration
+	Timeout           time.Duration
+
+	Table   string
+	IDField string
+	Fields  []idk.RawField
+}
+
+// ValidateConfig validates the config is usable.
+func ValidateConfig(c Config) error {
+	if c.Table == "" {
+		return errors.Errorf("table is required")
+	} else if len(c.Topics) == 0 {
+		return errors.Errorf("at least one topic is required")
+	} else if len(c.Fields) > 0 {
+		// We only need to do these checks if any fields are specified at all.
+		// If no fields are specified, that's ok because then we default to
+		// using fields based off the existing table.
+		if len(c.Fields) < 2 {
+			return errors.Errorf("at least two fields are required (one should be a primary key)")
+		} else {
+			var found int
+			for i := range c.Fields {
+				if c.Fields[i].PrimaryKey {
+					found++
+				}
+				if c.Fields[i].Name == "" {
+					return errors.Errorf("a name attribute (which isn't equal to \"\") should exist for all fields")
+				}
+			}
+			if found != 1 {
+				return errors.Errorf("exactly one primary key field is required")
+			}
+		}
+	}
+	return nil
+}
+
+// ConvertConfig converts a Config to one that suitable for IDK.
+func ConvertConfig(c Config) (ConfigForIDK, error) {
+	// Set a default kafka host in case one isn't provided.
+	hosts := []string{"localhost:9092"}
+	if len(c.Hosts) > 0 {
+		hosts = c.Hosts
+	}
+
+	// Copy all the shared members from Config to ConfigForIDK.
+	out := ConfigForIDK{
+		Hosts:             hosts,
+		Group:             c.Group,
+		Topics:            c.Topics,
+		BatchSize:         c.BatchSize,
+		BatchMaxStaleness: c.BatchMaxStaleness,
+		Timeout:           c.Timeout,
+		Table:             c.Table,
+	}
+
+	if len(c.Fields) == 0 {
+		return out, errors.New("fields cannot be empty")
+	}
+
+	// rawFields wil be the same as c.Fields, but possibly enhanced.
+	rawFields := make([]idk.RawField, 0, len(c.Fields))
+
+	var foundPK bool
+	for _, fld := range c.Fields {
+		if fld.PrimaryKey {
+			out.IDField = fld.Name
+			foundPK = true
+		}
+
+		typ, quals, err := dax.SplitFieldType(fld.SourceType)
+		if err != nil {
+			return out, errors.Wrap(err, "getting base type")
+		}
+
+		rawFld := idk.RawField{
+			Name: fld.Name,
+			Type: string(typ),
+			Path: fld.SourcePath,
+		}
+		// If a SourcePath wasn't provided, default to using the field name.
+		if len(rawFld.Path) == 0 {
+			rawFld.Path = []string{fld.Name}
+		}
+
+		switch typ {
+		case dax.BaseTypeInt:
+			// We don't have to handle min/max because we don't create the table.
+		case dax.BaseTypeDecimal:
+			if len(quals) != 1 {
+				return out, errors.Errorf("expected decimal scale")
+			}
+			rawFld.Config = []byte(fmt.Sprintf(`{"scale":%d}`, quals[0]))
+		case dax.BaseTypeID:
+			rawFld.Config = []byte("{\"mutex\":true}")
+		case dax.BaseTypeIDSet:
+			rawFld.Type = "ids"
+		case dax.BaseTypeString:
+			rawFld.Config = []byte("{\"mutex\":true}")
+		case dax.BaseTypeStringSet:
+			rawFld.Type = "strings"
+		case dax.BaseTypeTimestamp:
+			// No timestamp options are handled for now.
+		}
+
+		rawFields = append(rawFields, rawFld)
+	}
+	if !foundPK {
+		return out, errors.New("primary-key not found in fields")
+	}
+
+	out.Fields = rawFields
+
+	return out, nil
+}
+
+// ConfigToFields returns a list of *dax.Field based on the IDField and Fields
+// in the Config.
+func ConfigToFields(c Config) ([]*dax.Field, error) {
+	// We don't know if a primary key will be found, so we can't set the
+	// capacity to `len(c.Fields)-1`.
+	out := make([]*dax.Field, 0, len(c.Fields))
+
+	for _, fld := range c.Fields {
+		if fld.PrimaryKey {
+			continue
+		}
+		typ, quals, err := dax.SplitFieldType(fld.SourceType)
+		if err != nil {
+			return nil, errors.Wrap(err, "splitting field type")
+		}
+		dfld := &dax.Field{
+			Name: dax.FieldName(fld.Name),
+			Type: typ,
+		}
+		switch typ {
+		case dax.BaseTypeDecimal:
+			if len(quals) != 1 {
+				return nil, errors.Errorf("expected decimal scale")
+			}
+			scale, ok := quals[0].(int64)
+			if !ok {
+				return nil, errors.Errorf("invalid decimal scale: %v", quals[0])
+			}
+			dfld.Options.Scale = scale
+		}
+		out = append(out, dfld)
+	}
+
+	return out, nil
+}
+
+// FieldsToConfig returns a Config.Fields based on a list of *dax.Field.
+func FieldsToConfig(flds []*dax.Field) []Field {
+	out := make([]Field, 0, len(flds))
+	for _, fld := range flds {
+		out = append(out, Field{
+			Name:       string(fld.Name),
+			SourceType: fld.FullType(),
+			PrimaryKey: fld.IsPrimaryKey(),
+		})
+	}
+	return out
+}
+
+// CheckFieldCompatibility ensures that the fields provided in the kafka config
+// are compatible with the fields in the existing table. It returns a copy of
+// the kafka config fields with empty values defaulted to the table field
+// configuration.
+func CheckFieldCompatibility(cflds []Field, scr *featurebase.ShowColumnsResponse) ([]Field, error) {
+	out := make([]Field, len(cflds))
+	for i, cfld := range cflds {
+		out[i] = cfld
+		cfldName := dax.FieldName(cfld.Name)
+
+		// Primary key field.
+		if cfld.PrimaryKey {
+			f := scr.Field(dax.PrimaryKeyFieldName)
+			if f == nil {
+				return nil, dax.NewErrFieldDoesNotExist(dax.PrimaryKeyFieldName) // It should be impossible to hit this.
+			}
+			if out[i].SourceType == "" {
+				if f.StringKeys() {
+					out[i].SourceType = dax.BaseTypeString
+				} else {
+					out[i].SourceType = dax.BaseTypeID
+				}
+			}
+			continue
+		}
+
+		// Non primary key fields.
+		if cfldName == dax.PrimaryKeyFieldName {
+			return nil, errors.Errorf("field named '%s' must be a primary key", dax.PrimaryKeyFieldName)
+		}
+
+		f := scr.Field(cfldName)
+		if f == nil {
+			return nil, dax.NewErrFieldDoesNotExist(cfldName)
+		}
+
+		if out[i].SourceType == "" {
+			out[i].SourceType = f.FullType()
+		}
+	}
+	return out, nil
+}

cli/kafka/runner.go

@@ -0,0 +1,67 @@
+package kafka
+
+import (
+	"io"
+	"time"
+
+	fbbatch "github.com/featurebasedb/featurebase/v3/batch"
+	"github.com/featurebasedb/featurebase/v3/errors"
+	"github.com/featurebasedb/featurebase/v3/idk"
+	"github.com/featurebasedb/featurebase/v3/idk/kafka_static"
+	"github.com/featurebasedb/featurebase/v3/logger"
+)
+
+// Runner is a CLI-specific kafka consumer. It's similar to
+// idk.kafka_static.Main in that it embeds idk.Main and contains additional
+// functionality specific to its use case.
+type Runner struct {
+	idk.Main   `flag:"!embed"`
+	KafkaHosts []string       `help:"Comma separated list of host:port pairs for Kafka."`
+	Group      string         `help:"Kafka group."`
+	Topics     []string       `help:"Kafka topics to read from."`
+	Timeout    time.Duration  `help:"Time to wait for more records from Kafka before flushing a batch. 0 to disable."`
+	Header     []idk.RawField `help:"Header configuration."`
+}
+
+func NewRunner(cfg ConfigForIDK, batcher fbbatch.Batcher, logWriter io.Writer) *Runner {
+	idkMain := idk.NewMain()
+	idkMain.IDField = cfg.IDField
+	idkMain.Index = cfg.Table
+	idkMain.Batcher = batcher
+	idkMain.BatchSize = cfg.BatchSize
+	idkMain.BatchMaxStaleness = cfg.BatchMaxStaleness
+	idkMain.SetBasic()
+	idkMain.SetLog(logger.NewStandardLogger(logWriter))
+
+	kr := &Runner{
+		Main:       *idkMain,
+		KafkaHosts: cfg.Hosts,
+		Group:      cfg.Group,
+		Topics:     cfg.Topics,
+		Header:     cfg.Fields,
+		Timeout:    cfg.Timeout,
+	}
+	kr.OffsetMode = true
+	kr.Main.Namespace = "cli_kafka_runner"
+	kr.Main.Pprof = "" // don't initialize pprof until we actually use it in tests
+	kr.NewSource = func() (idk.Source, error) {
+		source := kafka_static.NewSource()
+		source.Hosts = kr.KafkaHosts
+		source.Group = kr.Group
+		source.Topics = kr.Topics
+		source.Log = kr.Main.Log()
+		// source.TLS = m.KafkaTLS
+		source.Timeout = kr.Timeout
+		// source.SkipOld = m.SkipOld
+		source.HeaderFields = kr.Header
+		// source.S3Region = m.S3Region
+		// source.AllowMissingFields = m.AllowMissingFields
+
+		err := source.Open()
+		if err != nil {
+			return nil, errors.Wrap(err, "opening source")
+		}
+		return source, nil
+	}
+	return kr
+}

cli/parts.go

@@ -0,0 +1,136 @@
+package cli
+
+import (
+	"fmt"
+	"io"
+	"os"
+	"strings"
+)
+
+// query is a collection of queryParts which, when applied together, make up an
+// executable SQL query.
+type query []queryPart
+
+func (q query) String() string {
+	var sb strings.Builder
+	for i := range q {
+		sb.WriteString(q[i].String())
+		if i < len(q)-1 {
+			sb.WriteRune('\n')
+		}
+	}
+	return sb.String()
+}
+
+// Reader returns the query as an io.Reader so that it can be passed to, for
+// example, http.Post().
+func (q query) Reader() io.Reader {
+	readers := make([]io.Reader, 0, len(q))
+	for i := range q {
+		readers = append(readers, q[i].Reader())
+	}
+	return io.MultiReader(readers...)
+}
+
+// queryPart is an interface representing anything which can use to build up a
+// query.
+type queryPart interface {
+	fmt.Stringer
+	Reader() io.Reader
+}
+
+func newRawQuery(s string) query {
+	return []queryPart{
+		newPartRaw(s),
+	}
+}
+
+// ////////////////////////////////////////////////////////////////////////////
+// raw
+// ////////////////////////////////////////////////////////////////////////////
+
+// Ensure type implements interface.
+var _ queryPart = (*partRaw)(nil)
+
+type partRaw struct {
+	raw string
+}
+
+func newPartRaw(s string) *partRaw {
+	return &partRaw{
+		raw: s,
+	}
+}
+
+func (p *partRaw) Reader() io.Reader {
+	return strings.NewReader(p.raw + "\n")
+}
+
+func (p *partRaw) String() string {
+	return p.raw
+}
+
+// ////////////////////////////////////////////////////////////////////////////
+// file
+// ////////////////////////////////////////////////////////////////////////////
+
+// Ensure type implements interface.
+var _ queryPart = (*partFile)(nil)
+
+type partFile struct {
+	file *os.File
+}
+
+func newPartFile(f *os.File) *partFile {
+	return &partFile{
+		file: f,
+	}
+}
+
+func (p *partFile) Reader() io.Reader {
+	return p.file
+}
+
+func (p *partFile) String() string {
+	return fmt.Sprintf("[file: %s]", p.file.Name())
+}
+
+// ////////////////////////////////////////////////////////////////////////////
+// batch file
+// ////////////////////////////////////////////////////////////////////////////
+
+// Ensure type implements interface.
+var _ queryPart = (*partBatchFile)(nil)
+
+type partBatchFile struct {
+	file *os.File
+}
+
+func (p *partBatchFile) Reader() io.Reader {
+	return p.file
+}
+
+func (p *partBatchFile) String() string {
+	return p.file.Name()
+}
+
+// ////////////////////////////////////////////////////////////////////////////
+// terminator (i.e. ";")
+// ////////////////////////////////////////////////////////////////////////////
+
+// Ensure type implements interface.
+var _ queryPart = (*partTerminator)(nil)
+
+type partTerminator struct{}
+
+func newPartTerminator() *partTerminator {
+	return &partTerminator{}
+}
+
+func (p *partTerminator) Reader() io.Reader {
+	return nil
+}
+
+func (p *partTerminator) String() string {
+	return terminationChar
+}

cli/queryer.go

@@ -1,20 +1,26 @@
 package cli
 
 import (
-	"bytes"
-	"encoding/json"
 	"fmt"
 	"io"
 	"net/http"
+	"time"
 
 	featurebase "github.com/featurebasedb/featurebase/v3"
-	"github.com/featurebasedb/featurebase/v3/dax"
-	queryerhttp "github.com/featurebasedb/featurebase/v3/dax/queryer/http"
 	"github.com/pkg/errors"
 )
 
 type Queryer interface {
-	Query(org, db, sql string) (*featurebase.WireQueryResponse, error)
+	Query(org string, db string, sql io.Reader) (*featurebase.WireQueryResponse, error)
+}
+
+// Ensure type implements interface.
+var _ Queryer = (*nopQueryer)(nil)
+
+type nopQueryer struct{}
+
+func (qryr *nopQueryer) Query(org string, db string, sql io.Reader) (*featurebase.WireQueryResponse, error) {
+	return nil, errors.Errorf("no-op queryer")
 }
 
 // Ensure type implements interface.
@@ -27,13 +33,10 @@ type standardQueryer struct {
 	Port string
 }
 
-func (qryr *standardQueryer) Query(org, db, sql string) (*featurebase.WireQueryResponse, error) {
-	buf := bytes.Buffer{}
+func (qryr *standardQueryer) Query(org string, db string, sql io.Reader) (*featurebase.WireQueryResponse, error) {
 	url := fmt.Sprintf("%s/sql", hostPort(qryr.Host, qryr.Port))
 
-	buf.Write([]byte(sql))
-
-	resp, err := http.Post(url, "application/json", &buf)
+	resp, err := http.Post(url, "application/json", sql)
 	if err != nil {
 		return nil, errors.Wrapf(err, "posting query")
 	}
@@ -42,8 +45,10 @@ func (qryr *standardQueryer) Query(org, db, sql string) (*featurebase.WireQueryR
 	if err != nil {
 		return nil, errors.Wrap(err, "reading response")
 	}
+
 	sqlResponse := &featurebase.WireQueryResponse{}
 	// TODO(tlt): switch this back once all responses are typed
+	// TODO(twg) 2023/03/01 using json.Number to decode large ints so care must be made
 	// if err := json.Unmarshal(fullbod, sqlResponse); err != nil {
 	if err := sqlResponse.UnmarshalJSONTyped(fullbod, true); err != nil {
 		return nil, errors.Wrapf(err, "unmarshaling query response, body:\n'%s'\n", fullbod)
@@ -53,32 +58,39 @@ func (qryr *standardQueryer) Query(org, db, sql string) (*featurebase.WireQueryR
 }
 
 // Ensure type implements interface.
-var _ Queryer = (*daxQueryer)(nil)
+var _ Queryer = (*serverlessQueryer)(nil)
 
-// daxQueryer is similar to the standardQueryer except that it hits a different
-// endpoint, and its payload is a json object which includes, in addition to the
-// sql statement, things like org and db.
-type daxQueryer struct {
+// serverlessQueryer is similar to the standardQueryer except that it hits a
+// different endpoint, and its payload is database-aware.
+type serverlessQueryer struct {
 	Host string
 	Port string
 }
 
-func (qryr *daxQueryer) Query(org, db, sql string) (*featurebase.WireQueryResponse, error) {
-	buf := bytes.Buffer{}
-	url := fmt.Sprintf("%s/queryer/sql", hostPort(qryr.Host, qryr.Port))
-
-	sqlReq := &queryerhttp.SQLRequest{
-		OrganizationID: dax.OrganizationID(org),
-		DatabaseID:     dax.DatabaseID(db),
-		SQL:            sql,
-	}
-	if err := json.NewEncoder(&buf).Encode(sqlReq); err != nil {
-		return nil, errors.Wrapf(err, "encoding sql request: %s", sql)
+func (qryr *serverlessQueryer) Query(org string, db string, sql io.Reader) (*featurebase.WireQueryResponse, error) {
+	if org == "" {
+		return nil, NewErrOrganizationRequired()
 	}
 
-	resp, err := http.Post(url, "application/json", &buf)
+	url := fmt.Sprintf("%s/queryer/databases/%s/sql", hostPort(qryr.Host, qryr.Port), db)
+	if db == "" {
+		url = fmt.Sprintf("%s/queryer/sql", hostPort(qryr.Host, qryr.Port))
+	}
+
+	client := &http.Client{
+		Timeout: time.Second * 30,
+	}
+
+	req, err := http.NewRequest(http.MethodPost, url, sql)
 	if err != nil {
-		return nil, errors.Wrapf(err, "posting query")
+		return nil, errors.Wrap(err, "creating new post request")
+	}
+	req.Header.Add("Content-Type", "text/plain")
+	req.Header.Add("OrganizationID", org)
+
+	var resp *http.Response
+	if resp, err = client.Do(req); err != nil {
+		return nil, errors.Wrap(err, "executing post request")
 	}
 
 	fullbod, err := io.ReadAll(resp.Body)

cli/replacer.go

@@ -0,0 +1,109 @@
+package cli
+
+import (
+	"strings"
+
+	"github.com/benhoyt/goawk/lexer"
+)
+
+// replacer can replace parts of a string based on some rules and the provided
+// map[string]string. For example, the Command can replace strings with values
+// in its `variables` map.
+type replacer struct {
+	m map[string]string
+}
+
+func newReplacer(m map[string]string) *replacer {
+	return &replacer{
+		m: m,
+	}
+}
+
+// replace replaces all instances of the string pattern `:key` with the value at
+// m[key]. For example we want something like this:
+//
+// GIVEN: `start :one,:'two', :"three" ::four ::`
+//
+// with map
+//
+//	map[string]string{
+//		"one":   "repl1",
+//		"three": "repl3",
+//	}
+//
+// WANT: `start repl1,:'two', "repl3" ::four ::`
+func (r *replacer) replace(s string) string {
+	// If no variables have been added to the map, there's no need to parse the
+	// string for variable replacement.
+	if len(r.m) == 0 {
+		return s
+	}
+
+	line := []byte(s)
+	lex := lexer.NewLexer(line)
+
+	// finger contains the index into line at the start of non-variable text
+	// that we want to include, as-is in the output.
+	var finger int
+
+	// sb builds the string which will be the final output.
+	var sb strings.Builder
+	for {
+		pos, tok, _ := lex.Scan()
+
+		switch tok {
+		case lexer.COLON:
+			// last is the last normal character position before the colon.
+			last := pos.Column - 1
+
+			// Get the next byte to see if the colon value is quoted, and if so,
+			// whether its has single or double quotes.
+			b := lex.PeekByte()
+
+			// padding is the amount of padding we have to consider around the
+			// variable name. If the variable is not quoted, it doesn't require
+			// any padding. But if it has quotes, it needs 2 characters of
+			// paddings to accomodate the quotes.
+			padding := 0
+
+			// quote holds the character to use to quote the final, replaced
+			// output value. Because the lexer doesn't tell us how a certain
+			// `string` token was quoted, we need to keep track of that here so
+			// we can put them back.
+			quote := ""
+			switch b {
+			case byte('\''): // single quote
+				quote = `'`
+				padding = 2
+			case byte('"'): // double quote
+				quote = `"`
+				padding = 2
+			}
+
+			pos, tok, key := lex.Scan()
+			switch tok {
+			case lexer.NAME, lexer.STRING:
+				// Write the normal text up to the variable replacement
+				// position.
+				sb.Write(line[finger:last])
+
+				if v, ok := r.m[key]; ok {
+					// Write replaced variable with the quotes it had.
+					sb.WriteString(quote + v + quote)
+				} else {
+					// Since the variable was not found in the map, just write
+					// back what was already there.
+					sb.WriteString(":" + quote + key + quote)
+				}
+
+				// Reset finger to point to the next position after the
+				// variable.
+				finger = pos.Column + len(key) + padding - 1
+			}
+		case lexer.EOF:
+			// Write the remainder of the string and return.
+			sb.Write(line[finger:])
+			return sb.String()
+		}
+	}
+}

cli/replacer_test.go

@@ -0,0 +1,109 @@
+package cli
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestReplacer(t *testing.T) {
+	t.Run("general replace function", func(t *testing.T) {
+
+		m := map[string]string{
+			"v1": "newVone",
+			"v2": "newVtwo",
+		}
+
+		tests := []struct {
+			s   string
+			m   map[string]string
+			exp string
+		}{
+			{
+				// no variables present
+				s:   "foo",
+				m:   m,
+				exp: "foo",
+			},
+			{
+				// variable prefix, but not in map
+				s:   ":foo",
+				m:   m,
+				exp: ":foo",
+			},
+			{
+				// variable name match, but missing prefix
+				s:   "v1",
+				m:   m,
+				exp: "v1",
+			},
+			{
+				// variable name match
+				s:   ":v1",
+				m:   m,
+				exp: "newVone",
+			},
+			{
+				// two variables, the same, no space
+				s:   ":v1:v1",
+				m:   m,
+				exp: "newVonenewVone",
+			},
+			{
+				// two variables, different, no space
+				s:   ":v1:v2",
+				m:   m,
+				exp: "newVonenewVtwo",
+			},
+			{
+				// two variables, different, spaces
+				s:   ":v1  :v2",
+				m:   m,
+				exp: "newVone  newVtwo",
+			},
+			{
+				// one variable, one non-variable, no space
+				s:   ":v1:foo",
+				m:   m,
+				exp: "newVone:foo",
+			},
+			{
+				// one non-variable, one variable, no space
+				s:   "foo:v1",
+				m:   m,
+				exp: "foonewVone",
+			},
+			{
+				// two variables, different, comma
+				s:   ":v1, :v2",
+				m:   m,
+				exp: "newVone, newVtwo",
+			},
+			{
+				// single quotes
+				s:   ":'v1'",
+				m:   m,
+				exp: "'newVone'",
+			},
+			{
+				// double quotes
+				s:   `:"v2"`,
+				m:   m,
+				exp: `"newVtwo"`,
+			},
+			{
+				// more quotes
+				s:   `start :v1,:'two', :"v2" ::four :: `,
+				m:   m,
+				exp: `start newVone,:'two', "newVtwo" ::four :: `,
+			},
+		}
+		for i, test := range tests {
+			t.Run(fmt.Sprintf("test-%d", i), func(t *testing.T) {
+				replacer := newReplacer(test.m)
+				assert.Equal(t, test.exp, replacer.replace(test.s))
+			})
+		}
+	})
+}

cli/splitter.go

@@ -0,0 +1,133 @@
+package cli
+
+import (
+	"strings"
+
+	"github.com/pkg/errors"
+)
+
+// splitter is a line splitter which splits a line into queryParts and
+// metaCommands. It may not be necessary to have this be a separate struct since
+// it contains no members and just has the one `split()` method, but here we
+// are.
+type splitter struct {
+	replacer *replacer
+}
+
+func newSplitter(r *replacer) *splitter {
+	return &splitter{
+		replacer: r,
+	}
+}
+
+// split splits the given line into queryParts and metaCommands.
+// If a metaCommand is found, everything after that is considered either arguments to that
+// metaCommand, or additional metaCommands. In other words, queryParts can not follow
+// metaCommands in the same line.
+//
+// A line can contain any of the following patterns:
+// 1- [queryParts...]: "select * from tbl; select"
+// 2- [metaCommands...]: "\! pwd \q"
+// 3- [queryParts...][metaCommands...]: "select * from \i file.sql"
+func (s *splitter) split(line string) ([]queryPart, []metaCommand, error) {
+	// Look for a comment line.
+	if strings.HasPrefix(line, "--") {
+		return nil, nil, nil
+	}
+
+	// Look for a meta command.
+	parts := strings.SplitN(line, `\`, 2)
+
+	switch len(parts) {
+	case 1:
+		// slice of queryParts (pattern 1)
+		if qps, err := s.splitQueryParts(strings.TrimSpace(parts[0])); err != nil {
+			return nil, nil, errors.Wrap(err, "splitting query parts")
+		} else {
+			return qps, nil, nil
+		}
+	case 2:
+		// slice of parts + slice of meta commands (pattern 3)
+		// or
+		// slice of meta commands (pattern 2)
+		qps, err := s.splitQueryParts(strings.TrimSpace(parts[0]))
+		if err != nil {
+			return nil, nil, errors.Wrap(err, "splitting query parts")
+		}
+
+		mcs, err := s.splitMetaCommands(strings.TrimSpace(parts[1]))
+		if err != nil {
+			return nil, nil, errors.Wrap(err, "splitting meta commands")
+		}
+
+		return qps, mcs, nil
+	}
+
+	return nil, nil, nil
+}
+
+func (s *splitter) splitQueryParts(line string) ([]queryPart, error) {
+	if line == "" {
+		return nil, nil
+	}
+
+	// Look for a termination character;
+	parts := strings.Split(line, terminationChar)
+
+	// Do variable replacement.
+	for i := range parts {
+		parts[i] = s.replacer.replace(parts[i])
+	}
+
+	if len(parts) == 1 {
+		part0 := strings.TrimSpace(parts[0])
+		return []queryPart{
+			newPartRaw(part0),
+		}, nil
+	}
+
+	qps := make([]queryPart, 0)
+	for i := range parts {
+		part := strings.TrimSpace(parts[i])
+		if part == "" {
+			// If the line starts with a ";", treat it as a terminator for a
+			// previous line.
+			if i == 0 {
+				qps = append(qps, &partTerminator{})
+			}
+			continue
+		}
+		qps = append(qps, newPartRaw(part))
+		if i < len(parts)-1 {
+			qps = append(qps, &partTerminator{})
+		}
+	}
+
+	return qps, nil
+}
+
+func (s *splitter) splitMetaCommands(in string) ([]metaCommand, error) {
+	parts := strings.Split(in, `\`)
+	if len(parts) == 1 {
+		mc, err := splitMetaCommand(parts[0], s.replacer)
+		if err != nil {
+			return nil, errors.Wrapf(err, "splitting meta command: %s", parts[0])
+		}
+		return []metaCommand{mc}, nil
+	}
+
+	mcs := make([]metaCommand, 0)
+	for i := range parts {
+		part := strings.TrimSpace(parts[i])
+		if part == "" {
+			continue
+		}
+		mc, err := splitMetaCommand(part, s.replacer)
+		if err != nil {
+			return nil, errors.Wrapf(err, "splitting meta command: %s", part)
+		}
+		mcs = append(mcs, mc)
+	}
+
+	return mcs, nil
+}

cli/splitter_test.go

@@ -0,0 +1,146 @@
+package cli
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestSplitter(t *testing.T) {
+	s := newSplitter(newReplacer(nil))
+	t.Run("Split", func(t *testing.T) {
+		tests := []struct {
+			line            string
+			expQueryParts   []queryPart
+			expMetaCommands []metaCommand
+			expError        string
+		}{
+			{
+				line: `foo`,
+				expQueryParts: []queryPart{
+					newPartRaw("foo"),
+				},
+			},
+			{
+				line: `foo;`,
+				expQueryParts: []queryPart{
+					newPartRaw("foo"),
+					newPartTerminator(),
+				},
+			},
+			{
+				line: `foo; `,
+				expQueryParts: []queryPart{
+					newPartRaw("foo"),
+					newPartTerminator(),
+				},
+			},
+			{
+				line: `foo; ; `,
+				expQueryParts: []queryPart{
+					newPartRaw("foo"),
+					newPartTerminator(),
+				},
+			},
+			{
+				line: `foo; bar`,
+				expQueryParts: []queryPart{
+					newPartRaw("foo"),
+					newPartTerminator(),
+					newPartRaw("bar"),
+				},
+			},
+			{
+				line: `foo; bar;`,
+				expQueryParts: []queryPart{
+					newPartRaw("foo"),
+					newPartTerminator(),
+					newPartRaw("bar"),
+					newPartTerminator(),
+				},
+			},
+			{
+				line: `\q`,
+				expMetaCommands: []metaCommand{
+					&metaQuit{},
+				},
+			},
+			{
+				line: ` \p`,
+				expMetaCommands: []metaCommand{
+					&metaPrint{},
+				},
+			},
+			{
+				line: `\q \p`,
+				expMetaCommands: []metaCommand{
+					&metaQuit{},
+					&metaPrint{},
+				},
+			},
+			{
+				line: `\q \p arg1 arg2`,
+				expMetaCommands: []metaCommand{
+					&metaQuit{},
+					&metaPrint{},
+				},
+			},
+			{
+				line: `\set`,
+				expMetaCommands: []metaCommand{
+					&metaSet{
+						args: []string{},
+					},
+				},
+			},
+			{
+				line: `\set arg1 arg2`,
+				expMetaCommands: []metaCommand{
+					&metaSet{
+						args: []string{"arg1", "arg2"},
+					},
+				},
+			},
+			{
+				line: `\set 'arg1' 'arg2'`,
+				expMetaCommands: []metaCommand{
+					&metaSet{
+						args: []string{"arg1", "arg2"},
+					},
+				},
+			},
+			{
+				line: `\set 'arg1' '"arg2"'`,
+				expMetaCommands: []metaCommand{
+					&metaSet{
+						args: []string{"arg1", "\"arg2\""},
+					},
+				},
+			},
+			{
+				line:     `\`,
+				expError: "unsupported meta-command:",
+			},
+			{
+				line:     `\xyzxyz`,
+				expError: "unsupported meta-command:",
+			},
+		}
+		for i, tt := range tests {
+			t.Run(fmt.Sprintf("test-%d-%s", i, tt.line), func(t *testing.T) {
+				qps, mcs, err := s.split(tt.line)
+				if tt.expError != "" {
+					if assert.Error(t, err) {
+						assert.Contains(t, err.Error(), tt.expError)
+					}
+					return
+				}
+
+				assert.NoError(t, err)
+				assert.ElementsMatch(t, tt.expQueryParts, qps)
+				assert.ElementsMatch(t, tt.expMetaCommands, mcs)
+			})
+		}
+	})
+}

cli/testdata/database

@@ -0,0 +1,53 @@
+// Show databases now that we have set org.
+SEND:SHOW DATABASES;
+EXPECT:+-----+------+-------+------------+------------+------------+-------+-------------+
+EXPECT:| _id | name | owner | updated_by | created_at | updated_at | units | description |
+EXPECT:+-----+------+-------+------------+------------+------------+-------+-------------+
+EXPECT:+-----+------+-------+------------+------------+------------+-------+-------------+
+EXPECT:
+
+// Create db1.
+SEND:CREATE DATABASE db1 WITH UNITS 1;
+EXPECT:
+
+// List databases via SHOW DATABASES.
+SEND:SHOW DATABASES;
+EXPECT:+--------------------------------------+------+-------+------------+----------------------+----------------------+-------+-------------+
+EXPECT:| _id                                  | name | owner | updated_by | created_at           | updated_at           | units | description |
+EXPECT:+--------------------------------------+------+-------+------------+----------------------+----------------------+-------+-------------+
+EXPECTCOMP:WithFormat:| {uuid} | db1  |       |            | {timestamp} | {timestamp} |     1 |             |
+EXPECT:+--------------------------------------+------+-------+------------+----------------------+----------------------+-------+-------------+
+EXPECT:
+
+// List databases via SHOW DATABASES.
+SEND:\l
+EXPECT:+--------------------------------------+------+-------+------------+----------------------+----------------------+-------+-------------+
+EXPECT:| _id                                  | name | owner | updated_by | created_at           | updated_at           | units | description |
+EXPECT:+--------------------------------------+------+-------+------------+----------------------+----------------------+-------+-------------+
+EXPECTCOMP:WithFormat:| {uuid} | db1  |       |            | {timestamp} | {timestamp} |     1 |             |
+EXPECT:+--------------------------------------+------+-------+------------+----------------------+----------------------+-------+-------------+
+EXPECT:
+
+// Check database connection.
+SEND:\c
+EXPECT:You are not connected to a database.
+
+// Try connecting to an invalid database.
+SEND:\c invalid
+EXPECT:executing meta command: invalid database: invalid
+
+// Try connecting with too many arguments.
+SEND:\c db1 extra
+EXPECT:executing meta command: meta command 'connect' takes zero or one argument
+
+// Connect to a database.
+SEND:\c db1
+EXPECTCOMP:WithFormat:You are now connected to database "db1" ({uuid}).
+
+// Disconnect from the current database.
+SEND:\c -
+EXPECT:You are not connected to a database.
+
+// Connect to a database again.
+SEND:\c db1
+EXPECTCOMP:WithFormat:You are now connected to database "db1" ({uuid}).

cli/testdata/famous.csv

@@ -0,0 +1,10 @@
+
+"Id", "Name", "Short description", "Gender", "Country", "Occupation", "Birth year", "Death year", "Manner of death", "Age of death"
+1, "George Washington", "1st president of the United States (1732–1799)", "Male", "United States of America; Kingdom of Great Britain", "Politician", "1732", "1799", "natural causes", "67"
+2, "Douglas Adams", "English writer and humorist", "Male", "United Kingdom", "Artist", "1952", "2001", "natural causes", "49"
+3, "Abraham Lincoln", "16th president of the United States (1809-1865)", "Male", "United States of America", "Politician", "1809", "1865", "homicide", "56"
+4, "Wolfgang Amadeus Mozart", "Austrian composer of the Classical period", "Male", "Archduchy of Austria; Archbishopric of Salzburg", "Artist", "1756", "1791", "0", "35"
+5, "Ludwig van Beethoven", "German classical and romantic composer", "Male", "Holy Roman Empire; Austrian Empire", "Artist", "1770", "1827", "0", "57"
+6, "Jean-François Champollion", "French classical scholar", "Male", "Kingdom of France; First French Empire", "Egyptologist", "1790", "1832", "natural causes", "42"
+7, "Paul Morand", "French writer", "Male", "France", "Artist", "1888", "1976", "0", "88"
+8, "Claude Monet", "French impressionist painter (1840-1926)", "Male", "France", "Artist", "1840", "1926", "natural causes", "86"

cli/testdata/meta_bang

@@ -0,0 +1,8 @@
+SEND:\! echo 'foo'
+EXPECT:foo
+
+SEND:\! echo "foo"
+EXPECT:"foo"
+
+SEND:\!
+EXPECT:executing meta command: meta command '!' requires at least one argument

cli/testdata/meta_cd

@@ -0,0 +1,17 @@
+// Make a directory so we can test \cd'ing into it.
+SEND:\! mkdir cli-test-dir
+SEND:\cd cli-test-dir
+SEND:\cd ..
+SEND:\! rmdir cli-test-dir
+
+// TODO(tlt): before we do this, we should implement the ability to execute
+// commands in a \set like:
+// \set homedir `pwd`
+// then we can store what directory we're in so we can move back to it 
+// at the end of the test
+// Switch to home directory.
+// SEND:\cd
+
+// Expect error on extra argument to \cd.
+SEND:\cd dir extra
+EXPECT:executing meta command: meta command 'cd' takes zero or one argument

cli/testdata/meta_describe

@@ -0,0 +1,33 @@
+// TODO(tlt): we can't run this test until we get the system tables under control (i.e. sorted). Currently, fb_views is in a map with users, so the following can fail 50% of the time.
+// Show tables for database by calling describe with no args.
+// SEND:\d
+// EXPECT:+-------------------------+-------------------------+-------+------------+----------------------+----------------------+-------+------------+------------------------+
+// EXPECT:| _id                     | name                    | owner | updated_by | created_at           | updated_at           | keys  | space_used | description            |
+// EXPECT:+-------------------------+-------------------------+-------+------------+----------------------+----------------------+-------+------------+------------------------+
+// EXPECTCOMP:WithFormat:| fb_veiws                | fb_views                |       |            | {timestamp} | {timestamp} | true  |          0 | system table for views |
+// EXPECTCOMP:WithFormat:| users                   | users                   |       |            | {timestamp} | {timestamp} | false |          0 |                        |
+// EXPECTCOMP:WithFormat:| fb_____________________ | fb_____________________ |       |            | {timestamp} | {timestamp} | false |          0 |                        |
+// EXPECTCOMP:WithFormat:| fb_____________________ | fb_____________________ |       |            | {timestamp} | {timestamp} | false |          0 |                        |
+// EXPECTCOMP:WithFormat:| fb_____________________ | fb_____________________ |       |            | {timestamp} | {timestamp} | false |          0 |                        |
+// EXPECTCOMP:WithFormat:| fb_____________________ | fb_____________________ |       |            | {timestamp} | {timestamp} | false |          0 |                        |
+// EXPECTCOMP:WithFormat:| fb_____________________ | fb_____________________ |       |            | {timestamp} | {timestamp} | false |          0 |                        |
+// EXPECT:+-------------------------+-------------------------+-------+------------+----------------------+----------------------+-------+------------+------------------------+
+// EXPECT:
+
+// Show columns for table.
+SEND:\d users
+EXPECT:+------+------+--------+----------------------+-------+------------+------------+-------+----------------------+---------------------+----------+-------+-------------+-----+
+EXPECT:| _id  | name | type   | created_at           | keys  | cache_type | cache_size | scale |                  min |                 max | timeunit | epoch | timequantum | ttl |
+EXPECT:+------+------+--------+----------------------+-------+------------+------------+-------+----------------------+---------------------+----------+-------+-------------+-----+
+EXPECTCOMP:WithFormat:| _id  | _id  | id     | {timestamp} | false |            |          0 |     0 |                    0 |                   0 |          |     0 |             | 0s  |
+EXPECTCOMP:WithFormat:| name | name | string | {timestamp} | true  | ranked     |      50000 |     0 |                    0 |                   0 |          |     0 |             | 0s  |
+EXPECTCOMP:WithFormat:| age  | age  | int    | {timestamp} | false |            |          0 |     0 | -9223372036854775808 | 9223372036854775807 |          |     0 |             | 0s  |
+EXPECT:+------+------+--------+----------------------+-------+------------+------------+-------+----------------------+---------------------+----------+-------+-------------+-----+
+EXPECT:
+
+// Show columns for an invalid table.
+SEND:\d invalid
+EXPECT:Error: compiling plan: [1:19] table 'invalid' not found
+
+SEND:\d users extra
+EXPECT:executing meta command: meta command 'describe' takes zero or one argument

cli/testdata/meta_echo

@@ -0,0 +1,6 @@
+SEND:\echo
+EXPECT:
+
+// Simple \echo.
+SEND:\echo foo bar
+EXPECT:foo bar

cli/testdata/meta_file

@@ -0,0 +1,70 @@
+// Create a table.
+SEND:CREATE TABLE famous (
+SEND:    _id ID,
+SEND:    name STRING,
+SEND:    description STRING,
+SEND:    gender STRING,
+SEND:    country STRING,
+SEND:    occupation STRING,
+SEND:    birth_year INT min -32767 max 32767,
+SEND:    death_year INT min -32767 max 32767,
+SEND:    death_manner STRING,
+SEND:    birth_age INT min -32767 max 32767
+SEND:);
+EXPECT:
+
+// Open bulk insert.
+SEND:BULK INSERT
+SEND:INTO famous (_id, name, description, gender, country, occupation,
+SEND:    birth_year, death_year, death_manner, birth_age )
+SEND:MAP(0 INT,
+SEND:1 STRING,
+SEND:2 STRING,
+SEND:3 STRING,
+SEND:4 STRING,
+SEND:5 STRING,
+SEND:6 INT,
+SEND:7 INT,
+SEND:8 STRING,
+SEND:9 INT )
+SEND:FROM
+SEND:    x'
+
+// Call \file
+SEND:\file testdata/famous.csv
+
+// Close bulk insert.
+SEND:'
+SEND:WITH
+SEND:    BATCHSIZE 100000
+SEND:    FORMAT 'CSV'
+SEND:    INPUT 'STREAM'
+SEND:    HEADER_ROW;
+EXPECT:
+
+// Query table to ensure we have data.
+SEND:SELECT * FROM famous;
+EXPECT:+-----+---------------------------+-------------------------------------------------+--------+----------------------------------------------------+--------------+------------+------------+----------------+-----------+
+EXPECT:| _id | name                      | description                                     | gender | country                                            | occupation   | birth_year | death_year | death_manner   | birth_age |
+EXPECT:+-----+---------------------------+-------------------------------------------------+--------+----------------------------------------------------+--------------+------------+------------+----------------+-----------+
+EXPECT:|   1 | George Washington         | 1st president of the United States (1732–1799)  | Male   | United States of America; Kingdom of Great Britain | Politician   |       1732 |       1799 | natural causes |        67 |
+EXPECT:|   2 | Douglas Adams             | English writer and humorist                     | Male   | United Kingdom                                     | Artist       |       1952 |       2001 | natural causes |        49 |
+EXPECT:|   3 | Abraham Lincoln           | 16th president of the United States (1809-1865) | Male   | United States of America                           | Politician   |       1809 |       1865 | homicide       |        56 |
+EXPECT:|   4 | Wolfgang Amadeus Mozart   | Austrian composer of the Classical period       | Male   | Archduchy of Austria; Archbishopric of Salzburg    | Artist       |       1756 |       1791 | 0              |        35 |
+EXPECT:|   5 | Ludwig van Beethoven      | German classical and romantic composer          | Male   | Holy Roman Empire; Austrian Empire                 | Artist       |       1770 |       1827 | 0              |        57 |
+EXPECT:|   6 | Jean-François Champollion | French classical scholar                        | Male   | Kingdom of France; First French Empire             | Egyptologist |       1790 |       1832 | natural causes |        42 |
+EXPECT:|   7 | Paul Morand               | French writer                                   | Male   | France                                             | Artist       |       1888 |       1976 | 0              |        88 |
+EXPECT:|   8 | Claude Monet              | French impressionist painter (1840-1926)        | Male   | France                                             | Artist       |       1840 |       1926 | natural causes |        86 |
+EXPECT:+-----+---------------------------+-------------------------------------------------+--------+----------------------------------------------------+--------------+------------+------------+----------------+-----------+
+EXPECT:
+
+// TODO(tlt): dropping the table seems to cause problems.
+// Drop the table.
+//SEND:DROP TABLE famous;
+
+// Ensure that invalid aruments (none or too many) return an error.
+SEND:\file
+EXPECT:executing meta command: meta command 'file' requires exactly one argument
+SEND:\file filename extra
+EXPECT:executing meta command: meta command 'file' requires exactly one argument
+

cli/testdata/meta_include

@@ -0,0 +1,24 @@
+// Include with no argument should error.
+SEND:\i
+EXPECT:executing meta command: meta command 'include' requires exactly one argument
+
+// Include with too many arguments should error.
+SEND:\include testdata/people.sql extra
+EXPECT:executing meta command: meta command 'include' requires exactly one argument
+
+// Invalid file should error.
+SEND:\include invalid.file
+EXPECT:executing meta command: opening file: invalid.file: open invalid.file: no such file or directory
+
+SEND:\include testdata/people.sql
+EXPECT:
+EXPECT:
+EXPECT:+-----+------+-----+
+EXPECT:| _id | name | age |
+EXPECT:+-----+------+-----+
+EXPECT:|   1 | Amy  |  42 |
+EXPECT:|   2 | Bob  |  27 |
+EXPECT:|   3 | Carl |  33 |
+EXPECT:+-----+------+-----+
+EXPECT:
+EXPECT:mix in a meta command

cli/testdata/meta_output

@@ -0,0 +1,67 @@
+SEND:SELECT * FROM users;
+EXPECT:+-----+-------+-----+
+EXPECT:| _id | name  | age |
+EXPECT:+-----+-------+-----+
+EXPECT:|   1 | Anne  |  38 |
+EXPECT:|   2 | Bill  |  23 |
+EXPECT:|   3 | Cindy |  64 |
+EXPECT:+-----+-------+-----+
+EXPECT:
+
+// Redirect output to a file.
+SEND:\o test-output-file
+SEND:SELECT * FROM users;
+
+// Ensure the output went to the file.
+SEND:\! cat test-output-file
+EXPECT:+-----+-------+-----+
+EXPECT:| _id | name  | age |
+EXPECT:+-----+-------+-----+
+EXPECT:|   1 | Anne  |  38 |
+EXPECT:|   2 | Bill  |  23 |
+EXPECT:|   3 | Cindy |  64 |
+EXPECT:+-----+-------+-----+
+EXPECT:
+
+// Let's test some qecho stuff here while we're at it.
+SEND:\qecho string with "double quotes"
+SEND:\qecho -n one
+SEND:\qecho -n two
+SEND:\qecho three
+SEND:\qecho four
+
+SEND:\! cat test-output-file
+EXPECT:+-----+-------+-----+
+EXPECT:| _id | name  | age |
+EXPECT:+-----+-------+-----+
+EXPECT:|   1 | Anne  |  38 |
+EXPECT:|   2 | Bill  |  23 |
+EXPECT:|   3 | Cindy |  64 |
+EXPECT:+-----+-------+-----+
+EXPECT:
+EXPECT:string with "double quotes"
+EXPECT:onetwothree
+EXPECT:four
+
+// And \warn messages should still go to stderr, not the file.
+SEND:\warn a warning string
+EXPECT:a warning string
+
+// Remove the file.
+SEND:\! rm test-output-file
+
+// Set the output back to stdout.
+SEND:\o
+SEND:SELECT * FROM users;
+EXPECT:+-----+-------+-----+
+EXPECT:| _id | name  | age |
+EXPECT:+-----+-------+-----+
+EXPECT:|   1 | Anne  |  38 |
+EXPECT:|   2 | Bill  |  23 |
+EXPECT:|   3 | Cindy |  64 |
+EXPECT:+-----+-------+-----+
+EXPECT:
+
+// Ensure extra arguments to \output causes an error.
+SEND:\o filename extra
+EXPECT:executing meta command: meta command 'output' takes zero or one argument

cli/testdata/meta_pset_border

@@ -0,0 +1,52 @@
+SEND:\pset border 2
+EXPECT:Border style is 2.
+
+SEND:SELECT 1 as foo, 'baz' as bar;
+EXPECT:+-----+-----+
+EXPECT:| foo | bar |
+EXPECT:+-----+-----+
+EXPECT:|   1 | baz |
+EXPECT:+-----+-----+
+EXPECT:
+
+SEND:\pset border
+EXPECT:Border style is 2.
+
+SEND:\pset border 999
+EXPECT:Border style is 0.
+
+SEND:\pset border 1
+EXPECT:Border style is 1.
+
+SEND:SELECT 1 as foo, 'baz' as bar;
+EXPECT: foo | bar 
+EXPECT:-----+-----
+EXPECT:   1 | baz 
+EXPECT:
+
+SEND:\pset border 2
+EXPECT:Border style is 2.
+
+SEND:SELECT 1 as foo, 'baz' as bar;
+EXPECT:+-----+-----+
+EXPECT:| foo | bar |
+EXPECT:+-----+-----+
+EXPECT:|   1 | baz |
+EXPECT:+-----+-----+
+EXPECT:
+
+SEND:\pset border 0
+EXPECT:Border style is 0.
+
+SEND:SELECT 1 as foo, 'baz' as bar;
+EXPECT:foo bar
+EXPECT:--- ---
+EXPECT:  1 baz
+EXPECT:
+
+SEND:\pset border 1 extra
+EXPECT:executing meta command: meta command 'pset' takes zero, one, or two arguments
+
+// Set border back to the testing default.
+SEND:\pset border 2
+EXPECT:Border style is 2.

cli/testdata/meta_pset_expanded

@@ -0,0 +1,35 @@
+// Set to off.
+SEND:\pset expanded off
+EXPECT:Expanded display is off.
+
+// Set to on.
+SEND:\pset expanded on
+EXPECT:Expanded display is on.
+
+// Toggle to off.
+SEND:\pset expanded
+EXPECT:Expanded display is off.
+
+// Toggle to on.
+SEND:\pset expanded
+EXPECT:Expanded display is on.
+
+// Set to something invalid.
+SEND:\pset expanded invalid
+EXPECT:executing meta command: unrecognized value "invalid" for "expanded": Boolean expected
+
+// Ensure expanded shows results vertically.
+SEND:SELECT 1 as foo, 'baz' as bar;
+EXPECT:+-----+-----+
+EXPECT:| foo | 1   |
+EXPECT:| bar | baz |
+EXPECT:+-----+-----+
+EXPECT:
+
+// Set back to off as we started.
+SEND:\pset expanded off
+EXPECT:Expanded display is off.
+
+// make sure the \x meta-command returns expected errors
+SEND:\x on extra
+EXPECT:executing meta command: meta command 'expanded' takes zero or one argument

cli/testdata/meta_pset_format_csv

@@ -0,0 +1,47 @@
+SEND:\pset format csv
+EXPECT:Output format is csv.
+
+SEND:SELECT * FROM users;
+EXPECT:_id,name,age
+EXPECT:1,Anne,38
+EXPECT:2,Bill,23
+EXPECT:3,Cindy,64
+
+// Exclude headers.
+SEND:\t on
+EXPECT:Tuples only is on.
+
+SEND:SELECT * FROM users;
+EXPECT:1,Anne,38
+EXPECT:2,Bill,23
+EXPECT:3,Cindy,64
+
+// Reset headers.
+SEND:\t off
+EXPECT:Tuples only is off.
+
+// Set expanded to on.
+SEND:\x on
+EXPECT:Expanded display is on.
+
+SEND:SELECT * FROM users;
+EXPECT:_id,1
+EXPECT:name,Anne
+EXPECT:age,38
+EXPECT:_id,2
+EXPECT:name,Bill
+EXPECT:age,23
+EXPECT:_id,3
+EXPECT:name,Cindy
+EXPECT:age,64
+
+// Set expanded back to off.
+SEND:\x off
+EXPECT:Expanded display is off.
+
+// Set format back to aligned as we started.
+SEND:\pset format aligned
+EXPECT:Output format is aligned.
+
+SEND:\pset format invalid
+EXPECT:executing meta command: \pset: allowed formats are aligned, csv

cli/testdata/meta_pset_tuples_only

@@ -0,0 +1,34 @@
+// Set to off.
+SEND:\pset tuples_only off
+EXPECT:Tuples only is off.
+
+// Set to on.
+SEND:\pset tuples_only on
+EXPECT:Tuples only is on.
+
+// Toggle to off.
+SEND:\pset tuples_only
+EXPECT:Tuples only is off.
+
+// Toggle to on.
+SEND:\pset tuples_only
+EXPECT:Tuples only is on.
+
+// Set to something invalid.
+SEND:\pset tuples_only invalid
+EXPECT:executing meta command: unrecognized value "invalid" for "tuples_only": Boolean expected
+
+// Ensure tuples_only shows only tuples.
+SEND:SELECT 1 as foo, 'baz' as bar;
+EXPECT:+---+-----+
+EXPECT:| 1 | baz |
+EXPECT:+---+-----+
+EXPECT:
+
+// Set back to off as we started.
+SEND:\pset tuples_only off
+EXPECT:Tuples only is off.
+
+// make sure the \t meta-command returns expected errors
+SEND:\t off extra
+EXPECT:executing meta command: meta command 'tuples_only' takes zero or one argument

cli/testdata/meta_set

@@ -0,0 +1,31 @@
+SEND:\set
+
+SEND:\set var1 foo
+SEND:\set
+EXPECT:var1 = 'foo'
+
+SEND:\set var2 bar
+SEND:\set
+EXPECT:var1 = 'foo'
+EXPECT:var2 = 'bar'
+
+SEND:\set var3 zoo
+SEND:\set
+EXPECT:var1 = 'foo'
+EXPECT:var2 = 'bar'
+EXPECT:var3 = 'zoo'
+
+SEND:\unset
+EXPECT:\unset: missing required argument
+
+SEND:\unset non-existent-key
+
+SEND:\unset var1
+SEND:\set
+EXPECT:var2 = 'bar'
+EXPECT:var3 = 'zoo'
+
+SEND:\unset var2 extra
+EXPECT:\unset: extra argument "extra" ignored
+SEND:\set
+EXPECT:var3 = 'zoo'

cli/testdata/meta_timing

@@ -0,0 +1,50 @@
+// Start by ensuring timing is off.
+SEND:\timing off
+EXPECT:Timing is off.
+
+// Set timing on.
+SEND:\timing on
+EXPECT:Timing is on.
+
+// Toggle timing.
+SEND:\timing
+EXPECT:Timing is off.
+
+// Toggle timing again.
+SEND:\timing
+EXPECT:Timing is on.
+
+// Send extra argument to \timing.
+SEND:\timing on extra
+EXPECT:executing meta command: meta command 'timing' takes zero or one argument
+
+
+SEND:SELECT * FROM users;
+EXPECT:+-----+-------+-----+
+EXPECT:| _id | name  | age |
+EXPECT:+-----+-------+-----+
+EXPECT:|   1 | Anne  |  38 |
+EXPECT:|   2 | Bill  |  23 |
+EXPECT:|   3 | Cindy |  64 |
+EXPECT:+-----+-------+-----+
+EXPECT:
+EXPECTCOMP:HasPrefix:Execution time:
+
+// Turn timing back off.
+SEND:\timing off
+EXPECT:Timing is off.
+
+// Ensure we don't get timing.
+SEND:SELECT * FROM users;
+EXPECT:+-----+-------+-----+
+EXPECT:| _id | name  | age |
+EXPECT:+-----+-------+-----+
+EXPECT:|   1 | Anne  |  38 |
+EXPECT:|   2 | Bill  |  23 |
+EXPECT:|   3 | Cindy |  64 |
+EXPECT:+-----+-------+-----+
+EXPECT:
+
+// Ensure an invalid timing value returns an error.
+SEND:\timing invalid
+EXPECT:executing meta command: unrecognized value "invalid" for "\timing": Boolean expected

cli/testdata/meta_write

@@ -0,0 +1,24 @@
+// Make sure there's something in the query buffer.
+// This is left unterminated because we don't need to execute the query;
+// we just need there to be something in the buffer.
+SEND:SELECT * FROM invalid-table
+SEND:\write query-buffer-contents
+
+// Reset the buffer.
+SEND:\r
+EXPECT:Query buffer reset (cleared).
+
+// Read from the file.
+SEND:\! cat query-buffer-contents
+EXPECT:SELECT * FROM invalid-table
+
+// Remove the file.
+SEND:\! rm query-buffer-contents
+
+// Send \write with no arguments.
+SEND:\write
+EXPECT:\w: missing required argument
+
+// Send \write with extra arguments.
+SEND:\write filename extra
+EXPECT:executing meta command: meta command 'w' exactly one argument

cli/testdata/people.sql

@@ -0,0 +1,12 @@
+-- Create a table.
+create table people (_id id, name string, age int);
+
+-- Insert some values.
+insert into people values (1, 'Amy', 42), (2, 'Bob', 27), (3, 'Carl', 33);
+
+-- Get all rows from the table.
+select * from people;
+
+-- Mix in a meta-command to show that both are supported
+-- in the include file.
+\echo mix in a meta command

cli/testdata/query_buffer

@@ -0,0 +1,40 @@
+SEND:select 1 as foo;
+EXPECT:+-----+
+EXPECT:| foo |
+EXPECT:+-----+
+EXPECT:|   1 |
+EXPECT:+-----+
+EXPECT:
+
+SEND:\p
+EXPECT:select 1 as foo;
+
+SEND:select 2
+SEND:\p
+EXPECT:select 2
+SEND:\r
+EXPECT:Query buffer reset (cleared).
+
+SEND:\p
+EXPECT:select 1 as foo;
+
+SEND:select 3
+SEND:\p
+EXPECT:select 3
+
+SEND:as foo
+SEND:\p
+EXPECT:select 3
+EXPECT:as foo
+
+SEND:;
+EXPECT:+-----+
+EXPECT:| foo |
+EXPECT:+-----+
+EXPECT:|   3 |
+EXPECT:+-----+
+EXPECT:
+
+SEND:\p
+EXPECT:select 3
+EXPECT:as foo;

cli/testdata/setup

@@ -0,0 +1,51 @@
+// Startup splash.
+EXPECT:FeatureBase CLI ()
+EXPECT:Type "\q" to quit.
+EXPECT:Detected on-prem, serverless deployment.
+EXPECTCOMP:HasPrefix:Host: http://localhost:
+EXPECT:You are not connected to a database.
+
+// Show databases.
+SEND:SHOW DATABASES;
+EXPECT:Organization required. Use \org to set an organization.
+
+// Get current org.
+SEND:\org
+EXPECT:You have not set an organization.
+
+// Set org.
+SEND:\org acme
+EXPECT:You have set organization "acme".
+
+// Try to set org with too many arguments.
+SEND:\org acme extra
+EXPECT:executing meta command: meta command 'org' takes zero or one argument
+
+// Set location to UTC so that expected timestamp size is consistent.
+// Without this, a test running locally in may have a timestamp that
+// ends in a timezone offset such as `-06:00`, while one running as UTC
+// will have `Z`. Since these string lengths differ, our generic
+// {timestamp} comparison will fail.
+SEND:\pset location UTC
+EXPECT:Location is UTC.
+
+// Set an invalid location.
+SEND:\pset location invalid
+EXPECT:executing meta command: loading location: invalid: unknown time zone invalid
+
+// Try to set location with too many arguments.
+SEND:\pset location UTC extra
+EXPECT:executing meta command: meta command 'pset' takes zero, one, or two arguments
+
+// Set border to 2 for testing because it makes it easier to visually see
+// what the tests are expecting (because lines don't end in spaces).
+SEND:\pset border 2
+EXPECT:Border style is 2.
+
+// Check the state of pset.
+SEND:\pset
+EXPECT:border      2
+EXPECT:expanded    off
+EXPECT:format      aligned
+EXPECT:location    UTC
+EXPECT:tuples_only off

cli/testdata/table

@@ -0,0 +1,72 @@
+// Show tables for database using SHOW TABLES WITH SYSTEM.
+SEND:SHOW TABLES WITH SYSTEM;
+EXPECT:+-------------------------+-------------------------+-------+------------+----------------------+----------------------+-------+------------+-------------+
+EXPECT:| _id                     | name                    | owner | updated_by | created_at           | updated_at           | keys  | space_used | description |
+EXPECT:+-------------------------+-------------------------+-------+------------+----------------------+----------------------+-------+------------+-------------+
+EXPECTCOMP:WithFormat:| fb_____________________ | fb_____________________ |       |            | {timestamp} | {timestamp} | false |          0 |             |
+EXPECTCOMP:WithFormat:| fb_____________________ | fb_____________________ |       |            | {timestamp} | {timestamp} | false |          0 |             |
+EXPECTCOMP:WithFormat:| fb_____________________ | fb_____________________ |       |            | {timestamp} | {timestamp} | false |          0 |             |
+EXPECTCOMP:WithFormat:| fb_____________________ | fb_____________________ |       |            | {timestamp} | {timestamp} | false |          0 |             |
+EXPECTCOMP:WithFormat:| fb_____________________ | fb_____________________ |       |            | {timestamp} | {timestamp} | false |          0 |             |
+EXPECT:+-------------------------+-------------------------+-------+------------+----------------------+----------------------+-------+------------+-------------+
+EXPECT:
+
+// Show tables for database using \d.
+SEND:\d
+EXPECT:+-------------------------+-------------------------+-------+------------+----------------------+----------------------+-------+------------+-------------+
+EXPECT:| _id                     | name                    | owner | updated_by | created_at           | updated_at           | keys  | space_used | description |
+EXPECT:+-------------------------+-------------------------+-------+------------+----------------------+----------------------+-------+------------+-------------+
+EXPECTCOMP:WithFormat:| fb_____________________ | fb_____________________ |       |            | {timestamp} | {timestamp} | false |          0 |             |
+EXPECTCOMP:WithFormat:| fb_____________________ | fb_____________________ |       |            | {timestamp} | {timestamp} | false |          0 |             |
+EXPECTCOMP:WithFormat:| fb_____________________ | fb_____________________ |       |            | {timestamp} | {timestamp} | false |          0 |             |
+EXPECTCOMP:WithFormat:| fb_____________________ | fb_____________________ |       |            | {timestamp} | {timestamp} | false |          0 |             |
+EXPECTCOMP:WithFormat:| fb_____________________ | fb_____________________ |       |            | {timestamp} | {timestamp} | false |          0 |             |
+EXPECT:+-------------------------+-------------------------+-------+------------+----------------------+----------------------+-------+------------+-------------+
+EXPECT:
+
+// Show tables for database using SHOW TABLES.
+SEND:SHOW TABLES;
+EXPECT:+-----+------+-------+------------+------------+------------+------+------------+-------------+
+EXPECT:| _id | name | owner | updated_by | created_at | updated_at | keys | space_used | description |
+EXPECT:+-----+------+-------+------------+------------+------------+------+------------+-------------+
+EXPECT:+-----+------+-------+------------+------------+------------+------+------------+-------------+
+EXPECT:
+
+
+// Show tables for database using \dt.
+SEND:\dt
+EXPECT:+-----+------+-------+------------+------------+------------+------+------------+-------------+
+EXPECT:| _id | name | owner | updated_by | created_at | updated_at | keys | space_used | description |
+EXPECT:+-----+------+-------+------------+------------+------------+------+------------+-------------+
+EXPECT:+-----+------+-------+------------+------------+------------+------+------------+-------------+
+EXPECT:
+
+// Create a table. That can be used for general testing.
+SEND:CREATE TABLE users (_id id, name string, age int);
+EXPECT:
+SEND:INSERT INTO users VALUES (1, 'Anne', 38), (2, 'Bill', 23), (3, 'Cindy', 64);
+EXPECT:
+
+
+// Show tables for database to get the newly created table.
+SEND:\dt
+EXPECT:+-------+-------+-------+------------+----------------------+----------------------+-------+------------+-------------+
+EXPECT:| _id   | name  | owner | updated_by | created_at           | updated_at           | keys  | space_used | description |
+EXPECT:+-------+-------+-------+------------+----------------------+----------------------+-------+------------+-------------+
+EXPECTCOMP:WithFormat:| users | users |       |            | {timestamp} | {timestamp} | false |          0 |             |
+EXPECT:+-------+-------+-------+------------+----------------------+----------------------+-------+------------+-------------+
+EXPECT:
+
+// We don't select from users until AFTER we check SHOW TABLES above because
+// running this creates the fb_views sytem table which has a description.
+// And it's annoying to mask out all of the description fields because we
+// don't know which row fb_views will fall into.
+SEND:SELECT * FROM users;
+EXPECT:+-----+-------+-----+
+EXPECT:| _id | name  | age |
+EXPECT:+-----+-------+-----+
+EXPECT:|   1 | Anne  |  38 |
+EXPECT:|   2 | Bill  |  23 |
+EXPECT:|   3 | Cindy |  64 |
+EXPECT:+-----+-------+-----+
+EXPECT:

cli/workingdir.go

@@ -0,0 +1,21 @@
+package cli
+
+import (
+	"os"
+)
+
+// workingDir was originally set up with the intention of using it to maintain a
+// reference to the current working directory. But it turns out we haven't
+// really needed that so far. The `cd()` method is unsed in one of the meta
+// commands, but we could probably just call `os.Chdir()` directly there. With
+// that said, I'm leaving it here for now until we're abosolutely sure we don't
+// need to use this for other directory/file handling functionality.
+type workingDir struct{}
+
+func newWorkingDir() *workingDir {
+	return &workingDir{}
+}
+
+func (wd *workingDir) cd(dir string) error {
+	return os.Chdir(dir)
+}

cli/writer.go

@@ -0,0 +1,294 @@
+package cli
+
+import (
+	"encoding/csv"
+	"fmt"
+	"io"
+	"log"
+	"time"
+
+	featurebase "github.com/featurebasedb/featurebase/v3"
+	"github.com/jedib0t/go-pretty/table"
+	"github.com/jedib0t/go-pretty/text"
+	"github.com/pkg/errors"
+)
+
+// writeOptions contains user configuration options which describe how to write
+// the query output.
+type writeOptions struct {
+	border     int
+	expanded   bool
+	format     string
+	location   *time.Location
+	timing     bool
+	tuplesOnly bool
+}
+
+const (
+	formatAligned = "aligned"
+	formatCSV     = "csv"
+)
+
+func defaultWriteOptions() *writeOptions {
+	return &writeOptions{
+		border:     1,
+		expanded:   false,
+		format:     formatAligned,
+		location:   time.Local,
+		timing:     false,
+		tuplesOnly: false,
+	}
+}
+
+// writeOutput writes the query response, taking the format into consideration.
+// It sends query output to qOut, non-error informational output (such as query
+// timing) to wOut, and errors to wErr.
+func writeOutput(r *featurebase.WireQueryResponse, opts *writeOptions, qOut io.Writer, wOut io.Writer, wErr io.Writer) error {
+	if r == nil {
+		return errors.New("attempt to write out nil response")
+	}
+	if r.Error != "" {
+		if _, err := wErr.Write([]byte("Error: " + r.Error + "\n")); err != nil {
+			return errors.Wrapf(err, "writing error: %s", r.Error)
+		}
+		return writeWarnings(r, wErr)
+	}
+
+	switch opts.format {
+	case formatAligned:
+		if err := writeTable(r, opts, qOut); err != nil {
+			return errors.Wrap(err, "writing table")
+		}
+
+		// Add some white space after query results.
+		qOut.Write([]byte("\n"))
+
+	case formatCSV:
+		if err := writeCSV(r, opts, qOut); err != nil {
+			return errors.Wrap(err, "writing csv")
+		}
+
+	default:
+		return errors.Errorf("invalid format: %s", opts.format)
+	}
+
+	if err := writeWarnings(r, wErr); err != nil {
+		return err
+	}
+
+	// Timing.
+	if opts.timing {
+		if _, err := wOut.Write([]byte(fmt.Sprintf("Execution time: %dμs\n", r.ExecutionTime))); err != nil {
+			return errors.Wrapf(err, "writing execution time: %s", r.Error)
+		}
+	}
+
+	return nil
+}
+
+// writeCSV writes the WireQueryResponse to qOut as csv.
+func writeCSV(r *featurebase.WireQueryResponse, opts *writeOptions, qOut io.Writer) error {
+	w := csv.NewWriter(qOut)
+
+	if opts.expanded {
+		// Expanded csv
+
+		// rec is used to write the row as a slice of strings. It is reused to
+		// avoid unnecessary memory allocation.
+		rec := make([]string, 2)
+
+		for _, row := range r.Data {
+			cleanRow(row, opts)
+			for i, col := range r.Schema.Fields {
+				rec[0] = string(col.Name)
+				rec[1] = fmt.Sprintf("%v", row[i])
+
+				// Write the record.
+				if err := w.Write(rec); err != nil {
+					log.Fatalln("error writing expanded record to csv:", err)
+				}
+			}
+		}
+
+	} else {
+		// Normal csv (i.e. NOT expanded)
+
+		// Write the schema.
+		if !opts.tuplesOnly {
+			header := make([]string, 0, len(r.Schema.Fields))
+			for i := range r.Schema.Fields {
+				header = append(header, string(r.Schema.Fields[i].Name))
+			}
+			if err := w.Write(header); err != nil {
+				return errors.Wrapf(err, "error writing header to csv")
+			}
+		}
+
+		// Write the records.
+
+		// rec is used to write the row as a slice of strings. It is reused to
+		// avoid unnecessary memory allocation.
+		rec := make([]string, len(r.Schema.Fields))
+
+		for _, row := range r.Data {
+			cleanRow(row, opts)
+			for i := range row {
+				rec[i] = fmt.Sprintf("%v", row[i])
+			}
+			if err := w.Write(rec); err != nil {
+				log.Fatalln("error writing record to csv:", err)
+			}
+		}
+	}
+
+	// Write any buffered data to the underlying writer (standard output).
+	w.Flush()
+
+	return w.Error()
+}
+
+// writeTable writes the WireQueryResponse to qOut in a tabular format.
+func writeTable(r *featurebase.WireQueryResponse, opts *writeOptions, qOut io.Writer) error {
+	t := table.NewWriter()
+	t.SetOutputMirror(qOut)
+	switch opts.border {
+	case 0:
+		t.SetStyle(styleBorder0)
+	case 1:
+		t.SetStyle(styleBorder1)
+	default:
+		t.SetStyle(styleBorder2)
+		// In expanded mode with a border, we need borders between each record.
+		if opts.expanded {
+			t.Style().Options.SeparateRows = true
+		}
+	}
+
+	// Don't uppercase the header values.
+	t.Style().Format.Header = text.FormatDefault
+
+	if opts.expanded {
+		// Expanded table
+		for _, row := range r.Data {
+			cleanRow(row, opts)
+			colRow := make([]interface{}, 2)
+			scolRow := make([]string, 2)
+			div := "\n"
+			for i, col := range r.Schema.Fields {
+				if i == len(r.Schema.Fields)-1 {
+					div = ""
+				}
+				scolRow[0] += fmt.Sprintf("%s%s", col.Name, div)
+				scolRow[1] += fmt.Sprintf("%v%s", row[i], div)
+			}
+			colRow[0] = scolRow[0]
+			colRow[1] = scolRow[1]
+			t.AppendRow(table.Row(colRow[:]))
+		}
+	} else {
+		// Normal table (i.e. NOT expanded)
+		if !opts.tuplesOnly {
+			t.AppendHeader(schemaToRow(r.Schema))
+		}
+		for _, row := range r.Data {
+			cleanRow(row, opts)
+			t.AppendRow(table.Row(row))
+		}
+	}
+	t.Render()
+
+	return nil
+}
+
+// cleanRow loops through all the columns of row and modifies its value based on
+// type.
+//
+// If the value is nil, replace it with a null string; go-pretty doesn't expect
+// nil pointers in the data values.
+//
+// If the value is a time.Time, we want to print it using RFC3339Nano to be
+// consistent with everything else.
+func cleanRow(row []interface{}, opts *writeOptions) {
+	for i := range row {
+		switch v := row[i].(type) {
+		case nil:
+			row[i] = nullValue
+		case time.Time:
+			row[i] = v.In(opts.location).Format(time.RFC3339Nano)
+		}
+	}
+}
+
+func schemaToRow(schema featurebase.WireQuerySchema) []interface{} {
+	ret := make([]interface{}, len(schema.Fields))
+	for i, field := range schema.Fields {
+		ret[i] = field.Name
+	}
+	return ret
+}
+
+func writeWarnings(r *featurebase.WireQueryResponse, w io.Writer) error {
+	if len(r.Warnings) == 0 {
+		return nil
+	}
+
+	if _, err := w.Write([]byte("\n")); err != nil {
+		return errors.Wrapf(err, "writing line feed")
+	}
+	for _, warning := range r.Warnings {
+		if _, err := w.Write([]byte("Warning: " + warning + "\n")); err != nil {
+			return errors.Wrapf(err, "writing warning: %s", warning)
+		}
+	}
+	return nil
+}
+
+var styleBorder2 table.Style = table.StyleDefault
+
+var styleBorder1 table.Style = table.Style{
+	Name:   "StyleBorder1",
+	Box:    table.StyleBoxDefault,
+	Color:  table.ColorOptionsDefault,
+	Format: table.FormatOptionsDefault,
+	Options: table.Options{
+		DrawBorder:      false,
+		SeparateColumns: true,
+		SeparateFooter:  true,
+		SeparateHeader:  true,
+		SeparateRows:    false,
+	},
+	Title: table.TitleOptionsDefault,
+}
+
+var styleBorder0 table.Style = table.Style{
+	Name: "StyleBorder0",
+	Box: table.BoxStyle{
+		BottomLeft:       "+",
+		BottomRight:      "+",
+		BottomSeparator:  "+",
+		Left:             "|",
+		LeftSeparator:    "+",
+		MiddleHorizontal: "-",
+		MiddleSeparator:  " ",
+		MiddleVertical:   " ",
+		PaddingLeft:      "",
+		PaddingRight:     "",
+		PageSeparator:    "\n",
+		Right:            "|",
+		RightSeparator:   "+",
+		TopLeft:          "+",
+		TopRight:         "+",
+		TopSeparator:     "+",
+		UnfinishedRow:    " ~",
+	},
+	Color:  table.ColorOptionsDefault,
+	Format: table.FormatOptionsDefault,
+	Options: table.Options{
+		DrawBorder:      false,
+		SeparateColumns: true,
+		SeparateFooter:  true,
+		SeparateHeader:  true,
+		SeparateRows:    false,
+	},
+	Title: table.TitleOptionsDefault,
+}

cli/writer_test.go

@@ -0,0 +1,188 @@
+package cli
+
+import (
+	"bytes"
+	"fmt"
+	"strings"
+	"testing"
+
+	featurebase "github.com/featurebasedb/featurebase/v3"
+	dax "github.com/featurebasedb/featurebase/v3/dax"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestWriter(t *testing.T) {
+	t.Run("writeTable", func(t *testing.T) {
+		wqr := &featurebase.WireQueryResponse{
+			Schema: featurebase.WireQuerySchema{
+				Fields: []*featurebase.WireQueryField{
+					{Name: "_id", Type: dax.BaseTypeID},
+					{Name: "name", Type: dax.BaseTypeString},
+					{Name: "age", Type: dax.BaseTypeInt},
+				},
+			},
+			Data: [][]interface{}{
+				{1, "Amy", 44},
+				{2, "Bob", 32},
+				{3, "Cindy", 28},
+			},
+		}
+
+		// TODO(tlt): used for debugging
+		// format := defaultWriteOptions()
+		// assert.NoError(t, writeTable(wqr, format, os.Stdout, os.Stdout, os.Stdout))
+		// return
+
+		tests := []struct {
+			format  *writeOptions
+			expQOut string
+			expOut  string
+			expErr  string
+		}{
+			{
+				// default format
+				format: defaultWriteOptions(),
+				expQOut: stringOfLines(
+					" _id | name  | age ",
+					"-----+-------+-----",
+					"   1 | Amy   |  44 ",
+					"   2 | Bob   |  32 ",
+					"   3 | Cindy |  28 ",
+					"",
+				),
+				expOut: "",
+				expErr: "",
+			},
+			{
+				// timing on
+				format: &writeOptions{
+					border:     1,
+					expanded:   false,
+					format:     formatAligned,
+					timing:     true,
+					tuplesOnly: false,
+				},
+				expQOut: stringOfLines(
+					" _id | name  | age ",
+					"-----+-------+-----",
+					"   1 | Amy   |  44 ",
+					"   2 | Bob   |  32 ",
+					"   3 | Cindy |  28 ",
+					"",
+				),
+				expOut: "Execution time: 0μs\n",
+				expErr: "",
+			},
+			{
+				// format.border = 2 (or higher)
+				format: &writeOptions{
+					border:     2,
+					expanded:   false,
+					format:     formatAligned,
+					timing:     false,
+					tuplesOnly: false,
+				},
+				expQOut: stringOfLines(
+					"+-----+-------+-----+",
+					"| _id | name  | age |",
+					"+-----+-------+-----+",
+					"|   1 | Amy   |  44 |",
+					"|   2 | Bob   |  32 |",
+					"|   3 | Cindy |  28 |",
+					"+-----+-------+-----+",
+					"",
+				),
+				expOut: "",
+				expErr: "",
+			},
+			{
+				// format.border = 0
+				format: &writeOptions{
+					border:     0,
+					expanded:   false,
+					format:     formatAligned,
+					timing:     false,
+					tuplesOnly: false,
+				},
+				expQOut: stringOfLines(
+					"_id name  age",
+					"--- ----- ---",
+					"  1 Amy    44",
+					"  2 Bob    32",
+					"  3 Cindy  28",
+					"",
+				),
+				expOut: "",
+				expErr: "",
+			},
+			{
+				// format.tuplesOnly = true
+				format: &writeOptions{
+					border:     1,
+					expanded:   false,
+					format:     formatAligned,
+					timing:     false,
+					tuplesOnly: true,
+				},
+				expQOut: stringOfLines(
+					" 1 | Amy   | 44 ",
+					" 2 | Bob   | 32 ",
+					" 3 | Cindy | 28 ",
+					"",
+				),
+				expOut: "",
+				expErr: "",
+			},
+			{
+				// format.border = 2, expanded
+				format: &writeOptions{
+					border:     2,
+					expanded:   true,
+					format:     formatAligned,
+					timing:     false,
+					tuplesOnly: false,
+				},
+				expQOut: stringOfLines(
+					"+------+-------+",
+					"| _id  | 1     |",
+					"| name | Amy   |",
+					"| age  | 44    |",
+					"+------+-------+",
+					"| _id  | 2     |",
+					"| name | Bob   |",
+					"| age  | 32    |",
+					"+------+-------+",
+					"| _id  | 3     |",
+					"| name | Cindy |",
+					"| age  | 28    |",
+					"+------+-------+",
+					"",
+				),
+				expOut: "",
+				expErr: "",
+			},
+		}
+		for i, test := range tests {
+			t.Run(fmt.Sprintf("test-%d", i), func(t *testing.T) {
+				// Set up buffers to capture the output.
+				qOut := bytes.NewBuffer(make([]byte, 0, 100000))
+				wOut := bytes.NewBuffer(make([]byte, 0, 100000))
+				wErr := bytes.NewBuffer(make([]byte, 0, 100000))
+
+				assert.NoError(t, writeOutput(wqr, test.format, qOut, wOut, wErr))
+
+				assert.Equal(t, test.expQOut, qOut.String())
+				assert.Equal(t, test.expOut, wOut.String())
+				assert.Equal(t, test.expErr, wErr.String())
+			})
+		}
+	})
+}
+
+func stringOfLines(lines ...string) string {
+	var sb strings.Builder
+	for _, line := range lines {
+		sb.WriteString(line + "\n")
+	}
+	return sb.String()
+}

client/api.go

@@ -26,6 +26,26 @@ func NewSchemaAPI(c *Client) *schemaAPI {
 	}
 }
 
+func (s *schemaAPI) CreateDatabase(context.Context, *dax.Database) error {
+	return errors.Errorf("unimplemented: schemaAPI.CreateDatabase()")
+}
+func (s *schemaAPI) DropDatabase(context.Context, dax.DatabaseID) error {
+	return errors.Errorf("unimplemented: schemaAPI.DropDatabase()")
+}
+
+func (s *schemaAPI) DatabaseByName(ctx context.Context, dbname dax.DatabaseName) (*dax.Database, error) {
+	return nil, errors.Errorf("unimplemented: schemaAPI.DatabaseByName()")
+}
+func (s *schemaAPI) DatabaseByID(ctx context.Context, dbid dax.DatabaseID) (*dax.Database, error) {
+	return nil, errors.Errorf("unimplemented: schemaAPI.DatabaseByID()")
+}
+func (s *schemaAPI) SetDatabaseOption(ctx context.Context, dbid dax.DatabaseID, option string, value string) error {
+	return nil
+}
+func (s *schemaAPI) Databases(context.Context, ...dax.DatabaseID) ([]*dax.Database, error) {
+	return nil, errors.Errorf("unimplemented: schemaAPI.Databases()")
+}
+
 func (s *schemaAPI) TableByName(ctx context.Context, tname dax.TableName) (*dax.Table, error) {
 	return nil, errors.New(errors.ErrUncoded, "schemaAPI.TableByName not implemented")
 }

cluster.go

@@ -26,7 +26,7 @@ const (
 )
 
 // cluster represents a collection of nodes.
-type cluster struct { // nolint: maligned
+type cluster struct { //nolint: maligned
 	noder disco.Noder
 
 	id   string

cmd/auth_token.go

@@ -16,7 +16,7 @@ func newAuthTokenCommand(logdest logger.Logger) *cobra.Command {
 		Long: `
 Retrieves an auth-token for use in authenticating with FeatureBase from the configured identity provider.
 `,
-		RunE: usageErrorWrapper(cmd),
+		RunE: UsageErrorWrapper(cmd),
 	}
 
 	flags := ccmd.Flags()

cmd/backup.go

@@ -16,7 +16,7 @@ func newBackupCommand(logdest logger.Logger) *cobra.Command {
 		Long: `
 Backs up a FeatureBase server to a local, tar-formatted snapshot file.
 `,
-		RunE: usageErrorWrapper(cmd),
+		RunE: UsageErrorWrapper(cmd),
 	}
 
 	flags := ccmd.Flags()

cmd/backup_tar.go

@@ -16,7 +16,7 @@ func newBackupTarCommand(logdest io.Writer) *cobra.Command {
 		Long: `
 Backs up a FeatureBase server to a local, tar-formatted snapshot file.
 `,
-		RunE: usageErrorWrapper(cmd),
+		RunE: UsageErrorWrapper(cmd),
 	}
 
 	flags := ccmd.Flags()
@@ -28,6 +28,7 @@ Backs up a FeatureBase server to a local, tar-formatted snapshot file.
 	ctl.SetTLSConfig(flags, "", &cmd.TLS.CertificatePath, &cmd.TLS.CertificateKeyPath, &cmd.TLS.CACertPath, &cmd.TLS.SkipVerify, &cmd.TLS.EnableClientVerification)
 	flags.StringVar(&cmd.AuthToken, "auth-token", "", "Authentication token")
 	flags.StringVar(&cmd.HeaderTimeoutStr, "header-timeout", cmd.HeaderTimeoutStr, "Length of time to wait for initial HTTP response before giving up.")
+	flags.StringVar(&cmd.TempDir, "temp-dir", cmd.TempDir, "Location of temporary spillover files. The default is the system's default (usually /tmp)")
 
 	return ccmd
 }

cmd/chksum.go

@@ -3,13 +3,15 @@
 package cmd
 
 import (
+	"os"
+
 	"github.com/featurebasedb/featurebase/v3/ctl"
 	"github.com/featurebasedb/featurebase/v3/logger"
 	"github.com/spf13/cobra"
 )
 
 func newChkSumCommand(logdest logger.Logger) *cobra.Command {
-	cmd := ctl.NewChkSumCommand(logdest)
+	cmd := ctl.NewChkSumCommand(logdest, os.Stdout)
 	ccmd := &cobra.Command{
 		Use:   "chksum",
 		Short: "Digital signature of FeatureBase data",
@@ -17,7 +19,7 @@ func newChkSumCommand(logdest logger.Logger) *cobra.Command {
 			Generates a digital signature of all the data associated with a provided FeatureBase server
 			WARNING: could be slow if high cardinality fields exist
 `,
-		RunE: usageErrorWrapper(cmd),
+		RunE: UsageErrorWrapper(cmd),
 	}
 
 	flags := ccmd.Flags()

cmd/cli.go

@@ -1,35 +0,0 @@
-// Copyright 2021 Molecula Corp. All rights reserved.
-package cmd
-
-import (
-	"github.com/featurebasedb/featurebase/v3/cli"
-	"github.com/featurebasedb/featurebase/v3/logger"
-	"github.com/spf13/cobra"
-)
-
-var cliCmd *cli.CLICommand
-
-// newCLICommand runs the FeatureBase CLI subcommand for ingesting bulk data.
-func newCLICommand(logdest logger.Logger) *cobra.Command {
-	cliCmd = cli.NewCLICommand(logdest)
-	cobraCmd := &cobra.Command{
-		Use:   "cli",
-		Short: "Query FB with SQL3 from the command line",
-		Long:  ``,
-		RunE:  usageErrorWrapper(cliCmd),
-	}
-
-	flags := cobraCmd.Flags()
-	flags.StringVarP(&cliCmd.Host, "host", "", cliCmd.Host, "hostname of FeatureBase.")
-	flags.StringVarP(&cliCmd.Port, "port", "", cliCmd.Port, "port of FeatureBase.")
-	flags.StringVar(&cliCmd.HistoryPath, "history-path", cliCmd.HistoryPath, "path for history files.")
-	flags.StringVar(&cliCmd.OrganizationID, "org-id", cliCmd.OrganizationID, "OrganizationID.")
-	flags.StringVar(&cliCmd.DatabaseID, "db-id", cliCmd.DatabaseID, "DatabaseID.")
-
-	flags.StringVar(&cliCmd.ClientID, "client-id", cliCmd.ClientID, "Cognito Client ID for FeatureBase Cloud access.")
-	flags.StringVar(&cliCmd.Region, "region", cliCmd.Region, "Cloud region for FeatureBase Cloud access (e.g. us-east-2).")
-	flags.StringVar(&cliCmd.Email, "email", cliCmd.Email, "Email address for FeatureBase Cloud access.")
-	flags.StringVar(&cliCmd.Password, "password", cliCmd.Password, "Password for FeatureBase Cloud access.")
-
-	return cobraCmd
-}

cmd/dataframe-csv-loader.go

@@ -15,7 +15,7 @@ func newDataframeCsvLoaderCommand(logdest logger.Logger) *cobra.Command {
 		Short: "load dataframe integer and floating point values into featurebase",
 		Long: `
 `,
-		RunE: usageErrorWrapper(cmd),
+		RunE: UsageErrorWrapper(cmd),
 	}
 	flags := loaderCmd.Flags()
 	flags.StringVar(&cmd.Path, "csv", "", "path to csv input file")
@@ -24,6 +24,7 @@ func newDataframeCsvLoaderCommand(logdest logger.Logger) *cobra.Command {
 	flags.StringVar(&cmd.AuthToken, "auth-token", "", "Authentication token")
 	flags.StringVar(&cmd.Index, "index", "", "Destination Index. ")
 	flags.IntVar(&cmd.MaxCapacity, "buffer", 0, "Maximum size of of the line buffer defaults to go bufio default ")
+	flags.IntVar(&cmd.BatchSize, "batch-size", 1048576, "Maximum number of records to send in a single batch ")
 	ctl.SetTLSConfig(
 		flags, "",
 		&cmd.TLS.CertificatePath,

cmd/export.go

@@ -26,7 +26,7 @@ The format of the CSV file is:
 
 The file does not contain any headers.
 `,
-		RunE: usageErrorWrapper(Exporter),
+		RunE: UsageErrorWrapper(Exporter),
 	}
 	flags := exportCmd.Flags()
 

cmd/fbsql/main.go

@@ -0,0 +1,66 @@
+// Copyright 2021 Molecula Corp. All rights reserved.
+package main
+
+import (
+	"io"
+	"os"
+
+	"github.com/featurebasedb/featurebase/v3/cli"
+	"github.com/featurebasedb/featurebase/v3/cmd"
+	"github.com/featurebasedb/featurebase/v3/logger"
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+)
+
+func main() {
+	command := newCLICommand(os.Stderr)
+	command.Execute()
+}
+
+// newCLICommand runs the FeatureBase CLI subcommand.
+func newCLICommand(stderr io.Writer) *cobra.Command {
+	logdest := logger.NewStandardLogger(stderr)
+	cliCmd := cli.NewCommand(logdest)
+	cobraCmd := &cobra.Command{
+		Use:   "fbsql",
+		Short: "Query FeatureBase with SQL from the command line",
+		Long:  ``,
+		RunE:  cmd.UsageErrorWrapper(cliCmd),
+		PersistentPreRunE: func(cobraCmd *cobra.Command, args []string) error {
+			v := viper.New()
+			return cmd.SetAllConfig(v, cobraCmd.Flags(), "FBSQL")
+		},
+		SilenceErrors: true,
+	}
+
+	// Attach flags to the command.
+	buildFlags(cobraCmd, cliCmd)
+	return cobraCmd
+}
+
+// buildFlags attaches a set of flags to the command for a cli instance.
+func buildFlags(cmd *cobra.Command, cliCmd *cli.Command) {
+	flags := cmd.Flags()
+
+	// Base struct flags.
+	flags.StringSliceVarP(&cliCmd.Commands, "command", "c", cliCmd.Commands, "Command to run in non-interactive mode. Provide multiple flags to execute more than one command. All `--command` flags run before all `--file` flags.")
+	flags.StringSliceVarP(&cliCmd.Files, "file", "f", cliCmd.Files, "File to run in non-interactive mode. Provide multiple flags to execute more than one file. All `--command` flags run before all `--file` flags.")
+
+	// Config flags.
+	flags.StringVarP(&cliCmd.Config.Host, "host", "", cliCmd.Config.Host, "hostname of FeatureBase.")
+	flags.StringVarP(&cliCmd.Config.Port, "port", "p", cliCmd.Config.Port, "port of FeatureBase.")
+	flags.StringVar(&cliCmd.Config.HistoryPath, "history-path", cliCmd.Config.HistoryPath, "path for history files.")
+	flags.StringVar(&cliCmd.Config.OrganizationID, "org-id", cliCmd.Config.OrganizationID, "OrganizationID.")
+	flags.StringVarP(&cliCmd.Config.Database, "dbname", "d", cliCmd.Config.Database, "Name of the database to connect to.")
+
+	flags.StringVar(&cliCmd.Config.CloudAuth.ClientID, "client-id", cliCmd.Config.CloudAuth.ClientID, "Cognito Client ID for FeatureBase Cloud access.")
+	flags.StringVar(&cliCmd.Config.CloudAuth.Region, "region", cliCmd.Config.CloudAuth.Region, "Cloud region for FeatureBase Cloud access (e.g. us-east-2).")
+	flags.StringVar(&cliCmd.Config.CloudAuth.Email, "email", cliCmd.Config.CloudAuth.Email, "Email address for FeatureBase Cloud access.")
+	flags.StringVar(&cliCmd.Config.CloudAuth.Password, "password", cliCmd.Config.CloudAuth.Password, "Password for FeatureBase Cloud access.")
+
+	flags.StringVar(&cliCmd.Config.KafkaConfig, "kafka-config", cliCmd.Config.KafkaConfig, "Kafka configuration file to read from.")
+	flags.BoolVar(&cliCmd.Config.CSV, "csv", cliCmd.Config.CSV, "CSV (Comma-Separated Values) table output mode.")
+	flags.StringSliceVar(&cliCmd.Config.PSets, "pset", cliCmd.Config.PSets, "Set printing option VAR to ARG (see \\pset command). Use form: --pset=VAR[=ARG]")
+
+	flags.String("config", "", "Configuration file to read from.")
+}

cmd/generate_config.go

@@ -18,7 +18,7 @@ func newGenerateConfigCommand(logdest logger.Logger) *cobra.Command {
 		Short: "Print the default configuration.",
 		Long: `generate-config prints the default configuration to stdout
 `,
-		RunE: usageErrorWrapper(generateConf),
+		RunE: UsageErrorWrapper(generateConf),
 	}
 
 	return confCmd

cmd/keygen.go

@@ -16,7 +16,7 @@ func newKeygenCommand(logdest logger.Logger) *cobra.Command {
 		Long: `
 Generate secret key to configure FeatureBase for Authentication.
 `,
-		RunE: usageErrorWrapper(cmd),
+		RunE: UsageErrorWrapper(cmd),
 	}
 
 	flags := ccmd.Flags()

cmd/parquet-info.go

@@ -0,0 +1,33 @@
+// Copyright 2022 Molecula Corp. (DBA FeatureBase).
+// SPDX-License-Identifier: Apache-2.0
+package cmd
+
+import (
+	"fmt"
+
+	"github.com/featurebasedb/featurebase/v3/ctl"
+	"github.com/featurebasedb/featurebase/v3/logger"
+	"github.com/spf13/cobra"
+)
+
+func newParquetInfoCommand(logdest logger.Logger) *cobra.Command {
+	c := ctl.NewParquetInfoCommand(logdest)
+	cmd := &cobra.Command{
+		Use:   "parquet-info PATH|URL",
+		Short: "Inspect Parquet Files.",
+		Long: `
+Displays schema and sample data from the specified file
+`,
+		Args: func(cmd *cobra.Command, args []string) error {
+			if len(args) == 0 {
+				return fmt.Errorf("data directory path required")
+			} else if len(args) > 1 {
+				return fmt.Errorf("too many command line arguments")
+			}
+			c.Path = args[0]
+			return nil
+		},
+		RunE: UsageErrorWrapper(c),
+	}
+	return cmd
+}

cmd/presort.go

@@ -15,7 +15,7 @@ func newPreSortCommand(logdest logger.Logger) *cobra.Command {
 		Long: `
 Takes all input files and writes PartitionN numbered files to a directory, where each file contains only records that will go into the partition it is named for.
 `,
-		RunE: usageErrorWrapper(cmd),
+		RunE: UsageErrorWrapper(cmd),
 	}
 
 	flags := ccmd.Flags()

cmd/rbf.go

@@ -44,7 +44,7 @@ Executes a consistency check on an RBF data directory.
 			c.Path = args[0]
 			return nil
 		},
-		RunE: usageErrorWrapper(c),
+		RunE: UsageErrorWrapper(c),
 	}
 	return cmd
 }
@@ -76,7 +76,7 @@ Dumps the raw hex data for one or more RBF pages.
 
 			return nil
 		},
-		RunE: usageErrorWrapper(c),
+		RunE: UsageErrorWrapper(c),
 	}
 	return cmd
 }
@@ -98,7 +98,7 @@ Prints a line for every page in the database with its type/status.
 			c.Path = args[0]
 			return nil
 		},
-		RunE: usageErrorWrapper(c),
+		RunE: UsageErrorWrapper(c),
 	}
 
 	flags := cmd.Flags()
@@ -133,7 +133,7 @@ Prints the header & cell data for one or more pages.
 
 			return nil
 		},
-		RunE: usageErrorWrapper(c),
+		RunE: UsageErrorWrapper(c),
 	}
 	return cmd
 }

cmd/restore.go

@@ -16,7 +16,7 @@ func newRestoreCommand(logdest logger.Logger) *cobra.Command {
 		Long: `
 The Restore command will take a backup archive and restore it to a new, clean cluster.
 `,
-		RunE: usageErrorWrapper(cmd),
+		RunE: UsageErrorWrapper(cmd),
 	}
 	flags := restoreCmd.Flags()
 	flags.StringVarP(&cmd.Path, "source", "s", "", "backup file; specify '-' to restore from stdin tar stream")

cmd/restore_tar.go

@@ -15,7 +15,7 @@ func newRestoreTarCommand(logdest logger.Logger) *cobra.Command {
 		Long: `
 The Restore command will take a tar-formatted backup archive and restore it to a new, clean cluster.
 `,
-		RunE: usageErrorWrapper(cmd),
+		RunE: UsageErrorWrapper(cmd),
 	}
 	flags := restoreCmd.Flags()
 	flags.StringVarP(&cmd.Path, "source", "s", "", "backup file; specify '-' to restore from stdin tar stream")
@@ -23,6 +23,8 @@ The Restore command will take a tar-formatted backup archive and restore it to a
 	flags.DurationVar(&cmd.RetryPeriod, "retry-period", cmd.RetryPeriod, "Length of time after HTTP request failure to continue retrying request.")
 	flags.StringVar(&cmd.Pprof, "pprof", cmd.Pprof, "host:port to listen for profiling requests at /debug/pprof and /debug/fgprof.")
 	flags.StringVar(&cmd.AuthToken, "auth-token", "", "Authentication token")
+	flags.StringVar(&cmd.TempDir, "temp-dir", cmd.TempDir, "Location of temporary spillover files. The default is the system's default (usually /tmp)")
+
 	ctl.SetTLSConfig(
 		flags, "",
 		&cmd.TLS.CertificatePath,

cmd/root.go

@@ -25,11 +25,11 @@ type runner interface {
 	Run(context.Context) error
 }
 
-// usageErrorWrapper takes a thing with a Run(context) error, and produces
+// UsageErrorWrapper takes a thing with a Run(context) error, and produces
 // a func(*cobra.Command, []string) error from it which will run that
 // command, and then set Cobra's SilenceUsage flag unless the returned
 // error errors.Is() a ctl.UsageError.
-func usageErrorWrapper(inner runner) func(*cobra.Command, []string) error {
+func UsageErrorWrapper(inner runner) func(*cobra.Command, []string) error {
 	return func(c *cobra.Command, args []string) error {
 		return considerUsageError(c, inner.Run(context.Background()))
 	}
@@ -41,7 +41,7 @@ func usageErrorWrapper(inner runner) func(*cobra.Command, []string) error {
 // wrappers.
 func considerUsageError(cmd *cobra.Command, err error) error {
 	cmd.SilenceErrors = true
-	if !errors.Is(err, ctl.UsageError) {
+	if !errors.Is(err, ctl.ErrUsage) {
 		cmd.SilenceUsage = true
 	}
 	return err
@@ -63,11 +63,12 @@ at https://docs.featurebase.com/.
 ` + pilosa.VersionInfo(true) + "\n",
 		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
 			v := viper.New()
-			if cmd.Use == "dax" {
+
+			switch cmd.Use {
+			case "dax":
 				v.Set("future.rename", true) // always use FEATUREBASE env for dax
 			}
-			err := setAllConfig(v, cmd.Flags())
-			if err != nil {
+			if err := SetAllConfig(v, cmd.Flags(), ""); err != nil {
 				return err
 			}
 
@@ -104,36 +105,37 @@ at https://docs.featurebase.com/.
 	rc.AddCommand(newServeCmd(stderr))
 	rc.AddCommand(newHolderCmd(stderr))
 	rc.AddCommand(newKeygenCommand(logdest))
-	rc.AddCommand(newCLICommand(logdest))
 	rc.AddCommand(newDAXCommand(stderr))
 	rc.AddCommand(newDataframeCsvLoaderCommand(logdest))
 	rc.AddCommand(newPreSortCommand(logdest))
+	rc.AddCommand(newParquetInfoCommand(logdest))
 
 	rc.SetOutput(stderr)
 	return rc
 }
 
-// setAllConfig takes a FlagSet to be the definition of all configuration
+// SetAllConfig takes a FlagSet to be the definition of all configuration
 // options, as well as their defaults. It then reads from the command line, the
 // environment, and a config file (if specified), and applies the configuration
 // in that priority order. Since each flag in the set contains a pointer to
-// where its value should be stored, setAllConfig can directly modify the value
+// where its value should be stored, SetAllConfig can directly modify the value
 // of each config variable.
 //
-// setAllConfig looks for environment variables which are capitalized versions
+// SetAllConfig looks for environment variables which are capitalized versions
 // of the flag names with dashes replaced by underscores, and prefixed with
 // envPrefix plus an underscore.
-func setAllConfig(v *viper.Viper, flags *pflag.FlagSet) error { // nolint: unparam
+func SetAllConfig(v *viper.Viper, flags *pflag.FlagSet, envPrefix string) error { // nolint: unparam
 	// add cmd line flag def to viper
 	err := v.BindPFlags(flags)
 	if err != nil {
 		return err
 	}
 
-	envPrefix := "PILOSA"
-	rename := v.GetBool("future.rename")
-	if rename {
-		envPrefix = "FEATUREBASE"
+	if envPrefix == "" {
+		envPrefix = "PILOSA"
+		if v.GetBool("future.rename") {
+			envPrefix = "FEATUREBASE"
+		}
 	}
 
 	// add env to viper

cmd/server.go

@@ -5,18 +5,10 @@ package cmd
 import (
 	"io"
 
-	"gopkg.in/DataDog/dd-trace-go.v1/ddtrace/opentracer"
-	"gopkg.in/DataDog/dd-trace-go.v1/ddtrace/tracer"
-
 	"github.com/featurebasedb/featurebase/v3/ctl"
 	"github.com/featurebasedb/featurebase/v3/server"
-	"github.com/featurebasedb/featurebase/v3/tracing"
-	"github.com/featurebasedb/featurebase/v3/tracing/opentracing"
 	"github.com/pkg/errors"
 	"github.com/spf13/cobra"
-	jaegercfg "github.com/uber/jaeger-client-go/config"
-
-	"gopkg.in/DataDog/dd-trace-go.v1/profiler"
 )
 
 // Server is global so that tests can control and verify it.
@@ -56,7 +48,6 @@ func newServeCmd(stderr io.Writer) *cobra.Command {
 		Use:   "server",
 		Short: "Run FeatureBase.",
 		Long: `featurebase server runs FeatureBase.
-
 It will load existing data from the configured
 directory and start listening for client connections
 on the configured port.`,
@@ -65,64 +56,7 @@ on the configured port.`,
 			if err := Server.Start(); err != nil {
 				return considerUsageError(cmd, errors.Wrap(err, "running server"))
 			}
-			// anything past here is definitely not a usage error
-			cmd.SilenceErrors = true
-			cmd.SilenceUsage = true
-			if Server.Config.DataDog.Enable {
-				opts := make([]profiler.ProfileType, 0)
-				if Server.Config.DataDog.CPUProfile {
-					opts = append(opts, profiler.CPUProfile)
-				}
-				if Server.Config.DataDog.HeapProfile {
-					opts = append(opts, profiler.HeapProfile)
-				}
-				if Server.Config.DataDog.BlockProfile {
-					opts = append(opts, profiler.BlockProfile)
-				}
-				if Server.Config.DataDog.GoroutineProfile {
-					opts = append(opts, profiler.GoroutineProfile)
-				}
-				if Server.Config.DataDog.MutexProfile {
-					opts = append(opts, profiler.MutexProfile)
-				}
-				err := profiler.Start(
-					profiler.WithService(Server.Config.DataDog.Service),
-					profiler.WithEnv(Server.Config.DataDog.Env),
-					profiler.WithVersion(Server.Config.DataDog.Version),
-					profiler.WithTags(Server.Config.DataDog.Tags),
-					profiler.WithProfileTypes(
-						opts...,
-					),
-				)
-				if err != nil {
-					return errors.Wrap(err, "starting datadog")
-				}
-				defer profiler.Stop()
-			}
 
-			if Server.Config.Tracing.SamplerType != "off" {
-				// Initialize tracing in the command since it is global.
-				var cfg jaegercfg.Configuration
-				cfg.ServiceName = "pilosa"
-				cfg.Sampler = &jaegercfg.SamplerConfig{
-					Type:  Server.Config.Tracing.SamplerType,
-					Param: Server.Config.Tracing.SamplerParam,
-				}
-				cfg.Reporter = &jaegercfg.ReporterConfig{
-					LocalAgentHostPort: Server.Config.Tracing.AgentHostPort,
-				}
-				tracer, closer, err := cfg.NewTracer()
-				if err != nil {
-					return errors.Wrap(err, "initializing jaeger tracer")
-				}
-				defer closer.Close()
-				tracing.GlobalTracer = opentracing.NewTracer(tracer, Server.Logger())
-
-			} else if Server.Config.DataDog.EnableTracing { // Give preference to legacy support of jaeger
-				t := opentracer.New(tracer.WithServiceName(Server.Config.DataDog.Service))
-				defer tracer.Stop()
-				tracing.GlobalTracer = opentracing.NewTracer(t, Server.Logger())
-			}
 			return errors.Wrap(Server.Wait(), "waiting on Server")
 		},
 	}

ctl/backup.go

@@ -10,7 +10,6 @@ import (
 	"fmt"
 	"io"
 	"io/fs"
-	"log"
 	"os"
 	"path/filepath"
 	"time"
@@ -95,13 +94,13 @@ func (cmd *BackupCommand) Run(ctx context.Context) (err error) {
 
 	// Validate arguments.
 	if cmd.OutputDir == "" {
-		return fmt.Errorf("%w: -o flag required", UsageError)
+		return fmt.Errorf("%w: -o flag required", ErrUsage)
 	} else if cmd.Concurrency <= 0 {
-		return fmt.Errorf("%w: concurrency must be at least one", UsageError)
+		return fmt.Errorf("%w: concurrency must be at least one", ErrUsage)
 	}
 	if cmd.HeaderTimeoutStr != "" {
 		if dur, err := time.ParseDuration(cmd.HeaderTimeoutStr); err != nil {
-			return fmt.Errorf("%w: could not parse '%s' as a duration: %v", UsageError, cmd.HeaderTimeoutStr, err)
+			return fmt.Errorf("%w: could not parse '%s' as a duration: %v", ErrUsage, cmd.HeaderTimeoutStr, err)
 		} else {
 			cmd.HeaderTimeout = dur
 		}
@@ -549,7 +548,7 @@ func (cmd *BackupCommand) backupShardDataframe(ctx context.Context, indexName st
 	resp, err := client.GetDataframeShard(ctx, indexName, shard)
 	// no error if doesn't exist
 	if err != nil {
-		log.Fatal(err)
+		return fmt.Errorf("getting dataframe: %w", err)
 	}
 	defer resp.Body.Close()
 	if resp.StatusCode == 404 {

ctl/backup_tar.go

@@ -3,13 +3,11 @@ package ctl
 
 import (
 	"archive/tar"
-	"bytes"
 	"context"
 	"crypto/tls"
 	"encoding/json"
 	"fmt"
 	"io"
-	"log"
 	"os"
 	"path"
 	"path/filepath"
@@ -17,6 +15,7 @@ import (
 
 	pilosa "github.com/featurebasedb/featurebase/v3"
 	"github.com/featurebasedb/featurebase/v3/authn"
+	"github.com/featurebasedb/featurebase/v3/buffer"
 	"github.com/featurebasedb/featurebase/v3/disco"
 	"github.com/featurebasedb/featurebase/v3/encoding/proto"
 	"github.com/featurebasedb/featurebase/v3/logger"
@@ -37,6 +36,9 @@ type BackupTarCommand struct { // nolint: maligned
 	// Path to write the backup to.
 	OutputPath string
 
+	// TempDir location of scratch files
+	TempDir string
+
 	// Amount of time after first failed request to continue retrying.
 	RetryPeriod time.Duration `json:"retry-period"`
 
@@ -80,7 +82,7 @@ func (cmd *BackupTarCommand) Run(ctx context.Context) (err error) {
 	logdest := cmd.Logger()
 	// Validate arguments.
 	if cmd.OutputPath == "" {
-		return fmt.Errorf("%w: -o flag required", UsageError)
+		return fmt.Errorf("%w: -o flag required", ErrUsage)
 	}
 	useStdout := cmd.OutputPath == "-"
 	if useStdout && cmd.logwriter == os.Stdout {
@@ -98,7 +100,7 @@ func (cmd *BackupTarCommand) Run(ctx context.Context) (err error) {
 
 	if cmd.HeaderTimeoutStr != "" {
 		if dur, err := time.ParseDuration(cmd.HeaderTimeoutStr); err != nil {
-			return fmt.Errorf("%w: could not parse '%s' as a duration: %v", UsageError, cmd.HeaderTimeoutStr, err)
+			return fmt.Errorf("%w: could not parse '%s' as a duration: %v", ErrUsage, cmd.HeaderTimeoutStr, err)
 		} else {
 			cmd.HeaderTimeout = dur
 		}
@@ -229,25 +231,7 @@ func (cmd *BackupTarCommand) backupTarIDAllocData(ctx context.Context, tw *tar.W
 	}
 	defer rc.Close()
 
-	// Read to buffer to determine size.
-	var buf bytes.Buffer
-	if _, err := buf.ReadFrom(rc); err != nil {
-		return fmt.Errorf("copying id alloc data to memory: %w", err)
-	}
-
-	// Build header & copy data to archive.
-	if err = tw.WriteHeader(&tar.Header{
-		Name:    "idalloc",
-		Mode:    0o666,
-		Size:    int64(buf.Len()),
-		ModTime: time.Now(),
-	}); err != nil {
-		return err
-	} else if _, err := io.Copy(tw, &buf); err != nil {
-		return fmt.Errorf("copying id alloc data to archive: %w", err)
-	}
-
-	return nil
+	return writeToTar(tw, "idalloc", rc, cmd.TempDir)
 }
 
 // backupTarIndex backs up all shards for a given index.
@@ -268,7 +252,7 @@ func (cmd *BackupTarCommand) backupTarIndex(ctx context.Context, tw *tar.Writer,
 	}
 
 	if ii.Options.Keys {
-		// Back up translation data after bitmap data so we ensure we can translate all data.
+		// Back up translation data after bitmap data so we ensurean translate all data.
 		if err := cmd.backupTarIndexTranslateData(ctx, tw, ii.Name); err != nil {
 			return err
 		}
@@ -330,27 +314,7 @@ func (cmd *BackupTarCommand) backupTarShardNode(ctx context.Context, tw *tar.Wri
 		return fmt.Errorf("fetching shard reader: %w", err)
 	}
 	defer rc.Close()
-
-	// Read to buffer to determine size.
-	// TODO: Provide size via the reader itself.
-	var buf bytes.Buffer
-	if _, err := buf.ReadFrom(rc); err != nil {
-		return fmt.Errorf("copying shard data to memory: %w", err)
-	}
-
-	// Build header & copy data to archive.
-	if err = tw.WriteHeader(&tar.Header{
-		Name:    filename,
-		Mode:    0o666,
-		Size:    int64(buf.Len()),
-		ModTime: time.Now(),
-	}); err != nil {
-		return err
-	} else if _, err := io.Copy(tw, &buf); err != nil {
-		return fmt.Errorf("copying shard data to archive: %w", err)
-	}
-
-	return nil
+	return writeToTar(tw, filename, rc, cmd.TempDir)
 }
 
 func (cmd *BackupTarCommand) backupTarShardDataframe(ctx context.Context, tw *tar.Writer, indexName string, shard uint64, node *disco.Node) error {
@@ -364,7 +328,7 @@ func (cmd *BackupTarCommand) backupTarShardDataframe(ctx context.Context, tw *ta
 	resp, err := client.GetDataframeShard(ctx, indexName, shard)
 	// no error if doesn't exist
 	if err != nil {
-		log.Fatal(err)
+		return fmt.Errorf("getting dataframe: %w", err)
 	}
 	defer resp.Body.Close()
 	if resp.StatusCode == 404 {
@@ -373,24 +337,7 @@ func (cmd *BackupTarCommand) backupTarShardDataframe(ctx context.Context, tw *ta
 	}
 
 	filename := filepath.Join("indexes", indexName, "dataframe", fmt.Sprintf("%04d", shard))
-	logger.Printf("writing %v", filename)
-	var buf bytes.Buffer
-	if _, err := buf.ReadFrom(resp.Body); err != nil {
-		return fmt.Errorf("copying shard data to memory: %w", err)
-	}
-
-	// Build header & copy data to archive.
-	if err = tw.WriteHeader(&tar.Header{
-		Name:    filename,
-		Mode:    0o666,
-		Size:    int64(buf.Len()),
-		ModTime: time.Now(),
-	}); err != nil {
-		return err
-	} else if _, err := io.Copy(tw, &buf); err != nil {
-		return fmt.Errorf("copying shard data to archive: %w", err)
-	}
-	return nil
+	return writeToTar(tw, filename, resp.Body, cmd.TempDir)
 }
 
 func (cmd *BackupTarCommand) backupTarIndexTranslateData(ctx context.Context, tw *tar.Writer, name string) error {
@@ -416,25 +363,7 @@ func (cmd *BackupTarCommand) backupTarIndexPartitionTranslateData(ctx context.Co
 	}
 	defer rc.Close()
 
-	// Read to buffer to determine size.
-	var buf bytes.Buffer
-	if _, err := buf.ReadFrom(rc); err != nil {
-		return fmt.Errorf("copying translate data to memory: %w", err)
-	}
-
-	// Build header & copy data to archive.
-	if err = tw.WriteHeader(&tar.Header{
-		Name:    path.Join("indexes", name, "translate", fmt.Sprintf("%04d", partitionID)),
-		Mode:    0o666,
-		Size:    int64(buf.Len()),
-		ModTime: time.Now(),
-	}); err != nil {
-		return err
-	} else if _, err := io.Copy(tw, &buf); err != nil {
-		return fmt.Errorf("copying translate data to archive: %w", err)
-	}
-
-	return nil
+	return writeToTar(tw, path.Join("indexes", name, "translate", fmt.Sprintf("%04d", partitionID)), rc, cmd.TempDir)
 }
 
 func (cmd *BackupTarCommand) backupTarFieldTranslateData(ctx context.Context, tw *tar.Writer, indexName, fieldName string) error {
@@ -448,27 +377,42 @@ func (cmd *BackupTarCommand) backupTarFieldTranslateData(ctx context.Context, tw
 		return fmt.Errorf("fetching translate data reader: %w", err)
 	}
 	defer rc.Close()
-
-	// Read to buffer to determine size.
-	var buf bytes.Buffer
-	if _, err := buf.ReadFrom(rc); err != nil {
-		return fmt.Errorf("copying translate data to memory: %w", err)
-	}
-
-	// Build header & copy data to archive.
-	if err = tw.WriteHeader(&tar.Header{
-		Name:    path.Join("indexes", indexName, "fields", fieldName, "translate"),
-		Mode:    0o666,
-		Size:    int64(buf.Len()),
-		ModTime: time.Now(),
-	}); err != nil {
-		return err
-	} else if _, err := io.Copy(tw, &buf); err != nil {
-		return fmt.Errorf("copying translate data to archive: %w", err)
-	}
-	return nil
+	return writeToTar(tw, path.Join("indexes", indexName, "fields", fieldName, "translate"), rc, cmd.TempDir)
 }
 
 func (cmd *BackupTarCommand) TLSHost() string { return cmd.Host }
 
 func (cmd *BackupTarCommand) TLSConfiguration() server.TLSConfig { return cmd.TLS }
+
+func writeToTar(tw *tar.Writer, entryName string, rc io.Reader, tmpDir string) error {
+	spillFile, err := os.CreateTemp("", "spill")
+	if err != nil {
+		return fmt.Errorf("creating temp file : %w", err)
+	}
+	defer func() {
+		spillFile.Close()
+		os.Remove(spillFile.Name())
+	}()
+	mb512 := 2 << 29
+	buf := buffer.NewFileBuffer(mb512, tmpDir)
+	defer buf.Close()
+
+	n, err := io.Copy(buf, rc)
+	// Read to buffer to determine size.
+	if err != nil {
+		return fmt.Errorf("copying translate data to memory: %w", err)
+	}
+
+	// Build header & copy data to archive.
+	if err = tw.WriteHeader(&tar.Header{
+		Name:    entryName,
+		Mode:    0o666,
+		Size:    n,
+		ModTime: time.Now(),
+	}); err != nil {
+		return err
+	} else if _, err := io.Copy(tw, buf); err != nil {
+		return fmt.Errorf("copying translate data to archive: %w", err)
+	}
+	return nil
+}

ctl/backup_test.go

@@ -15,19 +15,19 @@ func TestBackupCommand_Run(t *testing.T) {
 	cm := NewBackupCommand(cmLog)
 	cm.OutputDir = ""
 	err := cm.Run(context.Background())
-	if !errors.Is(err, UsageError) {
+	if !errors.Is(err, ErrUsage) {
 		t.Fatalf("expected usage error, got %v", err)
 	}
 	cm.OutputDir = "foo"
 	cm.Concurrency = 0
 	err = cm.Run(context.Background())
-	if !errors.Is(err, UsageError) {
+	if !errors.Is(err, ErrUsage) {
 		t.Fatalf("expected usage error, got %v", err)
 	}
 	cm.Concurrency = 1
 	cm.HeaderTimeoutStr = "until the cat wakes up"
 	err = cm.Run(context.Background())
-	if !errors.Is(err, UsageError) {
+	if !errors.Is(err, ErrUsage) {
 		t.Fatalf("expected usage error, got %v", err)
 	}
 }

ctl/chksum.go

@@ -7,7 +7,6 @@ import (
 	"crypto/tls"
 	"fmt"
 	"io"
-	"os"
 
 	"github.com/cespare/xxhash"
 	pilosa "github.com/featurebasedb/featurebase/v3"
@@ -38,9 +37,9 @@ func (cmd *ChkSumCommand) Logger() logger.Logger {
 }
 
 // NewChkSumCommand returns a new instance of BackupCommand.
-func NewChkSumCommand(logdest logger.Logger) *ChkSumCommand {
+func NewChkSumCommand(logdest logger.Logger, stdout io.Writer) *ChkSumCommand {
 	return &ChkSumCommand{
-		stdout:  os.Stdout,
+		stdout:  stdout,
 		logDest: logdest,
 	}
 }

ctl/chksum_test.go

@@ -13,9 +13,8 @@ import (
 
 func TestChkSumCommand_Run(t *testing.T) {
 	cmLog := logger.NewStandardLogger(os.Stderr)
-	cm := NewChkSumCommand(cmLog)
 	buf := &bytes.Buffer{}
-	cm.stdout = buf
+	cm := NewChkSumCommand(cmLog, buf)
 
 	cluster := test.MustRunCluster(t, 1)
 	defer cluster.Close()

ctl/dataframe-csv-loader.go

@@ -36,7 +36,7 @@ func init() {
 
 // TODO(rdp): add refresh token to this as well
 
-// NewDataframeCsvLoaderCommand
+// DataframeCsvLoaderCommand is used to load a dataframe from CSV.
 type DataframeCsvLoaderCommand struct {
 	tlsConfig *tls.Config
 
@@ -50,6 +50,9 @@ type DataframeCsvLoaderCommand struct {
 	// max line length of csv file
 	MaxCapacity int
 
+	// Batch Size
+	BatchSize int
+
 	// Host:port on which to listen for pprof.
 	Pprof string `json:"pprof"`
 
@@ -146,7 +149,7 @@ func (cmd *DataframeCsvLoaderCommand) Run(ctx context.Context) (err error) {
 
 	// Validate arguments.
 	if cmd.Path == "" {
-		return fmt.Errorf("%w: --csv flag required", UsageError)
+		return fmt.Errorf("%w: --csv flag required", ErrUsage)
 	}
 
 	readFile, err := os.Open(cmd.Path)
@@ -234,7 +237,9 @@ func (cmd *DataframeCsvLoaderCommand) Run(ctx context.Context) (err error) {
 	fileScanner.Scan() // skip the header
 	cmd.Logger().Infof("Build the dataframe input package in memory")
 	id := uint64(0)
+	recordCounter := 0
 	for fileScanner.Scan() {
+		recordCounter++
 		records := strings.Split(fileScanner.Text(), ",")
 		if cmd.needTranslation {
 			id = lookup[records[0]]
@@ -278,13 +283,23 @@ func (cmd *DataframeCsvLoaderCommand) Run(ctx context.Context) (err error) {
 				}
 			}
 		}
+		if recordCounter > cmd.BatchSize {
+			cmd.Logger().Infof("sending package to featurebase")
+			err = sharder.Store(arrowSchema, cmd.client)
+			if err != nil {
+				return err
+			}
+			sharder.Reset()
+			recordCounter = 0
+		}
 	}
-	cmd.Logger().Infof("sending package to featurebase")
-	err = sharder.Store(arrowSchema, cmd.client)
-	if err != nil {
-		return err
+	if recordCounter > 0 {
+		err = sharder.Store(arrowSchema, cmd.client)
+		if err != nil {
+			return err
+		}
 	}
-	return err
+	return nil
 }
 
 type pair struct {
@@ -378,6 +393,10 @@ type Sharder struct {
 	log logger.Logger
 }
 
+func (s *Sharder) Reset() {
+	s.shards = make(map[uint64]*ShardDiff)
+}
+
 func (s *Sharder) GetShard(shard uint64) (*ShardDiff, error) {
 	f, ok := s.shards[shard]
 	if ok {

ctl/dax.go

@@ -14,15 +14,27 @@ func BuildDAXFlags(cmd *cobra.Command, srv *server.Command) {
 	flags.BoolVar(&srv.Config.Verbose, "verbose", srv.Config.Verbose, "Enable verbose logging")
 	flags.StringVar(&srv.Config.LogPath, "log-path", srv.Config.LogPath, "Log path")
 
-	// MDS
-	flags.BoolVar(&srv.Config.MDS.Run, "mds.run", srv.Config.MDS.Run, "Run the MDS service in process.")
-	flags.DurationVar(&srv.Config.MDS.Config.RegistrationBatchTimeout, "mds.config.registration-batch-timeout", srv.Config.MDS.Config.RegistrationBatchTimeout, "Timeout for node registration batches.")
-	flags.StringVar(&srv.Config.MDS.Config.DataDir, "mds.config.data-dir", srv.Config.MDS.Config.DataDir, "MDS directory to use in process.")
-	flags.DurationVar(&srv.Config.MDS.Config.SnappingTurtleTimeout, "mds.config.snapping-turtle-timeout", srv.Config.MDS.Config.SnappingTurtleTimeout, "Period for running automatic snapshotting routine.")
+	// Controller
+	flags.BoolVar(&srv.Config.Controller.Run, "controller.run", srv.Config.Controller.Run, "Run the Controller service in process.")
+	flags.DurationVar(&srv.Config.Controller.Config.RegistrationBatchTimeout, "controller.config.registration-batch-timeout", srv.Config.Controller.Config.RegistrationBatchTimeout, "Timeout for node registration batches.")
+	flags.StringVar(&srv.Config.Controller.Config.StorageMethod, "controller.config.storage-method", srv.Config.Controller.Config.StorageMethod, "Backing store. boltdb or sqldb.")
+	flags.DurationVar(&srv.Config.Controller.Config.SnappingTurtleTimeout, "controller.config.snapping-turtle-timeout", srv.Config.Controller.Config.SnappingTurtleTimeout, "Period for running automatic snapshotting routine.")
+
+	// Controller.SQLDB
+	flags.StringVar(&srv.Config.Controller.Config.SQLDB.Database, "controller.config.sqldb.database", srv.Config.Controller.Config.SQLDB.Database, "Database name.")
+	flags.StringVar(&srv.Config.Controller.Config.SQLDB.Host, "controller.config.sqldb.host", srv.Config.Controller.Config.SQLDB.Host, "Hostname of SQL Database")
+	flags.StringVar(&srv.Config.Controller.Config.SQLDB.Port, "controller.config.sqldb.port", srv.Config.Controller.Config.SQLDB.Port, "Port of SQL Database")
+	flags.StringVar(&srv.Config.Controller.Config.SQLDB.User, "controller.config.sqldb.user", srv.Config.Controller.Config.SQLDB.User, "Username for connection to SQL Database")
+	flags.StringVar(&srv.Config.Controller.Config.SQLDB.Password, "controller.config.sqldb.password", srv.Config.Controller.Config.SQLDB.Password, "Password for connection to SQL Database")
+	flags.StringVar(&srv.Config.Controller.Config.SQLDB.URL, "controller.config.sqldb.url", srv.Config.Controller.Config.SQLDB.URL, "URL for connection to SQL Database (supersedes host/port/username etc)")
+	flags.IntVar(&srv.Config.Controller.Config.SQLDB.Pool, "controller.config.sqldb.pool", srv.Config.Controller.Config.SQLDB.Pool, "Max number of open connections to database. 0=unlimited")
+	flags.IntVar(&srv.Config.Controller.Config.SQLDB.IdlePool, "controller.config.sqldb.idle-pool", srv.Config.Controller.Config.SQLDB.IdlePool, "Maximum number of idle connections to database.")
+	flags.DurationVar(&srv.Config.Controller.Config.SQLDB.ConnMaxLifetime, "controller.config.sqldb.conn-max-lifetime", srv.Config.Controller.Config.SQLDB.ConnMaxLifetime, "See https://golang.org/pkg/database/sql/#DB.SetConnMaxLifetime")
+	flags.DurationVar(&srv.Config.Controller.Config.SQLDB.ConnMaxIdleTime, "controller.config.sqldb.conn-max-idle-time", srv.Config.Controller.Config.SQLDB.ConnMaxIdleTime, "See https://golang.org/pkg/database/sql/#DB.SetConnMaxIdletime")
 
 	// Queryer
 	flags.BoolVar(&srv.Config.Queryer.Run, "queryer.run", srv.Config.Queryer.Run, "Run the Queryer service in process.")
-	flags.StringVar(&srv.Config.Queryer.Config.MDSAddress, "queryer.config.mds-address", srv.Config.Queryer.Config.MDSAddress, "Address of remote MDS process.")
+	flags.StringVar(&srv.Config.Queryer.Config.ControllerAddress, "queryer.config.controller-address", srv.Config.Queryer.Config.ControllerAddress, "Address of remote Controller process.")
 
 	// Computer
 	flags.BoolVar(&srv.Config.Computer.Run, "computer.run", srv.Config.Computer.Run, "Run the Computer service in process.")

ctl/export.go

@@ -50,9 +50,9 @@ func (cmd *ExportCommand) Run(ctx context.Context) error {
 
 	// Validate arguments.
 	if cmd.Index == "" {
-		return fmt.Errorf("%w: %v", UsageError, pilosa.ErrIndexRequired)
+		return fmt.Errorf("%w: %v", ErrUsage, pilosa.ErrIndexRequired)
 	} else if cmd.Field == "" {
-		return fmt.Errorf("%w: %v", UsageError, pilosa.ErrFieldRequired)
+		return fmt.Errorf("%w: %v", ErrUsage, pilosa.ErrFieldRequired)
 	}
 
 	// Use output file, if specified.

ctl/import.go

@@ -86,11 +86,11 @@ func (cmd *ImportCommand) Run(ctx context.Context) error {
 	// Validate arguments.
 	// Index and field are validated early before the files are parsed.
 	if cmd.Index == "" {
-		return fmt.Errorf("%w: %v", UsageError, pilosa.ErrIndexRequired)
+		return fmt.Errorf("%w: %v", ErrUsage, pilosa.ErrIndexRequired)
 	} else if cmd.Field == "" {
-		return fmt.Errorf("%w: %v", UsageError, pilosa.ErrFieldRequired)
+		return fmt.Errorf("%w: %v", ErrUsage, pilosa.ErrFieldRequired)
 	} else if len(cmd.Paths) == 0 {
-		return fmt.Errorf("%w: path required", UsageError)
+		return fmt.Errorf("%w: path required", ErrUsage)
 	}
 	// Create a client to the server.
 	client, err := commandClient(cmd)

ctl/keygen.go

@@ -12,7 +12,7 @@ import (
 	"github.com/gorilla/securecookie"
 )
 
-// Keygen represents a command for generating a cryptographic key.
+// KeygenCommand represents a command for generating a cryptographic key.
 type KeygenCommand struct {
 	stdout    io.Writer
 	logDest   logger.Logger

ctl/parquet-info.go

@@ -0,0 +1,118 @@
+// Copyright 2022 Molecula Corp. (DBA FeatureBase).
+// SPDX-License-Identifier: Apache-2.0
+package ctl
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"os"
+
+	"github.com/apache/arrow/go/v10/arrow/memory"
+	"github.com/apache/arrow/go/v10/parquet/file"
+	"github.com/apache/arrow/go/v10/parquet/pqarrow"
+	pilosa "github.com/featurebasedb/featurebase/v3"
+	"github.com/featurebasedb/featurebase/v3/logger"
+)
+
+// ParquetInfoCommand represents a command for displaying info about a parquet file
+type ParquetInfoCommand struct {
+	// Filepath or URL  to the parquet file.
+	Path string
+
+	// Standard input/output
+	stdout  io.Writer
+	logDest logger.Logger
+}
+
+// NewParquetInfoCommand returns a new instance of ParquetInfoCommand.
+func NewParquetInfoCommand(logdest logger.Logger) *ParquetInfoCommand {
+	return &ParquetInfoCommand{
+		stdout:  os.Stdout,
+		logDest: logdest,
+	}
+}
+
+// Run displays schema and samples data from a parquet file
+func (cmd *ParquetInfoCommand) Run(ctx context.Context) error {
+	// Open database.
+	var f *os.File
+	_, err := url.ParseRequestURI(cmd.Path)
+	if err == nil { // treat as a URL
+		response, err := http.Get(cmd.Path)
+		if err != nil {
+			return err
+		}
+		if response.StatusCode != 200 {
+			return fmt.Errorf("unexpected response %d", response.StatusCode)
+		}
+		defer response.Body.Close()
+		// download to temp file first
+		f, err = os.CreateTemp("", "BulkParquetFile.parquet")
+		if err != nil {
+			return fmt.Errorf("error creating tempfile %v", err)
+		}
+		_, err = io.Copy(f, response.Body)
+		if err != nil {
+			return fmt.Errorf("error downloading url %v %v", cmd.Path, err)
+		}
+		defer os.Remove(f.Name())
+
+		_, err = f.Seek(0, io.SeekStart)
+		if err != nil {
+			return fmt.Errorf("error reseting file for reading %v ", err)
+		}
+	} else {
+		f, err = os.Open(cmd.Path)
+		if err != nil {
+			return err
+		}
+	}
+
+	pf, err := file.NewParquetReader(f)
+	if err != nil {
+		return err
+	}
+	mem := memory.NewGoAllocator()
+	reader, err := pqarrow.NewFileReader(pf, pqarrow.ArrowReadProperties{}, mem)
+	if err != nil {
+		return err
+	}
+	table, err := reader.ReadTable(ctx)
+	if err != nil {
+		return err
+	}
+	// print file name
+	fmt.Printf("\n\nName:%v\n", cmd.Path)
+	// print schema
+	schema := table.Schema()
+	fields := schema.Fields()
+	for i, field := range fields {
+		fmt.Printf("%v. Name: %v\n", i, field.Name)
+		fmt.Printf("%v. Type: %v\n", i, field.Type)
+		fmt.Printf("%v. Nullable: %v\n\n", i, field.Nullable)
+	}
+	bt := pilosa.BasicTableFromArrow(table, mem)
+	// print num rows
+	numRows := int(bt.NumRows())
+	fmt.Printf("Number of rows:%v\n", numRows)
+	if numRows > 10 {
+		numRows = 10
+	}
+	fmt.Println("Sample:")
+	// print at most 10 sample rows in table format
+	for _, field := range fields {
+		fmt.Printf("%v\t", field.Name)
+	}
+	fmt.Println("")
+	for i := 0; i < numRows; i++ {
+		for j := 0; j < len(fields); j++ {
+			fmt.Printf("%v\t", bt.Get(j, i))
+		}
+		fmt.Println("")
+	}
+
+	return nil
+}

ctl/restore.go

@@ -83,9 +83,9 @@ func (cmd *RestoreCommand) Run(ctx context.Context) (err error) {
 
 	// Validate arguments.
 	if cmd.Path == "" {
-		return fmt.Errorf("%w: -s flag required", UsageError)
+		return fmt.Errorf("%w: -s flag required", ErrUsage)
 	} else if cmd.Concurrency <= 0 {
-		return fmt.Errorf("%w: concurrency must be at least one", UsageError)
+		return fmt.Errorf("%w: concurrency must be at least one", ErrUsage)
 	}
 
 	// Parse TLS configuration for node-specific clients.

ctl/restore_tar.go

@@ -3,13 +3,13 @@ package ctl
 
 import (
 	"archive/tar"
-	"bytes"
 	"compress/gzip"
 	"context"
 	"crypto/tls"
 	"fmt"
 	"io"
-	gohttp "net/http"
+	"io/ioutil"
+	"net/http"
 	"os"
 	"strconv"
 	"strings"
@@ -17,6 +17,7 @@ import (
 
 	pilosa "github.com/featurebasedb/featurebase/v3"
 	"github.com/featurebasedb/featurebase/v3/authn"
+	"github.com/featurebasedb/featurebase/v3/buffer"
 	"github.com/featurebasedb/featurebase/v3/disco"
 	"github.com/featurebasedb/featurebase/v3/logger"
 	"github.com/featurebasedb/featurebase/v3/server"
@@ -48,6 +49,9 @@ type RestoreTarCommand struct {
 	TLS server.TLSConfig
 
 	AuthToken string
+
+	// TempDir location of scratch files
+	TempDir string
 }
 
 // Logger returns the command's associated Logger to maintain CommandWithTLSSupport interface compatibility
@@ -75,7 +79,7 @@ func (cmd *RestoreTarCommand) Run(ctx context.Context) (err error) {
 
 	// Validate arguments.
 	if cmd.Path == "" {
-		return fmt.Errorf("%w: -s flag required", UsageError)
+		return fmt.Errorf("%w: -s flag required", ErrUsage)
 	}
 	useStdin := cmd.Path == "-"
 
@@ -133,8 +137,13 @@ func (cmd *RestoreTarCommand) Run(ctx context.Context) (err error) {
 	if primary == nil {
 		return errors.New("no primary")
 	}
-	c := &gohttp.Client{}
+	c := &http.Client{}
+	// buf := new(bytes.Buffer)
+	mb512 := 2 << 29
+	buf := buffer.NewFileBuffer(mb512, cmd.TempDir)
+	defer buf.Reset()
 	for {
+		buf.Reset()
 		header, err := tarReader.Next()
 		if err == io.EOF {
 			break
@@ -175,21 +184,21 @@ func (cmd *RestoreTarCommand) Run(ctx context.Context) (err error) {
 			} else if len(fragmentNodes) == 0 {
 				return fmt.Errorf("no fragmentNodes available")
 			}
-
-			shardBytes, err := io.ReadAll(tarReader) // this feels wrong but works for now
+			_, err = io.Copy(buf, tarReader)
 			if err != nil {
-				return err
+				return errors.Wrap(err, "copying")
 			}
 			g, _ := errgroup.WithContext(ctx)
 			for _, node := range fragmentNodes {
 				node := node
+				rd, err := buf.NewReader()
+				if err != nil {
+					return err
+				}
 				g.Go(func() error {
-					client := &gohttp.Client{}
-					rd := bytes.NewReader(shardBytes)
 					logger.Printf("shard %v %v", shard, indexName)
 					url := node.URI.Path(fmt.Sprintf("/internal/restore/%v/%v", indexName, shard))
-					_, err = client.Post(url, "application/octet-stream", rd)
-					return err
+					return Post(ctx, url, "application/octet-stream", rd, nil)
 				})
 			}
 			if err := g.Wait(); err != nil {
@@ -207,20 +216,22 @@ func (cmd *RestoreTarCommand) Run(ctx context.Context) (err error) {
 				return fmt.Errorf("no fragmentNodes available")
 			}
 
-			shardBytes, err := io.ReadAll(tarReader) // this feels wrong but works for now
+			_, err = io.Copy(buf, tarReader)
 			if err != nil {
-				return err
+				return errors.Wrap(err, "copying")
 			}
+
 			g, _ := errgroup.WithContext(ctx)
 			for _, node := range fragmentNodes {
 				node := node
+				rd, err := buf.NewReader()
+				if err != nil {
+					return err
+				}
 				g.Go(func() error {
-					client := &gohttp.Client{}
-					rd := bytes.NewReader(shardBytes)
 					logger.Printf("dataframe shard %v %v", shard, indexName)
 					url := node.URI.Path(fmt.Sprintf("/internal/dataframe/restore/%v/%v", indexName, shard))
-					_, err = client.Post(url, "application/octet-stream", rd)
-					return err
+					return Post(ctx, url, "application/octet-stream", rd, nil)
 				})
 			}
 			if err := g.Wait(); err != nil {
@@ -236,20 +247,22 @@ func (cmd *RestoreTarCommand) Run(ctx context.Context) (err error) {
 			if err != nil {
 				return err
 			}
-			shardBytes, err := io.ReadAll(tarReader) // this feels wrong but works for now
+
+			_, err = io.Copy(buf, tarReader)
 			if err != nil {
-				return err
+				return errors.Wrap(err, "copying")
 			}
+
 			g, _ := errgroup.WithContext(ctx)
 			for _, node := range partitionNodes {
 				node := node
+				rd, err := buf.NewReader()
+				if err != nil {
+					return err
+				}
 				g.Go(func() error {
-					// rd := bytes.NewReader(shardBytes)
-					rd := func() (io.Reader, error) {
-						return bytes.NewReader(shardBytes), nil
-					}
-
-					return cmd.client.ImportIndexKeys(ctx, &node.URI, indexName, partitionID, false, rd)
+					url := node.URI.Path(fmt.Sprintf("/internal/translate/index/%s/%d", indexName, partitionID))
+					return Post(ctx, url, "application/octet-stream", rd, nil)
 				})
 			}
 			if err := g.Wait(); err != nil {
@@ -263,21 +276,20 @@ func (cmd *RestoreTarCommand) Run(ctx context.Context) (err error) {
 			switch action := record[4]; action {
 			case "translate":
 				logger.Printf("field keys %v %v", indexName, fieldName)
-				// needs to go to all nodes
-				shardBytes, err := io.ReadAll(tarReader) // this feels wrong but works for now
+				_, err = io.Copy(buf, tarReader)
 				if err != nil {
-					return err
+					return errors.Wrap(err, "copying")
 				}
 				g, _ := errgroup.WithContext(ctx)
 				for _, node := range nodes {
 					node := node
+					rd, err := buf.NewReader()
+					if err != nil {
+						return err
+					}
 					g.Go(func() error {
-						// rd := bytes.NewReader(shardBytes)
-						rd := func() (io.Reader, error) {
-							return bytes.NewReader(shardBytes), nil
-						}
-
-						return cmd.client.ImportFieldKeys(ctx, &node.URI, indexName, fieldName, false, rd)
+						url := node.URI.Path(fmt.Sprintf("/internal/translate/field/%s/%s", indexName, fieldName))
+						return Post(ctx, url, "application/octet-stream", rd, nil)
 					})
 				}
 				if err := g.Wait(); err != nil {
@@ -304,3 +316,33 @@ func (cmd *RestoreTarCommand) Run(ctx context.Context) (err error) {
 func (cmd *RestoreTarCommand) TLSHost() string { return cmd.Host }
 
 func (cmd *RestoreTarCommand) TLSConfiguration() server.TLSConfig { return cmd.TLS }
+
+func Post(ctx context.Context, url, contentType string, rd io.Reader, query map[string]string) error {
+	client := &http.Client{}
+	req, err := http.NewRequest(http.MethodPost, url, rd)
+	if err != nil {
+		return err
+	}
+	req.Header.Set("User-Agent", "pilosa/"+pilosa.Version)
+	req.Header.Set("Content-Type", contentType)
+	pilosa.AddAuthToken(ctx, &req.Header)
+
+	// appending to existing query args
+
+	q := req.URL.Query()
+	for k, v := range query {
+		q.Add(k, v)
+	}
+
+	// assign encoded query string to http request
+	req.URL.RawQuery = q.Encode()
+
+	resp, err := client.Do(req)
+	if err != nil {
+		fmt.Println("Errored when sending request to the server")
+		return err
+	}
+	defer resp.Body.Close()
+	_, err = ioutil.ReadAll(resp.Body) // drain the response
+	return err
+}

ctl/restore_tar_test.go

@@ -24,7 +24,7 @@ func TestRestoreTarCommand_Run(t *testing.T) {
 	cm.Host = hostport
 	cm.Path = ""
 	err := cm.Run(context.Background())
-	if !errors.Is(err, UsageError) {
+	if !errors.Is(err, ErrUsage) {
 		t.Fatalf("expected usage error with empty path, got %v", err)
 	}
 	cm.Path = "nonexistent-file"

ctl/restore_test.go

@@ -15,13 +15,13 @@ func TestRestoreCommand_Run(t *testing.T) {
 	cm := NewRestoreCommand(cmLog)
 	cm.Path = ""
 	err := cm.Run(context.Background())
-	if !errors.Is(err, UsageError) {
+	if !errors.Is(err, ErrUsage) {
 		t.Fatalf("expected usage error, got %v", err)
 	}
 	cm.Path = "foo"
 	cm.Concurrency = 0
 	err = cm.Run(context.Background())
-	if !errors.Is(err, UsageError) {
+	if !errors.Is(err, ErrUsage) {
 		t.Fatalf("expected usage error, got %v", err)
 	}
 }