Update README.md

changed links to the community help repo

.github/workflows/ci.yml

@@ -36,7 +36,7 @@ jobs:
         with:
           # Optional: version of golangci-lint to use in form of v1.2 or v1.2.3 or `latest` to use the latest version
           # version: v1.29
-          args: --timeout=5m
+          args: --timeout=8m
 
   validate:
     name: Code Checks

.gitignore

@@ -84,3 +84,4 @@ dax/dax-data
 
 coverage-from-docker
 *.client_id.txt
+

.gitlab/.gitlab-ci.yml

@@ -12,10 +12,22 @@ include:
   - template: Security/License-Scanning.gitlab-ci.yml
   - template: Security/Dependency-Scanning.gitlab-ci.yml
 
+default:
+  retry:
+    max: 2 # This is confusing but this means "3 runs at max".
+    when:
+      - unknown_failure
+      - api_failure
+      - runner_system_failure
+      - job_execution_timeout
+      - stuck_or_timeout_failure
+
 variables:
   GOVERSION: "1.19.3"
   GOFUTURE: "latest"
   CI_IMAGE: "${CI_REGISTRY_IMAGE}/ci-builder:0.0.1"
+  CI_PRE_CLONE_SCRIPT: |
+    set -x
 
 stages:
   - ci_image_build
@@ -157,19 +169,52 @@ build featurebase:
     - go install github.com/rakyll/statik@v0.1.7
     - $GOPATH/bin/statik -src=lattice
     - export SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct)
-    - GOOS="linux" GOARCH="amd64" make build FLAGS="-o featurebase_linux_amd64"
-    - GOOS="linux" GOARCH="arm64" make build FLAGS="-o featurebase_linux_arm64"
-    - GOOS="darwin" GOARCH="amd64" make build FLAGS="-o featurebase_darwin_amd64"
-    - GOOS="darwin" GOARCH="arm64" make build FLAGS="-o featurebase_darwin_arm64"
+    - |
+      for goos in "darwin" "linux"; do
+        for goarch in "amd64" "arm64"; do
+          GOOS="${goos}" GOARCH="${goarch}" make build FLAGS="-o featurebase_${goos}_${goarch}"
+        done
+      done
   artifacts:
     paths:
-      - featurebase_linux_amd64
-      - featurebase_linux_arm64
-      - featurebase_darwin_amd64
-      - featurebase_darwin_arm64
+      - featurebase_*
   needs:
     - job: build lattice
 
+build fbsql amd64:
+  stage: test
+  variables:
+    BUILD_NAME: build_${CI_COMMIT_SHA}_${CI_CONCURRENT_ID}
+  tags:
+    - shell
+  rules:
+    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
+  script:
+    - export SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct)
+    - date
+    - GOOS="linux" GOARCH="amd64" make docker-build-fbsql BUILD_CGO=1
+    - GOOS="darwin" GOARCH="amd64" make docker-build-fbsql
+  artifacts:
+    paths:
+      - ./build/fbsql_*
+
+build fbsql arm64:
+  stage: test
+  variables:
+    BUILD_NAME: build_${CI_COMMIT_SHA}_${CI_CONCURRENT_ID}
+  tags:
+    - shell-arm64
+  rules:
+    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
+  script:
+    - export SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct)
+    - date
+    - GOOS="linux" GOARCH="arm64" make docker-build-fbsql BUILD_CGO=1
+    - GOOS="darwin" GOARCH="arm64" make docker-build-fbsql
+  artifacts:
+    paths:
+      - ./build/fbsql_*
+
 build amd container fb:
   stage: test
   tags:
@@ -210,13 +255,27 @@ run go tests race:
   stage: nonblocking # don't let this job block any other jobs because it takes much longer than the other tests.
   image: golang:$GOVERSION
   extends: .go-cache
+  variables:
+    FEATUREBASE_CONTROLLER_CONFIG_SQLDB_DATABASE: run_go_tests_race
+    POSTGRES_DB: run_go_tests_race
+    FEATUREBASE_CONTROLLER_CONFIG_SQLDB_USER: postgres
+    POSTGRES_USER: postgres
+    FEATUREBASE_CONTROLLER_CONFIG_SQLDB_PASSWORD: $POSTGRES_PASSWORD
+    POSTGRES_PASSWORD: $POSTGRES_PASSWORD
+    FEATUREBASE_CONTROLLER_CONFIG_SQLDB_HOST: postgres
+  services:
+    - postgres:14.7
   rules:
     - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
   needs: ["smoke build"] # we do block on smoke build though bc it's pretty dumb to test stuff if it doesn't build
   script:
     - echo "Running featurebase race tests..."
     - PKG_LIST=$(go list ./... | grep -Ev 'internal/clustertests|simulacraData|batch|idk|v3/dax/test/dax' | paste -s -d, -)
-    - RAMDISK=/mnt/ramdisk go test -race -v -timeout=10m ${PKG_LIST//,/ }
+    - export TMPDIR=/mnt/ramdisk/test-$CI_JOB_ID
+    - mkdir -p $TMPDIR
+    - go test -race -v -timeout=10m ${PKG_LIST//,/ }
+  after_script:
+    - rm -rf /mnt/ramdisk/test-$CI_JOB_ID
   tags:
     - docker
 
@@ -227,12 +286,26 @@ run go tests:
   stage: test
   image: golang:$GOVERSION
   extends: .go-cache
+  variables:
+    FEATUREBASE_CONTROLLER_CONFIG_SQLDB_DATABASE: run_go_tests
+    POSTGRES_DB: run_go_tests
+    FEATUREBASE_CONTROLLER_CONFIG_SQLDB_USER: postgres
+    POSTGRES_USER: postgres
+    FEATUREBASE_CONTROLLER_CONFIG_SQLDB_PASSWORD: $POSTGRES_PASSWORD
+    POSTGRES_PASSWORD: $POSTGRES_PASSWORD
+    FEATUREBASE_CONTROLLER_CONFIG_SQLDB_HOST: postgres
+  services:
+    - postgres:14.7
   rules:
     - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
   script:
     - echo "Running featurebase unit tests..."
     - PKG_LIST=$(go list ./... | grep -Ev 'internal/clustertests|simulacraData|batch|idk|v3/dax/test/dax' | paste -s -d, -)
-    - RAMDISK=/mnt/ramdisk go test -tags=shardwidth22 -timeout=10m -coverprofile=coverage.out -covermode=atomic -coverpkg=${PKG_LIST} ${PKG_LIST//,/ }
+    - export TMPDIR=/mnt/ramdisk/test-$CI_JOB_ID
+    - mkdir -p $TMPDIR
+    - go test -tags=shardwidth22 -timeout=10m -coverprofile=coverage.out -covermode=atomic -coverpkg=${PKG_LIST} ${PKG_LIST//,/ }
+  after_script:
+    - rm -rf /mnt/ramdisk/test-$CI_JOB_ID
   artifacts:
     paths:
       - coverage.out
@@ -242,42 +315,45 @@ run go tests:
 run go tests dax/test/dax:
   stage: test
   image: golang:$GOVERSION
+  extends: .go-cache
   tags:
-    - aws
+    - docker
+  variables:
+    FEATUREBASE_CONTROLLER_CONFIG_SQLDB_DATABASE: run_go_tests_dax
+    POSTGRES_DB: run_go_tests_dax
+    FEATUREBASE_CONTROLLER_CONFIG_SQLDB_USER: postgres
+    POSTGRES_USER: postgres
+    FEATUREBASE_CONTROLLER_CONFIG_SQLDB_PASSWORD: $POSTGRES_PASSWORD
+    POSTGRES_PASSWORD: $POSTGRES_PASSWORD
+    FEATUREBASE_CONTROLLER_CONFIG_SQLDB_HOST: postgres
+  services:
+    - postgres:14.7
   rules:
     - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
   script:
     - echo "Building FB and Datagen docker images for DAX tests"
     - PKG_LIST=$(go list ./... | grep -Ev 'internal/clustertests|simulacraData' | paste -s -d, -)
+    - export TMPDIR=/mnt/ramdisk/test-$CI_JOB_ID
+    - mkdir -p $TMPDIR
     - go test -coverprofile=coverage-dax-integration.out -covermode=atomic -coverpkg=${PKG_LIST} -timeout=20m ./dax/test/dax
+  after_script:
+    - rm -rf /mnt/ramdisk/test-$CI_JOB_ID
   artifacts:
     paths:
       - coverage-dax-integration.out
 
-.idk_changed:
-  rules:
-    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
-      changes:
-        compare_to: refs/heads/master
-        paths:
-          - idk/**
-          - client/**
-          - batch/**
-
 # idk tests
 run go tests idk race:
   variables:
     PROJECT: race_${CI_CONCURRENT_ID}
   stage: nonblocking
   retry: 1
-  rules:
-    - !reference [.idk_changed, rules]
   script:
     - echo "Running test-all-race"
     - cd ./idk/
     - echo $PROJECT
     - echo $CI_COMMIT_REF_SLUG
-    - BRANCH_NAME=${CI_COMMIT_REF_SLUG} make test-all-race
+    - BRANCH_NAME=${CI_COMMIT_REF_SLUG} IDK_FEATUREBASE_TAG=${CI_COMMIT_TAG} IDK_FEATUREBASE_HASH=${CI_COMMIT_SHA} make test-all-race
   after_script:
     - cd ./idk/
     - make save-pilosa-logs
@@ -297,14 +373,12 @@ run go tests idk shard transactional:
     PROJECT: shardttrans_${CI_CONCURRENT_ID}
   stage: nonblocking
   retry: 1
-  rules:
-    - !reference [.idk_changed, rules]
   script:
     - echo "Running shard transactional tests"
     - cd ./idk/
     - echo $PROJECT
     - echo $CI_COMMIT_REF_SLUG
-    - BRANCH_NAME=${CI_COMMIT_REF_SLUG} make test-all
+    - BRANCH_NAME=${CI_COMMIT_REF_SLUG} IDK_FEATUREBASE_TAG=${CI_COMMIT_TAG} IDK_FEATUREBASE_HASH=${CI_COMMIT_SHA} make test-all
   after_script:
     - cd ./idk/
     - make save-pilosa-logs
@@ -330,13 +404,11 @@ run go tests idk 533:
     - cd ./idk/
     - echo $PROJECT
     - echo $CI_COMMIT_REF_SLUG
-    - CONFLUENT_VERSION=5.3.3 BRANCH_NAME=${CI_COMMIT_REF_SLUG} make test-all
+    - CONFLUENT_VERSION=5.3.3 BRANCH_NAME=${CI_COMMIT_REF_SLUG} IDK_FEATUREBASE_TAG=${CI_COMMIT_TAG} IDK_FEATUREBASE_HASH=${CI_COMMIT_SHA} make test-all
   after_script:
     - cd ./idk/
     - make save-pilosa-logs
     - make shutdown
-  rules:
-    - !reference [.idk_changed, rules]
   tags:
     - shell
     - aws
@@ -356,13 +428,11 @@ run go tests idk sasl:
     - cd ./idk/
     - echo $PROJECT
     - echo $CI_COMMIT_REF_SLUG
-    - BRANCH_NAME=${CI_COMMIT_REF_SLUG} make test-all-kafka-sasl
+    - BRANCH_NAME=${CI_COMMIT_REF_SLUG} IDK_FEATUREBASE_TAG=${CI_COMMIT_TAG} IDK_FEATUREBASE_HASH=${CI_COMMIT_SHA} make test-all-kafka-sasl
   after_script:
     - cd ./idk/
     - make save-pilosa-logs
     - make shutdown
-  rules:
-    - !reference [.idk_changed, rules]
   tags:
     - shell
     - aws
@@ -422,7 +492,8 @@ upload_artifacts_to_nexus:
   rules:
     - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
   script:
-    - find . -maxdepth 1 -name '*.rpm' -exec curl -v --user "$NEXUS_YUM_CREDS" --upload-file {} https://nexus.molecula.com/repository/molecula-yum/release/ \;
+    - curl -v --user "$NEXUS_YUM_CREDS" --upload-file *.arm64.rpm https://nexus.molecula.com/repository/molecula-yum/release/
+    - curl -v --user "$NEXUS_YUM_CREDS" --upload-file *.amd64.rpm https://nexus.molecula.com/repository/molecula-yum/release/
   dependencies:
     - package for linux amd64
     - package for linux arm64
@@ -624,16 +695,19 @@ s3 dump:
     - aws configure set aws_secret_access_key $AWS_FBCI_SECRET_ACCESS_KEY
     - aws configure set region "us-east-2"
     - aws configure set aws_profile $PROFILE
-    - aws s3 cp featurebase_linux_amd64 s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/${CI_COMMIT_SHORT_SHA}/featurebase_linux_amd64
-    - aws s3 cp featurebase_linux_amd64 s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/_latest/featurebase_linux_amd64
-    - aws s3 cp featurebase_linux_arm64 s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/${CI_COMMIT_SHORT_SHA}/featurebase_linux_arm64
-    - aws s3 cp featurebase_linux_arm64 s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/_latest/featurebase_linux_arm64
-    - aws s3 cp featurebase_darwin_amd64 s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/${CI_COMMIT_SHORT_SHA}/featurebase_darwin_amd64
-    - aws s3 cp featurebase_darwin_amd64 s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/_latest/featurebase_darwin_amd64
-    - aws s3 cp featurebase_darwin_arm64 s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/${CI_COMMIT_SHORT_SHA}/featurebase_darwin_arm64
-    - aws s3 cp featurebase_darwin_arm64 s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/_latest/featurebase_darwin_arm64
+    - |
+      for goos in "darwin" "linux"; do
+        for goarch in "amd64" "arm64"; do
+          aws s3 cp featurebase_${goos}_${goarch} s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/${CI_COMMIT_SHORT_SHA}/featurebase_${goos}_${goarch}
+          aws s3 cp featurebase_${goos}_${goarch} s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/_latest/featurebase_${goos}_${goarch}
+          aws s3 cp ./build/fbsql_${goos}_${goarch} s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/${CI_COMMIT_SHORT_SHA}/fbsql_${goos}_${goarch}
+          aws s3 cp ./build/fbsql_${goos}_${goarch} s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/_latest/fbsql_${goos}_${goarch}
+        done
+      done
   needs:
     - job: build featurebase
+    - job: build fbsql amd64
+    - job: build fbsql arm64
 
 s3 dump tag:
   stage: post build
@@ -659,6 +733,7 @@ s3 dump tag:
           echo "Directory ${dir}"
           mkdir $dir
           mv featurebase_${goos}_${goarch} ${dir}/featurebase
+          mv ./build/fbsql_${goos}_${goarch} ${dir}/fbsql
           cp NOTICE install/featurebase.conf install/featurebase.*.service ${dir}/
           tar cvzf ${dir}.tar.gz ${dir}
           aws s3 cp ${dir} s3://${LOCATION}/${CI_COMMIT_TAG}/${dir}/ --recursive
@@ -668,3 +743,5 @@ s3 dump tag:
 
   needs:
     - job: build featurebase
+    - job: build fbsql amd64
+    - job: build fbsql arm64

.golangci.yml

@@ -10,9 +10,26 @@ run:
     - pql/pql.peg.go
 linters:
   enable:
+    # Recommended to be enabled by default (https://golangci-lint.run).
+    # - errcheck (lots to fix)
+    - gosimple
     - govet
-    - gofmt
+    - ineffassign
     - staticcheck
+    - typecheck
+    # - unused (about 20 to fix)
+
+    # Additional linters we choose to enable.
+    # - bodyclose (lots to fix, but we should)
+    - errchkjson
+    - errname
+    - gofmt
+    # - misspell (lots to fix, but we should)
+    - prealloc
+    # - predeclared (20 to fix)
+    # - stylecheck (quite a lot to fix, but we should definitely work on this)
+    - stylecheck
+    # - unconvert (not at all critical, but makes for cleaner code)
   enable-all: false
   disable-all: true
 
@@ -50,6 +67,12 @@ linters-settings:
       - shadow
     disable-all: false
 
+  stylecheck:
+    # ST1000: at least one file in a package should have a package comment
+    # ST1003: golang naming standards
+    # ST1016: methods on the same type should have the same receiver name
+    # ST1020: comment on exported function
+    checks: ["all", "-ST1000", "-ST1003", "-ST1016", "-ST1020"]
 
 issues:
   exclude-use-default: false

Dockerfile-fbsql

@@ -0,0 +1,48 @@
+ARG GO_VERSION=1.19
+
+FROM golang:1.19-buster as builder
+
+WORKDIR /
+RUN apt-get update -y -qq && apt-get install -y -qq \
+  build-essential \
+  git \
+  musl-tools \
+  netcat \
+  unixodbc \
+  unixodbc-dev \
+  && rm -rf /var/lib/apt/lists/*
+RUN ["git", "clone", "https://github.com/edenhill/librdkafka.git"]
+WORKDIR /librdkafka
+RUN ./configure --prefix /usr && \
+  make && \
+  make install
+
+WORKDIR /featurebase
+
+COPY . .
+
+ARG MAKE_FLAGS
+ARG GO_BUILD_FLAGS
+ARG SOURCE_DATE_EPOCH
+
+WORKDIR /featurebase/
+
+ENV SOURCE_DATE_EPOCH=${SOURCE_DATE_EPOCH}
+RUN make build-fbsql GO_BUILD_FLAGS="-mod=vendor ${GO_BUILD_FLAGS}" ${MAKE_FLAGS}
+
+FROM ubuntu:20.04 as runner
+
+RUN apt-get update -y -qq && apt-get install -y -qq \
+  ca-certificates \
+  musl-tools \
+  netcat \
+  unixodbc-dev \
+  && rm -rf /var/lib/apt/lists/*
+
+COPY --from=builder /featurebase/fbsql /usr/local/bin/
+
+# Verify that the linker can find everything.
+FROM runner as linkcheck
+RUN if [ -e /usr/local/bin/fbsql ] ; then ldd /usr/local/bin/fbsql; fi
+
+FROM runner

Makefile

@@ -1,4 +1,4 @@
-.PHONY: build clean build-lattice cover cover-viz default docker docker-build docker-tag-push generate generate-protoc generate-pql generate-statik generate-stringer install install-protoc-gen-gofast install-protoc install-statik install-peg test docker-login
+.PHONY: build clean build-lattice cover cover-viz default docker docker-build docker-build-fbsql docker-tag-push generate generate-protoc generate-pql generate-statik generate-stringer install install-protoc-gen-gofast install-protoc install-statik install-peg test docker-login
 
 SHELL := /bin/bash
 VERSION := $(shell git describe --tags 2> /dev/null || echo unknown)
@@ -19,11 +19,15 @@ SHARD_WIDTH = 20
 COMMIT := $(shell git describe --exact-match >/dev/null 2>&1 || git rev-parse --short HEAD)
 LDFLAGS="-X github.com/featurebasedb/featurebase/v3.Version=$(VERSION) -X github.com/featurebasedb/featurebase/v3.BuildTime=$(BUILD_TIME) -X github.com/featurebasedb/featurebase/v3.Variant=$(VARIANT) -X github.com/featurebasedb/featurebase/v3.Commit=$(COMMIT) -X github.com/featurebasedb/featurebase/v3.TrialDeadline=$(TRIAL_DEADLINE)"
 GO_VERSION=1.19
+GO_BUILD_FLAGS=
 DOCKER_BUILD= # set to 1 to use `docker-build` instead of `build` when creating a release
 BUILD_TAGS += 
 TEST_TAGS = roaringparanoia
 TEST_TIMEOUT=10m
 RACE_TEST_TIMEOUT=10m
+# size in GB to use for ramdisk, ?= so you can override it with env
+# 4GB is not enough for `make test`, 8GB usually is.
+RAMDISK_SIZE ?= 8
 
 export GO111MODULE=on
 export GOPRIVATE=github.com/molecula
@@ -47,11 +51,11 @@ version:
 
 # We build a list of packages that omits the IDK and batch packages because
 # those packages require fancy environment setup.
-GOPACKAGES := $(shell $(GO) list ./... | grep -v "/idk" | grep -v "/batch")
+GOPACKAGES := $(shell $(GO) list ./... | grep -v "/v3/idk" | grep -v "/v3/batch")
 
 # Run test suite
 test:
-	$(GO) test $(GOPACKAGES) -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) -v -timeout $(TEST_TIMEOUT)
+	$(GO) test $(GOPACKAGES) -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) -v -timeout $(TEST_TIMEOUT) -count=1
 
 # Run test suite with race flag
 test-race:
@@ -77,13 +81,18 @@ testvsub:
 			echo; echo "999 done testing subpkg $$pkg"; \
 		done
 
-# make a 2GB RAMDisk. Speed up tests by running them with RAMDISK=/mnt/ramdisk
+# make a $(RAMDISK_SIZE)GB RAMDisk. Speed up tests by running
+# them with TMPDIR=/mnt/ramdisk.
 ramdisk-linux:
-	mount -o size=2G -t tmpfs none /mnt/ramdisk
+	mount -o size=$(RAMDISK__SIZE)G -t tmpfs none /mnt/ramdisk
 
-# make a 2GB RAMDisk. Speed up tests by running them with RAMDISK=/Volumes/RAMDisk
+# make a $(RAMDISK_SIZE)GB RAMDisk. Speed up tests by running
+# them with TMPDIR=/Volumes/RAMDisk. This is more important on
+# OS X than it is on Linux, because there's performance issues
+# with fsync on OS X that can make the SSD slow down to moving-platters
+# drive speeds. Oops.
 ramdisk-osx:
-	diskutil erasevolume HFS+ 'RAMDisk' `hdiutil attach -nobrowse -nomount ram://4194304`
+	diskutil erasevolume HFS+ 'RAMDisk' $$(hdiutil attach -nobrowse -nomount ram://$$(expr 2097152 \* $(RAMDISK_SIZE)))
 
 detach-ramdisk-osx:
 	hdiutil detach /Volumes/RAMDisk
@@ -108,12 +117,13 @@ cover:
 cover-viz: cover
 	$(GO) tool cover -html=build/coverage.out
 
-# Compile Pilosa
+# Build featurebase
 build:
 	$(GO) build -tags='$(BUILD_TAGS)' -ldflags $(LDFLAGS) $(FLAGS) ./cmd/featurebase
 
 package:
-	GOOS=$(GOOS) GOARCH=$(GOARCH) FLAGS="-o featurebase" $(MAKE) build
+	GOOS=$(GOOS) GOARCH=$(GOARCH) $(MAKE) build
+	GOOS=$(GOOS) GOARCH=$(GOARCH) $(MAKE) build-fbsql
 	GOARCH=$(GOARCH) VERSION=$(VERSION) nfpm package --packager deb --target featurebase.$(VERSION).$(GOARCH).deb
 	GOARCH=$(GOARCH) VERSION=$(VERSION) nfpm package --packager rpm --target featurebase.$(VERSION).$(GOARCH).rpm
 
@@ -144,7 +154,7 @@ authclustertests: vendor
 	CLUSTERTESTS_FB_ARGS=$(AUTH_ARGS) $(DOCKER_COMPOSE) -f internal/clustertests/docker-compose.yml down
 
 # Install FeatureBase and IDK
-install: install-featurebase install-idk
+install: install-featurebase install-idk install-fbsql
 
 install-featurebase:
 	$(GO) install -tags='$(BUILD_TAGS)' -ldflags $(LDFLAGS) $(FLAGS) ./cmd/featurebase
@@ -152,6 +162,9 @@ install-featurebase:
 install-idk:
 	$(MAKE) -C ./idk install
 
+install-fbsql:
+	CGO_ENABLED=1 $(GO) install ./cmd/fbsql
+
 # Build the lattice assets
 build-lattice:
 	docker build -t lattice:build ./lattice
@@ -220,6 +233,13 @@ docker-image-featurebase: vendor
 		--file Dockerfile-dax \
 		--tag dax/featurebase .
 
+docker-image-featurebase-linux-amd64: vendor
+	docker build \
+	    --build-arg GO_VERSION=$(GO_VERSION) \
+	    --platform linux/amd64 \
+	    --file Dockerfile-dax \
+	    --tag dax/featurebase .
+
 docker-image-featurebase-test: vendor
 	docker build \
 	    --build-arg GO_VERSION=$(GO_VERSION) \
@@ -246,7 +266,12 @@ docker-image-featurebase-quick: build-for-quick
 docker-image-datagen: vendor
 	docker build --tag dax/datagen --file Dockerfile-datagen .
 
+get-account-id:
+	$(eval AWS_ACCOUNTID := $(shell aws sts get-caller-identity --output=json | jq -r .Account))
+
+
 ecr-push-featurebase: docker-login
+	echo "Pushing to account $(AWS_ACCOUNTID), profile $(AWS_PROFILE)"
 	docker tag dax/featurebase:latest $(AWS_ACCOUNTID).dkr.ecr.us-east-2.amazonaws.com/dax/featurebase:latest
 	docker push $(AWS_ACCOUNTID).dkr.ecr.us-east-2.amazonaws.com/dax/featurebase:latest
 
@@ -254,7 +279,7 @@ ecr-push-datagen: docker-login
 	docker tag dax/datagen:latest $(AWS_ACCOUNTID).dkr.ecr.us-east-2.amazonaws.com/dax/datagen:latest
 	docker push $(AWS_ACCOUNTID).dkr.ecr.us-east-2.amazonaws.com/dax/datagen:latest
 
-docker-login:
+docker-login: get-account-id
 	aws ecr get-login-password --region us-east-2 | docker login --username AWS --password-stdin $(AWS_ACCOUNTID).dkr.ecr.us-east-2.amazonaws.com
 
 # Create docker image (alias)
@@ -328,3 +353,53 @@ test-external-lookup:
 
 bnf:
 	ebnf2railroad --no-overview-diagram  --no-optimizations ./sql3/sql3.ebnf
+
+#################################
+# fbsql builds in docker
+#################################
+
+# This allows multiple concurrent builds to happen in CI without
+# creating container name conflicts and such. (different BUILD_NAMEs
+# are passed in from gitlab-ci.yml)
+BUILD_NAME ?= fbsql-build
+
+LDFLAGS_STATIC="-linkmode external -extldflags \"-static\" -X 'github.com/featurebasedb/featurebase/v3/fbsql.Version=$(VERSION)' -X 'github.com/featurebasedb/featurebase/v3/fbsql.BuildTime=$(BUILD_TIME)' "
+
+UNAME_P := $(shell uname -p)
+BUILD_CGO ?= 0
+
+# Build fbsql
+build-fbsql:
+	@echo GOOS=$(GOOS) GOARCH=$(GOARCH) uname -p=$(UNAME_P) build_cgo=$(BUILD_CGO)
+ifeq ($(BUILD_CGO), 0)
+	make build-fbsql-non-cgo
+endif
+ifeq ($(BUILD_CGO), 1)
+	make build-fbsql-cgo
+endif
+
+build-fbsql-non-cgo:
+	CGO_ENABLED=0 $(GO) build -ldflags $(LDFLAGS) $(GO_BUILD_FLAGS) -o fbsql ./cmd/fbsql
+
+build-fbsql-cgo:
+ifeq ($(GOARCH), arm64)
+	CGO_ENABLED=1 $(GO) build -tags dynamic $(GO_BUILD_FLAGS) -o fbsql ./cmd/fbsql
+endif
+ifeq ($(GOARCH), amd64)
+	CC=/usr/bin/musl-gcc CGO_ENABLED=1 $(GO) build -tags "musl static" -ldflags $(LDFLAGS_STATIC) $(GO_BUILD_FLAGS) -o fbsql ./cmd/fbsql
+endif
+
+docker-build-fbsql: vendor
+	DOCKER_BUILDKIT=0 docker build \
+		--file Dockerfile-fbsql \
+	    --build-arg GO_VERSION=$(GO_VERSION) \
+	    --build-arg MAKE_FLAGS="GOOS=$(GOOS) GOARCH=$(GOARCH) BUILD_CGO=$(BUILD_CGO)" \
+	    --build-arg GO_BUILD_FLAGS=$(GO_BUILD_FLAGS) \
+		--build-arg SOURCE_DATE_EPOCH=$(SOURCE_DATE_EPOCH) \
+	    --target builder \
+	    --tag fbsql:$(BUILD_NAME) .
+	mkdir -p build
+	docker create --name $(BUILD_NAME) fbsql:$(BUILD_NAME)
+	docker cp $(BUILD_NAME):/featurebase/fbsql ./build/fbsql_$(GOOS)_$(GOARCH)
+	docker rm $(BUILD_NAME)
+

OPENSOURCE.md

@@ -0,0 +1,45 @@
+## User Contribution Guidelines for FeatureBase
+
+Thank you for your interest in contributing to FeatureBase! We appreciate your support in making this open-source project even better. Here are some guidelines to help you get started with contributing to FeatureBase:
+
+1. Familiarize Yourself with the Project:
+   - Visit the FeatureBase website at www.featurebase.com to understand the project's goals, capabilities, and features.
+   - Read the documentation available on the website, including the installation guide, configuration options, and data modeling concepts.
+   - Explore the codebase by cloning the repository and reviewing the source code.
+
+2. Join the Community:
+   - Visit the FeatureBase community page at https://www.featurebase.com/community to learn more about the project's community and how to get involved.
+   - Join the Discord server at https://discord.gg/FBn2vEp7Na to chat with other contributors and users, ask questions, and share your ideas.
+
+3. Set Up Your Development Environment:
+   - Ensure you have Go installed on your machine. Make sure your shell's search path includes the go/bin directory.
+   - Clone the FeatureBase repository or download it as a zip file from the repository's page.
+   - Follow the "Build FeatureBase Server from source" instructions in the README file to compile the server binary and the ingester binaries.
+
+4. Choose a Contribution Area:
+   - Identify the area you'd like to contribute to, such as bug fixes, new features, performance improvements, documentation updates, or community support.
+   - Check the issue tracker on the repository or the FeatureBase community for open issues or feature requests that align with your interests and skills. Alternatively, propose your own idea by creating a new issue.
+
+5. Create a New Branch:
+   - Before making any changes, create a new branch in the repository's Git repository. This branch will contain your contributions.
+   - Give your branch a descriptive name that reflects the nature of your contribution.
+
+6. Make Your Changes:
+   - Follow the coding style and conventions used in the existing codebase.
+   - Write clear and concise commit messages for each logical change.
+   - If you're introducing new features or modifying existing behavior, make sure to update the documentation to reflect the changes.
+
+7. Test Your Changes:
+   - Run the existing test suite to ensure that your modifications do not introduce any regressions.
+   - If applicable, write additional tests to cover the changes you made.
+   - Document any new testing procedures required for your contribution.
+
+8. Submitting Your Contribution:
+   - Push your branch to the main repository or create a fork and submit a pull request to the main repository.
+   - Provide a detailed description of your changes, including the problem you solved and the approach you took.
+   - Be responsive to any feedback or suggestions provided by the project maintainers or other contributors.
+   - Once your contribution is approved, it will be reviewed and merged into the main codebase.
+
+Please note that by contributing to FeatureBase, you agree that your contributions will be licensed under the Apache 2.0 license, which governs the project.
+
+Thank you for considering contributing to FeatureBase! Your contributions are valuable and help improve the project for everyone.

README.md

@@ -1,4 +1,10 @@
-# FeatureBase
+# FeatureBase Community
+
+FeatureBase Community is now archived and no longer maintained.
+
+* [FeatureBase Community Help](https://github.com/FeatureBaseDB/FB-community-help)
+
+
 
 ## Pilosa is now FeatureBase
 
@@ -10,6 +16,8 @@ For more information about FeatureBase, please visit [www.featurebase.com][HomeP
 
 ## Getting Started
 
+* [Learn how to install FeatureBase Community](https://github.com/FeatureBaseDB/FB-community-help/blob/main/docs/community/com-getstart/com-getstart-home.md)
+
 ### Build FeatureBase Server from source
 
 0. Install go. Ensure that your shell's search path includes the go/bin directory.
@@ -19,42 +27,20 @@ For more information about FeatureBase, please visit [www.featurebase.com][HomeP
 4. Run `featurebase server --handler.allowed-origins=http://localhost:3000` to run FeatureBase server with default settings (learn more about configuring FeatureBase at the link below). The `--handler.allowed-origins` parameter allows the standalone web UI to talk to the server; this can be omitted if the web UI is not needed.
 5. Run `curl localhost:10101/status` to verify the server is running and accessible.
 
-### Ingest Data and Query
-
-1. Run 
-```
-molecula-consumer-csv \
-    --index repository \
-    --header "language__ID_F,project_id__ID_F" \
-    --id-field project_id \
-    --batch-size 1000 \
-    --files example.csv
-```
-
-This will ingest the `example.csv` file into a FeatureBase table called `repository`. If the table does not exist, it will be automatically created. Learn more about [ingesting data into FeatureBase][Ingest]
-
-2. Query your data. 
-```
-curl localhost:10101/index/repository/query \
-     -X POST \
-     -d 'Row(example=5)'
-```
-Learn about supported [SQL][SQL], native [Pilosa Query Language (PQL)][PQL].
-
 ### Data Model
 
 Because FeatureBase is built on bitmaps, there is bit of a learning curve to grasp how your data is represented. 
-[Learn about Data Modeling][DataModel].
 
-### More Information
+* [Learn about Data Modeling](https://github.com/FeatureBaseDB/FB-community-help/blob/main/docs/concepts/concepts-home.md)
 
-[Installation][Install]
 
-[Configuration][Config]
+### Ingest Data and Query
+
+* [Learn how to ingest data from multiple data sources](https://github.com/FeatureBaseDB/FB-community-help/blob/main/docs/community/com-ingest/com-ingest-manage.md)
 
 ## Community
 
-You can email us at community@featurebase.com or learn more about contributing at [https://www.featurebase.com/community][Community].
+You can email us at community@featurebase.com and [learn more about contributing](https://github.com/FeatureBaseDB/featurebase/blob/master/OPENSOURCE.md).
 
 Chat with us: [https://discord.gg/FBn2vEp7Na][Discord]
 
@@ -73,13 +59,14 @@ A lot has changed since the days of Pilosa. This list highlights some new capabi
 
 FeatureBase is licensed under the [Apache License, Version 2.0][License]
 
-[Community]: http://www.featurebase.com/community?utm_campaign=Open%20Source&utm_source=GitHub
-[Config]: https://docs.featurebase.com/docs/community/com-config/old-config-flags/?utm_campaign=Open%20Source&utm_source=GitHub
-[DataModel]: https://docs.featurebase.com/docs/concepts/overview-data-modeling/?utm_campaign=Open%20Source&utm_source=GitHub
+[Community]: https://github.com/FeatureBaseDB/FB-community-help/tree/main
+[Install]:https://github.com/FeatureBaseDB/FB-community-help/blob/main/docs/community/com-getstart/com-getstart-home.md
+[Config]: https://github.com/FeatureBaseDB/FB-community-help/tree/main/docs/community/com-config
+[DataModel]: https://github.com/FeatureBaseDB/FB-community-help/blob/main/docs/concepts/concepts-home.md
 [Discord]: https://discord.gg/FBn2vEp7Na
 [HomePage]: http://featurebase.com?utm_campaign=Open%20Source&utm_source=GitHub
-[Ingest]: https://docs.featurebase.com/docs/community/com-ingest/old-ingesters/?utm_campaign=Open%20Source&utm_source=GitHub
-[Install]: https://docs.featurebase.com/docs/community/com-home/#install-featurebase-community?utm_campaign=Open%20Source&utm_source=GitHub
+[Ingest]: https://github.com/FeatureBaseDB/FB-community-help/blob/main/docs/community/com-ingest/com-ingest-manage.md
+ 
 [License]: http://www.apache.org/licenses/LICENSE-2.0
 [PQL]: https://docs.featurebase.com/docs/pql-guide/pql-home/?utm_campaign=Open%20Source&utm_source=GitHub
 [SQL]: https://docs.featurebase.com/docs/sql-guide/sql-guide-home/?utm_campaign=Open%20Source&utm_source=GitHub

api.go

@@ -367,8 +367,8 @@ func (api *API) DeleteIndex(ctx context.Context, indexName string) error {
 }
 
 // CreateField makes the named field in the named index with the given options.
-// This method currently only takes a single functional option, but that may be
-// changed in the future to support multiple options.
+//
+// The resulting field will always have TrackExistence set.
 func (api *API) CreateField(ctx context.Context, indexName string, fieldName string, opts ...FieldOption) (*Field, error) {
 	span, _ := tracing.StartSpanFromContext(ctx, "API.CreateField")
 	defer span.Finish()
@@ -381,6 +381,11 @@ func (api *API) CreateField(ctx context.Context, indexName string, fieldName str
 	// authN/Z info
 	requestUserID, _ := fbcontext.UserID(ctx) // requestUserID is "" if not in ctx
 
+	// newFieldOptions is also used in the path through the index creating
+	// a field from an update from DAX, so it can't assume it can always
+	// override this. But we're the call path for creating new fields, and
+	// new fields should always have TrackExistence on.
+	opts = append(opts, OptFieldTrackExistence())
 	// Apply and validate functional options.
 	fo, err := newFieldOptions(opts...)
 	if err != nil {
@@ -494,16 +499,9 @@ func importWorker(importWork chan importJob) {
 	for j := range importWork {
 		err := func() (err0 error) {
 			for viewName, viewData := range j.req.Views {
-				// The logic here corresponds to the logic in fragment.cleanViewName().
-				// Unfortunately, the logic in that method is not completely exclusive
-				// (i.e. an "other" view named with format YYYYMMDD would be handled
-				// incorrectly). One way to address this would be to change the logic
-				// overall so there weren't conflicts. For now, we just
-				// rely on the field type to inform the intended view name.
-				if viewName == "" {
-					viewName = viewStandard
-				} else if j.field.Type() == FieldTypeTime {
-					viewName = fmt.Sprintf("%s_%s", viewStandard, viewName)
+				viewName, err0 = j.field.cleanupViewName(viewName)
+				if err0 != nil {
+					return err0
 				}
 				if len(viewData) == 0 {
 					return fmt.Errorf("no data to import for view: %s", viewName)
@@ -1316,7 +1314,6 @@ type ImportOptions struct {
 	Clear          bool
 	IgnoreKeyCheck bool
 	Presorted      bool
-	fullySorted    bool // format-aware sorting, internal use only please.
 	suppressLog    bool
 
 	// test Tx atomicity if > 0
@@ -1523,7 +1520,6 @@ func (api *API) ImportWithTx(ctx context.Context, qcx *Qcx, req *ImportRequest,
 		return errors.Wrap(err, "validating api method")
 	}
 
-	api.server.logger.Debugf("ImportWithTx: %v %v %v", req.Index, req.Field, req.Shard)
 	idx, field, err := api.indexField(req.Index, req.Field, req.Shard)
 	if err != nil {
 		return errors.Wrap(err, "getting index and field")
@@ -1642,6 +1638,12 @@ func (api *API) ImportWithTx(ctx context.Context, qcx *Qcx, req *ImportRequest,
 // across many fields in a single shard. It can both set and clear
 // bits and updates caches/bitDepth as appropriate, although only the
 // bitmap parts happen truly transactionally.
+//
+// This function does not attempt to do existence tracking, because
+// it can't; there's no way to distinguish empty sets from not setting
+// bits. As a result, users of this endpoint are responsible for
+// providing corrected existence views for fields with existence
+// tracking. Our batch API does that.
 func (api *API) ImportRoaringShard(ctx context.Context, indexName string, shard uint64, req *ImportRoaringShardRequest) error {
 	index, err := api.Index(ctx, indexName)
 	if err != nil {
@@ -1672,7 +1674,7 @@ func (api *API) ImportRoaringShard(ctx context.Context, indexName string, shard
 		}
 
 		fieldType := field.Options().Type
-		if err1 = cleanupView(fieldType, &viewUpdate); err1 != nil {
+		if viewUpdate.View, err1 = field.cleanupViewName(viewUpdate.View); err1 != nil {
 			return err1
 		}
 
@@ -1764,27 +1766,6 @@ func (api *API) ImportRoaringShard(ctx context.Context, indexName string, shard
 	return nil
 }
 
-func cleanupView(fieldType string, viewUpdate *RoaringUpdate) error {
-	// TODO wouldn't hurt to have consolidated logic somewhere for validating view names.
-	switch fieldType {
-	case FieldTypeSet, FieldTypeTime:
-		if viewUpdate.View == "" {
-			viewUpdate.View = "standard"
-		}
-		// add 'standard_' if we just have a time... this is how IDK works by default
-		if fieldType == FieldTypeTime && !strings.HasPrefix(viewUpdate.View, viewStandard) {
-			viewUpdate.View = fmt.Sprintf("%s_%s", viewStandard, viewUpdate.View)
-		}
-	case FieldTypeInt, FieldTypeDecimal, FieldTypeTimestamp:
-		if viewUpdate.View == "" {
-			viewUpdate.View = "bsig_" + viewUpdate.Field
-		} else if viewUpdate.View != "bsig_"+viewUpdate.Field {
-			return NewBadRequestError(errors.Errorf("invalid view name (%s) for field %s of type %s", viewUpdate.View, viewUpdate.Field, fieldType))
-		}
-	}
-	return nil
-}
-
 // ImportValue is a wrapper around the common code in ImportValueWithTx, which
 // currently just translates req.Clear into a clear ImportOption.
 func (api *API) ImportValue(ctx context.Context, qcx *Qcx, req *ImportValueRequest, opts ...ImportOption) error {
@@ -2038,21 +2019,20 @@ func (api *API) ImportValueWithTx(ctx context.Context, qcx *Qcx, req *ImportValu
 	return nil
 }
 
-func importExistenceColumns(qcx *Qcx, index *Index, columnIDs []uint64, shard uint64) error {
+func importExistenceColumns(qcx *Qcx, index *Index, columnIDs []uint64, shard uint64) (err0 error) {
 	ef := index.existenceField()
 	if ef == nil {
 		return nil
 	}
-
-	existenceRowIDs := make([]uint64, len(columnIDs))
-	// If we don't gratuitously hand-duplicate things in field.Import,
-	// the fact that fragment.bulkImport rewrites its row and column
-	// lists can burn us if we don't make a copy before doing the
-	// existence field write.
-	columnCopy := make([]uint64, len(columnIDs))
-	copy(columnCopy, columnIDs)
-	options := ImportOptions{}
-	return ef.Import(qcx, existenceRowIDs, columnCopy, nil, shard, &options)
+	tx, finisher, err := qcx.GetTx(Txo{Write: true, Index: index, Shard: shard})
+	if err != nil {
+		return err
+	}
+	defer finisher(&err0)
+	// markExistingInView is simpler/faster than Import, but unusually, we use the
+	// standard view of the existence field, instead of the existence view of
+	// a specific field, when doing the index-wide update.
+	return ef.markExistingInView(tx, columnIDs, viewStandard, shard)
 }
 
 // ShardDistribution returns an object representing the distribution of shards
@@ -3377,6 +3357,7 @@ func (n *NopSchemaAPI) DeleteField(ctx context.Context, tname dax.TableName, fna
 
 type ClusterNode struct {
 	ID        string
+	Type      string
 	State     string
 	URI       string
 	GRPCURI   string

api_directive.go

@@ -35,6 +35,22 @@ func (api *API) ApplyDirective(ctx context.Context, d *dax.Directive) error {
 	// Handle the operations based on the directive method.
 	switch d.Method {
 	case dax.DirectiveMethodDiff:
+		// In order to prevent adding too much code specific to handling a diff
+		// directive (e.g. adding something like an `enactDirectiveDiff()`
+		// method), we are instead going to build a full Directive based on the
+		// diff, and then proceed normally as if we had received a full
+		// Directive. We do that by copying the previous Directive and then
+		// applying the diffs to the copy.
+		newD := previousDirective.Copy()
+
+		// Apply the diffs from the incoming Directive to the new, copied
+		// Directive.
+		newD.ApplyDiff(d)
+
+		// Now proceed with the new diff as if we had received it as a full diff.
+		d = newD
+
+	case dax.DirectiveMethodFull:
 		// pass: normal operation
 
 	case dax.DirectiveMethodReset:
@@ -964,7 +980,7 @@ func createField(idx *Index, fld *dax.Field) error {
 		return errors.Wrapf(err, "creating field options from field: %s", fld.Name)
 	}
 
-	if _, err := idx.CreateField(string(fld.Name), "", opts...); err != nil {
+	if _, err := idx.createNullableField(string(fld.Name), "", opts...); err != nil {
 		return errors.Wrapf(err, "creating field on index: %s", fld.Name)
 	}
 	return nil

api_directive_test.go

@@ -30,7 +30,7 @@ func TestAPI_Directive(t *testing.T) {
 		// Empty directive (and empty holder).
 		{
 			d := &dax.Directive{
-				Method:  dax.DirectiveMethodDiff,
+				Method:  dax.DirectiveMethodFull,
 				Version: 1,
 			}
 			err := api.ApplyDirective(ctx, d)
@@ -41,7 +41,7 @@ func TestAPI_Directive(t *testing.T) {
 		// Add a new table.
 		{
 			d := &dax.Directive{
-				Method: dax.DirectiveMethodDiff,
+				Method: dax.DirectiveMethodFull,
 				Tables: []*dax.QualifiedTable{
 					tbl1,
 				},
@@ -55,7 +55,7 @@ func TestAPI_Directive(t *testing.T) {
 		// Add a new table, and keep the existing table.
 		{
 			d := &dax.Directive{
-				Method: dax.DirectiveMethodDiff,
+				Method: dax.DirectiveMethodFull,
 				Tables: []*dax.QualifiedTable{
 					tbl1,
 					tbl2,
@@ -70,7 +70,7 @@ func TestAPI_Directive(t *testing.T) {
 		// Add a new table and remove one of the existing tables.
 		{
 			d := &dax.Directive{
-				Method: dax.DirectiveMethodDiff,
+				Method: dax.DirectiveMethodFull,
 				Tables: []*dax.QualifiedTable{
 					tbl2,
 					tbl3,

api_test.go

@@ -837,7 +837,7 @@ func TestAPI_IDAlloc(t *testing.T) {
 			t.Fatalf("obtaining random bytes: %v", err)
 		}
 		ids3, err := primary.ReserveIDs(key, session, 0, 2)
-		var esync pilosa.ErrIDOffsetDesync
+		var esync pilosa.IDOffsetDesyncError
 		if errors.As(err, &esync) {
 			if esync.Requested != 0 {
 				t.Errorf("incorrect requested offset in error: provided %d but got %d", 0, esync.Requested)

arrow.go

@@ -478,7 +478,7 @@ func readTableArrow(filename string, mem memory.Allocator) (arrow.Table, error)
 		return nil, err
 	}
 	defer rr.Close()
-	records := make([]arrow.Record, rr.NumRecords(), rr.NumRecords())
+	records := make([]arrow.Record, rr.NumRecords())
 	i := 0
 	for {
 		rec, err := rr.Read()

batch/batch.go

@@ -23,6 +23,7 @@ import (
 const (
 	DefaultKeyTranslateBatchSize = 100000
 	existenceFieldName           = "_exists"
+	existenceViewName            = "existence" // this should match top level featurebase viewExistence
 )
 
 // TODO if using column translation, column ids might get way out of
@@ -573,7 +574,11 @@ func (b *Batch) Add(rec Row) error {
 		case int64:
 			b.values[field.Name] = append(b.values[field.Name], val)
 		case []string:
-			if len(val) == 0 {
+			// note that a length of 0 can be valid, and represents an
+			// empty set. an empty set counts as a non-NULL value for
+			// SQL purposes -- it means the existence view bit should
+			// get set.
+			if val == nil {
 				continue
 			}
 			rowIDSets, ok := b.rowIDSets[field.Name]
@@ -608,7 +613,11 @@ func (b *Batch) Add(rec Row) error {
 			}
 			b.rowIDSets[field.Name] = append(rowIDSets, rowIDs)
 		case []uint64:
-			if len(val) == 0 {
+			// note that a length of 0 can be valid, and represents an
+			// empty set. an empty set counts as a non-NULL value for
+			// SQL purposes -- it means the existence view bit should
+			// get set.
+			if val == nil {
 				continue
 			}
 			rowIDSets, ok := b.rowIDSets[field.Name]
@@ -663,6 +672,9 @@ func (b *Batch) Add(rec Row) error {
 
 	for i, uval := range rec.Clears {
 		field := b.header[i]
+		if field.Options.Type == featurebase.FieldTypeMutex && uval != nil {
+			return errors.Errorf("individual-bit clears not allowed on mutex fields; use nil to clear a mutex")
+		}
 		if _, ok := b.clearRowIDs[i]; !ok {
 			b.clearRowIDs[i] = make(map[int]uint64)
 		}
@@ -1245,7 +1257,7 @@ func (b *Batch) doImport(frags, clearFrags fragments) error {
 			}
 
 			ferr := b.importer.ImportRoaringBitmap(ctx, b.tbl.ID, fld, shard, viewMap, false)
-			b.log.Debugf("imp-roar    field: %s, shard:%d, views:%d %v", field, shard, len(clearViewMap), time.Since(starty))
+			b.log.Debugf("imp-roar    field: %s, shard:%d, views:%d %v", field, shard, len(viewMap), time.Since(starty))
 			return errors.Wrapf(ferr, "importing data for %s", field)
 		})
 	}
@@ -1343,6 +1355,7 @@ func (b *Batch) makeFragments(frags, clearFrags fragments) (fragments, fragments
 		curShard := ^uint64(0) // impossible sentinel value for shard.
 		var curBM *roaring.Bitmap
 		var clearBM *roaring.Bitmap
+		var existCurBM *roaring.Bitmap
 		for j := range b.ids {
 			col := b.ids[j]
 			row := nilSentinel
@@ -1355,8 +1368,12 @@ func (b *Batch) makeFragments(frags, clearFrags fragments) (fragments, fragments
 
 			if col/shardWidth != curShard {
 				curShard = col / shardWidth
+				// the API treats "" as standard
 				curBM = frags.GetOrCreate(curShard, field.Name, "")
 				clearBM = clearFrags.GetOrCreate(curShard, field.Name, "")
+				if opts.ActuallyTrackingExistence() {
+					existCurBM = frags.GetOrCreate(curShard, field.Name, existenceViewName)
+				}
 			}
 			if row != nilSentinel {
 				// TODO this is super ugly, but we want to avoid setting
@@ -1366,6 +1383,9 @@ func (b *Batch) makeFragments(frags, clearFrags fragments) (fragments, fragments
 				// the NoStandardView case would be great.
 				if !(opts.Type == featurebase.FieldTypeTime && opts.NoStandardView) {
 					curBM.DirectAdd(row*shardWidth + (col % shardWidth))
+					if opts.ActuallyTrackingExistence() {
+						existCurBM.DirectAdd(col % shardWidth)
+					}
 				}
 				if opts.Type == featurebase.FieldTypeTime {
 					views, err := b.times[j].views(opts.TimeQuantum)
@@ -1386,6 +1406,16 @@ func (b *Batch) makeFragments(frags, clearFrags fragments) (fragments, fragments
 				// we want to make sure that at this point, the "set"
 				// fragments don't contain the bit that we're clearing
 				curBM.DirectRemoveN(clearRow*shardWidth + (col % shardWidth))
+				// Because this is RowIDs, not RowIDSets, there's only one
+				// bit. We should not be setting the existence bit based on
+				// this value, if we're actually clearing it. This doesn't
+				// mean we will clear an existing existence bit, though.
+				// The case where we would clear an existence bit is the
+				// case where someone specified row[mutexField].Clears = nil,
+				// which is far from here.
+				if opts.ActuallyTrackingExistence() {
+					existCurBM.DirectRemoveN(col % shardWidth)
+				}
 			}
 		}
 	}
@@ -1404,14 +1434,23 @@ func (b *Batch) makeFragments(frags, clearFrags fragments) (fragments, fragments
 		opts := field.Options
 		curShard := ^uint64(0) // impossible sentinel value for shard.
 		var curBM *roaring.Bitmap
+		var existCurBM *roaring.Bitmap
 		for j := range b.ids {
 			col, rowIDs := b.ids[j], rowIDSets[j]
-			if len(rowIDs) == 0 {
-				continue
-			}
 			if col/shardWidth != curShard {
 				curShard = col / shardWidth
 				curBM = frags.GetOrCreate(curShard, fname, "")
+				if opts.ActuallyTrackingExistence() {
+					existCurBM = frags.GetOrCreate(curShard, fname, existenceViewName)
+				}
+			}
+			if len(rowIDs) == 0 {
+				// you can validly specify an empty set, which is not the same as a null,
+				// but which still ought to set the existence bit if we're tracking that.
+				if opts.ActuallyTrackingExistence() && rowIDs != nil {
+					existCurBM.DirectAdd(col % shardWidth)
+				}
+				continue
 			}
 			// TODO this is super ugly, but we want to avoid setting
 			// bits on the standard view in the specific case when
@@ -1422,6 +1461,9 @@ func (b *Batch) makeFragments(frags, clearFrags fragments) (fragments, fragments
 				for _, row := range rowIDs {
 					curBM.DirectAdd(row*shardWidth + (col % shardWidth))
 				}
+				if opts.ActuallyTrackingExistence() {
+					existCurBM.DirectAdd(col % shardWidth)
+				}
 			}
 			if opts.Type == featurebase.FieldTypeTime {
 				views, err := b.times[j].views(opts.TimeQuantum)
@@ -1549,6 +1591,11 @@ func (b *Batch) makeSingleValFragments(frags, clearFrags fragments) (fragments,
 		shard := ids[0] / shardWidth
 		bitmap := frags.GetOrCreate(shard, field.Name, "standard")
 		clearBM := clearFrags.GetOrCreate(shard, field.Name, "standard")
+		var existBM, existClearBM *roaring.Bitmap
+		if field.Options.ActuallyTrackingExistence() {
+			existBM = frags.GetOrCreate(shard, field.Name, existenceViewName)
+			existClearBM = clearFrags.GetOrCreate(shard, field.Name, existenceViewName)
+		}
 		for i, id := range ids {
 			if i+1 < len(ids) {
 				// we only want the last value set for each id
@@ -1561,6 +1608,10 @@ func (b *Batch) makeSingleValFragments(frags, clearFrags fragments) (fragments,
 				shard = id / shardWidth
 				bitmap = frags.GetOrCreate(shard, field.Name, "standard")
 				clearBM = clearFrags.GetOrCreate(shard, field.Name, "standard")
+				if field.Options.ActuallyTrackingExistence() {
+					existBM = frags.GetOrCreate(shard, field.Name, existenceViewName)
+					existClearBM = clearFrags.GetOrCreate(shard, field.Name, existenceViewName)
+				}
 			}
 			fragmentColumn := id % shardWidth
 			clearBM.Add(fragmentColumn) // Will use this to clear columns.
@@ -1568,6 +1619,11 @@ func (b *Batch) makeSingleValFragments(frags, clearFrags fragments) (fragments,
 				// clearSentinel is used for deletion
 				// so this value should only be added if its not clearSentinel
 				bitmap.Add(row*shardWidth + fragmentColumn)
+				if field.Options.ActuallyTrackingExistence() {
+					existBM.Add(fragmentColumn)
+				}
+			} else if field.Options.ActuallyTrackingExistence() {
+				existClearBM.Add(fragmentColumn)
 			}
 		}
 	}
@@ -1596,6 +1652,11 @@ func (b *Batch) makeSingleValFragments(frags, clearFrags fragments) (fragments,
 			fragmentColumn := recID % shardWidth
 
 			clearBM.Add(fragmentColumn)
+			if field.Options.ActuallyTrackingExistence() {
+				existClearBM := clearFrags.GetOrCreate(shard, field.Name, existenceViewName)
+
+				existClearBM.Add(fragmentColumn)
+			}
 		}
 	}
 
@@ -1618,6 +1679,10 @@ func (b *Batch) makeSingleValFragments(frags, clearFrags fragments) (fragments,
 
 			fragmentColumn := recID % shardWidth
 			clearBM.Add(fragmentColumn)
+			if field.Options.ActuallyTrackingExistence() {
+				exist := frags.GetOrCreate(shard, field.Name, existenceViewName)
+				exist.Add(fragmentColumn)
+			}
 
 			if boolVal {
 				bitmap.Add(trueRowOffset + fragmentColumn)

batch/batch_test.go

@@ -103,6 +103,12 @@ func testStringSliceCombos(t *testing.T, importer featurebase.Importer, sapi fea
 		Index: idx.Name,
 		Query: "TopN(a1, n=10)",
 	})
+	if resp.Err != nil {
+		t.Fatalf("unexpected error from TopN query: %v", resp.Err)
+	}
+	if len(resp.Results) < 1 {
+		t.Fatalf("expected non-empty result set, got empty results")
+	}
 	pairsField, ok := resp.Results[0].(*featurebase.PairsField)
 	assert.True(t, ok, "wrong return type: %T", resp.Results[0])
 
@@ -508,10 +514,11 @@ func testStringSliceEmptyAndNil(t *testing.T, importer featurebase.Importer, sap
 			{
 				Name: "strslice",
 				Options: featurebase.FieldOptions{
-					Type:      featurebase.FieldTypeSet,
-					Keys:      true,
-					CacheType: featurebase.CacheTypeRanked,
-					CacheSize: 100,
+					Type:           featurebase.FieldTypeSet,
+					Keys:           true,
+					CacheType:      featurebase.CacheTypeRanked,
+					CacheSize:      100,
+					TrackExistence: true,
 				},
 			},
 		},
@@ -611,6 +618,14 @@ func testStringSliceEmptyAndNil(t *testing.T, importer featurebase.Importer, sap
 			pql: "Row(strslice='z')",
 			exp: []uint64{2},
 		},
+		{
+			pql: "Row(strslice==null)",
+			exp: []uint64{1},
+		},
+		{
+			pql: "Row(strslice!=null)",
+			exp: []uint64{0, 2, 3, 4},
+		},
 	}
 	for i, test := range tests {
 		t.Run(fmt.Sprintf("test-%d", i), func(t *testing.T) {
@@ -2045,7 +2060,7 @@ func mutexClearRegression(t *testing.T, importer featurebase.Importer, sapi feat
 	}
 
 	col := uint64(0)
-	row := uint64(1)
+	row := uint64(0)
 	for i := uint64(0); i <= 21; i++ {
 		col = (i%2+1)*featurebase.ShardWidth + i%5
 		row = i % 3
@@ -2126,7 +2141,7 @@ func mutexNilClearID(t *testing.T, importer featurebase.Importer, sapi featureba
 	}
 
 	col := uint64(0)
-	row := uint64(1)
+	row := uint64(0)
 	// populate mutex with some data
 	for i := uint64(0); i < 11; i++ {
 		col = (i%2+1)*featurebase.ShardWidth + i%5

batch/batcher.go

@@ -0,0 +1,20 @@
+package batch
+
+import (
+	"time"
+
+	"github.com/featurebasedb/featurebase/v3/dax"
+)
+
+// Batcher is an interface implemented by anything which can allocate new
+// batches.
+type Batcher interface {
+	NewBatch(cfg Config, tbl *dax.Table, fields []*dax.Field) (RecordBatch, error)
+}
+
+// Config is the configuration options passed to NewBatch for any implementation
+// of the Batcher interface.
+type Config struct {
+	Size         int
+	MaxStaleness time.Duration
+}

batch/egpool/egpool.go

@@ -58,11 +58,11 @@ func (eg *Group) err(err error) {
 	eg.errs = append(eg.errs, err)
 }
 
-type ErrPanic struct {
+type PanicError struct {
 	Value interface{}
 }
 
-func (p ErrPanic) Error() string {
+func (p PanicError) Error() string {
 	return fmt.Sprintf("panic: %v", p.Value)
 }
 
@@ -77,7 +77,7 @@ func (eg *Group) processJobs() {
 	defer func() {
 		if !finished {
 			if p := recover(); p != nil {
-				eg.err(ErrPanic{p})
+				eg.err(PanicError{p})
 			} else {
 				eg.err(ErrGoexit)
 			}

buffer/filebuffer.go

@@ -0,0 +1,110 @@
+package buffer
+
+import (
+	"bytes"
+	"io"
+	"io/ioutil"
+	"os"
+	"sync"
+)
+
+// NewFileBuffer returns a file buffer which will use an in-memory buffer, until `max` bytes have been written, at which point it will write the contents of memory to a file, and continue writing future data to the file.
+// The file will be written to `temp` directory. The buffer fulfills the io.Reader and io.Writer interface
+func NewFileBuffer(max int, temp string) *FileBuffer {
+	return &FileBuffer{max: max, tempDir: temp}
+}
+
+type FileBuffer struct {
+	max     int
+	buf     bytes.Buffer
+	file    *os.File
+	tempDir string
+	reading bool
+	files   []*os.File
+	mu      sync.Mutex
+}
+
+func (fb *FileBuffer) Write(p []byte) (n int, err error) {
+	if fb.reading {
+		panic("cannot write after read")
+	}
+	if fb.file != nil {
+		return fb.file.Write(p)
+	}
+	n, err = fb.buf.Write(p)
+	if err != nil {
+		return
+	}
+	if fb.buf.Len() > fb.max {
+		fb.file, err = ioutil.TempFile(fb.tempDir, "filebuffer-")
+		if err != nil {
+			return
+		}
+		_, err = io.Copy(fb.file, &fb.buf)
+		fb.buf.Reset()
+	}
+	return
+}
+
+func (fb *FileBuffer) Len() (int64, error) {
+	if fb.file == nil {
+		return int64(fb.buf.Len()), nil
+	}
+	fi, err := fb.file.Stat()
+	if err != nil {
+		return 0, err
+	}
+
+	return fi.Size(), nil
+}
+
+func (fb *FileBuffer) Read(p []byte) (n int, err error) {
+	if fb.file != nil {
+		if !fb.reading {
+			fb.reading = true
+			_, err = fb.file.Seek(0, 0)
+			if err != nil {
+				return
+			}
+		}
+		return fb.file.Read(p)
+	}
+	fb.reading = true
+	return fb.buf.Read(p)
+}
+
+func (fb *FileBuffer) Close() error {
+	if fb.file != nil {
+		name := fb.file.Name()
+		if err := fb.file.Close(); err != nil {
+			return err
+		}
+		for _, f := range fb.files {
+			f.Close()
+		}
+		fb.files = fb.files[:0]
+		fb.file = nil
+		return os.Remove(name)
+	}
+	return nil
+}
+
+func (fb *FileBuffer) Reset() error {
+	fb.mu.Lock()
+	defer fb.mu.Unlock()
+	fb.reading = false
+	fb.buf.Reset()
+	return fb.Close()
+}
+
+func (fb *FileBuffer) NewReader() (io.Reader, error) {
+	fb.mu.Lock()
+	defer fb.mu.Unlock()
+	fb.reading = true
+	if fb.file == nil {
+		return bytes.NewReader(fb.buf.Bytes()), nil
+	}
+	f, err := os.OpenFile(fb.file.Name(), os.O_RDONLY, 0)
+	fb.files = append(fb.files, f)
+	return f, err
+}

bufferpool/page.go

@@ -53,10 +53,10 @@ const PAGE_PREV_POINTER_OFFSET = 12 // offset 12, length 4, end 16
 const PAGE_NEXT_POINTER_OFFSET = 16 // offset 16, length 4, end 20
 const PAGE_SLOTS_START_OFFSET = 20  // offset 20
 
-// page slots
+// PAGE_SLOT_LENGTH is the size of the page slot key/value.
 //
-//	key offset int16    	//offset 0, length 2, end 2
-//	value offset int16	    //offset 2, length 2, end 4
+// key offset int16    	//offset 0, length 2, end 2
+// value offset int16	    //offset 2, length 2, end 4
 const PAGE_SLOT_LENGTH = 4
 
 // Page represents a page on disk

cache_test.go

@@ -6,7 +6,7 @@ import (
 	"reflect"
 	"testing"
 
-	"github.com/featurebasedb/featurebase/v3"
+	pilosa "github.com/featurebasedb/featurebase/v3"
 )
 
 // Ensure cache stays constrained to its configured size.
@@ -62,7 +62,7 @@ func TestCache_Rank_Dirty(t *testing.T) {
 		cache.Add(v.ID, v.Count)
 	}
 
-	var got []pair
+	var got []pair //nolint:prealloc
 	for _, p := range cache.Top() {
 		got = append(got, pair(p))
 	}

catcher.go

@@ -124,6 +124,17 @@ func (c *catcherTx) Remove(index, field, view string, shard uint64, a ...uint64)
 	return c.b.Remove(index, field, view, shard, a...)
 }
 
+func (c *catcherTx) Removed(index, field, view string, shard uint64, a ...uint64) (changed []uint64, err error) {
+
+	defer func() {
+		if r := recover(); r != nil {
+			vprint.AlwaysPrintf("see Removed() PanicOn '%v' at '%v'", r, vprint.Stack())
+			vprint.PanicOn(r)
+		}
+	}()
+	return c.b.Removed(index, field, view, shard, a...)
+}
+
 func (c *catcherTx) Contains(index, field, view string, shard uint64, key uint64) (exists bool, err error) {
 
 	defer func() {

cli/Makefile

@@ -0,0 +1,15 @@
+.PHONY: test testv test-integration testv-integration
+
+GO=go
+
+test:
+	$(GO) test ./... -short
+
+testv:
+	$(GO) test -v ./... -short
+
+test-integration:
+	$(GO) test . -count 1 -timeout 20m -run TestCLIIntegration/$(RUN)
+
+testv-integration:
+	$(GO) test -v . -count 1 -timeout 20m -run TestCLIIntegration/$(RUN)

cli/batch/inserter.go

@@ -0,0 +1,8 @@
+package batch
+
+// Inserter can be implemented by anything which can handle a SQL statement
+// representing a write operation. An example is `BULK INSERT`. The Insert()
+// method on this interface does not return any results other than an error.
+type Inserter interface {
+	Insert(sql string) error
+}

cli/batch/sql.go

@@ -0,0 +1,215 @@
+package batch
+
+import (
+	"encoding/json"
+	"fmt"
+	"strings"
+	"time"
+
+	fbbatch "github.com/featurebasedb/featurebase/v3/batch"
+	"github.com/featurebasedb/featurebase/v3/dax"
+	"github.com/featurebasedb/featurebase/v3/errors"
+	"github.com/featurebasedb/featurebase/v3/pql"
+)
+
+// Ensure type implements interface.
+var _ fbbatch.Batcher = (*sqlBatcher)(nil)
+
+type sqlBatcher struct {
+	inserter Inserter
+	fields   []*dax.Field
+}
+
+func NewSQLBatcher(i Inserter, flds []*dax.Field) *sqlBatcher {
+	return &sqlBatcher{
+		inserter: i,
+		fields:   flds,
+	}
+}
+
+func (b *sqlBatcher) NewBatch(cfg fbbatch.Config, tbl *dax.Table, flds []*dax.Field) (fbbatch.RecordBatch, error) {
+	fields := flds
+	if b.fields != nil {
+		fields = b.fields
+	}
+	return &sqlBatch{
+		table:        tbl,
+		fields:       fields,
+		size:         cfg.Size,
+		maxStaleness: cfg.MaxStaleness,
+		ids:          make([]interface{}, 0, cfg.Size),
+		rows:         make([][]interface{}, 0, cfg.Size),
+		inserter:     b.inserter,
+	}, nil
+}
+
+// Ensure type implements interface.
+var _ fbbatch.RecordBatch = (*sqlBatch)(nil)
+
+type sqlBatch struct {
+	table  *dax.Table
+	fields []*dax.Field
+	size   int
+
+	ids  []interface{}
+	rows [][]interface{}
+
+	// staleTime tracks the time the first record of the batch was inserted
+	// plus the maxStaleness, in order to raise ErrBatchNowStale if the
+	// maxStaleness has elapsed
+	staleTime    time.Time
+	maxStaleness time.Duration
+
+	// inserter handles SQL INSERT statements generated for each batch.
+	inserter Inserter
+}
+
+func (b *sqlBatch) Add(rec fbbatch.Row) error {
+	// Clear rec.Values and rec.Clears upon return.
+	defer func() {
+		for i := range rec.Values {
+			rec.Values[i] = nil
+		}
+		for k := range rec.Clears {
+			delete(rec.Clears, k)
+		}
+	}()
+
+	if len(b.ids) == cap(b.ids) {
+		return fbbatch.ErrBatchAlreadyFull
+	}
+	if len(rec.Values) != len(b.fields) {
+		return errors.Errorf("record needs to match up with batch fields, got %d fields and %d record", len(b.fields), len(rec.Values))
+	}
+
+	// Append the ID to b.ids.
+	b.ids = append(b.ids, rec.ID)
+
+	// Convert decimal fields (which come in as int64, along with the scale in
+	// field) to pql.Decimal.
+	for i, fld := range b.fields {
+		switch b.fields[i].Type {
+		case dax.BaseTypeDecimal:
+			if val, ok := rec.Values[i].(int64); ok {
+				rec.Values[i] = pql.NewDecimal(val, fld.Options.Scale)
+			}
+		case dax.BaseTypeTimestamp:
+			if val, ok := rec.Values[i].(int64); ok {
+				ts := time.Unix(val, 0)
+				rec.Values[i] = ts.Format(time.RFC3339)
+			}
+		}
+	}
+
+	// Append the record to b.rows.
+	vals := make([]interface{}, 0, len(rec.Values))
+	vals = append(vals, rec.Values...)
+	b.rows = append(b.rows, vals)
+
+	// Check for batch full or stale.
+	if len(b.ids) == cap(b.ids) {
+		return fbbatch.ErrBatchNowFull
+	}
+	if b.maxStaleness != time.Duration(0) { // set maxStaleness to 0 to disable staleness checking
+		if len(b.ids) == 1 {
+			b.staleTime = time.Now().Add(b.maxStaleness)
+		} else if time.Now().After(b.staleTime) {
+			return fbbatch.ErrBatchNowStale
+		}
+	}
+	return nil
+}
+
+func (b *sqlBatch) Import() error {
+	if len(b.rows) == 0 {
+		return nil
+	}
+
+	// Construct the BULK INSERT statement based on the table and fields.
+	sql, err := buildBulkInsert(b.table, b.fields, b.ids, b.rows)
+	if err != nil {
+		return errors.Wrap(err, "building bulk insert statement")
+	}
+
+	// Reset batch data.
+	b.reset()
+
+	// Submit the SQL statement.
+	return b.inserter.Insert(sql)
+}
+
+func (b *sqlBatch) reset() {
+	b.ids = b.ids[:0]
+	b.rows = b.rows[:0]
+}
+
+func (b *sqlBatch) Len() int {
+	return len(b.rows)
+}
+
+func (b *sqlBatch) Flush() error {
+	return nil
+}
+
+func buildBulkInsert(tbl *dax.Table, fields []*dax.Field, ids []interface{}, rows [][]interface{}) (string, error) {
+	// Validation.
+	if tbl.Name == "" {
+		return "", errors.New(errors.ErrUncoded, "table name is required")
+	} else if len(fields) == 0 {
+		return "", errors.New(errors.ErrUncoded, "at least one field is required")
+	}
+
+	var sb strings.Builder
+
+	sb.WriteString(`BULK INSERT INTO `)
+	sb.WriteString(string(tbl.Name))
+	sb.WriteString(` (_id,`)
+
+	flds := make([]string, 0, len(fields))
+	maps := make([]string, 0, len(fields))
+	for i := range fields {
+		flds = append(flds, string(fields[i].Name))
+		maps = append(maps, fmt.Sprintf("'$.col_%d' %s", i, fields[i].FullType()))
+	}
+	// Fields
+	sb.WriteString(strings.Join(flds, ","))
+
+	// MAP
+	keyType := dax.BaseTypeID
+	if tbl.StringKeys() {
+		keyType = dax.BaseTypeString
+	}
+	sb.WriteString(`) MAP ('$._id' `)
+	sb.WriteString(keyType)
+	sb.WriteString(`,`)
+	sb.WriteString(strings.Join(maps, ","))
+	sb.WriteString(`) FROM x'`)
+
+	// Row values.
+
+	// m is a map representing a single row to be marshalled and added to the
+	// bulk insert as one line in the NDJSON payload. We re-use the map for each
+	// row.
+	m := make(map[string]interface{})
+	for i := range rows {
+		// Write the ID value.
+		m[string(dax.PrimaryKeyFieldName)] = ids[i]
+		// Write the rest of the data values.
+		for col := range rows[i] {
+			m[fmt.Sprintf("col_%d", col)] = rows[i][col]
+		}
+
+		// Marshal the map to json and add to the sql statement.
+		if j, err := json.Marshal(m); err != nil {
+			return "", errors.Wrap(err, "marshalling row to json")
+		} else {
+			sb.Write(j)
+			sb.WriteString("\n")
+		}
+	}
+
+	// WITH
+	sb.WriteString(fmt.Sprintf(`' WITH BATCHSIZE %d FORMAT 'NDJSON' INPUT 'STREAM'`, len(rows)))
+
+	return sb.String(), nil
+}

cli/batch/sql_test.go

@@ -0,0 +1,47 @@
+package batch
+
+import (
+	"testing"
+
+	"github.com/featurebasedb/featurebase/v3/dax"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestBatchSQL(t *testing.T) {
+	tbl := &dax.Table{
+		Name: "foo",
+	}
+	fields := []*dax.Field{
+		{
+			Name: "name",
+			Type: dax.BaseTypeString,
+		},
+		{
+			Name: "age",
+			Type: dax.BaseTypeInt,
+		},
+	}
+	ids := []interface{}{
+		0, 1, 2,
+	}
+	rows := [][]interface{}{
+		{
+			[]interface{}{"Alice", int64(11)},
+		},
+		{
+			[]interface{}{"Bob", int64(22)},
+		},
+		{
+			[]interface{}{"Carl,Comma", int64(33)},
+		},
+	}
+
+	s, err := buildBulkInsert(tbl, fields, ids, rows)
+	assert.NoError(t, err)
+
+	exp := `BULK INSERT INTO foo (_id,name,age) MAP ('$._id' id,'$.col_0' string,'$.col_1' int) FROM x'{"_id":0,"col_0":["Alice",11]}
+{"_id":1,"col_0":["Bob",22]}
+{"_id":2,"col_0":["Carl,Comma",33]}
+' WITH BATCHSIZE 3 FORMAT 'NDJSON' INPUT 'STREAM'`
+	assert.Equal(t, exp, s)
+}

cli/cli.go

@@ -13,15 +13,16 @@ import (
 
 	"github.com/chzyer/readline"
 	featurebase "github.com/featurebasedb/featurebase/v3"
+	"github.com/featurebasedb/featurebase/v3/cli/batch"
 	"github.com/featurebasedb/featurebase/v3/cli/fbcloud"
+	"github.com/featurebasedb/featurebase/v3/errors"
 	"github.com/featurebasedb/featurebase/v3/logger"
-	"github.com/pkg/errors"
 )
 
 const (
 	defaultHost     string = "localhost"
-	promptBegin     string = "fbsql> "
-	promptMid       string = "    -> "
+	defaultClientID string = "6i2gs7mu215ab23cnvmshdoq6t" // production Cognito client ID
+	defaultRegion   string = "us-east-2"
 	terminationChar string = ";"
 	nullValue       string = "NULL"
 )
@@ -32,93 +33,210 @@ var (
 	Stderr io.Writer     = os.Stderr
 )
 
-var (
-	splash string = fmt.Sprintf(`FeatureBase CLI (%s)
+var splash string = fmt.Sprintf(`FeatureBase CLI (%s)
 Type "\q" to quit.
 `, featurebase.Version)
-)
 
-type CLICommand struct {
-	Host        string `json:"host"`
-	Port        string `json:"port"`
-	HistoryPath string `json:"history-path"`
+// Ensure type implments interfaces.
+var _ printer = (*Command)(nil)
+var _ batch.Inserter = (*Command)(nil)
 
-	// Cloud Auth
-	ClientID string `json:"client-id"`
-	Region   string `json:"region"`
-	Email    string `json:"email"`
-	Password string `json:"password"`
+type Command struct {
+	host string
+	port string
 
 	splitter   *splitter
 	buffer     *buffer
 	workingDir *workingDir
 
-	OrganizationID string `json:"org-id"`
-	Database       string `json:"db"`
+	organizationID string
+	database       string
 	databaseID     string
 	databaseName   string
 
 	Queryer Queryer `json:"-"`
 
-	Stdin  io.ReadCloser `json:"-"`
-	Stdout io.Writer     `json:"-"`
-	Stderr io.Writer     `json:"-"`
+	stdin  io.ReadCloser `json:"-"`
+	stdout io.Writer     `json:"-"`
+	stderr io.Writer     `json:"-"`
 
 	// output is where actual results are written. This might point to stdout,
 	// or to a file, based on the current configuration.
 	output       io.Writer `json:"-"`
 	writeOptions *writeOptions
 
+	Config *Config `json:"config"`
+
+	historyPath string
+
+	// Commands contains optional commands provided via one or more `-c` (or
+	// `--command`) flags. If this is non-empty, the cli will run in
+	// non-interactive mode; i.e. it will quit after the command is complete.
+	Commands []string `json:"commands"`
+
+	// Files contains optional files provided via one or more `-f` (or `--file`)
+	// flags. If this is non-empty, the cli will run in non-interactive mode;
+	// i.e. it will quit after the command is complete.
+	Files []string `json:"files"`
+
+	// variables holds the variables created with the \set meta-command.
+	variables map[string]string
+
+	// nonInteractiveMode is set to true when fbsql is running in
+	// non-ineracative mode. And example of this is when the user has provided a
+	// `-c` flag in the command line.
+	nonInteractiveMode bool
+
 	// quit gets closed when Run should stop listening for input.
 	quit chan struct{}
 }
 
-func NewCLICommand(logdest logger.Logger) *CLICommand {
-	return &CLICommand{
-		Host:        defaultHost,
-		HistoryPath: "",
+func NewCommand(logdest logger.Logger) *Command {
+	variables := make(map[string]string)
 
-		OrganizationID: "",
-		Database:       "",
+	return &Command{
+		Config: &Config{
+			Host: defaultHost,
+			Port: "",
+
+			OrganizationID: "",
+			Database:       "",
+
+			CloudAuth: CloudAuthConfig{
+				ClientID: defaultClientID,
+				Region:   defaultRegion,
+				Email:    "",
+				Password: "",
+			},
+
+			HistoryPath: "",
+
+			CSV: false,
+		},
 
-		splitter:   newSplitter(),
 		buffer:     newBuffer(),
+		splitter:   newSplitter(newReplacer(variables)),
 		workingDir: newWorkingDir(),
 
-		Stdin:  Stdin,
-		Stdout: Stdout,
-		Stderr: Stderr,
+		stdin:  Stdin,
+		stdout: Stdout,
+		stderr: Stderr,
 
 		output:       Stdout,
 		writeOptions: defaultWriteOptions(),
 
+		variables: variables,
+
 		quit: make(chan struct{}),
 	}
 }
 
-// Run is the main entry-point to the CLI. Currently it handles the interaction
-// with a user, as opposed to calling `featurebase cli` in a script.
-func (cmd *CLICommand) Run(ctx context.Context) error {
+// SetStdin sets stdin. This is useful for initial configuration in tests.
+func (cmd *Command) SetStdin(rc io.ReadCloser) {
+	cmd.stdin = rc
+}
+
+// SetStdout sets both stdout and output to the value provided. This is useful
+// for initial configuration in tests.
+func (cmd *Command) SetStdout(w io.Writer) {
+	cmd.stdout = w
+	cmd.output = w
+}
+
+// SetStderr sets stderr. This is useful for initial configuration in tests.
+func (cmd *Command) SetStderr(w io.Writer) {
+	cmd.stderr = w
+}
+
+// Run is the main entry-point to the CLI.
+func (cmd *Command) Run(ctx context.Context) error {
+	if err := cmd.run(ctx); err != nil {
+		cmd.Errorf(err.Error() + "\n")
+		return err
+	}
+	return nil
+}
+
+// run is effectively wrapped by the Run() method, but it's split out this way
+// so that run() can simply return errors, rather than worrying about how errors
+// should be printed; printing errors returned by run() is left up to the Run()
+// method.
+func (cmd *Command) run(ctx context.Context) error {
+	if err := cmd.setupConfig(); err != nil {
+		return errors.Wrap(err, "setting up config")
+	}
+
+	// Check to see if Command needs to run in non-interactive mode.
+	if len(cmd.Commands) > 0 ||
+		len(cmd.Files) > 0 ||
+		cmd.Config.KafkaConfig != "" ||
+		cmd.Config.CSV {
+		cmd.nonInteractiveMode = true
+	}
+
 	// Print the splash message.
-	cmd.Printf(splash)
-	cmd.setupHistory()
+	if !cmd.nonInteractiveMode {
+		cmd.Printf(splash)
+	}
+
 	if err := cmd.setupClient(); err != nil {
 		return errors.Wrap(err, "setting up client")
 	}
-	cmd.printConnInfo()
-	if err := cmd.connectToDatabase(cmd.Database); err != nil {
-		cmd.Errorf(errors.Wrap(err, "connecting to database").Error() + "\n")
+
+	// Print the connection info.
+	if !cmd.nonInteractiveMode {
+		cmd.printConnInfo()
 	}
 
+	if err := cmd.connectToDatabase(cmd.database); err != nil {
+		cmd.Errorf(errors.Wrap(err, "connecting to database").Error() + "\n")
+		// We intentionally do not return err here.
+	}
+
+	// Run in non-interactive mode based on flags and configuration.
+	// This includes either handling `-c` and/or `-f` flags, or handling a
+	// `--kafka-config` flag.
+	if len(cmd.Commands) > 0 || len(cmd.Files) > 0 {
+		// Run Commands.
+		for _, line := range cmd.Commands {
+			if err := cmd.handleLine(line); err != nil {
+				return errors.Wrapf(err, "handling line: %s", line)
+			}
+		}
+
+		// Run Files.
+		for _, fname := range cmd.Files {
+			if _, err := executeFile(cmd, fname); err != nil {
+				return errors.Wrapf(err, "executing file: %s", fname)
+			}
+		}
+
+		return nil
+	} else if cmd.Config.KafkaConfig != "" {
+		runner, err := cmd.newKafkaRunner(cmd.Config.KafkaConfig)
+		if err != nil {
+			return errors.Wrap(err, "getting new kafka runner")
+		}
+		if err := runner.Main.Run(); err != nil {
+			return errors.Wrap(err, "running kafka")
+		}
+		return nil
+	}
+
+	// From this point on, we should be in interactive mode.
+
+	// Set up history for saving user input.
+	cmd.setupHistory()
+
 	rl, err := readline.NewEx(&readline.Config{
-		Prompt:                 promptBegin,
-		HistoryFile:            cmd.HistoryPath,
+		Prompt:                 cmd.prompt(false),
+		HistoryFile:            cmd.historyPath,
 		HistoryLimit:           100000,
 		DisableAutoSaveHistory: true,
 
-		Stdin:  cmd.Stdin,
-		Stdout: cmd.Stdout,
-		Stderr: cmd.Stderr,
+		Stdin:  cmd.stdin,
+		Stdout: cmd.stdout,
+		Stderr: cmd.stderr,
 	})
 	if err != nil {
 		return errors.Wrap(err, "getting readline")
@@ -130,15 +248,15 @@ func (cmd *CLICommand) Run(ctx context.Context) error {
 	var inMidCommand bool
 
 	for {
-		if inMidCommand {
-			rl.SetPrompt(promptMid)
-		} else {
-			rl.SetPrompt(promptBegin)
-		}
+		rl.SetPrompt(cmd.prompt(inMidCommand))
 
 		// Read user provided input.
 		line, err := rl.Readline()
-		if err != nil {
+		if err == readline.ErrInterrupt {
+			inMidCommand = false
+			cmd.buffer.reset()
+			continue
+		} else if err != nil {
 			return errors.Wrap(err, "reading line")
 		}
 
@@ -229,56 +347,160 @@ func (cmd *CLICommand) Run(ctx context.Context) error {
 			}
 			return nil
 		default:
-			//pass
+			// pass
 		}
 	}
 }
 
+// prompt constructs the prompt that the user sees based on the currently
+// connected database and whether the user is in the middle of a sql statement.
+func (cmd *Command) prompt(mid bool) string {
+	db := "fbsql" // default prompt when a database is not set.
+	if cmd.databaseName != "" {
+		db = cmd.databaseName
+	}
+
+	if mid {
+		return strings.Repeat(" ", len(db)) + "-# "
+	}
+	return db + "=# "
+}
+
 // close is called upon quitting. It should close any remaining open file
 // handles used by the CLICommand.
-func (cmd *CLICommand) close() error {
+func (cmd *Command) close() error {
 	return cmd.closeOutput()
 }
 
-func (cmd *CLICommand) executeAndWriteQuery(qry query) error {
-	queryResponse, err := cmd.executeQuery(qry)
-	if err != nil {
-		return errors.Wrap(err, "making query")
+// setupConfig sets up private struct members based on values provided via the
+// configuration flags.
+func (cmd *Command) setupConfig() error {
+	if cmd.Config == nil {
+		return nil
 	}
 
-	if err := writeTable(queryResponse, cmd.writeOptions, cmd.output, cmd.Stdout, cmd.Stderr); err != nil {
+	cmd.host = cmd.Config.Host
+	cmd.port = cmd.Config.Port
+
+	cmd.organizationID = cmd.Config.OrganizationID
+	cmd.database = cmd.Config.Database
+
+	cmd.historyPath = cmd.Config.HistoryPath
+
+	// Apply any pset flag arguments.
+	for _, pset := range cmd.Config.PSets {
+		if err := cmd.applyPSet(pset); err != nil {
+			return errors.Wrapf(err, "applying pset: %s", pset)
+		}
+	}
+
+	// If running with the `--csv` flag, configure things to ensure the output
+	// is correct (i.e. that it's just the csv).
+	if cmd.Config.CSV {
+		cmd.writeOptions.format = formatCSV
+	}
+
+	return nil
+}
+
+// applyPSet takes a pset string of the form `arg` or `arg=val` and applies it
+// as if the user had run `\pset arg val`. The only difference is that applying
+// pset here suppresses any output to stdout.
+func (cmd *Command) applyPSet(pset string) error {
+	// We expect arg to be one of the folowing formats:
+	// arg
+	// arg=val
+	args := strings.SplitN(pset, "=", 2)
+
+	// This is kind of hacky, but until we re-think the metaCommand interface to
+	// take a printer interface somewhere (so we can pass in the nopPrinter
+	// here), we're just going to discard stdout for the duration of this apply,
+	// and then set stdout back to its previous writer after the apply.
+	hold := cmd.stdout
+	cmd.stdout = io.Discard
+	defer func() {
+		cmd.stdout = hold
+	}()
+
+	_, err := newMetaPSet(args).execute(cmd)
+	return err
+}
+
+func (cmd *Command) executeAndWriteQuery(qry query) error {
+	queryResponse, err := cmd.executeQuery(qry)
+	if err != nil {
+		if errors.Is(err, ErrOrganizationRequired) {
+			// Print an error message and return nil, effectively aborting any
+			// further writes for this query.
+			cmd.Errorf("Organization required. Use \\org to set an organization.\n")
+			return nil
+		}
+		return errors.Wrap(err, "making query")
+	}
+	if err := writeOutput(queryResponse, cmd.writeOptions, cmd.output, cmd.stdout, cmd.stderr); err != nil {
 		return errors.Wrap(err, "writing out response")
 	}
 
 	return nil
 }
 
-func (cmd *CLICommand) executeQuery(qry query) (*featurebase.WireQueryResponse, error) {
-	return cmd.Queryer.Query(cmd.OrganizationID, cmd.databaseID, qry.Reader())
+func (cmd *Command) executeQuery(qry query) (*featurebase.WireQueryResponse, error) {
+	wqr, err := cmd.Queryer.Query(cmd.organizationID, cmd.databaseID, qry.Reader())
+	if err != nil {
+		return nil, errors.Wrap(err, "executing query")
+	}
+
+	// If we're running in non-interactive mode, we need to check the error that
+	// comes back in the WireQueryResponse. If there's an error, we want to
+	// return it now (rather than just printing it later) so that we immediately
+	// stop any further execution of commands.
+	if cmd.nonInteractiveMode && wqr.Error != "" {
+		return nil, errors.Errorf(wqr.Error)
+	}
+
+	return wqr, nil
 }
 
+// printer is an interface which encapsulates the methods used to print output
+// to the various io.Writers.
+type printer interface {
+	Printf(format string, a ...any)
+	Outputf(format string, a ...any)
+	Errorf(format string, a ...any)
+}
+
+type nopPrinter struct{}
+
+func newNopPrinter() *nopPrinter {
+	return &nopPrinter{}
+}
+
+func (n *nopPrinter) Printf(format string, a ...any)  {}
+func (n *nopPrinter) Outputf(format string, a ...any) {}
+func (n *nopPrinter) Errorf(format string, a ...any)  {}
+
 // Printf is a helper method which sends the given payload to stdout.
-func (cmd *CLICommand) Printf(format string, a ...any) {
+func (cmd *Command) Printf(format string, a ...any) {
 	out := fmt.Sprintf(format, a...)
-	cmd.Stdout.Write([]byte(out))
+	cmd.stdout.Write([]byte(out))
 }
 
 // Outputf is a helper method which sends the given payload to output.
-func (cmd *CLICommand) Outputf(format string, a ...any) {
+func (cmd *Command) Outputf(format string, a ...any) {
 	out := fmt.Sprintf(format, a...)
 	cmd.output.Write([]byte(out))
 }
 
 // Errorf is a helper method which sends the given payload to stderr.
-func (cmd *CLICommand) Errorf(format string, a ...any) {
+func (cmd *Command) Errorf(format string, a ...any) {
 	out := fmt.Sprintf(format, a...)
-	cmd.Stderr.Write([]byte(out))
+	cmd.stderr.Write([]byte(out))
 }
 
-func (cmd *CLICommand) setupHistory() {
+func (cmd *Command) setupHistory() {
 	// If HistoryPath has already been configured (i.e. with a command flag),
 	// don't bother setting up the default in the home directory.
-	if cmd.HistoryPath != "" {
+	if cmd.historyPath != "" {
 		return
 	}
 
@@ -291,76 +513,87 @@ func (cmd *CLICommand) setupHistory() {
 		if err != nil {
 			cmd.Errorf("Creating directory for history: %v\n", err)
 		} else {
-			historyPath = filepath.Join(historyDir, "cli_history")
+			historyPath = filepath.Join(historyDir, "fbsql_history")
 		}
 	}
-	cmd.HistoryPath = historyPath
+	cmd.historyPath = historyPath
 }
 
 // printConnInfo displays the currently set host.
 // TODO(tlt): extend this to be the output of the /conninfo meta-command.
-func (cmd *CLICommand) printConnInfo() {
-	cmd.Printf("Host: %s\n", hostPort(cmd.Host, cmd.Port))
+func (cmd *Command) printConnInfo() {
+	cmd.Printf("Host: %s\n", hostPort(cmd.host, cmd.port))
 }
 
-func (cmd *CLICommand) connectToDatabase(dbName string) error {
-	if dbName == "" {
+func (cmd *Command) connectToDatabase(dbName string) error {
+	var p printer = cmd
+	if cmd.nonInteractiveMode {
+		p = newNopPrinter()
+	}
+
+	// Providing a blank ("") or hyphen ("-") dbName is the equivalent of
+	// disconnecting from the current database. We support the hyphen option
+	// because calling the `\c` meta-command without an argument is how you
+	// print the current connection.
+	switch dbName {
+	case "-", "":
 		cmd.databaseID = ""
 		cmd.databaseName = ""
-		cmd.Printf(cmd.connectionMessage())
+		p.Printf(cmd.connectionMessage())
 		return nil
 	}
 
 	// Look up dbID based on dbName.
-	qry := []queryPart{
-		newPartRaw("SHOW DATABASES"),
-	}
-
-	qr, err := cmd.executeQuery(qry)
+	wqr, err := cmd.executeQuery(newRawQuery("SHOW DATABASES"))
 	if err != nil {
 		return errors.Wrap(err, "executing query")
 	}
 
-	for _, db := range qr.Data {
+	for _, db := range wqr.Data {
 		// 0: _id
 		// 1: name
 		if db[1] == dbName {
 			cmd.databaseName = dbName
 			cmd.databaseID = db[0].(string)
-			cmd.Printf(cmd.connectionMessage())
+			p.Printf(cmd.connectionMessage())
 			return nil
 		}
 	}
 	return errors.Errorf("invalid database: %s", dbName)
 }
 
-func (cmd *CLICommand) orgMessage() string {
-	if cmd.OrganizationID == "" {
+func (cmd *Command) orgMessage() string {
+	if cmd.organizationID == "" {
 		return "You have not set an organization.\n"
 	}
-	return fmt.Sprintf("You have set organization \"%s\".\n", cmd.OrganizationID)
+	return fmt.Sprintf("You have set organization \"%s\".\n", cmd.organizationID)
 }
 
-func (cmd *CLICommand) connectionMessage() string {
+func (cmd *Command) connectionMessage() string {
 	if cmd.databaseName == "" {
 		return "You are not connected to a database.\n"
 	}
-	return fmt.Sprintf("You are now connected to database \"%s\" (%s) as user \"???\".\n", cmd.databaseName, cmd.databaseID)
+	return fmt.Sprintf("You are now connected to database \"%s\" (%s).\n", cmd.databaseName, cmd.databaseID)
 }
 
-func (cmd *CLICommand) setupClient() error {
+func (cmd *Command) setupClient() error {
 	// If the Queryer has already been set (in tests for example), don't bother
 	// trying to detect it.
 	if cmd.Queryer != nil {
 		return nil
 	}
 
-	if strings.TrimSpace(cmd.Host) == "" {
+	var p printer = cmd
+	if cmd.nonInteractiveMode {
+		p = newNopPrinter()
+	}
+
+	if strings.TrimSpace(cmd.host) == "" {
 		return errors.Errorf("no host provided\n")
 	}
 
-	if !strings.HasPrefix(cmd.Host, "http") {
-		cmd.Host = "http://" + cmd.Host
+	if !strings.HasPrefix(cmd.host, "http") {
+		cmd.host = "http://" + cmd.host
 	}
 
 	typ, err := cmd.detectFBType()
@@ -370,39 +603,40 @@ func (cmd *CLICommand) setupClient() error {
 
 	switch typ {
 	case featurebaseTypeOnPremClassic:
-		cmd.Printf("Detected on-prem, classic deployment.\n")
+		p.Printf("Detected on-prem, classic deployment.\n")
 		cmd.Queryer = &standardQueryer{
-			Host: cmd.Host,
-			Port: cmd.Port,
+			Host: cmd.host,
+			Port: cmd.port,
 		}
 	case featurebaseTypeOnPremServerless:
-		cmd.Printf("Detected on-prem, serverless deployment.\n")
+		p.Printf("Detected on-prem, serverless deployment.\n")
 		cmd.Queryer = &serverlessQueryer{
-			Host: cmd.Host,
-			Port: cmd.Port,
+			Host: cmd.host,
+			Port: cmd.port,
 		}
 	case featurebaseTypeCloud:
-		cmd.Printf("Detected cloud deployment.\n")
-		cmd.Queryer = &fbcloud.Queryer{
-			Host: hostPort(cmd.Host, cmd.Port),
+		p.Printf("Detected cloud deployment.\n")
 
-			ClientID: cmd.ClientID,
-			Region:   cmd.Region,
-			Email:    cmd.Email,
-			Password: cmd.Password,
+		cmd.Queryer = &fbcloud.Queryer{
+			Host: hostPort(cmd.host, cmd.port),
+
+			ClientID: cmd.Config.CloudAuth.ClientID,
+			Region:   cmd.Config.CloudAuth.Region,
+			Email:    cmd.Config.CloudAuth.Email,
+			Password: cmd.Config.CloudAuth.Password,
 		}
 	case featurebaseTypeUnknown:
-		cmd.Printf("Could not detect deployment\n")
+		p.Printf("Could not detect deployment\n")
 		// cmd.Queryer = &nopQueryer{}
 		// Instead of using a no-op queryer when the type can't be detected, we
 		// default to using a cloud queryer.
 		cmd.Queryer = &fbcloud.Queryer{
-			Host: hostPort(cmd.Host, cmd.Port),
+			Host: hostPort(cmd.host, cmd.port),
 
-			ClientID: cmd.ClientID,
-			Region:   cmd.Region,
-			Email:    cmd.Email,
-			Password: cmd.Password,
+			ClientID: cmd.Config.CloudAuth.ClientID,
+			Region:   cmd.Config.CloudAuth.Region,
+			Email:    cmd.Config.CloudAuth.Email,
+			Password: cmd.Config.CloudAuth.Password,
 		}
 	default:
 		return errors.Errorf("unknown type: %s", typ)
@@ -428,7 +662,7 @@ func hostPort(host, port string) string {
 
 // detectFBType determines if we're talking to standalone FeatureBase
 // or FeatureBase Cloud
-func (cmd *CLICommand) detectFBType() (featurebaseType, error) {
+func (cmd *Command) detectFBType() (featurebaseType, error) {
 	type trial struct {
 		port   string
 		health string
@@ -440,23 +674,23 @@ func (cmd *CLICommand) detectFBType() (featurebaseType, error) {
 	trials := []trial{}
 
 	var clientTimeout time.Duration
-	if cmd.Port != "" {
+	if cmd.port != "" {
 		clientTimeout = 100 * time.Millisecond
 		trials = append(trials,
 			// on-prem, serverless
 			trial{
-				port:   cmd.Port,
+				port:   cmd.port,
 				health: "/queryer/health",
 				typ:    featurebaseTypeOnPremServerless,
 			},
 			// on-prem, classic
 			trial{
-				port:   cmd.Port,
+				port:   cmd.port,
 				health: "/status",
 				typ:    featurebaseTypeOnPremClassic,
 			},
 		)
-	} else if strings.HasPrefix(cmd.Host, "https") {
+	} else if strings.HasPrefix(cmd.host, "https") {
 		// https suggesting we might be connecting to a cloud host
 		clientTimeout = 1 * time.Second
 		trials = append(trials,
@@ -490,11 +724,11 @@ func (cmd *CLICommand) detectFBType() (featurebaseType, error) {
 		Timeout: clientTimeout,
 	}
 	for _, trial := range trials {
-		url := hostPort(cmd.Host, trial.port) + trial.health
+		url := hostPort(cmd.host, trial.port) + trial.health
 		if resp, err := client.Get(url); err != nil {
 			continue
 		} else if resp.StatusCode/100 == 2 {
-			cmd.Port = trial.port
+			cmd.port = trial.port
 			return trial.typ, nil
 		}
 	}
@@ -502,7 +736,7 @@ func (cmd *CLICommand) detectFBType() (featurebaseType, error) {
 	return featurebaseTypeUnknown, nil
 }
 
-func (cmd *CLICommand) closeOutput() error {
+func (cmd *Command) closeOutput() error {
 	if cmd.output == nil {
 		return nil
 	}
@@ -513,3 +747,70 @@ func (cmd *CLICommand) closeOutput() error {
 
 	return nil
 }
+
+func (cmd *Command) handleLine(line string) error {
+	// For single-line command handling, we handle either a meta-command, or
+	// query parts, but not both. The logic is that any line which begins with
+	// "\" will be handled as a meta-command, otherwise it will be handled as a
+	// query.
+	if len(line) == 0 {
+		return nil
+	} else if line[0] == byte('\\') {
+		return cmd.handleLineAsMetaCommand(line)
+	} else {
+		return cmd.handleLineAsQueryParts(line)
+	}
+}
+
+func (cmd *Command) handleLineAsMetaCommand(line string) error {
+	_, mcs, err := cmd.splitter.split(line)
+	if err != nil {
+		return errors.Wrapf(err, "splitting line")
+	}
+
+	for i := range mcs {
+		_, err := mcs[i].execute(cmd)
+		if err != nil {
+			return errors.Wrap(err, "executing meta command")
+		}
+	}
+
+	return nil
+}
+
+func (cmd *Command) handleLineAsQueryParts(line string) error {
+	qps, mcs, err := cmd.splitter.split(line)
+	if err != nil {
+		return errors.Wrapf(err, "splitting line")
+	} else if len(mcs) > 0 {
+		return errors.Errorf("--command does not support meta-commands")
+	}
+
+	// Add a termintor part to the end of []queryPart. We do this because the
+	// command is coming in from the --command flag, it may not end with a
+	// semi-colon, but we still want to execute it.
+	if len(qps) > 0 {
+		if _, ok := qps[len(qps)-1].(*partTerminator); !ok {
+			qps = append(qps, newPartTerminator())
+		}
+	}
+
+	for i := range qps {
+		if qry, err := cmd.buffer.addPart(qps[i]); err != nil {
+			return errors.Wrap(err, "adding part to buffer")
+		} else if qry != nil {
+			if err := cmd.executeAndWriteQuery(qry); err != nil {
+				return errors.Wrap(err, "executing query")
+			}
+		}
+	}
+	return nil
+}
+
+func (cmd *Command) Insert(sql string) error {
+	wqr, err := cmd.executeQuery(newRawQuery(sql))
+	if wqr.Error != "" {
+		return errors.Errorf(wqr.Error)
+	}
+	return err
+}

cli/cli_integration_test.go

@@ -0,0 +1,257 @@
+package cli_test
+
+import (
+	"bufio"
+	"context"
+	"fmt"
+	"os"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/featurebasedb/featurebase/v3/cli"
+	"github.com/featurebasedb/featurebase/v3/dax/server/test"
+	"github.com/featurebasedb/featurebase/v3/logger"
+	"github.com/stretchr/testify/require"
+)
+
+func TestCLIIntegration(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test")
+	}
+
+	ctx := context.Background()
+
+	t.Run("Stubbed Framework", func(t *testing.T) {
+		mc := test.MustRunManagedCommand(t)
+		defer mc.Close()
+
+		addr := mc.Address()
+
+		capture := newCapture(t)
+
+		comparer := newComparer(t)
+		comparer.run()
+
+		fbsql := cli.NewCommand(logger.StderrLogger)
+		fbsql.SetStdin(capture)
+		fbsql.SetStdout(comparer)
+		fbsql.SetStderr(comparer)
+
+		fbsql.Config = &cli.Config{
+			Host: addr.Host(),
+			Port: fmt.Sprintf("%d", addr.Port()),
+		}
+
+		// Run fbsql in a goroutine so we can continue to send it commands
+		// below.
+		didQuit := make(chan struct{})
+		go func() {
+			require.NoError(t, fbsql.Run(ctx))
+			close(didQuit)
+		}()
+
+		// testFiles reference files located in the cli/testdata directory. All
+		// tests should be placed there; other than adding another test file to
+		// this list, you probably shouldn't be editing this file unless you are
+		// trying to modify the way the test framework itself works.
+		testFiles := []string{
+			"setup",
+			"database",
+			"table",
+			// the tests below may be dependent on the previous tests, which do
+			// setup and some shared database and table creation.
+			"query_buffer",
+			// meta commands
+			"meta_bang",
+			"meta_cd",
+			"meta_echo",
+			"meta_describe",
+			"meta_file",
+			"meta_pset_border",
+			"meta_pset_expanded",
+			"meta_pset_format_csv",
+			"meta_pset_tuples_only",
+			"meta_include",
+			"meta_output",
+			"meta_set",
+			"meta_timing",
+			"meta_write",
+		}
+
+		for _, testFile := range testFiles {
+			t.Run(testFile, func(t *testing.T) {
+				f, err := os.Open("testdata/" + testFile)
+				require.NoError(t, err)
+
+				scanner := bufio.NewScanner(f)
+				var lineNo int
+				for scanner.Scan() {
+					line := scanner.Text()
+					lineNo++
+
+					// Empty lines and comments (//) are ignored.
+					if line == "" {
+						continue
+					} else if strings.HasPrefix(line, "//") {
+						continue
+					}
+
+					parts := strings.SplitN(line, ":", 2)
+
+					switch parts[0] {
+					case "SEND":
+						v := ""
+						if len(parts) == 2 {
+							v = parts[1]
+						}
+						capture.sendLine(v)
+					case "EXPECT":
+						v := ""
+						if len(parts) == 2 {
+							v = parts[1]
+						}
+						comparer.expectLine(v, testFile, lineNo)
+					case "EXPECTCOMP":
+						if len(parts) == 2 {
+							comps := strings.SplitN(parts[1], ":", 2)
+							v := ""
+							if len(comps) == 2 {
+								v = comps[1]
+							}
+							comparer.expectLineComp(comparator(comps[0]), v, testFile, lineNo)
+						} else {
+							t.Errorf("unexpected line: %s[%d]:%s", testFile, lineNo, line)
+						}
+					default:
+						t.Errorf("unexpected line: %s[%d]:%s", testFile, lineNo, line)
+					}
+				}
+				require.NoError(t, scanner.Err())
+			})
+		}
+
+		// End with quit to ensure that fbsql closes without error.
+		capture.sendLine(`\q`)
+
+		// Ensure fbsql quits cleanly.
+		select {
+		case <-didQuit:
+		case <-time.After(time.Second):
+			t.Fatalf("expected fbsql to quit")
+		}
+	})
+}
+
+// compare is used to compare fbsql output written to its Stdout with expected
+// lines.
+type comparer struct {
+	t       *testing.T
+	out     chan byte
+	outline chan []byte
+	exp     chan []byte
+}
+
+func newComparer(t *testing.T) *comparer {
+	return &comparer{
+		t:       t,
+		out:     make(chan byte, 1024),
+		outline: make(chan []byte, 128),
+		exp:     make(chan []byte, 1024),
+	}
+}
+
+func (c *comparer) run() {
+	// Read bytes off output, and for every line (designated by a line feed "\n"),
+	// push the line onto the outline channel.
+	go func() {
+		var line []byte
+		for {
+			b := <-c.out
+			if b == byte('\n') {
+				c.outline <- line
+				line = []byte{}
+				continue
+			}
+			line = append(line, b)
+		}
+	}()
+}
+
+type comparator string
+
+const (
+	compEquals     = "Equals"
+	compHasPrefix  = "HasPrefix"
+	compWithFormat = "WithFormat"
+)
+
+// expectLine is a convenience method which calls expectLineComp with the compEq
+// comparator and the given line.
+func (c *comparer) expectLine(line string, fileName string, lineNo int) {
+	c.expectLineComp(compEquals, line, fileName, lineNo)
+}
+
+// expectLineComp reads the next line from the outline channel and compares it
+// with the given `line`. A comparator can be provided to inform how the lines
+// should be compared (for example, the compHasPrefix comparator will just
+// compare the beginning part of the outline).
+func (c *comparer) expectLineComp(comp comparator, line string, fileName string, lineNo int) {
+	var outline []byte
+	select {
+	case outline = <-c.outline:
+	case <-time.After(10 * time.Second):
+		// TODO(tlt): this is 10 seconds to account for the fb_views creation on
+		// a local mac. This should really be something like 2 seconds. Put this
+		// back to 2 once fb_views issue is addressed.
+		c.t.Fatalf("expected output line %s[%d]: >%s<", fileName, lineNo, line)
+	}
+
+	// msg is included in any require which fails.
+	msg := []interface{}{"exp: %s[%d], got: >%s<", fileName, lineNo, outline}
+
+	switch comp {
+	case compEquals:
+		require.Equal(c.t, []byte(line), outline, msg...)
+	case compHasPrefix:
+		require.True(c.t, strings.HasPrefix(string(outline), line), msg...)
+	case compWithFormat:
+		require.True(c.t, compareByteSlices(outline, []byte(line)), msg...)
+	default:
+		c.t.Fatalf("invalid comparator: %s", comp)
+	}
+}
+
+func (c *comparer) Write(b []byte) (n int, err error) {
+	for i := range b {
+		c.out <- b[i]
+	}
+	return len(b), err
+}
+
+// compareByteSlices compares a byte slice s with another byte slice format and
+// returns true if they are the same. It will accept underscore as a
+// single-character wildcard anywhere in slice format.
+func compareByteSlices(s, format []byte) bool {
+	// Replace some helpers in format before comparing.
+	f := string(format)
+	f = strings.ReplaceAll(f, `{uuid}`, `________-____-____-____-____________`)
+	f = strings.ReplaceAll(f, `{timestamp}`, `____-__-__T__:__:__Z`)
+	format = []byte(f)
+
+	if len(s) != len(format) {
+		return false
+	}
+
+	for i := range s {
+		if format[i] == '_' {
+			continue
+		}
+		if s[i] != format[i] {
+			// log.Printf("DEBUG: characters differ: (%d): '%v' != '%v'", i, s[i], format[i])
+			return false
+		}
+	}
+
+	return true
+}

cli/cli_test.go

@@ -21,9 +21,9 @@ func TestCLI(t *testing.T) {
 
 		capture := newCapture(t)
 
-		cli := cli.NewCLICommand(logger.StderrLogger)
-		cli.Stdin = capture
-		cli.Stdout = capture
+		cli := cli.NewCommand(logger.StderrLogger)
+		cli.SetStdin(capture)
+		cli.SetStdout(capture)
 		cli.Queryer = capture
 
 		go func() {
@@ -74,7 +74,7 @@ var _ cli.Queryer = (*capture)(nil)
 
 // capture implements the various CLI interfaces in order to capture test input
 // and submit it as though that input were being read from the command line. It
-// also captures calls made to the Queryer.Query method and ensures the sql the
+// also captures calls made to the Queryer.Query method and ensures the sql they
 // contain is expected.
 type capture struct {
 	t *testing.T

cli/config.go

@@ -0,0 +1,31 @@
+package cli
+
+// Config represents the configuration for the command.
+type Config struct {
+	Host string `json:"host"`
+	Port string `json:"port"`
+
+	OrganizationID string `json:"org-id"`
+	Database       string `json:"db"`
+
+	// CloudAuth
+	CloudAuth CloudAuthConfig `json:"cloud-auth"`
+
+	// Kafka
+	KafkaConfig string `json:"kafka-config"`
+
+	HistoryPath string `json:"history-path"`
+
+	// CSV (Comma-Separated Values) table output mode.
+	CSV bool `json:"csv"`
+
+	// PSet takes one or more pset arguments of the form: `--pset=VAR[=ARG]`.
+	PSets []string `json:"pset"`
+}
+
+type CloudAuthConfig struct {
+	ClientID string `json:"client-id"`
+	Region   string `json:"region"`
+	Email    string `json:"email"`
+	Password string `json:"password"`
+}

cli/errors.go

@@ -0,0 +1,16 @@
+package cli
+
+import (
+	"github.com/featurebasedb/featurebase/v3/errors"
+)
+
+const (
+	ErrOrganizationRequired errors.Code = "OrganizationRequired"
+)
+
+func NewErrOrganizationRequired() error {
+	return errors.New(
+		ErrOrganizationRequired,
+		"organization required",
+	)
+}

cli/kafka.go

@@ -0,0 +1,70 @@
+package cli
+
+import (
+	"fmt"
+
+	"github.com/featurebasedb/featurebase/v3/cli/batch"
+	"github.com/featurebasedb/featurebase/v3/cli/kafka"
+	"github.com/featurebasedb/featurebase/v3/errors"
+	"github.com/spf13/viper"
+)
+
+func (cmd *Command) newKafkaRunner(cfgFile string) (*kafka.Runner, error) {
+	// Read the kafka config file.
+	v := viper.New()
+	v.SetConfigFile(cfgFile)
+	v.SetConfigType("toml")
+	err := v.ReadInConfig()
+	if err != nil {
+		return nil, fmt.Errorf("error reading configuration file '%s': %v", cfgFile, err)
+	}
+
+	cfg := kafka.Config{}
+	if err := v.Unmarshal(&cfg); err != nil {
+		return nil, errors.Wrap(err, "unmarshalling config")
+	}
+
+	if err := kafka.ValidateConfig(cfg); err != nil {
+		return nil, errors.Wrap(err, "validating config")
+	}
+
+	// Create a new config with defaults.
+
+	// Look up fields based on table provided in the config.
+	wqr, err := cmd.executeQuery(newRawQuery("SHOW COLUMNS FROM " + cfg.Table))
+	if err != nil {
+		return nil, errors.Wrap(err, "executing query")
+	}
+
+	scr, err := wqr.ShowColumnsResponse()
+	if err != nil {
+		return nil, errors.Wrap(err, "getting show columns from wire query response")
+	}
+
+	// If no fields were provided in the config, use the fields defined on the
+	// table and assume a 1-to-1 mapping of source to destination.
+	if len(cfg.Fields) == 0 {
+		cfg.Fields = kafka.FieldsToConfig(scr.Fields)
+	} else {
+		cfg.Fields, err = kafka.CheckFieldCompatibility(cfg.Fields, scr)
+		if err != nil {
+			return nil, errors.Wrap(err, "validating config fields")
+		}
+	}
+
+	idkCfg, err := kafka.ConvertConfig(cfg)
+	if err != nil {
+		return nil, errors.Wrap(err, "cleaning config")
+	}
+
+	flds, err := kafka.ConfigToFields(cfg)
+	if err != nil {
+		return nil, errors.Wrap(err, "getting fields from config")
+	}
+
+	return kafka.NewRunner(
+		idkCfg,
+		batch.NewSQLBatcher(cmd, flds),
+		cmd.stderr,
+	), nil
+}

cli/kafka/config.go

@@ -0,0 +1,251 @@
+package kafka
+
+import (
+	"fmt"
+	"time"
+
+	featurebase "github.com/featurebasedb/featurebase/v3"
+	"github.com/featurebasedb/featurebase/v3/dax"
+	"github.com/featurebasedb/featurebase/v3/idk"
+	"github.com/pkg/errors"
+)
+
+// Config is the user-facing configuration for kafka support in the CLI. This is
+// unmarshalled from the the toml config file supplied by the user.
+type Config struct {
+	Hosts  []string `mapstructure:"hosts" help:"Kafka hosts."`
+	Group  string   `mapstructure:"group" help:"Kafka group."`
+	Topics []string `mapstructure:"topics" help:"Kafka topics to read from."`
+
+	BatchSize         int           `mapstructure:"batch-size" help:"Batch size."`
+	BatchMaxStaleness time.Duration `mapstructure:"batch-max-staleness" help:"Maximum length of time that the oldest record in a batch can exist before flushing the batch. Note that this can potentially stack with timeouts waiting for the source."`
+	Timeout           time.Duration `mapstructure:"timeout" help:"Time to wait for more records from Kafka before flushing a batch. 0 to disable."`
+
+	Table  string  `mapstructure:"table" help:"Destination table name."`
+	Fields []Field `mapstructure:"fields"`
+}
+
+// Field is a user-facing configuration field.
+type Field struct {
+	Name       string   `mapstructure:"name"`
+	SourceType string   `mapstructure:"source-type"`
+	SourcePath []string `mapstructure:"source-path"`
+	PrimaryKey bool     `mapstructure:"primary-key"`
+}
+
+// ConfigForIDK represents Config converted to values suitable for IDK. In
+// particular, the idk.RawField is used in parsing the schema in IDK.
+type ConfigForIDK struct {
+	Hosts  []string
+	Group  string
+	Topics []string
+
+	BatchSize         int
+	BatchMaxStaleness time.Duration
+	Timeout           time.Duration
+
+	Table   string
+	IDField string
+	Fields  []idk.RawField
+}
+
+// ValidateConfig validates the config is usable.
+func ValidateConfig(c Config) error {
+	if c.Table == "" {
+		return errors.Errorf("table is required")
+	} else if len(c.Topics) == 0 {
+		return errors.Errorf("at least one topic is required")
+	} else if len(c.Fields) > 0 {
+		// We only need to do these checks if any fields are specified at all.
+		// If no fields are specified, that's ok because then we default to
+		// using fields based off the existing table.
+		if len(c.Fields) < 2 {
+			return errors.Errorf("at least two fields are required (one should be a primary key)")
+		} else {
+			var found int
+			for i := range c.Fields {
+				if c.Fields[i].PrimaryKey {
+					found++
+				}
+				if c.Fields[i].Name == "" {
+					return errors.Errorf("a name attribute (which isn't equal to \"\") should exist for all fields")
+				}
+			}
+			if found != 1 {
+				return errors.Errorf("exactly one primary key field is required")
+			}
+		}
+	}
+	return nil
+}
+
+// ConvertConfig converts a Config to one that suitable for IDK.
+func ConvertConfig(c Config) (ConfigForIDK, error) {
+	// Set a default kafka host in case one isn't provided.
+	hosts := []string{"localhost:9092"}
+	if len(c.Hosts) > 0 {
+		hosts = c.Hosts
+	}
+
+	// Copy all the shared members from Config to ConfigForIDK.
+	out := ConfigForIDK{
+		Hosts:             hosts,
+		Group:             c.Group,
+		Topics:            c.Topics,
+		BatchSize:         c.BatchSize,
+		BatchMaxStaleness: c.BatchMaxStaleness,
+		Timeout:           c.Timeout,
+		Table:             c.Table,
+	}
+
+	if len(c.Fields) == 0 {
+		return out, errors.New("fields cannot be empty")
+	}
+
+	// rawFields wil be the same as c.Fields, but possibly enhanced.
+	rawFields := make([]idk.RawField, 0, len(c.Fields))
+
+	var foundPK bool
+	for _, fld := range c.Fields {
+		if fld.PrimaryKey {
+			out.IDField = fld.Name
+			foundPK = true
+		}
+
+		typ, quals, err := dax.SplitFieldType(fld.SourceType)
+		if err != nil {
+			return out, errors.Wrap(err, "getting base type")
+		}
+
+		rawFld := idk.RawField{
+			Name: fld.Name,
+			Type: string(typ),
+			Path: fld.SourcePath,
+		}
+		// If a SourcePath wasn't provided, default to using the field name.
+		if len(rawFld.Path) == 0 {
+			rawFld.Path = []string{fld.Name}
+		}
+
+		switch typ {
+		case dax.BaseTypeInt:
+			// We don't have to handle min/max because we don't create the table.
+		case dax.BaseTypeDecimal:
+			if len(quals) != 1 {
+				return out, errors.Errorf("expected decimal scale")
+			}
+			rawFld.Config = []byte(fmt.Sprintf(`{"scale":%d}`, quals[0]))
+		case dax.BaseTypeID:
+			rawFld.Config = []byte("{\"mutex\":true}")
+		case dax.BaseTypeIDSet:
+			rawFld.Type = "ids"
+		case dax.BaseTypeString:
+			rawFld.Config = []byte("{\"mutex\":true}")
+		case dax.BaseTypeStringSet:
+			rawFld.Type = "strings"
+		case dax.BaseTypeTimestamp:
+			// No timestamp options are handled for now.
+		}
+
+		rawFields = append(rawFields, rawFld)
+	}
+	if !foundPK {
+		return out, errors.New("primary-key not found in fields")
+	}
+
+	out.Fields = rawFields
+
+	return out, nil
+}
+
+// ConfigToFields returns a list of *dax.Field based on the IDField and Fields
+// in the Config.
+func ConfigToFields(c Config) ([]*dax.Field, error) {
+	// We don't know if a primary key will be found, so we can't set the
+	// capacity to `len(c.Fields)-1`.
+	out := make([]*dax.Field, 0, len(c.Fields))
+
+	for _, fld := range c.Fields {
+		if fld.PrimaryKey {
+			continue
+		}
+		typ, quals, err := dax.SplitFieldType(fld.SourceType)
+		if err != nil {
+			return nil, errors.Wrap(err, "splitting field type")
+		}
+		dfld := &dax.Field{
+			Name: dax.FieldName(fld.Name),
+			Type: typ,
+		}
+		switch typ {
+		case dax.BaseTypeDecimal:
+			if len(quals) != 1 {
+				return nil, errors.Errorf("expected decimal scale")
+			}
+			scale, ok := quals[0].(int64)
+			if !ok {
+				return nil, errors.Errorf("invalid decimal scale: %v", quals[0])
+			}
+			dfld.Options.Scale = scale
+		}
+		out = append(out, dfld)
+	}
+
+	return out, nil
+}
+
+// FieldsToConfig returns a Config.Fields based on a list of *dax.Field.
+func FieldsToConfig(flds []*dax.Field) []Field {
+	out := make([]Field, 0, len(flds))
+	for _, fld := range flds {
+		out = append(out, Field{
+			Name:       string(fld.Name),
+			SourceType: fld.FullType(),
+			PrimaryKey: fld.IsPrimaryKey(),
+		})
+	}
+	return out
+}
+
+// CheckFieldCompatibility ensures that the fields provided in the kafka config
+// are compatible with the fields in the existing table. It returns a copy of
+// the kafka config fields with empty values defaulted to the table field
+// configuration.
+func CheckFieldCompatibility(cflds []Field, scr *featurebase.ShowColumnsResponse) ([]Field, error) {
+	out := make([]Field, len(cflds))
+	for i, cfld := range cflds {
+		out[i] = cfld
+		cfldName := dax.FieldName(cfld.Name)
+
+		// Primary key field.
+		if cfld.PrimaryKey {
+			f := scr.Field(dax.PrimaryKeyFieldName)
+			if f == nil {
+				return nil, dax.NewErrFieldDoesNotExist(dax.PrimaryKeyFieldName) // It should be impossible to hit this.
+			}
+			if out[i].SourceType == "" {
+				if f.StringKeys() {
+					out[i].SourceType = dax.BaseTypeString
+				} else {
+					out[i].SourceType = dax.BaseTypeID
+				}
+			}
+			continue
+		}
+
+		// Non primary key fields.
+		if cfldName == dax.PrimaryKeyFieldName {
+			return nil, errors.Errorf("field named '%s' must be a primary key", dax.PrimaryKeyFieldName)
+		}
+
+		f := scr.Field(cfldName)
+		if f == nil {
+			return nil, dax.NewErrFieldDoesNotExist(cfldName)
+		}
+
+		if out[i].SourceType == "" {
+			out[i].SourceType = f.FullType()
+		}
+	}
+	return out, nil
+}

cli/kafka/runner.go

@@ -0,0 +1,67 @@
+package kafka
+
+import (
+	"io"
+	"time"
+
+	fbbatch "github.com/featurebasedb/featurebase/v3/batch"
+	"github.com/featurebasedb/featurebase/v3/errors"
+	"github.com/featurebasedb/featurebase/v3/idk"
+	"github.com/featurebasedb/featurebase/v3/idk/kafka_static"
+	"github.com/featurebasedb/featurebase/v3/logger"
+)
+
+// Runner is a CLI-specific kafka consumer. It's similar to
+// idk.kafka_static.Main in that it embeds idk.Main and contains additional
+// functionality specific to its use case.
+type Runner struct {
+	idk.Main   `flag:"!embed"`
+	KafkaHosts []string       `help:"Comma separated list of host:port pairs for Kafka."`
+	Group      string         `help:"Kafka group."`
+	Topics     []string       `help:"Kafka topics to read from."`
+	Timeout    time.Duration  `help:"Time to wait for more records from Kafka before flushing a batch. 0 to disable."`
+	Header     []idk.RawField `help:"Header configuration."`
+}
+
+func NewRunner(cfg ConfigForIDK, batcher fbbatch.Batcher, logWriter io.Writer) *Runner {
+	idkMain := idk.NewMain()
+	idkMain.IDField = cfg.IDField
+	idkMain.Index = cfg.Table
+	idkMain.Batcher = batcher
+	idkMain.BatchSize = cfg.BatchSize
+	idkMain.BatchMaxStaleness = cfg.BatchMaxStaleness
+	idkMain.SetBasic()
+	idkMain.SetLog(logger.NewStandardLogger(logWriter))
+
+	kr := &Runner{
+		Main:       *idkMain,
+		KafkaHosts: cfg.Hosts,
+		Group:      cfg.Group,
+		Topics:     cfg.Topics,
+		Header:     cfg.Fields,
+		Timeout:    cfg.Timeout,
+	}
+	kr.OffsetMode = true
+	kr.Main.Namespace = "cli_kafka_runner"
+	kr.Main.Pprof = "" // don't initialize pprof until we actually use it in tests
+	kr.NewSource = func() (idk.Source, error) {
+		source := kafka_static.NewSource()
+		source.Hosts = kr.KafkaHosts
+		source.Group = kr.Group
+		source.Topics = kr.Topics
+		source.Log = kr.Main.Log()
+		// source.TLS = m.KafkaTLS
+		source.Timeout = kr.Timeout
+		// source.SkipOld = m.SkipOld
+		source.HeaderFields = kr.Header
+		// source.S3Region = m.S3Region
+		// source.AllowMissingFields = m.AllowMissingFields
+
+		err := source.Open()
+		if err != nil {
+			return nil, errors.Wrap(err, "opening source")
+		}
+		return source, nil
+	}
+	return kr
+}

cli/meta.go

@@ -2,11 +2,13 @@ package cli
 
 import (
 	"bufio"
+	"fmt"
 	"io"
 	"os"
 	"os/exec"
 	"os/signal"
 	"path/filepath"
+	"sort"
 	"strconv"
 	"strings"
 	"syscall"
@@ -16,10 +18,11 @@ import (
 	"github.com/featurebasedb/featurebase/v3/errors"
 )
 
-// action is used to indicate how CLICommand should respond after execution a
-// given metaCommand. For example, an action of type "reset" tells CLICommand
-// that the buffer has been reset and it needs to change its user prompt.
-type action string
+// responseAction is used to indicate how CLICommand should respond after
+// executing a given metaCommand. For example, a responseAction of type "reset"
+// tells CLICommand that the buffer has been reset and it needs to change its
+// user prompt.
+type responseAction string
 
 const (
 	actionNone  = ""
@@ -29,7 +32,7 @@ const (
 
 // metaCommand is the interface for any type responding to a "\" meta-command.
 type metaCommand interface {
-	execute(cmd *CLICommand) (action, error)
+	execute(cmd *Command) (responseAction, error)
 }
 
 // Ensure type implements interface.
@@ -37,6 +40,7 @@ var _ metaCommand = (*metaBang)(nil)
 var _ metaCommand = (*metaBorder)(nil)
 var _ metaCommand = (*metaChangeDirectory)(nil)
 var _ metaCommand = (*metaConnect)(nil)
+var _ metaCommand = (*metaDescribe)(nil)
 var _ metaCommand = (*metaEcho)(nil)
 var _ metaCommand = (*metaExpanded)(nil)
 var _ metaCommand = (*metaFile)(nil)
@@ -44,6 +48,7 @@ var _ metaCommand = (*metaHelp)(nil)
 var _ metaCommand = (*metaInclude)(nil)
 var _ metaCommand = (*metaListDatabases)(nil)
 var _ metaCommand = (*metaListTables)(nil)
+var _ metaCommand = (*metaListViews)(nil)
 var _ metaCommand = (*metaOrg)(nil)
 var _ metaCommand = (*metaOutput)(nil)
 var _ metaCommand = (*metaPrint)(nil)
@@ -54,6 +59,7 @@ var _ metaCommand = (*metaReset)(nil)
 var _ metaCommand = (*metaSet)(nil)
 var _ metaCommand = (*metaTiming)(nil)
 var _ metaCommand = (*metaTuplesOnly)(nil)
+var _ metaCommand = (*metaUnset)(nil)
 var _ metaCommand = (*metaWarn)(nil)
 var _ metaCommand = (*metaWatch)(nil)
 var _ metaCommand = (*metaWrite)(nil)
@@ -71,13 +77,13 @@ func newMetaBang(args []string) *metaBang {
 	}
 }
 
-func (m *metaBang) execute(cmd *CLICommand) (action, error) {
+func (m *metaBang) execute(cmd *Command) (responseAction, error) {
 	if len(m.args) == 0 {
 		return actionNone, errors.Errorf("meta command '!' requires at least one argument")
 	}
 	c := exec.Command(m.args[0])
 	c.Args = m.args
-	c.Stdout = cmd.Stdout
+	c.Stdout = cmd.stdout
 	err := c.Run()
 	return actionNone, errors.Wrap(err, "running bang command")
 }
@@ -95,7 +101,7 @@ func newMetaBorder(args []string) *metaBorder {
 	}
 }
 
-func (m *metaBorder) execute(cmd *CLICommand) (action, error) {
+func (m *metaBorder) execute(cmd *Command) (responseAction, error) {
 	switch len(m.args) {
 	case 0:
 		// pass
@@ -129,12 +135,23 @@ func newMetaChangeDirectory(args []string) *metaChangeDirectory {
 	}
 }
 
-func (m *metaChangeDirectory) execute(cmd *CLICommand) (action, error) {
-	if len(m.args) != 1 {
-		return actionNone, errors.Errorf("meta command 'cd' requires exactly one argument")
+func (m *metaChangeDirectory) execute(cmd *Command) (responseAction, error) {
+	var dir string
+	switch len(m.args) {
+	case 0:
+		if d, err := os.UserHomeDir(); err != nil {
+			return actionNone, errors.Wrapf(err, "getting home directory")
+		} else {
+			dir = d
+		}
+	case 1:
+		dir = m.args[0]
+	default:
+		return actionNone, errors.Errorf("meta command 'cd' takes zero or one argument")
 	}
-	err := cmd.workingDir.cd(m.args[0])
-	return actionNone, errors.Wrap(err, "running cd command")
+
+	err := cmd.workingDir.cd(dir)
+	return actionNone, errors.Wrapf(err, "changing directory to: %s", dir)
 }
 
 // ////////////////////////////////////////////////////////////////////////////
@@ -150,7 +167,7 @@ func newMetaConnect(args []string) *metaConnect {
 	}
 }
 
-func (m *metaConnect) execute(cmd *CLICommand) (action, error) {
+func (m *metaConnect) execute(cmd *Command) (responseAction, error) {
 	switch len(m.args) {
 	case 0:
 		cmd.Printf(cmd.connectionMessage())
@@ -158,7 +175,6 @@ func (m *metaConnect) execute(cmd *CLICommand) (action, error) {
 	case 1:
 		err := cmd.connectToDatabase(m.args[0])
 		return actionNone, err
-
 	default:
 		return actionNone, errors.Errorf("meta command 'connect' takes zero or one argument")
 	}
@@ -177,11 +193,11 @@ func newMetaEcho(args []string) *metaEcho {
 	}
 }
 
-func (m *metaEcho) execute(cmd *CLICommand) (action, error) {
-	return echo(m.args, cmd.Stdout)
+func (m *metaEcho) execute(cmd *Command) (responseAction, error) {
+	return echo(m.args, cmd.stdout)
 }
 
-func echo(args []string, w io.Writer) (action, error) {
+func echo(args []string, w io.Writer) (responseAction, error) {
 	switch len(args) {
 	case 0:
 		w.Write([]byte("\n"))
@@ -218,7 +234,7 @@ func newMetaExpanded(args []string) *metaExpanded {
 	}
 }
 
-func (m *metaExpanded) execute(cmd *CLICommand) (action, error) {
+func (m *metaExpanded) execute(cmd *Command) (responseAction, error) {
 	switch len(m.args) {
 	case 0:
 		cmd.writeOptions.expanded = !cmd.writeOptions.expanded
@@ -257,7 +273,7 @@ func newMetaFile(args []string) *metaFile {
 	}
 }
 
-func (m *metaFile) execute(cmd *CLICommand) (action, error) {
+func (m *metaFile) execute(cmd *Command) (responseAction, error) {
 	if len(m.args) != 1 {
 		return actionNone, errors.Errorf("meta command 'file' requires exactly one argument")
 	}
@@ -279,6 +295,40 @@ func (m *metaFile) execute(cmd *CLICommand) (action, error) {
 	return actionNone, errors.Wrap(err, "adding part file")
 }
 
+// ////////////////////////////////////////////////////////////////////////////
+// format (sub-command of pset)
+// ////////////////////////////////////////////////////////////////////////////
+type metaFormat struct {
+	args []string
+}
+
+func newMetaFormat(args []string) *metaFormat {
+	return &metaFormat{
+		args: args,
+	}
+}
+
+func (m *metaFormat) execute(cmd *Command) (responseAction, error) {
+	switch len(m.args) {
+	case 0:
+		// pass
+	case 1:
+		switch m.args[0] {
+		case formatAligned:
+			cmd.writeOptions.format = formatAligned
+		case formatCSV:
+			cmd.writeOptions.format = formatCSV
+		default:
+			return actionNone, errors.Errorf("\\pset: allowed formats are aligned, csv")
+		}
+	default:
+		return actionNone, errors.Errorf("meta command 'format' takes zero or one argument")
+	}
+
+	cmd.Printf("Output format is %s.\n", cmd.writeOptions.format)
+	return actionNone, nil
+}
+
 // ////////////////////////////////////////////////////////////////////////////
 // help (?)
 // ////////////////////////////////////////////////////////////////////////////
@@ -292,7 +342,7 @@ func newMetaHelp(args []string) *metaHelp {
 	}
 }
 
-func (m *metaHelp) execute(cmd *CLICommand) (action, error) {
+func (m *metaHelp) execute(cmd *Command) (responseAction, error) {
 	helpText := `General
   \q[uit]                quit psql
   \watch [SEC]           execute query every SEC seconds
@@ -303,36 +353,42 @@ Help
 Query Buffer
   \p[rint]               show the contents of the query buffer
   \r[eset]               reset (clear) the query buffer
-  \w FILE                write query buffer to file
+  \w[rite] FILE          write query buffer to file
 
 Input/Output
   \echo [-n] [STRING]    write string to standard output (-n for no newline)
   \file ...              reference a local file to stream to the server
   \i[nclude] FILE        execute commands from file
-  \o [FILE]              send all query results to file
+  \o[ut] [FILE]          send all query results to file
   \qecho [-n] [STRING]   write string to \o output stream (-n for no newline)
   \warn [-n] [STRING]    write string to standard error (-n for no newline)
 
 Informational
-  \d                     list tables and views
+  \d                     list tables, including system tables
+  \d NAME                describe table
   \dt                    list tables
   \dv                    list views
-  \l                     list databases
+  \l[ist]                list databases
 
 Formatting
   \pset [NAME [VALUE]]   set table output option
-                         (border|expanded|tuples_only)
+                         (border|expanded|format|location|tuples_only)
   \t [on|off]            show only rows
-  \x [on|off|auto]       toggle expanded output
+  \x [on|off]            toggle expanded output
 
 Connection
   \c[onnect] [DBNAME]    connect to new database
+                         disconnect by sending DBNAME "-"
   \org [ORGNAME]         set organization id
 
 Operating System
   \cd [DIR]              change the current working directory
   \timing [on|off]       toggle timing of commands
   \! [COMMAND]           execute command in shell or start interactive shell
+
+Variables
+  \set [NAME [VALUE]]    set internal variable, or list all if no parameters
+  \unset NAME            unset (delete) internal variable
 `
 	cmd.Printf("%s\n", helpText)
 
@@ -352,18 +408,22 @@ func newMetaInclude(args []string) *metaInclude {
 	}
 }
 
-func (m *metaInclude) execute(cmd *CLICommand) (action, error) {
+func (m *metaInclude) execute(cmd *Command) (responseAction, error) {
 	if len(m.args) != 1 {
 		return actionNone, errors.Errorf("meta command 'include' requires exactly one argument")
 	}
 
-	file, err := os.Open(m.args[0])
+	return executeFile(cmd, m.args[0])
+}
+
+func executeFile(cmd *Command, fileName string) (responseAction, error) {
+	file, err := os.Open(fileName)
 	if err != nil {
-		return actionNone, errors.Wrapf(err, "opening file: %s", m.args[0])
+		return actionNone, errors.Wrapf(err, "opening file: %s", fileName)
 	}
 	defer file.Close()
 
-	splitter := newSplitter()
+	splitter := newSplitter(newReplacer(cmd.variables))
 	buffer := newBuffer()
 
 	// Read the file by line, pushing the lines into a new line splitter, then
@@ -375,8 +435,6 @@ func (m *metaInclude) execute(cmd *CLICommand) (action, error) {
 		qps, mcs, err := splitter.split(line)
 		if err != nil {
 			return actionNone, errors.Wrapf(err, "splitting lines")
-		} else if len(mcs) > 0 {
-			return actionNone, errors.Errorf("include does  not support meta-commands")
 		}
 
 		for i := range qps {
@@ -388,9 +446,21 @@ func (m *metaInclude) execute(cmd *CLICommand) (action, error) {
 				}
 			}
 		}
+
+		for i := range mcs {
+			action, err := mcs[i].execute(cmd)
+			if err != nil {
+				return actionNone, errors.Wrap(err, "executing meta command")
+			}
+			switch action {
+			case actionQuit:
+				close(cmd.quit)
+				return action, nil
+			}
+		}
 	}
 	if err := sc.Err(); err != nil {
-		return actionNone, errors.Wrapf(err, "scanning file: %s", m.args[0])
+		return actionNone, errors.Wrapf(err, "scanning file: %s", fileName)
 	}
 
 	return actionReset, nil
@@ -405,7 +475,7 @@ func newMetaListDatabases() *metaListDatabases {
 	return &metaListDatabases{}
 }
 
-func (m *metaListDatabases) execute(cmd *CLICommand) (action, error) {
+func (m *metaListDatabases) execute(cmd *Command) (responseAction, error) {
 	qry := []queryPart{
 		newPartRaw("SHOW DATABASES"),
 	}
@@ -418,7 +488,53 @@ func (m *metaListDatabases) execute(cmd *CLICommand) (action, error) {
 }
 
 // ////////////////////////////////////////////////////////////////////////////
-// list tables (d or dt)
+// describe (d)
+// ////////////////////////////////////////////////////////////////////////////
+type metaDescribe struct {
+	args []string
+}
+
+func newMetaDescribe(args []string) *metaDescribe {
+	return &metaDescribe{
+		args: args,
+	}
+}
+
+func (m *metaDescribe) execute(cmd *Command) (responseAction, error) {
+	switch len(m.args) {
+	case 0:
+		// Describe with no args should list all relations (tables, views,
+		// etc.). For now, we're just going to list the tables, including system
+		// tables.
+		qry := []queryPart{
+			newPartRaw("SHOW TABLES WITH SYSTEM"),
+		}
+
+		if err := cmd.executeAndWriteQuery(qry); err != nil {
+			return actionNone, errors.Wrap(err, "executing query")
+		}
+
+		return actionReset, nil
+
+	case 1:
+		// Describe with a single arg will assume the arg is a table name, so it
+		// runs a `SHOW COLUMNS` for that table.
+		qry := []queryPart{
+			newPartRaw(fmt.Sprintf(`SHOW COLUMNS FROM "%s"`, m.args[0])),
+		}
+
+		if err := cmd.executeAndWriteQuery(qry); err != nil {
+			return actionNone, errors.Wrap(err, "executing query")
+		}
+
+		return actionReset, nil
+	default:
+		return actionNone, errors.Errorf("meta command 'describe' takes zero or one argument")
+	}
+}
+
+// ////////////////////////////////////////////////////////////////////////////
+// describe (dt)
 // ////////////////////////////////////////////////////////////////////////////
 type metaListTables struct{}
 
@@ -426,7 +542,7 @@ func newMetaListTables() *metaListTables {
 	return &metaListTables{}
 }
 
-func (m *metaListTables) execute(cmd *CLICommand) (action, error) {
+func (m *metaListTables) execute(cmd *Command) (responseAction, error) {
 	qry := []queryPart{
 		newPartRaw("SHOW TABLES"),
 	}
@@ -438,6 +554,58 @@ func (m *metaListTables) execute(cmd *CLICommand) (action, error) {
 	return actionReset, nil
 }
 
+// ////////////////////////////////////////////////////////////////////////////
+// describe views (dv)
+// ////////////////////////////////////////////////////////////////////////////
+type metaListViews struct{}
+
+func newMetaListViews() *metaListViews {
+	return &metaListViews{}
+}
+
+func (m *metaListViews) execute(cmd *Command) (responseAction, error) {
+	qry := []queryPart{
+		newPartRaw("SELECT * FROM fb_views"),
+	}
+
+	if err := cmd.executeAndWriteQuery(qry); err != nil {
+		return actionNone, errors.Wrap(err, "executing query")
+	}
+
+	return actionReset, nil
+}
+
+// ////////////////////////////////////////////////////////////////////////////
+// location (sub-command of pset)
+// ////////////////////////////////////////////////////////////////////////////
+type metaLocation struct {
+	args []string
+}
+
+func newMetaLocation(args []string) *metaLocation {
+	return &metaLocation{
+		args: args,
+	}
+}
+
+func (m *metaLocation) execute(cmd *Command) (responseAction, error) {
+	switch len(m.args) {
+	case 0:
+		// pass
+	case 1:
+		loc, err := time.LoadLocation(m.args[0])
+		if err != nil {
+			return actionNone, errors.Wrapf(err, "loading location: %s", m.args[0])
+		}
+		cmd.writeOptions.location = loc
+	default:
+		return actionNone, errors.Errorf("meta command 'location' takes zero or one argument")
+	}
+
+	cmd.Printf("Location is %s.\n", cmd.writeOptions.location)
+	return actionNone, nil
+}
+
 // ////////////////////////////////////////////////////////////////////////////
 // org
 // ////////////////////////////////////////////////////////////////////////////
@@ -451,14 +619,14 @@ func newMetaOrg(args []string) *metaOrg {
 	}
 }
 
-func (m *metaOrg) execute(cmd *CLICommand) (action, error) {
+func (m *metaOrg) execute(cmd *Command) (responseAction, error) {
 	switch len(m.args) {
 
 	case 0:
 		cmd.Printf(cmd.orgMessage())
 		return actionNone, nil
 	case 1:
-		cmd.OrganizationID = m.args[0]
+		cmd.organizationID = m.args[0]
 		cmd.Printf(cmd.orgMessage())
 		return actionNone, nil
 
@@ -480,7 +648,7 @@ func newMetaOutput(args []string) *metaOutput {
 	}
 }
 
-func (m *metaOutput) execute(cmd *CLICommand) (action, error) {
+func (m *metaOutput) execute(cmd *Command) (responseAction, error) {
 	switch len(m.args) {
 	case 0:
 		// Close cmd.output (if closable).
@@ -489,7 +657,7 @@ func (m *metaOutput) execute(cmd *CLICommand) (action, error) {
 		}
 
 		// Set cmd.output to cmd.Stdout.
-		cmd.output = cmd.Stdout
+		cmd.output = cmd.stdout
 
 		return actionNone, nil
 
@@ -522,7 +690,7 @@ func newMetaPrint() *metaPrint {
 	return &metaPrint{}
 }
 
-func (m *metaPrint) execute(cmd *CLICommand) (action, error) {
+func (m *metaPrint) execute(cmd *Command) (responseAction, error) {
 	cmd.Printf(cmd.buffer.print() + "\n")
 	return actionNone, nil
 }
@@ -540,7 +708,7 @@ func newMetaPSet(args []string) *metaPSet {
 	}
 }
 
-func (m *metaPSet) print(cmd *CLICommand) {
+func (m *metaPSet) print(cmd *Command) {
 	onOff := func(b bool) string {
 		if b {
 			return "on"
@@ -550,18 +718,22 @@ func (m *metaPSet) print(cmd *CLICommand) {
 
 	fmt := `border      %d
 expanded    %s
+format      %s
+location    %s
 tuples_only %s
 `
 
 	cmd.Printf(fmt,
 		cmd.writeOptions.border,
 		onOff(cmd.writeOptions.expanded),
+		cmd.writeOptions.format,
+		cmd.writeOptions.location,
 		onOff(cmd.writeOptions.tuplesOnly),
 	)
 
 }
 
-func (m *metaPSet) execute(cmd *CLICommand) (action, error) {
+func (m *metaPSet) execute(cmd *Command) (responseAction, error) {
 	switch len(m.args) {
 	case 0:
 		m.print(cmd)
@@ -571,10 +743,16 @@ func (m *metaPSet) execute(cmd *CLICommand) (action, error) {
 		case "border":
 			sub := newMetaBorder(m.args[1:])
 			return sub.execute(cmd)
-		case "expanded":
+		case "expanded", "x":
 			sub := newMetaExpanded(m.args[1:])
 			return sub.execute(cmd)
-		case "tuples_only":
+		case "format":
+			sub := newMetaFormat(m.args[1:])
+			return sub.execute(cmd)
+		case "location":
+			sub := newMetaLocation(m.args[1:])
+			return sub.execute(cmd)
+		case "tuples_only", "t":
 			sub := newMetaTuplesOnly(m.args[1:])
 			return sub.execute(cmd)
 		default:
@@ -598,7 +776,7 @@ func newMetaQEcho(args []string) *metaQEcho {
 	}
 }
 
-func (m *metaQEcho) execute(cmd *CLICommand) (action, error) {
+func (m *metaQEcho) execute(cmd *Command) (responseAction, error) {
 	return echo(m.args, cmd.output)
 }
 
@@ -611,7 +789,7 @@ func newMetaQuit() *metaQuit {
 	return &metaQuit{}
 }
 
-func (m *metaQuit) execute(cmd *CLICommand) (action, error) {
+func (m *metaQuit) execute(cmd *Command) (responseAction, error) {
 	return actionQuit, nil
 }
 
@@ -624,8 +802,8 @@ func newMetaReset() *metaReset {
 	return &metaReset{}
 }
 
-func (m *metaReset) execute(cmd *CLICommand) (action, error) {
-	cmd.Printf(cmd.buffer.reset())
+func (m *metaReset) execute(cmd *Command) (responseAction, error) {
+	cmd.Printf(cmd.buffer.reset() + "\n")
 	return actionReset, nil
 }
 
@@ -642,8 +820,30 @@ func newMetaSet(args []string) *metaSet {
 	}
 }
 
-func (m *metaSet) execute(cmd *CLICommand) (action, error) {
-	// TODO: set the variable (or clear it, etc)
+func (m *metaSet) execute(cmd *Command) (responseAction, error) {
+	switch len(m.args) {
+	case 0:
+		// Sort the variables before printing them.
+		keys := make([]string, 0, len(cmd.variables))
+		for k := range cmd.variables {
+			keys = append(keys, k)
+		}
+		sort.Strings(keys)
+
+		// Print out the variables.
+		for _, k := range keys {
+			cmd.Printf("%s = '%s'\n", k, cmd.variables[k])
+		}
+		cmd.writeOptions.timing = !cmd.writeOptions.timing
+	default:
+		// The first arg is the key, the remaining args are concatenated together to form the values
+		// For example:
+		// \set one two three
+		// will result in `one = 'twothree'`
+		k := m.args[0]
+		v := strings.Join(m.args[1:], "")
+		cmd.variables[k] = v
+	}
 	return actionNone, nil
 }
 
@@ -660,7 +860,7 @@ func newMetaTiming(args []string) *metaTiming {
 	}
 }
 
-func (m *metaTiming) execute(cmd *CLICommand) (action, error) {
+func (m *metaTiming) execute(cmd *Command) (responseAction, error) {
 	switch len(m.args) {
 	case 0:
 		cmd.writeOptions.timing = !cmd.writeOptions.timing
@@ -671,7 +871,7 @@ func (m *metaTiming) execute(cmd *CLICommand) (action, error) {
 		case "off":
 			cmd.writeOptions.timing = false
 		default:
-			return actionNone, errors.Errorf("unrecognized value \"%s\" for \"\timing\": Boolean expected", m.args[0])
+			return actionNone, errors.Errorf("unrecognized value \"%s\" for \"\\timing\": Boolean expected", m.args[0])
 		}
 	default:
 		return actionNone, errors.Errorf("meta command 'timing' takes zero or one argument")
@@ -699,7 +899,7 @@ func newMetaTuplesOnly(args []string) *metaTuplesOnly {
 	}
 }
 
-func (m *metaTuplesOnly) execute(cmd *CLICommand) (action, error) {
+func (m *metaTuplesOnly) execute(cmd *Command) (responseAction, error) {
 	switch len(m.args) {
 	case 0:
 		cmd.writeOptions.tuplesOnly = !cmd.writeOptions.tuplesOnly
@@ -725,6 +925,35 @@ func (m *metaTuplesOnly) execute(cmd *CLICommand) (action, error) {
 	return actionNone, nil
 }
 
+// ////////////////////////////////////////////////////////////////////////////
+// unset
+// ////////////////////////////////////////////////////////////////////////////
+type metaUnset struct {
+	args []string
+}
+
+func newMetaUnset(args []string) *metaUnset {
+	return &metaUnset{
+		args: args,
+	}
+}
+
+func (m *metaUnset) execute(cmd *Command) (responseAction, error) {
+	switch len(m.args) {
+	case 0:
+		cmd.Printf("\\unset: missing required argument\n")
+		return actionNone, nil
+	default:
+		if len(m.args) > 1 {
+			for _, s := range m.args[1:] {
+				cmd.Printf("\\unset: extra argument \"%s\" ignored\n", s)
+			}
+		}
+		delete(cmd.variables, m.args[0])
+	}
+	return actionNone, nil
+}
+
 // ////////////////////////////////////////////////////////////////////////////
 // warn
 // ////////////////////////////////////////////////////////////////////////////
@@ -738,8 +967,8 @@ func newMetaWarn(args []string) *metaWarn {
 	}
 }
 
-func (m *metaWarn) execute(cmd *CLICommand) (action, error) {
-	return echo(m.args, cmd.Stderr)
+func (m *metaWarn) execute(cmd *Command) (responseAction, error) {
+	return echo(m.args, cmd.stderr)
 }
 
 // ////////////////////////////////////////////////////////////////////////////
@@ -755,7 +984,7 @@ func newMetaWatch(args []string) *metaWatch {
 	}
 }
 
-func (m *metaWatch) execute(cmd *CLICommand) (action, error) {
+func (m *metaWatch) execute(cmd *Command) (responseAction, error) {
 	period := 2 * time.Second
 
 	qry := cmd.buffer.lastQuery
@@ -809,7 +1038,7 @@ func newMetaWrite(args []string) *metaWrite {
 	}
 }
 
-func (m *metaWrite) execute(cmd *CLICommand) (action, error) {
+func (m *metaWrite) execute(cmd *Command) (responseAction, error) {
 	switch len(m.args) {
 	case 0:
 		cmd.Errorf(`\w: missing required argument` + "\n")
@@ -850,7 +1079,7 @@ func (m *metaWrite) execute(cmd *CLICommand) (action, error) {
 //	`cmd 'arg1' arg2 'arg three'`
 //
 // It returns the metaCommand which maps to `cmd`.
-func splitMetaCommand(in string) (metaCommand, error) {
+func splitMetaCommand(in string, replacer *replacer) (metaCommand, error) {
 	parts := strings.SplitN(in, ` `, 2)
 	key := strings.TrimRightFunc(parts[0], unicode.IsSpace)
 
@@ -873,6 +1102,11 @@ func splitMetaCommand(in string) (metaCommand, error) {
 		}
 	}
 
+	// Do variable replacement.
+	for i := range args {
+		args[i] = replacer.replace(args[i])
+	}
+
 	switch key {
 	case "!":
 		return newMetaBang(args), nil
@@ -880,8 +1114,12 @@ func splitMetaCommand(in string) (metaCommand, error) {
 		return newMetaChangeDirectory(args), nil
 	case "c", "connect":
 		return newMetaConnect(args), nil
-	case "d", "dt":
+	case "d":
+		return newMetaDescribe(args), nil
+	case "dt":
 		return newMetaListTables(), nil
+	case "dv":
+		return newMetaListViews(), nil
 	case "echo":
 		return newMetaEcho(args), nil
 	case "file":
@@ -890,9 +1128,9 @@ func splitMetaCommand(in string) (metaCommand, error) {
 		return newMetaHelp(args), nil
 	case "i", "include":
 		return newMetaInclude(args), nil
-	case "l":
+	case "l", "list":
 		return newMetaListDatabases(), nil
-	case "o":
+	case "o", "out":
 		return newMetaOutput(args), nil
 	case "org":
 		return newMetaOrg(args), nil
@@ -912,11 +1150,13 @@ func splitMetaCommand(in string) (metaCommand, error) {
 		return newMetaTuplesOnly(args), nil
 	case "timing":
 		return newMetaTiming(args), nil
+	case "unset":
+		return newMetaUnset(args), nil
 	case "warn":
 		return newMetaWarn(args), nil
 	case "watch":
 		return newMetaWatch(args), nil
-	case "w":
+	case "w", "write":
 		return newMetaWrite(args), nil
 	case "x":
 		return newMetaExpanded(args), nil

cli/parts.go

@@ -39,6 +39,12 @@ type queryPart interface {
 	Reader() io.Reader
 }
 
+func newRawQuery(s string) query {
+	return []queryPart{
+		newPartRaw(s),
+	}
+}
+
 // ////////////////////////////////////////////////////////////////////////////
 // raw
 // ////////////////////////////////////////////////////////////////////////////

cli/queryer.go

@@ -45,8 +45,10 @@ func (qryr *standardQueryer) Query(org string, db string, sql io.Reader) (*featu
 	if err != nil {
 		return nil, errors.Wrap(err, "reading response")
 	}
+
 	sqlResponse := &featurebase.WireQueryResponse{}
 	// TODO(tlt): switch this back once all responses are typed
+	// TODO(twg) 2023/03/01 using json.Number to decode large ints so care must be made
 	// if err := json.Unmarshal(fullbod, sqlResponse); err != nil {
 	if err := sqlResponse.UnmarshalJSONTyped(fullbod, true); err != nil {
 		return nil, errors.Wrapf(err, "unmarshaling query response, body:\n'%s'\n", fullbod)
@@ -66,7 +68,10 @@ type serverlessQueryer struct {
 }
 
 func (qryr *serverlessQueryer) Query(org string, db string, sql io.Reader) (*featurebase.WireQueryResponse, error) {
-	// buf := bytes.Buffer{}
+	if org == "" {
+		return nil, NewErrOrganizationRequired()
+	}
+
 	url := fmt.Sprintf("%s/queryer/databases/%s/sql", hostPort(qryr.Host, qryr.Port), db)
 	if db == "" {
 		url = fmt.Sprintf("%s/queryer/sql", hostPort(qryr.Host, qryr.Port))

cli/replacer.go

@@ -0,0 +1,109 @@
+package cli
+
+import (
+	"strings"
+
+	"github.com/benhoyt/goawk/lexer"
+)
+
+// replacer can replace parts of a string based on some rules and the provided
+// map[string]string. For example, the Command can replace strings with values
+// in its `variables` map.
+type replacer struct {
+	m map[string]string
+}
+
+func newReplacer(m map[string]string) *replacer {
+	return &replacer{
+		m: m,
+	}
+}
+
+// replace replaces all instances of the string pattern `:key` with the value at
+// m[key]. For example we want something like this:
+//
+// GIVEN: `start :one,:'two', :"three" ::four ::`
+//
+// with map
+//
+//	map[string]string{
+//		"one":   "repl1",
+//		"three": "repl3",
+//	}
+//
+// WANT: `start repl1,:'two', "repl3" ::four ::`
+func (r *replacer) replace(s string) string {
+	// If no variables have been added to the map, there's no need to parse the
+	// string for variable replacement.
+	if len(r.m) == 0 {
+		return s
+	}
+
+	line := []byte(s)
+	lex := lexer.NewLexer(line)
+
+	// finger contains the index into line at the start of non-variable text
+	// that we want to include, as-is in the output.
+	var finger int
+
+	// sb builds the string which will be the final output.
+	var sb strings.Builder
+	for {
+		pos, tok, _ := lex.Scan()
+
+		switch tok {
+		case lexer.COLON:
+			// last is the last normal character position before the colon.
+			last := pos.Column - 1
+
+			// Get the next byte to see if the colon value is quoted, and if so,
+			// whether its has single or double quotes.
+			b := lex.PeekByte()
+
+			// padding is the amount of padding we have to consider around the
+			// variable name. If the variable is not quoted, it doesn't require
+			// any padding. But if it has quotes, it needs 2 characters of
+			// paddings to accomodate the quotes.
+			padding := 0
+
+			// quote holds the character to use to quote the final, replaced
+			// output value. Because the lexer doesn't tell us how a certain
+			// `string` token was quoted, we need to keep track of that here so
+			// we can put them back.
+			quote := ""
+			switch b {
+			case byte('\''): // single quote
+				quote = `'`
+				padding = 2
+			case byte('"'): // double quote
+				quote = `"`
+				padding = 2
+			}
+
+			pos, tok, key := lex.Scan()
+			switch tok {
+			case lexer.NAME, lexer.STRING:
+				// Write the normal text up to the variable replacement
+				// position.
+				sb.Write(line[finger:last])
+
+				if v, ok := r.m[key]; ok {
+					// Write replaced variable with the quotes it had.
+					sb.WriteString(quote + v + quote)
+				} else {
+					// Since the variable was not found in the map, just write
+					// back what was already there.
+					sb.WriteString(":" + quote + key + quote)
+				}
+
+				// Reset finger to point to the next position after the
+				// variable.
+				finger = pos.Column + len(key) + padding - 1
+			}
+		case lexer.EOF:
+			// Write the remainder of the string and return.
+			sb.Write(line[finger:])
+			return sb.String()
+		}
+	}
+}

cli/replacer_test.go

@@ -0,0 +1,109 @@
+package cli
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestReplacer(t *testing.T) {
+	t.Run("general replace function", func(t *testing.T) {
+
+		m := map[string]string{
+			"v1": "newVone",
+			"v2": "newVtwo",
+		}
+
+		tests := []struct {
+			s   string
+			m   map[string]string
+			exp string
+		}{
+			{
+				// no variables present
+				s:   "foo",
+				m:   m,
+				exp: "foo",
+			},
+			{
+				// variable prefix, but not in map
+				s:   ":foo",
+				m:   m,
+				exp: ":foo",
+			},
+			{
+				// variable name match, but missing prefix
+				s:   "v1",
+				m:   m,
+				exp: "v1",
+			},
+			{
+				// variable name match
+				s:   ":v1",
+				m:   m,
+				exp: "newVone",
+			},
+			{
+				// two variables, the same, no space
+				s:   ":v1:v1",
+				m:   m,
+				exp: "newVonenewVone",
+			},
+			{
+				// two variables, different, no space
+				s:   ":v1:v2",
+				m:   m,
+				exp: "newVonenewVtwo",
+			},
+			{
+				// two variables, different, spaces
+				s:   ":v1  :v2",
+				m:   m,
+				exp: "newVone  newVtwo",
+			},
+			{
+				// one variable, one non-variable, no space
+				s:   ":v1:foo",
+				m:   m,
+				exp: "newVone:foo",
+			},
+			{
+				// one non-variable, one variable, no space
+				s:   "foo:v1",
+				m:   m,
+				exp: "foonewVone",
+			},
+			{
+				// two variables, different, comma
+				s:   ":v1, :v2",
+				m:   m,
+				exp: "newVone, newVtwo",
+			},
+			{
+				// single quotes
+				s:   ":'v1'",
+				m:   m,
+				exp: "'newVone'",
+			},
+			{
+				// double quotes
+				s:   `:"v2"`,
+				m:   m,
+				exp: `"newVtwo"`,
+			},
+			{
+				// more quotes
+				s:   `start :v1,:'two', :"v2" ::four :: `,
+				m:   m,
+				exp: `start newVone,:'two', "newVtwo" ::four :: `,
+			},
+		}
+		for i, test := range tests {
+			t.Run(fmt.Sprintf("test-%d", i), func(t *testing.T) {
+				replacer := newReplacer(test.m)
+				assert.Equal(t, test.exp, replacer.replace(test.s))
+			})
+		}
+	})
+}

cli/splitter.go

@@ -10,10 +10,14 @@ import (
 // metaCommands. It may not be necessary to have this be a separate struct since
 // it contains no members and just has the one `split()` method, but here we
 // are.
-type splitter struct{}
+type splitter struct {
+	replacer *replacer
+}
 
-func newSplitter() *splitter {
-	return &splitter{}
+func newSplitter(r *replacer) *splitter {
+	return &splitter{
+		replacer: r,
+	}
 }
 
 // split splits the given line into queryParts and metaCommands.
@@ -26,7 +30,12 @@ func newSplitter() *splitter {
 // 2- [metaCommands...]: "\! pwd \q"
 // 3- [queryParts...][metaCommands...]: "select * from \i file.sql"
 func (s *splitter) split(line string) ([]queryPart, []metaCommand, error) {
-	// Look for a meta command
+	// Look for a comment line.
+	if strings.HasPrefix(line, "--") {
+		return nil, nil, nil
+	}
+
+	// Look for a meta command.
 	parts := strings.SplitN(line, `\`, 2)
 
 	switch len(parts) {
@@ -65,6 +74,11 @@ func (s *splitter) splitQueryParts(line string) ([]queryPart, error) {
 	// Look for a termination character;
 	parts := strings.Split(line, terminationChar)
 
+	// Do variable replacement.
+	for i := range parts {
+		parts[i] = s.replacer.replace(parts[i])
+	}
+
 	if len(parts) == 1 {
 		part0 := strings.TrimSpace(parts[0])
 		return []queryPart{
@@ -95,7 +109,7 @@ func (s *splitter) splitQueryParts(line string) ([]queryPart, error) {
 func (s *splitter) splitMetaCommands(in string) ([]metaCommand, error) {
 	parts := strings.Split(in, `\`)
 	if len(parts) == 1 {
-		mc, err := splitMetaCommand(parts[0])
+		mc, err := splitMetaCommand(parts[0], s.replacer)
 		if err != nil {
 			return nil, errors.Wrapf(err, "splitting meta command: %s", parts[0])
 		}
@@ -108,7 +122,7 @@ func (s *splitter) splitMetaCommands(in string) ([]metaCommand, error) {
 		if part == "" {
 			continue
 		}
-		mc, err := splitMetaCommand(part)
+		mc, err := splitMetaCommand(part, s.replacer)
 		if err != nil {
 			return nil, errors.Wrapf(err, "splitting meta command: %s", part)
 		}

cli/splitter_test.go

@@ -8,7 +8,7 @@ import (
 )
 
 func TestSplitter(t *testing.T) {
-	s := newSplitter()
+	s := newSplitter(newReplacer(nil))
 	t.Run("Split", func(t *testing.T) {
 		tests := []struct {
 			line            string

cli/testdata/database

@@ -0,0 +1,53 @@
+// Show databases now that we have set org.
+SEND:SHOW DATABASES;
+EXPECT:+-----+------+-------+------------+------------+------------+-------+-------------+
+EXPECT:| _id | name | owner | updated_by | created_at | updated_at | units | description |
+EXPECT:+-----+------+-------+------------+------------+------------+-------+-------------+
+EXPECT:+-----+------+-------+------------+------------+------------+-------+-------------+
+EXPECT:
+
+// Create db1.
+SEND:CREATE DATABASE db1 WITH UNITS 1;
+EXPECT:
+
+// List databases via SHOW DATABASES.
+SEND:SHOW DATABASES;
+EXPECT:+--------------------------------------+------+-------+------------+----------------------+----------------------+-------+-------------+
+EXPECT:| _id                                  | name | owner | updated_by | created_at           | updated_at           | units | description |
+EXPECT:+--------------------------------------+------+-------+------------+----------------------+----------------------+-------+-------------+
+EXPECTCOMP:WithFormat:| {uuid} | db1  |       |            | {timestamp} | {timestamp} |     1 |             |
+EXPECT:+--------------------------------------+------+-------+------------+----------------------+----------------------+-------+-------------+
+EXPECT:
+
+// List databases via SHOW DATABASES.
+SEND:\l
+EXPECT:+--------------------------------------+------+-------+------------+----------------------+----------------------+-------+-------------+
+EXPECT:| _id                                  | name | owner | updated_by | created_at           | updated_at           | units | description |
+EXPECT:+--------------------------------------+------+-------+------------+----------------------+----------------------+-------+-------------+
+EXPECTCOMP:WithFormat:| {uuid} | db1  |       |            | {timestamp} | {timestamp} |     1 |             |
+EXPECT:+--------------------------------------+------+-------+------------+----------------------+----------------------+-------+-------------+
+EXPECT:
+
+// Check database connection.
+SEND:\c
+EXPECT:You are not connected to a database.
+
+// Try connecting to an invalid database.
+SEND:\c invalid
+EXPECT:executing meta command: invalid database: invalid
+
+// Try connecting with too many arguments.
+SEND:\c db1 extra
+EXPECT:executing meta command: meta command 'connect' takes zero or one argument
+
+// Connect to a database.
+SEND:\c db1
+EXPECTCOMP:WithFormat:You are now connected to database "db1" ({uuid}).
+
+// Disconnect from the current database.
+SEND:\c -
+EXPECT:You are not connected to a database.
+
+// Connect to a database again.
+SEND:\c db1
+EXPECTCOMP:WithFormat:You are now connected to database "db1" ({uuid}).

cli/testdata/famous.csv

@@ -0,0 +1,10 @@
+
+"Id", "Name", "Short description", "Gender", "Country", "Occupation", "Birth year", "Death year", "Manner of death", "Age of death"
+1, "George Washington", "1st president of the United States (1732–1799)", "Male", "United States of America; Kingdom of Great Britain", "Politician", "1732", "1799", "natural causes", "67"
+2, "Douglas Adams", "English writer and humorist", "Male", "United Kingdom", "Artist", "1952", "2001", "natural causes", "49"
+3, "Abraham Lincoln", "16th president of the United States (1809-1865)", "Male", "United States of America", "Politician", "1809", "1865", "homicide", "56"
+4, "Wolfgang Amadeus Mozart", "Austrian composer of the Classical period", "Male", "Archduchy of Austria; Archbishopric of Salzburg", "Artist", "1756", "1791", "0", "35"
+5, "Ludwig van Beethoven", "German classical and romantic composer", "Male", "Holy Roman Empire; Austrian Empire", "Artist", "1770", "1827", "0", "57"
+6, "Jean-François Champollion", "French classical scholar", "Male", "Kingdom of France; First French Empire", "Egyptologist", "1790", "1832", "natural causes", "42"
+7, "Paul Morand", "French writer", "Male", "France", "Artist", "1888", "1976", "0", "88"
+8, "Claude Monet", "French impressionist painter (1840-1926)", "Male", "France", "Artist", "1840", "1926", "natural causes", "86"

cli/testdata/meta_bang

@@ -0,0 +1,8 @@
+SEND:\! echo 'foo'
+EXPECT:foo
+
+SEND:\! echo "foo"
+EXPECT:"foo"
+
+SEND:\!
+EXPECT:executing meta command: meta command '!' requires at least one argument

cli/testdata/meta_cd

@@ -0,0 +1,17 @@
+// Make a directory so we can test \cd'ing into it.
+SEND:\! mkdir cli-test-dir
+SEND:\cd cli-test-dir
+SEND:\cd ..
+SEND:\! rmdir cli-test-dir
+
+// TODO(tlt): before we do this, we should implement the ability to execute
+// commands in a \set like:
+// \set homedir `pwd`
+// then we can store what directory we're in so we can move back to it 
+// at the end of the test
+// Switch to home directory.
+// SEND:\cd
+
+// Expect error on extra argument to \cd.
+SEND:\cd dir extra
+EXPECT:executing meta command: meta command 'cd' takes zero or one argument

cli/testdata/meta_describe

@@ -0,0 +1,33 @@
+// TODO(tlt): we can't run this test until we get the system tables under control (i.e. sorted). Currently, fb_views is in a map with users, so the following can fail 50% of the time.
+// Show tables for database by calling describe with no args.
+// SEND:\d
+// EXPECT:+-------------------------+-------------------------+-------+------------+----------------------+----------------------+-------+------------+------------------------+
+// EXPECT:| _id                     | name                    | owner | updated_by | created_at           | updated_at           | keys  | space_used | description            |
+// EXPECT:+-------------------------+-------------------------+-------+------------+----------------------+----------------------+-------+------------+------------------------+
+// EXPECTCOMP:WithFormat:| fb_veiws                | fb_views                |       |            | {timestamp} | {timestamp} | true  |          0 | system table for views |
+// EXPECTCOMP:WithFormat:| users                   | users                   |       |            | {timestamp} | {timestamp} | false |          0 |                        |
+// EXPECTCOMP:WithFormat:| fb_____________________ | fb_____________________ |       |            | {timestamp} | {timestamp} | false |          0 |                        |
+// EXPECTCOMP:WithFormat:| fb_____________________ | fb_____________________ |       |            | {timestamp} | {timestamp} | false |          0 |                        |
+// EXPECTCOMP:WithFormat:| fb_____________________ | fb_____________________ |       |            | {timestamp} | {timestamp} | false |          0 |                        |
+// EXPECTCOMP:WithFormat:| fb_____________________ | fb_____________________ |       |            | {timestamp} | {timestamp} | false |          0 |                        |
+// EXPECTCOMP:WithFormat:| fb_____________________ | fb_____________________ |       |            | {timestamp} | {timestamp} | false |          0 |                        |
+// EXPECT:+-------------------------+-------------------------+-------+------------+----------------------+----------------------+-------+------------+------------------------+
+// EXPECT:
+
+// Show columns for table.
+SEND:\d users
+EXPECT:+------+------+--------+----------------------+-------+------------+------------+-------+----------------------+---------------------+----------+-------+-------------+-----+
+EXPECT:| _id  | name | type   | created_at           | keys  | cache_type | cache_size | scale |                  min |                 max | timeunit | epoch | timequantum | ttl |
+EXPECT:+------+------+--------+----------------------+-------+------------+------------+-------+----------------------+---------------------+----------+-------+-------------+-----+
+EXPECTCOMP:WithFormat:| _id  | _id  | id     | {timestamp} | false |            |          0 |     0 |                    0 |                   0 |          |     0 |             | 0s  |
+EXPECTCOMP:WithFormat:| name | name | string | {timestamp} | true  | ranked     |      50000 |     0 |                    0 |                   0 |          |     0 |             | 0s  |
+EXPECTCOMP:WithFormat:| age  | age  | int    | {timestamp} | false |            |          0 |     0 | -9223372036854775808 | 9223372036854775807 |          |     0 |             | 0s  |
+EXPECT:+------+------+--------+----------------------+-------+------------+------------+-------+----------------------+---------------------+----------+-------+-------------+-----+
+EXPECT:
+
+// Show columns for an invalid table.
+SEND:\d invalid
+EXPECT:Error: compiling plan: [1:19] table 'invalid' not found
+
+SEND:\d users extra
+EXPECT:executing meta command: meta command 'describe' takes zero or one argument

cli/testdata/meta_echo

@@ -0,0 +1,6 @@
+SEND:\echo
+EXPECT:
+
+// Simple \echo.
+SEND:\echo foo bar
+EXPECT:foo bar

cli/testdata/meta_file

@@ -0,0 +1,70 @@
+// Create a table.
+SEND:CREATE TABLE famous (
+SEND:    _id ID,
+SEND:    name STRING,
+SEND:    description STRING,
+SEND:    gender STRING,
+SEND:    country STRING,
+SEND:    occupation STRING,
+SEND:    birth_year INT min -32767 max 32767,
+SEND:    death_year INT min -32767 max 32767,
+SEND:    death_manner STRING,
+SEND:    birth_age INT min -32767 max 32767
+SEND:);
+EXPECT:
+
+// Open bulk insert.
+SEND:BULK INSERT
+SEND:INTO famous (_id, name, description, gender, country, occupation,
+SEND:    birth_year, death_year, death_manner, birth_age )
+SEND:MAP(0 INT,
+SEND:1 STRING,
+SEND:2 STRING,
+SEND:3 STRING,
+SEND:4 STRING,
+SEND:5 STRING,
+SEND:6 INT,
+SEND:7 INT,
+SEND:8 STRING,
+SEND:9 INT )
+SEND:FROM
+SEND:    x'
+
+// Call \file
+SEND:\file testdata/famous.csv
+
+// Close bulk insert.
+SEND:'
+SEND:WITH
+SEND:    BATCHSIZE 100000
+SEND:    FORMAT 'CSV'
+SEND:    INPUT 'STREAM'
+SEND:    HEADER_ROW;
+EXPECT:
+
+// Query table to ensure we have data.
+SEND:SELECT * FROM famous;
+EXPECT:+-----+---------------------------+-------------------------------------------------+--------+----------------------------------------------------+--------------+------------+------------+----------------+-----------+
+EXPECT:| _id | name                      | description                                     | gender | country                                            | occupation   | birth_year | death_year | death_manner   | birth_age |
+EXPECT:+-----+---------------------------+-------------------------------------------------+--------+----------------------------------------------------+--------------+------------+------------+----------------+-----------+
+EXPECT:|   1 | George Washington         | 1st president of the United States (1732–1799)  | Male   | United States of America; Kingdom of Great Britain | Politician   |       1732 |       1799 | natural causes |        67 |
+EXPECT:|   2 | Douglas Adams             | English writer and humorist                     | Male   | United Kingdom                                     | Artist       |       1952 |       2001 | natural causes |        49 |
+EXPECT:|   3 | Abraham Lincoln           | 16th president of the United States (1809-1865) | Male   | United States of America                           | Politician   |       1809 |       1865 | homicide       |        56 |
+EXPECT:|   4 | Wolfgang Amadeus Mozart   | Austrian composer of the Classical period       | Male   | Archduchy of Austria; Archbishopric of Salzburg    | Artist       |       1756 |       1791 | 0              |        35 |
+EXPECT:|   5 | Ludwig van Beethoven      | German classical and romantic composer          | Male   | Holy Roman Empire; Austrian Empire                 | Artist       |       1770 |       1827 | 0              |        57 |
+EXPECT:|   6 | Jean-François Champollion | French classical scholar                        | Male   | Kingdom of France; First French Empire             | Egyptologist |       1790 |       1832 | natural causes |        42 |
+EXPECT:|   7 | Paul Morand               | French writer                                   | Male   | France                                             | Artist       |       1888 |       1976 | 0              |        88 |
+EXPECT:|   8 | Claude Monet              | French impressionist painter (1840-1926)        | Male   | France                                             | Artist       |       1840 |       1926 | natural causes |        86 |
+EXPECT:+-----+---------------------------+-------------------------------------------------+--------+----------------------------------------------------+--------------+------------+------------+----------------+-----------+
+EXPECT:
+
+// TODO(tlt): dropping the table seems to cause problems.
+// Drop the table.
+//SEND:DROP TABLE famous;
+
+// Ensure that invalid aruments (none or too many) return an error.
+SEND:\file
+EXPECT:executing meta command: meta command 'file' requires exactly one argument
+SEND:\file filename extra
+EXPECT:executing meta command: meta command 'file' requires exactly one argument
+

cli/testdata/meta_include

@@ -0,0 +1,24 @@
+// Include with no argument should error.
+SEND:\i
+EXPECT:executing meta command: meta command 'include' requires exactly one argument
+
+// Include with too many arguments should error.
+SEND:\include testdata/people.sql extra
+EXPECT:executing meta command: meta command 'include' requires exactly one argument
+
+// Invalid file should error.
+SEND:\include invalid.file
+EXPECT:executing meta command: opening file: invalid.file: open invalid.file: no such file or directory
+
+SEND:\include testdata/people.sql
+EXPECT:
+EXPECT:
+EXPECT:+-----+------+-----+
+EXPECT:| _id | name | age |
+EXPECT:+-----+------+-----+
+EXPECT:|   1 | Amy  |  42 |
+EXPECT:|   2 | Bob  |  27 |
+EXPECT:|   3 | Carl |  33 |
+EXPECT:+-----+------+-----+
+EXPECT:
+EXPECT:mix in a meta command

cli/testdata/meta_output

@@ -0,0 +1,67 @@
+SEND:SELECT * FROM users;
+EXPECT:+-----+-------+-----+
+EXPECT:| _id | name  | age |
+EXPECT:+-----+-------+-----+
+EXPECT:|   1 | Anne  |  38 |
+EXPECT:|   2 | Bill  |  23 |
+EXPECT:|   3 | Cindy |  64 |
+EXPECT:+-----+-------+-----+
+EXPECT:
+
+// Redirect output to a file.
+SEND:\o test-output-file
+SEND:SELECT * FROM users;
+
+// Ensure the output went to the file.
+SEND:\! cat test-output-file
+EXPECT:+-----+-------+-----+
+EXPECT:| _id | name  | age |
+EXPECT:+-----+-------+-----+
+EXPECT:|   1 | Anne  |  38 |
+EXPECT:|   2 | Bill  |  23 |
+EXPECT:|   3 | Cindy |  64 |
+EXPECT:+-----+-------+-----+
+EXPECT:
+
+// Let's test some qecho stuff here while we're at it.
+SEND:\qecho string with "double quotes"
+SEND:\qecho -n one
+SEND:\qecho -n two
+SEND:\qecho three
+SEND:\qecho four
+
+SEND:\! cat test-output-file
+EXPECT:+-----+-------+-----+
+EXPECT:| _id | name  | age |
+EXPECT:+-----+-------+-----+
+EXPECT:|   1 | Anne  |  38 |
+EXPECT:|   2 | Bill  |  23 |
+EXPECT:|   3 | Cindy |  64 |
+EXPECT:+-----+-------+-----+
+EXPECT:
+EXPECT:string with "double quotes"
+EXPECT:onetwothree
+EXPECT:four
+
+// And \warn messages should still go to stderr, not the file.
+SEND:\warn a warning string
+EXPECT:a warning string
+
+// Remove the file.
+SEND:\! rm test-output-file
+
+// Set the output back to stdout.
+SEND:\o
+SEND:SELECT * FROM users;
+EXPECT:+-----+-------+-----+
+EXPECT:| _id | name  | age |
+EXPECT:+-----+-------+-----+
+EXPECT:|   1 | Anne  |  38 |
+EXPECT:|   2 | Bill  |  23 |
+EXPECT:|   3 | Cindy |  64 |
+EXPECT:+-----+-------+-----+
+EXPECT:
+
+// Ensure extra arguments to \output causes an error.
+SEND:\o filename extra
+EXPECT:executing meta command: meta command 'output' takes zero or one argument

cli/testdata/meta_pset_border

@@ -0,0 +1,52 @@
+SEND:\pset border 2
+EXPECT:Border style is 2.
+
+SEND:SELECT 1 as foo, 'baz' as bar;
+EXPECT:+-----+-----+
+EXPECT:| foo | bar |
+EXPECT:+-----+-----+
+EXPECT:|   1 | baz |
+EXPECT:+-----+-----+
+EXPECT:
+
+SEND:\pset border
+EXPECT:Border style is 2.
+
+SEND:\pset border 999
+EXPECT:Border style is 0.
+
+SEND:\pset border 1
+EXPECT:Border style is 1.
+
+SEND:SELECT 1 as foo, 'baz' as bar;
+EXPECT: foo | bar 
+EXPECT:-----+-----
+EXPECT:   1 | baz 
+EXPECT:
+
+SEND:\pset border 2
+EXPECT:Border style is 2.
+
+SEND:SELECT 1 as foo, 'baz' as bar;
+EXPECT:+-----+-----+
+EXPECT:| foo | bar |
+EXPECT:+-----+-----+
+EXPECT:|   1 | baz |
+EXPECT:+-----+-----+
+EXPECT:
+
+SEND:\pset border 0
+EXPECT:Border style is 0.
+
+SEND:SELECT 1 as foo, 'baz' as bar;
+EXPECT:foo bar
+EXPECT:--- ---
+EXPECT:  1 baz
+EXPECT:
+
+SEND:\pset border 1 extra
+EXPECT:executing meta command: meta command 'pset' takes zero, one, or two arguments
+
+// Set border back to the testing default.
+SEND:\pset border 2
+EXPECT:Border style is 2.

cli/testdata/meta_pset_expanded

@@ -0,0 +1,35 @@
+// Set to off.
+SEND:\pset expanded off
+EXPECT:Expanded display is off.
+
+// Set to on.
+SEND:\pset expanded on
+EXPECT:Expanded display is on.
+
+// Toggle to off.
+SEND:\pset expanded
+EXPECT:Expanded display is off.
+
+// Toggle to on.
+SEND:\pset expanded
+EXPECT:Expanded display is on.
+
+// Set to something invalid.
+SEND:\pset expanded invalid
+EXPECT:executing meta command: unrecognized value "invalid" for "expanded": Boolean expected
+
+// Ensure expanded shows results vertically.
+SEND:SELECT 1 as foo, 'baz' as bar;
+EXPECT:+-----+-----+
+EXPECT:| foo | 1   |
+EXPECT:| bar | baz |
+EXPECT:+-----+-----+
+EXPECT:
+
+// Set back to off as we started.
+SEND:\pset expanded off
+EXPECT:Expanded display is off.
+
+// make sure the \x meta-command returns expected errors
+SEND:\x on extra
+EXPECT:executing meta command: meta command 'expanded' takes zero or one argument

cli/testdata/meta_pset_format_csv

@@ -0,0 +1,47 @@
+SEND:\pset format csv
+EXPECT:Output format is csv.
+
+SEND:SELECT * FROM users;
+EXPECT:_id,name,age
+EXPECT:1,Anne,38
+EXPECT:2,Bill,23
+EXPECT:3,Cindy,64
+
+// Exclude headers.
+SEND:\t on
+EXPECT:Tuples only is on.
+
+SEND:SELECT * FROM users;
+EXPECT:1,Anne,38
+EXPECT:2,Bill,23
+EXPECT:3,Cindy,64
+
+// Reset headers.
+SEND:\t off
+EXPECT:Tuples only is off.
+
+// Set expanded to on.
+SEND:\x on
+EXPECT:Expanded display is on.
+
+SEND:SELECT * FROM users;
+EXPECT:_id,1
+EXPECT:name,Anne
+EXPECT:age,38
+EXPECT:_id,2
+EXPECT:name,Bill
+EXPECT:age,23
+EXPECT:_id,3
+EXPECT:name,Cindy
+EXPECT:age,64
+
+// Set expanded back to off.
+SEND:\x off
+EXPECT:Expanded display is off.
+
+// Set format back to aligned as we started.
+SEND:\pset format aligned
+EXPECT:Output format is aligned.
+
+SEND:\pset format invalid
+EXPECT:executing meta command: \pset: allowed formats are aligned, csv

cli/testdata/meta_pset_tuples_only

@@ -0,0 +1,34 @@
+// Set to off.
+SEND:\pset tuples_only off
+EXPECT:Tuples only is off.
+
+// Set to on.
+SEND:\pset tuples_only on
+EXPECT:Tuples only is on.
+
+// Toggle to off.
+SEND:\pset tuples_only
+EXPECT:Tuples only is off.
+
+// Toggle to on.
+SEND:\pset tuples_only
+EXPECT:Tuples only is on.
+
+// Set to something invalid.
+SEND:\pset tuples_only invalid
+EXPECT:executing meta command: unrecognized value "invalid" for "tuples_only": Boolean expected
+
+// Ensure tuples_only shows only tuples.
+SEND:SELECT 1 as foo, 'baz' as bar;
+EXPECT:+---+-----+
+EXPECT:| 1 | baz |
+EXPECT:+---+-----+
+EXPECT:
+
+// Set back to off as we started.
+SEND:\pset tuples_only off
+EXPECT:Tuples only is off.
+
+// make sure the \t meta-command returns expected errors
+SEND:\t off extra
+EXPECT:executing meta command: meta command 'tuples_only' takes zero or one argument

cli/testdata/meta_set

@@ -0,0 +1,31 @@
+SEND:\set
+
+SEND:\set var1 foo
+SEND:\set
+EXPECT:var1 = 'foo'
+
+SEND:\set var2 bar
+SEND:\set
+EXPECT:var1 = 'foo'
+EXPECT:var2 = 'bar'
+
+SEND:\set var3 zoo
+SEND:\set
+EXPECT:var1 = 'foo'
+EXPECT:var2 = 'bar'
+EXPECT:var3 = 'zoo'
+
+SEND:\unset
+EXPECT:\unset: missing required argument
+
+SEND:\unset non-existent-key
+
+SEND:\unset var1
+SEND:\set
+EXPECT:var2 = 'bar'
+EXPECT:var3 = 'zoo'
+
+SEND:\unset var2 extra
+EXPECT:\unset: extra argument "extra" ignored
+SEND:\set
+EXPECT:var3 = 'zoo'

cli/testdata/meta_timing

@@ -0,0 +1,50 @@
+// Start by ensuring timing is off.
+SEND:\timing off
+EXPECT:Timing is off.
+
+// Set timing on.
+SEND:\timing on
+EXPECT:Timing is on.
+
+// Toggle timing.
+SEND:\timing
+EXPECT:Timing is off.
+
+// Toggle timing again.
+SEND:\timing
+EXPECT:Timing is on.
+
+// Send extra argument to \timing.
+SEND:\timing on extra
+EXPECT:executing meta command: meta command 'timing' takes zero or one argument
+
+
+SEND:SELECT * FROM users;
+EXPECT:+-----+-------+-----+
+EXPECT:| _id | name  | age |
+EXPECT:+-----+-------+-----+
+EXPECT:|   1 | Anne  |  38 |
+EXPECT:|   2 | Bill  |  23 |
+EXPECT:|   3 | Cindy |  64 |
+EXPECT:+-----+-------+-----+
+EXPECT:
+EXPECTCOMP:HasPrefix:Execution time:
+
+// Turn timing back off.
+SEND:\timing off
+EXPECT:Timing is off.
+
+// Ensure we don't get timing.
+SEND:SELECT * FROM users;
+EXPECT:+-----+-------+-----+
+EXPECT:| _id | name  | age |
+EXPECT:+-----+-------+-----+
+EXPECT:|   1 | Anne  |  38 |
+EXPECT:|   2 | Bill  |  23 |
+EXPECT:|   3 | Cindy |  64 |
+EXPECT:+-----+-------+-----+
+EXPECT:
+
+// Ensure an invalid timing value returns an error.
+SEND:\timing invalid
+EXPECT:executing meta command: unrecognized value "invalid" for "\timing": Boolean expected

cli/testdata/meta_write

@@ -0,0 +1,24 @@
+// Make sure there's something in the query buffer.
+// This is left unterminated because we don't need to execute the query;
+// we just need there to be something in the buffer.
+SEND:SELECT * FROM invalid-table
+SEND:\write query-buffer-contents
+
+// Reset the buffer.
+SEND:\r
+EXPECT:Query buffer reset (cleared).
+
+// Read from the file.
+SEND:\! cat query-buffer-contents
+EXPECT:SELECT * FROM invalid-table
+
+// Remove the file.
+SEND:\! rm query-buffer-contents
+
+// Send \write with no arguments.
+SEND:\write
+EXPECT:\w: missing required argument
+
+// Send \write with extra arguments.
+SEND:\write filename extra
+EXPECT:executing meta command: meta command 'w' exactly one argument

cli/testdata/people.sql

@@ -0,0 +1,12 @@
+-- Create a table.
+create table people (_id id, name string, age int);
+
+-- Insert some values.
+insert into people values (1, 'Amy', 42), (2, 'Bob', 27), (3, 'Carl', 33);
+
+-- Get all rows from the table.
+select * from people;
+
+-- Mix in a meta-command to show that both are supported
+-- in the include file.
+\echo mix in a meta command

cli/testdata/query_buffer

@@ -0,0 +1,40 @@
+SEND:select 1 as foo;
+EXPECT:+-----+
+EXPECT:| foo |
+EXPECT:+-----+
+EXPECT:|   1 |
+EXPECT:+-----+
+EXPECT:
+
+SEND:\p
+EXPECT:select 1 as foo;
+
+SEND:select 2
+SEND:\p
+EXPECT:select 2
+SEND:\r
+EXPECT:Query buffer reset (cleared).
+
+SEND:\p
+EXPECT:select 1 as foo;
+
+SEND:select 3
+SEND:\p
+EXPECT:select 3
+
+SEND:as foo
+SEND:\p
+EXPECT:select 3
+EXPECT:as foo
+
+SEND:;
+EXPECT:+-----+
+EXPECT:| foo |
+EXPECT:+-----+
+EXPECT:|   3 |
+EXPECT:+-----+
+EXPECT:
+
+SEND:\p
+EXPECT:select 3
+EXPECT:as foo;

cli/testdata/setup

@@ -0,0 +1,51 @@
+// Startup splash.
+EXPECT:FeatureBase CLI ()
+EXPECT:Type "\q" to quit.
+EXPECT:Detected on-prem, serverless deployment.
+EXPECTCOMP:HasPrefix:Host: http://localhost:
+EXPECT:You are not connected to a database.
+
+// Show databases.
+SEND:SHOW DATABASES;
+EXPECT:Organization required. Use \org to set an organization.
+
+// Get current org.
+SEND:\org
+EXPECT:You have not set an organization.
+
+// Set org.
+SEND:\org acme
+EXPECT:You have set organization "acme".
+
+// Try to set org with too many arguments.
+SEND:\org acme extra
+EXPECT:executing meta command: meta command 'org' takes zero or one argument
+
+// Set location to UTC so that expected timestamp size is consistent.
+// Without this, a test running locally in may have a timestamp that
+// ends in a timezone offset such as `-06:00`, while one running as UTC
+// will have `Z`. Since these string lengths differ, our generic
+// {timestamp} comparison will fail.
+SEND:\pset location UTC
+EXPECT:Location is UTC.
+
+// Set an invalid location.
+SEND:\pset location invalid
+EXPECT:executing meta command: loading location: invalid: unknown time zone invalid
+
+// Try to set location with too many arguments.
+SEND:\pset location UTC extra
+EXPECT:executing meta command: meta command 'pset' takes zero, one, or two arguments
+
+// Set border to 2 for testing because it makes it easier to visually see
+// what the tests are expecting (because lines don't end in spaces).
+SEND:\pset border 2
+EXPECT:Border style is 2.
+
+// Check the state of pset.
+SEND:\pset
+EXPECT:border      2
+EXPECT:expanded    off
+EXPECT:format      aligned
+EXPECT:location    UTC
+EXPECT:tuples_only off

cli/testdata/table

@@ -0,0 +1,72 @@
+// Show tables for database using SHOW TABLES WITH SYSTEM.
+SEND:SHOW TABLES WITH SYSTEM;
+EXPECT:+-------------------------+-------------------------+-------+------------+----------------------+----------------------+-------+------------+-------------+
+EXPECT:| _id                     | name                    | owner | updated_by | created_at           | updated_at           | keys  | space_used | description |
+EXPECT:+-------------------------+-------------------------+-------+------------+----------------------+----------------------+-------+------------+-------------+
+EXPECTCOMP:WithFormat:| fb_____________________ | fb_____________________ |       |            | {timestamp} | {timestamp} | false |          0 |             |
+EXPECTCOMP:WithFormat:| fb_____________________ | fb_____________________ |       |            | {timestamp} | {timestamp} | false |          0 |             |
+EXPECTCOMP:WithFormat:| fb_____________________ | fb_____________________ |       |            | {timestamp} | {timestamp} | false |          0 |             |
+EXPECTCOMP:WithFormat:| fb_____________________ | fb_____________________ |       |            | {timestamp} | {timestamp} | false |          0 |             |
+EXPECTCOMP:WithFormat:| fb_____________________ | fb_____________________ |       |            | {timestamp} | {timestamp} | false |          0 |             |
+EXPECT:+-------------------------+-------------------------+-------+------------+----------------------+----------------------+-------+------------+-------------+
+EXPECT:
+
+// Show tables for database using \d.
+SEND:\d
+EXPECT:+-------------------------+-------------------------+-------+------------+----------------------+----------------------+-------+------------+-------------+
+EXPECT:| _id                     | name                    | owner | updated_by | created_at           | updated_at           | keys  | space_used | description |
+EXPECT:+-------------------------+-------------------------+-------+------------+----------------------+----------------------+-------+------------+-------------+
+EXPECTCOMP:WithFormat:| fb_____________________ | fb_____________________ |       |            | {timestamp} | {timestamp} | false |          0 |             |
+EXPECTCOMP:WithFormat:| fb_____________________ | fb_____________________ |       |            | {timestamp} | {timestamp} | false |          0 |             |
+EXPECTCOMP:WithFormat:| fb_____________________ | fb_____________________ |       |            | {timestamp} | {timestamp} | false |          0 |             |
+EXPECTCOMP:WithFormat:| fb_____________________ | fb_____________________ |       |            | {timestamp} | {timestamp} | false |          0 |             |
+EXPECTCOMP:WithFormat:| fb_____________________ | fb_____________________ |       |            | {timestamp} | {timestamp} | false |          0 |             |
+EXPECT:+-------------------------+-------------------------+-------+------------+----------------------+----------------------+-------+------------+-------------+
+EXPECT:
+
+// Show tables for database using SHOW TABLES.
+SEND:SHOW TABLES;
+EXPECT:+-----+------+-------+------------+------------+------------+------+------------+-------------+
+EXPECT:| _id | name | owner | updated_by | created_at | updated_at | keys | space_used | description |
+EXPECT:+-----+------+-------+------------+------------+------------+------+------------+-------------+
+EXPECT:+-----+------+-------+------------+------------+------------+------+------------+-------------+
+EXPECT:
+
+
+// Show tables for database using \dt.
+SEND:\dt
+EXPECT:+-----+------+-------+------------+------------+------------+------+------------+-------------+
+EXPECT:| _id | name | owner | updated_by | created_at | updated_at | keys | space_used | description |
+EXPECT:+-----+------+-------+------------+------------+------------+------+------------+-------------+
+EXPECT:+-----+------+-------+------------+------------+------------+------+------------+-------------+
+EXPECT:
+
+// Create a table. That can be used for general testing.
+SEND:CREATE TABLE users (_id id, name string, age int);
+EXPECT:
+SEND:INSERT INTO users VALUES (1, 'Anne', 38), (2, 'Bill', 23), (3, 'Cindy', 64);
+EXPECT:
+
+
+// Show tables for database to get the newly created table.
+SEND:\dt
+EXPECT:+-------+-------+-------+------------+----------------------+----------------------+-------+------------+-------------+
+EXPECT:| _id   | name  | owner | updated_by | created_at           | updated_at           | keys  | space_used | description |
+EXPECT:+-------+-------+-------+------------+----------------------+----------------------+-------+------------+-------------+
+EXPECTCOMP:WithFormat:| users | users |       |            | {timestamp} | {timestamp} | false |          0 |             |
+EXPECT:+-------+-------+-------+------------+----------------------+----------------------+-------+------------+-------------+
+EXPECT:
+
+// We don't select from users until AFTER we check SHOW TABLES above because
+// running this creates the fb_views sytem table which has a description.
+// And it's annoying to mask out all of the description fields because we
+// don't know which row fb_views will fall into.
+SEND:SELECT * FROM users;
+EXPECT:+-----+-------+-----+
+EXPECT:| _id | name  | age |
+EXPECT:+-----+-------+-----+
+EXPECT:|   1 | Anne  |  38 |
+EXPECT:|   2 | Bill  |  23 |
+EXPECT:|   3 | Cindy |  64 |
+EXPECT:+-----+-------+-----+
+EXPECT:

cli/writer.go

@@ -1,8 +1,11 @@
 package cli
 
 import (
+	"encoding/csv"
 	"fmt"
 	"io"
+	"log"
+	"time"
 
 	featurebase "github.com/featurebasedb/featurebase/v3"
 	"github.com/jedib0t/go-pretty/table"
@@ -15,23 +18,32 @@ import (
 type writeOptions struct {
 	border     int
 	expanded   bool
+	format     string
+	location   *time.Location
 	timing     bool
 	tuplesOnly bool
 }
 
+const (
+	formatAligned = "aligned"
+	formatCSV     = "csv"
+)
+
 func defaultWriteOptions() *writeOptions {
 	return &writeOptions{
 		border:     1,
 		expanded:   false,
-		timing:     true,
+		format:     formatAligned,
+		location:   time.Local,
+		timing:     false,
 		tuplesOnly: false,
 	}
 }
 
-// writeTable writes the query response, taking the format into consideration.
+// writeOutput writes the query response, taking the format into consideration.
 // It sends query output to qOut, non-error informational output (such as query
 // timing) to wOut, and errors to wErr.
-func writeTable(r *featurebase.WireQueryResponse, format *writeOptions, qOut io.Writer, wOut io.Writer, wErr io.Writer) error {
+func writeOutput(r *featurebase.WireQueryResponse, opts *writeOptions, qOut io.Writer, wOut io.Writer, wErr io.Writer) error {
 	if r == nil {
 		return errors.New("attempt to write out nil response")
 	}
@@ -42,9 +54,104 @@ func writeTable(r *featurebase.WireQueryResponse, format *writeOptions, qOut io.
 		return writeWarnings(r, wErr)
 	}
 
+	switch opts.format {
+	case formatAligned:
+		if err := writeTable(r, opts, qOut); err != nil {
+			return errors.Wrap(err, "writing table")
+		}
+
+		// Add some white space after query results.
+		qOut.Write([]byte("\n"))
+
+	case formatCSV:
+		if err := writeCSV(r, opts, qOut); err != nil {
+			return errors.Wrap(err, "writing csv")
+		}
+
+	default:
+		return errors.Errorf("invalid format: %s", opts.format)
+	}
+
+	if err := writeWarnings(r, wErr); err != nil {
+		return err
+	}
+
+	// Timing.
+	if opts.timing {
+		if _, err := wOut.Write([]byte(fmt.Sprintf("Execution time: %dμs\n", r.ExecutionTime))); err != nil {
+			return errors.Wrapf(err, "writing execution time: %s", r.Error)
+		}
+	}
+
+	return nil
+}
+
+// writeCSV writes the WireQueryResponse to qOut as csv.
+func writeCSV(r *featurebase.WireQueryResponse, opts *writeOptions, qOut io.Writer) error {
+	w := csv.NewWriter(qOut)
+
+	if opts.expanded {
+		// Expanded csv
+
+		// rec is used to write the row as a slice of strings. It is reused to
+		// avoid unnecessary memory allocation.
+		rec := make([]string, 2)
+
+		for _, row := range r.Data {
+			cleanRow(row, opts)
+			for i, col := range r.Schema.Fields {
+				rec[0] = string(col.Name)
+				rec[1] = fmt.Sprintf("%v", row[i])
+
+				// Write the record.
+				if err := w.Write(rec); err != nil {
+					log.Fatalln("error writing expanded record to csv:", err)
+				}
+			}
+		}
+
+	} else {
+		// Normal csv (i.e. NOT expanded)
+
+		// Write the schema.
+		if !opts.tuplesOnly {
+			header := make([]string, 0, len(r.Schema.Fields))
+			for i := range r.Schema.Fields {
+				header = append(header, string(r.Schema.Fields[i].Name))
+			}
+			if err := w.Write(header); err != nil {
+				return errors.Wrapf(err, "error writing header to csv")
+			}
+		}
+
+		// Write the records.
+
+		// rec is used to write the row as a slice of strings. It is reused to
+		// avoid unnecessary memory allocation.
+		rec := make([]string, len(r.Schema.Fields))
+
+		for _, row := range r.Data {
+			cleanRow(row, opts)
+			for i := range row {
+				rec[i] = fmt.Sprintf("%v", row[i])
+			}
+			if err := w.Write(rec); err != nil {
+				log.Fatalln("error writing record to csv:", err)
+			}
+		}
+	}
+
+	// Write any buffered data to the underlying writer (standard output).
+	w.Flush()
+
+	return w.Error()
+}
+
+// writeTable writes the WireQueryResponse to qOut in a tabular format.
+func writeTable(r *featurebase.WireQueryResponse, opts *writeOptions, qOut io.Writer) error {
 	t := table.NewWriter()
 	t.SetOutputMirror(qOut)
-	switch format.border {
+	switch opts.border {
 	case 0:
 		t.SetStyle(styleBorder0)
 	case 1:
@@ -52,7 +159,7 @@ func writeTable(r *featurebase.WireQueryResponse, format *writeOptions, qOut io.
 	default:
 		t.SetStyle(styleBorder2)
 		// In expanded mode with a border, we need borders between each record.
-		if format.expanded {
+		if opts.expanded {
 			t.Style().Options.SeparateRows = true
 		}
 	}
@@ -60,9 +167,10 @@ func writeTable(r *featurebase.WireQueryResponse, format *writeOptions, qOut io.
 	// Don't uppercase the header values.
 	t.Style().Format.Header = text.FormatDefault
 
-	if format.expanded {
+	if opts.expanded {
 		// Expanded table
 		for _, row := range r.Data {
+			cleanRow(row, opts)
 			colRow := make([]interface{}, 2)
 			scolRow := make([]string, 2)
 			div := "\n"
@@ -71,11 +179,7 @@ func writeTable(r *featurebase.WireQueryResponse, format *writeOptions, qOut io.
 					div = ""
 				}
 				scolRow[0] += fmt.Sprintf("%s%s", col.Name, div)
-				if row[i] == nil {
-					scolRow[1] += fmt.Sprintf("%s%s", nullValue, div)
-				} else {
-					scolRow[1] += fmt.Sprintf("%v%s", row[i], div)
-				}
+				scolRow[1] += fmt.Sprintf("%v%s", row[i], div)
 			}
 			colRow[0] = scolRow[0]
 			colRow[1] = scolRow[1]
@@ -83,37 +187,36 @@ func writeTable(r *featurebase.WireQueryResponse, format *writeOptions, qOut io.
 		}
 	} else {
 		// Normal table (i.e. NOT expanded)
-		if !format.tuplesOnly {
+		if !opts.tuplesOnly {
 			t.AppendHeader(schemaToRow(r.Schema))
 		}
 		for _, row := range r.Data {
-			// If the value is nil, replace it with a null string; go-pretty doesn't
-			// expect nil pointers in the data values.
-			for i := range row {
-				if row[i] == nil {
-					row[i] = nullValue
-				}
-			}
+			cleanRow(row, opts)
 			t.AppendRow(table.Row(row))
 		}
 	}
 	t.Render()
 
-	if err := writeWarnings(r, wErr); err != nil {
-		return err
-	}
+	return nil
+}
 
-	// Add some white space after query results.
-	qOut.Write([]byte("\n"))
-
-	// Timing.
-	if format.timing {
-		if _, err := wOut.Write([]byte(fmt.Sprintf("Execution time: %dμs\n", r.ExecutionTime))); err != nil {
-			return errors.Wrapf(err, "writing execution time: %s", r.Error)
+// cleanRow loops through all the columns of row and modifies its value based on
+// type.
+//
+// If the value is nil, replace it with a null string; go-pretty doesn't expect
+// nil pointers in the data values.
+//
+// If the value is a time.Time, we want to print it using RFC3339Nano to be
+// consistent with everything else.
+func cleanRow(row []interface{}, opts *writeOptions) {
+	for i := range row {
+		switch v := row[i].(type) {
+		case nil:
+			row[i] = nullValue
+		case time.Time:
+			row[i] = v.In(opts.location).Format(time.RFC3339Nano)
 		}
 	}
-
-	return nil
 }
 
 func schemaToRow(schema featurebase.WireQuerySchema) []interface{} {

cli/writer_test.go

@@ -50,6 +50,26 @@ func TestWriter(t *testing.T) {
 					"   3 | Cindy |  28 ",
 					"",
 				),
+				expOut: "",
+				expErr: "",
+			},
+			{
+				// timing on
+				format: &writeOptions{
+					border:     1,
+					expanded:   false,
+					format:     formatAligned,
+					timing:     true,
+					tuplesOnly: false,
+				},
+				expQOut: stringOfLines(
+					" _id | name  | age ",
+					"-----+-------+-----",
+					"   1 | Amy   |  44 ",
+					"   2 | Bob   |  32 ",
+					"   3 | Cindy |  28 ",
+					"",
+				),
 				expOut: "Execution time: 0μs\n",
 				expErr: "",
 			},
@@ -58,6 +78,7 @@ func TestWriter(t *testing.T) {
 				format: &writeOptions{
 					border:     2,
 					expanded:   false,
+					format:     formatAligned,
 					timing:     false,
 					tuplesOnly: false,
 				},
@@ -79,6 +100,7 @@ func TestWriter(t *testing.T) {
 				format: &writeOptions{
 					border:     0,
 					expanded:   false,
+					format:     formatAligned,
 					timing:     false,
 					tuplesOnly: false,
 				},
@@ -98,6 +120,7 @@ func TestWriter(t *testing.T) {
 				format: &writeOptions{
 					border:     1,
 					expanded:   false,
+					format:     formatAligned,
 					timing:     false,
 					tuplesOnly: true,
 				},
@@ -115,6 +138,7 @@ func TestWriter(t *testing.T) {
 				format: &writeOptions{
 					border:     2,
 					expanded:   true,
+					format:     formatAligned,
 					timing:     false,
 					tuplesOnly: false,
 				},
@@ -145,7 +169,7 @@ func TestWriter(t *testing.T) {
 				wOut := bytes.NewBuffer(make([]byte, 0, 100000))
 				wErr := bytes.NewBuffer(make([]byte, 0, 100000))
 
-				assert.NoError(t, writeTable(wqr, test.format, qOut, wOut, wErr))
+				assert.NoError(t, writeOutput(wqr, test.format, qOut, wOut, wErr))
 
 				assert.Equal(t, test.expQOut, qOut.String())
 				assert.Equal(t, test.expOut, wOut.String())

cmd/auth_token.go

@@ -16,7 +16,7 @@ func newAuthTokenCommand(logdest logger.Logger) *cobra.Command {
 		Long: `
 Retrieves an auth-token for use in authenticating with FeatureBase from the configured identity provider.
 `,
-		RunE: usageErrorWrapper(cmd),
+		RunE: UsageErrorWrapper(cmd),
 	}
 
 	flags := ccmd.Flags()

cmd/backup.go

@@ -16,7 +16,7 @@ func newBackupCommand(logdest logger.Logger) *cobra.Command {
 		Long: `
 Backs up a FeatureBase server to a local, tar-formatted snapshot file.
 `,
-		RunE: usageErrorWrapper(cmd),
+		RunE: UsageErrorWrapper(cmd),
 	}
 
 	flags := ccmd.Flags()

cmd/backup_tar.go

@@ -16,7 +16,7 @@ func newBackupTarCommand(logdest io.Writer) *cobra.Command {
 		Long: `
 Backs up a FeatureBase server to a local, tar-formatted snapshot file.
 `,
-		RunE: usageErrorWrapper(cmd),
+		RunE: UsageErrorWrapper(cmd),
 	}
 
 	flags := ccmd.Flags()
@@ -28,6 +28,7 @@ Backs up a FeatureBase server to a local, tar-formatted snapshot file.
 	ctl.SetTLSConfig(flags, "", &cmd.TLS.CertificatePath, &cmd.TLS.CertificateKeyPath, &cmd.TLS.CACertPath, &cmd.TLS.SkipVerify, &cmd.TLS.EnableClientVerification)
 	flags.StringVar(&cmd.AuthToken, "auth-token", "", "Authentication token")
 	flags.StringVar(&cmd.HeaderTimeoutStr, "header-timeout", cmd.HeaderTimeoutStr, "Length of time to wait for initial HTTP response before giving up.")
+	flags.StringVar(&cmd.TempDir, "temp-dir", cmd.TempDir, "Location of temporary spillover files. The default is the system's default (usually /tmp)")
 
 	return ccmd
 }

cmd/chksum.go

@@ -3,13 +3,15 @@
 package cmd
 
 import (
+	"os"
+
 	"github.com/featurebasedb/featurebase/v3/ctl"
 	"github.com/featurebasedb/featurebase/v3/logger"
 	"github.com/spf13/cobra"
 )
 
 func newChkSumCommand(logdest logger.Logger) *cobra.Command {
-	cmd := ctl.NewChkSumCommand(logdest)
+	cmd := ctl.NewChkSumCommand(logdest, os.Stdout)
 	ccmd := &cobra.Command{
 		Use:   "chksum",
 		Short: "Digital signature of FeatureBase data",
@@ -17,7 +19,7 @@ func newChkSumCommand(logdest logger.Logger) *cobra.Command {
 			Generates a digital signature of all the data associated with a provided FeatureBase server
 			WARNING: could be slow if high cardinality fields exist
 `,
-		RunE: usageErrorWrapper(cmd),
+		RunE: UsageErrorWrapper(cmd),
 	}
 
 	flags := ccmd.Flags()

cmd/cli.go

@@ -1,35 +0,0 @@
-// Copyright 2021 Molecula Corp. All rights reserved.
-package cmd
-
-import (
-	"github.com/featurebasedb/featurebase/v3/cli"
-	"github.com/featurebasedb/featurebase/v3/logger"
-	"github.com/spf13/cobra"
-)
-
-var cliCmd *cli.CLICommand
-
-// newCLICommand runs the FeatureBase CLI subcommand for ingesting bulk data.
-func newCLICommand(logdest logger.Logger) *cobra.Command {
-	cliCmd = cli.NewCLICommand(logdest)
-	cobraCmd := &cobra.Command{
-		Use:   "cli",
-		Short: "Query FeatureBase with SQL from the command line",
-		Long:  ``,
-		RunE:  usageErrorWrapper(cliCmd),
-	}
-
-	flags := cobraCmd.Flags()
-	flags.StringVarP(&cliCmd.Host, "host", "", cliCmd.Host, "hostname of FeatureBase.")
-	flags.StringVarP(&cliCmd.Port, "port", "", cliCmd.Port, "port of FeatureBase.")
-	flags.StringVar(&cliCmd.HistoryPath, "history-path", cliCmd.HistoryPath, "path for history files.")
-	flags.StringVar(&cliCmd.OrganizationID, "org-id", cliCmd.OrganizationID, "OrganizationID.")
-	flags.StringVar(&cliCmd.Database, "db", cliCmd.Database, "Name of the database to connect to.")
-
-	flags.StringVar(&cliCmd.ClientID, "client-id", cliCmd.ClientID, "Cognito Client ID for FeatureBase Cloud access.")
-	flags.StringVar(&cliCmd.Region, "region", cliCmd.Region, "Cloud region for FeatureBase Cloud access (e.g. us-east-2).")
-	flags.StringVar(&cliCmd.Email, "email", cliCmd.Email, "Email address for FeatureBase Cloud access.")
-	flags.StringVar(&cliCmd.Password, "password", cliCmd.Password, "Password for FeatureBase Cloud access.")
-
-	return cobraCmd
-}

cmd/dataframe-csv-loader.go

@@ -15,7 +15,7 @@ func newDataframeCsvLoaderCommand(logdest logger.Logger) *cobra.Command {
 		Short: "load dataframe integer and floating point values into featurebase",
 		Long: `
 `,
-		RunE: usageErrorWrapper(cmd),
+		RunE: UsageErrorWrapper(cmd),
 	}
 	flags := loaderCmd.Flags()
 	flags.StringVar(&cmd.Path, "csv", "", "path to csv input file")
@@ -24,6 +24,7 @@ func newDataframeCsvLoaderCommand(logdest logger.Logger) *cobra.Command {
 	flags.StringVar(&cmd.AuthToken, "auth-token", "", "Authentication token")
 	flags.StringVar(&cmd.Index, "index", "", "Destination Index. ")
 	flags.IntVar(&cmd.MaxCapacity, "buffer", 0, "Maximum size of of the line buffer defaults to go bufio default ")
+	flags.IntVar(&cmd.BatchSize, "batch-size", 1048576, "Maximum number of records to send in a single batch ")
 	ctl.SetTLSConfig(
 		flags, "",
 		&cmd.TLS.CertificatePath,

cmd/export.go

@@ -26,7 +26,7 @@ The format of the CSV file is:
 
 The file does not contain any headers.
 `,
-		RunE: usageErrorWrapper(Exporter),
+		RunE: UsageErrorWrapper(Exporter),
 	}
 	flags := exportCmd.Flags()
 

cmd/fbsql/main.go

@@ -0,0 +1,66 @@
+// Copyright 2021 Molecula Corp. All rights reserved.
+package main
+
+import (
+	"io"
+	"os"
+
+	"github.com/featurebasedb/featurebase/v3/cli"
+	"github.com/featurebasedb/featurebase/v3/cmd"
+	"github.com/featurebasedb/featurebase/v3/logger"
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+)
+
+func main() {
+	command := newCLICommand(os.Stderr)
+	command.Execute()
+}
+
+// newCLICommand runs the FeatureBase CLI subcommand.
+func newCLICommand(stderr io.Writer) *cobra.Command {
+	logdest := logger.NewStandardLogger(stderr)
+	cliCmd := cli.NewCommand(logdest)
+	cobraCmd := &cobra.Command{
+		Use:   "fbsql",
+		Short: "Query FeatureBase with SQL from the command line",
+		Long:  ``,
+		RunE:  cmd.UsageErrorWrapper(cliCmd),
+		PersistentPreRunE: func(cobraCmd *cobra.Command, args []string) error {
+			v := viper.New()
+			return cmd.SetAllConfig(v, cobraCmd.Flags(), "FBSQL")
+		},
+		SilenceErrors: true,
+	}
+
+	// Attach flags to the command.
+	buildFlags(cobraCmd, cliCmd)
+	return cobraCmd
+}
+
+// buildFlags attaches a set of flags to the command for a cli instance.
+func buildFlags(cmd *cobra.Command, cliCmd *cli.Command) {
+	flags := cmd.Flags()
+
+	// Base struct flags.
+	flags.StringSliceVarP(&cliCmd.Commands, "command", "c", cliCmd.Commands, "Command to run in non-interactive mode. Provide multiple flags to execute more than one command. All `--command` flags run before all `--file` flags.")
+	flags.StringSliceVarP(&cliCmd.Files, "file", "f", cliCmd.Files, "File to run in non-interactive mode. Provide multiple flags to execute more than one file. All `--command` flags run before all `--file` flags.")
+
+	// Config flags.
+	flags.StringVarP(&cliCmd.Config.Host, "host", "", cliCmd.Config.Host, "hostname of FeatureBase.")
+	flags.StringVarP(&cliCmd.Config.Port, "port", "p", cliCmd.Config.Port, "port of FeatureBase.")
+	flags.StringVar(&cliCmd.Config.HistoryPath, "history-path", cliCmd.Config.HistoryPath, "path for history files.")
+	flags.StringVar(&cliCmd.Config.OrganizationID, "org-id", cliCmd.Config.OrganizationID, "OrganizationID.")
+	flags.StringVarP(&cliCmd.Config.Database, "dbname", "d", cliCmd.Config.Database, "Name of the database to connect to.")
+
+	flags.StringVar(&cliCmd.Config.CloudAuth.ClientID, "client-id", cliCmd.Config.CloudAuth.ClientID, "Cognito Client ID for FeatureBase Cloud access.")
+	flags.StringVar(&cliCmd.Config.CloudAuth.Region, "region", cliCmd.Config.CloudAuth.Region, "Cloud region for FeatureBase Cloud access (e.g. us-east-2).")
+	flags.StringVar(&cliCmd.Config.CloudAuth.Email, "email", cliCmd.Config.CloudAuth.Email, "Email address for FeatureBase Cloud access.")
+	flags.StringVar(&cliCmd.Config.CloudAuth.Password, "password", cliCmd.Config.CloudAuth.Password, "Password for FeatureBase Cloud access.")
+
+	flags.StringVar(&cliCmd.Config.KafkaConfig, "kafka-config", cliCmd.Config.KafkaConfig, "Kafka configuration file to read from.")
+	flags.BoolVar(&cliCmd.Config.CSV, "csv", cliCmd.Config.CSV, "CSV (Comma-Separated Values) table output mode.")
+	flags.StringSliceVar(&cliCmd.Config.PSets, "pset", cliCmd.Config.PSets, "Set printing option VAR to ARG (see \\pset command). Use form: --pset=VAR[=ARG]")
+
+	flags.String("config", "", "Configuration file to read from.")
+}

cmd/generate_config.go

@@ -18,7 +18,7 @@ func newGenerateConfigCommand(logdest logger.Logger) *cobra.Command {
 		Short: "Print the default configuration.",
 		Long: `generate-config prints the default configuration to stdout
 `,
-		RunE: usageErrorWrapper(generateConf),
+		RunE: UsageErrorWrapper(generateConf),
 	}
 
 	return confCmd

cmd/keygen.go

@@ -16,7 +16,7 @@ func newKeygenCommand(logdest logger.Logger) *cobra.Command {
 		Long: `
 Generate secret key to configure FeatureBase for Authentication.
 `,
-		RunE: usageErrorWrapper(cmd),
+		RunE: UsageErrorWrapper(cmd),
 	}
 
 	flags := ccmd.Flags()

cmd/parquet-info.go

@@ -27,7 +27,7 @@ Displays schema and sample data from the specified file
 			c.Path = args[0]
 			return nil
 		},
-		RunE: usageErrorWrapper(c),
+		RunE: UsageErrorWrapper(c),
 	}
 	return cmd
 }

cmd/presort.go

@@ -15,7 +15,7 @@ func newPreSortCommand(logdest logger.Logger) *cobra.Command {
 		Long: `
 Takes all input files and writes PartitionN numbered files to a directory, where each file contains only records that will go into the partition it is named for.
 `,
-		RunE: usageErrorWrapper(cmd),
+		RunE: UsageErrorWrapper(cmd),
 	}
 
 	flags := ccmd.Flags()

cmd/rbf.go

@@ -44,7 +44,7 @@ Executes a consistency check on an RBF data directory.
 			c.Path = args[0]
 			return nil
 		},
-		RunE: usageErrorWrapper(c),
+		RunE: UsageErrorWrapper(c),
 	}
 	return cmd
 }
@@ -76,7 +76,7 @@ Dumps the raw hex data for one or more RBF pages.
 
 			return nil
 		},
-		RunE: usageErrorWrapper(c),
+		RunE: UsageErrorWrapper(c),
 	}
 	return cmd
 }
@@ -98,7 +98,7 @@ Prints a line for every page in the database with its type/status.
 			c.Path = args[0]
 			return nil
 		},
-		RunE: usageErrorWrapper(c),
+		RunE: UsageErrorWrapper(c),
 	}
 
 	flags := cmd.Flags()
@@ -133,7 +133,7 @@ Prints the header & cell data for one or more pages.
 
 			return nil
 		},
-		RunE: usageErrorWrapper(c),
+		RunE: UsageErrorWrapper(c),
 	}
 	return cmd
 }

cmd/restore.go

@@ -16,7 +16,7 @@ func newRestoreCommand(logdest logger.Logger) *cobra.Command {
 		Long: `
 The Restore command will take a backup archive and restore it to a new, clean cluster.
 `,
-		RunE: usageErrorWrapper(cmd),
+		RunE: UsageErrorWrapper(cmd),
 	}
 	flags := restoreCmd.Flags()
 	flags.StringVarP(&cmd.Path, "source", "s", "", "backup file; specify '-' to restore from stdin tar stream")

cmd/restore_tar.go

@@ -15,7 +15,7 @@ func newRestoreTarCommand(logdest logger.Logger) *cobra.Command {
 		Long: `
 The Restore command will take a tar-formatted backup archive and restore it to a new, clean cluster.
 `,
-		RunE: usageErrorWrapper(cmd),
+		RunE: UsageErrorWrapper(cmd),
 	}
 	flags := restoreCmd.Flags()
 	flags.StringVarP(&cmd.Path, "source", "s", "", "backup file; specify '-' to restore from stdin tar stream")
@@ -23,6 +23,8 @@ The Restore command will take a tar-formatted backup archive and restore it to a
 	flags.DurationVar(&cmd.RetryPeriod, "retry-period", cmd.RetryPeriod, "Length of time after HTTP request failure to continue retrying request.")
 	flags.StringVar(&cmd.Pprof, "pprof", cmd.Pprof, "host:port to listen for profiling requests at /debug/pprof and /debug/fgprof.")
 	flags.StringVar(&cmd.AuthToken, "auth-token", "", "Authentication token")
+	flags.StringVar(&cmd.TempDir, "temp-dir", cmd.TempDir, "Location of temporary spillover files. The default is the system's default (usually /tmp)")
+
 	ctl.SetTLSConfig(
 		flags, "",
 		&cmd.TLS.CertificatePath,

cmd/root.go

@@ -25,11 +25,11 @@ type runner interface {
 	Run(context.Context) error
 }
 
-// usageErrorWrapper takes a thing with a Run(context) error, and produces
+// UsageErrorWrapper takes a thing with a Run(context) error, and produces
 // a func(*cobra.Command, []string) error from it which will run that
 // command, and then set Cobra's SilenceUsage flag unless the returned
 // error errors.Is() a ctl.UsageError.
-func usageErrorWrapper(inner runner) func(*cobra.Command, []string) error {
+func UsageErrorWrapper(inner runner) func(*cobra.Command, []string) error {
 	return func(c *cobra.Command, args []string) error {
 		return considerUsageError(c, inner.Run(context.Background()))
 	}
@@ -41,7 +41,7 @@ func usageErrorWrapper(inner runner) func(*cobra.Command, []string) error {
 // wrappers.
 func considerUsageError(cmd *cobra.Command, err error) error {
 	cmd.SilenceErrors = true
-	if !errors.Is(err, ctl.UsageError) {
+	if !errors.Is(err, ctl.ErrUsage) {
 		cmd.SilenceUsage = true
 	}
 	return err
@@ -63,11 +63,12 @@ at https://docs.featurebase.com/.
 ` + pilosa.VersionInfo(true) + "\n",
 		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
 			v := viper.New()
-			if cmd.Use == "dax" {
+
+			switch cmd.Use {
+			case "dax":
 				v.Set("future.rename", true) // always use FEATUREBASE env for dax
 			}
-			err := setAllConfig(v, cmd.Flags())
-			if err != nil {
+			if err := SetAllConfig(v, cmd.Flags(), ""); err != nil {
 				return err
 			}
 
@@ -104,7 +105,6 @@ at https://docs.featurebase.com/.
 	rc.AddCommand(newServeCmd(stderr))
 	rc.AddCommand(newHolderCmd(stderr))
 	rc.AddCommand(newKeygenCommand(logdest))
-	rc.AddCommand(newCLICommand(logdest))
 	rc.AddCommand(newDAXCommand(stderr))
 	rc.AddCommand(newDataframeCsvLoaderCommand(logdest))
 	rc.AddCommand(newPreSortCommand(logdest))
@@ -114,27 +114,28 @@ at https://docs.featurebase.com/.
 	return rc
 }
 
-// setAllConfig takes a FlagSet to be the definition of all configuration
+// SetAllConfig takes a FlagSet to be the definition of all configuration
 // options, as well as their defaults. It then reads from the command line, the
 // environment, and a config file (if specified), and applies the configuration
 // in that priority order. Since each flag in the set contains a pointer to
-// where its value should be stored, setAllConfig can directly modify the value
+// where its value should be stored, SetAllConfig can directly modify the value
 // of each config variable.
 //
-// setAllConfig looks for environment variables which are capitalized versions
+// SetAllConfig looks for environment variables which are capitalized versions
 // of the flag names with dashes replaced by underscores, and prefixed with
 // envPrefix plus an underscore.
-func setAllConfig(v *viper.Viper, flags *pflag.FlagSet) error { // nolint: unparam
+func SetAllConfig(v *viper.Viper, flags *pflag.FlagSet, envPrefix string) error { // nolint: unparam
 	// add cmd line flag def to viper
 	err := v.BindPFlags(flags)
 	if err != nil {
 		return err
 	}
 
-	envPrefix := "PILOSA"
-	rename := v.GetBool("future.rename")
-	if rename {
-		envPrefix = "FEATUREBASE"
+	if envPrefix == "" {
+		envPrefix = "PILOSA"
+		if v.GetBool("future.rename") {
+			envPrefix = "FEATUREBASE"
+		}
 	}
 
 	// add env to viper

cmd/server.go

@@ -48,7 +48,6 @@ func newServeCmd(stderr io.Writer) *cobra.Command {
 		Use:   "server",
 		Short: "Run FeatureBase.",
 		Long: `featurebase server runs FeatureBase.
-
 It will load existing data from the configured
 directory and start listening for client connections
 on the configured port.`,

ctl/backup.go

@@ -10,7 +10,6 @@ import (
 	"fmt"
 	"io"
 	"io/fs"
-	"log"
 	"os"
 	"path/filepath"
 	"time"
@@ -95,13 +94,13 @@ func (cmd *BackupCommand) Run(ctx context.Context) (err error) {
 
 	// Validate arguments.
 	if cmd.OutputDir == "" {
-		return fmt.Errorf("%w: -o flag required", UsageError)
+		return fmt.Errorf("%w: -o flag required", ErrUsage)
 	} else if cmd.Concurrency <= 0 {
-		return fmt.Errorf("%w: concurrency must be at least one", UsageError)
+		return fmt.Errorf("%w: concurrency must be at least one", ErrUsage)
 	}
 	if cmd.HeaderTimeoutStr != "" {
 		if dur, err := time.ParseDuration(cmd.HeaderTimeoutStr); err != nil {
-			return fmt.Errorf("%w: could not parse '%s' as a duration: %v", UsageError, cmd.HeaderTimeoutStr, err)
+			return fmt.Errorf("%w: could not parse '%s' as a duration: %v", ErrUsage, cmd.HeaderTimeoutStr, err)
 		} else {
 			cmd.HeaderTimeout = dur
 		}
@@ -549,7 +548,7 @@ func (cmd *BackupCommand) backupShardDataframe(ctx context.Context, indexName st
 	resp, err := client.GetDataframeShard(ctx, indexName, shard)
 	// no error if doesn't exist
 	if err != nil {
-		log.Fatal(err)
+		return fmt.Errorf("getting dataframe: %w", err)
 	}
 	defer resp.Body.Close()
 	if resp.StatusCode == 404 {

ctl/backup_tar.go

@@ -3,13 +3,11 @@ package ctl
 
 import (
 	"archive/tar"
-	"bytes"
 	"context"
 	"crypto/tls"
 	"encoding/json"
 	"fmt"
 	"io"
-	"log"
 	"os"
 	"path"
 	"path/filepath"
@@ -17,6 +15,7 @@ import (
 
 	pilosa "github.com/featurebasedb/featurebase/v3"
 	"github.com/featurebasedb/featurebase/v3/authn"
+	"github.com/featurebasedb/featurebase/v3/buffer"
 	"github.com/featurebasedb/featurebase/v3/disco"
 	"github.com/featurebasedb/featurebase/v3/encoding/proto"
 	"github.com/featurebasedb/featurebase/v3/logger"
@@ -37,6 +36,9 @@ type BackupTarCommand struct { // nolint: maligned
 	// Path to write the backup to.
 	OutputPath string
 
+	// TempDir location of scratch files
+	TempDir string
+
 	// Amount of time after first failed request to continue retrying.
 	RetryPeriod time.Duration `json:"retry-period"`
 
@@ -80,7 +82,7 @@ func (cmd *BackupTarCommand) Run(ctx context.Context) (err error) {
 	logdest := cmd.Logger()
 	// Validate arguments.
 	if cmd.OutputPath == "" {
-		return fmt.Errorf("%w: -o flag required", UsageError)
+		return fmt.Errorf("%w: -o flag required", ErrUsage)
 	}
 	useStdout := cmd.OutputPath == "-"
 	if useStdout && cmd.logwriter == os.Stdout {
@@ -98,7 +100,7 @@ func (cmd *BackupTarCommand) Run(ctx context.Context) (err error) {
 
 	if cmd.HeaderTimeoutStr != "" {
 		if dur, err := time.ParseDuration(cmd.HeaderTimeoutStr); err != nil {
-			return fmt.Errorf("%w: could not parse '%s' as a duration: %v", UsageError, cmd.HeaderTimeoutStr, err)
+			return fmt.Errorf("%w: could not parse '%s' as a duration: %v", ErrUsage, cmd.HeaderTimeoutStr, err)
 		} else {
 			cmd.HeaderTimeout = dur
 		}
@@ -229,25 +231,7 @@ func (cmd *BackupTarCommand) backupTarIDAllocData(ctx context.Context, tw *tar.W
 	}
 	defer rc.Close()
 
-	// Read to buffer to determine size.
-	var buf bytes.Buffer
-	if _, err := buf.ReadFrom(rc); err != nil {
-		return fmt.Errorf("copying id alloc data to memory: %w", err)
-	}
-
-	// Build header & copy data to archive.
-	if err = tw.WriteHeader(&tar.Header{
-		Name:    "idalloc",
-		Mode:    0o666,
-		Size:    int64(buf.Len()),
-		ModTime: time.Now(),
-	}); err != nil {
-		return err
-	} else if _, err := io.Copy(tw, &buf); err != nil {
-		return fmt.Errorf("copying id alloc data to archive: %w", err)
-	}
-
-	return nil
+	return writeToTar(tw, "idalloc", rc, cmd.TempDir)
 }
 
 // backupTarIndex backs up all shards for a given index.
@@ -268,7 +252,7 @@ func (cmd *BackupTarCommand) backupTarIndex(ctx context.Context, tw *tar.Writer,
 	}
 
 	if ii.Options.Keys {
-		// Back up translation data after bitmap data so we ensure we can translate all data.
+		// Back up translation data after bitmap data so we ensurean translate all data.
 		if err := cmd.backupTarIndexTranslateData(ctx, tw, ii.Name); err != nil {
 			return err
 		}
@@ -330,27 +314,7 @@ func (cmd *BackupTarCommand) backupTarShardNode(ctx context.Context, tw *tar.Wri
 		return fmt.Errorf("fetching shard reader: %w", err)
 	}
 	defer rc.Close()
-
-	// Read to buffer to determine size.
-	// TODO: Provide size via the reader itself.
-	var buf bytes.Buffer
-	if _, err := buf.ReadFrom(rc); err != nil {
-		return fmt.Errorf("copying shard data to memory: %w", err)
-	}
-
-	// Build header & copy data to archive.
-	if err = tw.WriteHeader(&tar.Header{
-		Name:    filename,
-		Mode:    0o666,
-		Size:    int64(buf.Len()),
-		ModTime: time.Now(),
-	}); err != nil {
-		return err
-	} else if _, err := io.Copy(tw, &buf); err != nil {
-		return fmt.Errorf("copying shard data to archive: %w", err)
-	}
-
-	return nil
+	return writeToTar(tw, filename, rc, cmd.TempDir)
 }
 
 func (cmd *BackupTarCommand) backupTarShardDataframe(ctx context.Context, tw *tar.Writer, indexName string, shard uint64, node *disco.Node) error {
@@ -364,7 +328,7 @@ func (cmd *BackupTarCommand) backupTarShardDataframe(ctx context.Context, tw *ta
 	resp, err := client.GetDataframeShard(ctx, indexName, shard)
 	// no error if doesn't exist
 	if err != nil {
-		log.Fatal(err)
+		return fmt.Errorf("getting dataframe: %w", err)
 	}
 	defer resp.Body.Close()
 	if resp.StatusCode == 404 {
@@ -373,24 +337,7 @@ func (cmd *BackupTarCommand) backupTarShardDataframe(ctx context.Context, tw *ta
 	}
 
 	filename := filepath.Join("indexes", indexName, "dataframe", fmt.Sprintf("%04d", shard))
-	logger.Printf("writing %v", filename)
-	var buf bytes.Buffer
-	if _, err := buf.ReadFrom(resp.Body); err != nil {
-		return fmt.Errorf("copying shard data to memory: %w", err)
-	}
-
-	// Build header & copy data to archive.
-	if err = tw.WriteHeader(&tar.Header{
-		Name:    filename,
-		Mode:    0o666,
-		Size:    int64(buf.Len()),
-		ModTime: time.Now(),
-	}); err != nil {
-		return err
-	} else if _, err := io.Copy(tw, &buf); err != nil {
-		return fmt.Errorf("copying shard data to archive: %w", err)
-	}
-	return nil
+	return writeToTar(tw, filename, resp.Body, cmd.TempDir)
 }
 
 func (cmd *BackupTarCommand) backupTarIndexTranslateData(ctx context.Context, tw *tar.Writer, name string) error {
@@ -416,25 +363,7 @@ func (cmd *BackupTarCommand) backupTarIndexPartitionTranslateData(ctx context.Co
 	}
 	defer rc.Close()
 
-	// Read to buffer to determine size.
-	var buf bytes.Buffer
-	if _, err := buf.ReadFrom(rc); err != nil {
-		return fmt.Errorf("copying translate data to memory: %w", err)
-	}
-
-	// Build header & copy data to archive.
-	if err = tw.WriteHeader(&tar.Header{
-		Name:    path.Join("indexes", name, "translate", fmt.Sprintf("%04d", partitionID)),
-		Mode:    0o666,
-		Size:    int64(buf.Len()),
-		ModTime: time.Now(),
-	}); err != nil {
-		return err
-	} else if _, err := io.Copy(tw, &buf); err != nil {
-		return fmt.Errorf("copying translate data to archive: %w", err)
-	}
-
-	return nil
+	return writeToTar(tw, path.Join("indexes", name, "translate", fmt.Sprintf("%04d", partitionID)), rc, cmd.TempDir)
 }
 
 func (cmd *BackupTarCommand) backupTarFieldTranslateData(ctx context.Context, tw *tar.Writer, indexName, fieldName string) error {
@@ -448,27 +377,42 @@ func (cmd *BackupTarCommand) backupTarFieldTranslateData(ctx context.Context, tw
 		return fmt.Errorf("fetching translate data reader: %w", err)
 	}
 	defer rc.Close()
-
-	// Read to buffer to determine size.
-	var buf bytes.Buffer
-	if _, err := buf.ReadFrom(rc); err != nil {
-		return fmt.Errorf("copying translate data to memory: %w", err)
-	}
-
-	// Build header & copy data to archive.
-	if err = tw.WriteHeader(&tar.Header{
-		Name:    path.Join("indexes", indexName, "fields", fieldName, "translate"),
-		Mode:    0o666,
-		Size:    int64(buf.Len()),
-		ModTime: time.Now(),
-	}); err != nil {
-		return err
-	} else if _, err := io.Copy(tw, &buf); err != nil {
-		return fmt.Errorf("copying translate data to archive: %w", err)
-	}
-	return nil
+	return writeToTar(tw, path.Join("indexes", indexName, "fields", fieldName, "translate"), rc, cmd.TempDir)
 }
 
 func (cmd *BackupTarCommand) TLSHost() string { return cmd.Host }
 
 func (cmd *BackupTarCommand) TLSConfiguration() server.TLSConfig { return cmd.TLS }
+
+func writeToTar(tw *tar.Writer, entryName string, rc io.Reader, tmpDir string) error {
+	spillFile, err := os.CreateTemp("", "spill")
+	if err != nil {
+		return fmt.Errorf("creating temp file : %w", err)
+	}
+	defer func() {
+		spillFile.Close()
+		os.Remove(spillFile.Name())
+	}()
+	mb512 := 2 << 29
+	buf := buffer.NewFileBuffer(mb512, tmpDir)
+	defer buf.Close()
+
+	n, err := io.Copy(buf, rc)
+	// Read to buffer to determine size.
+	if err != nil {
+		return fmt.Errorf("copying translate data to memory: %w", err)
+	}
+
+	// Build header & copy data to archive.
+	if err = tw.WriteHeader(&tar.Header{
+		Name:    entryName,
+		Mode:    0o666,
+		Size:    n,
+		ModTime: time.Now(),
+	}); err != nil {
+		return err
+	} else if _, err := io.Copy(tw, buf); err != nil {
+		return fmt.Errorf("copying translate data to archive: %w", err)
+	}
+	return nil
+}

ctl/backup_test.go

@@ -15,19 +15,19 @@ func TestBackupCommand_Run(t *testing.T) {
 	cm := NewBackupCommand(cmLog)
 	cm.OutputDir = ""
 	err := cm.Run(context.Background())
-	if !errors.Is(err, UsageError) {
+	if !errors.Is(err, ErrUsage) {
 		t.Fatalf("expected usage error, got %v", err)
 	}
 	cm.OutputDir = "foo"
 	cm.Concurrency = 0
 	err = cm.Run(context.Background())
-	if !errors.Is(err, UsageError) {
+	if !errors.Is(err, ErrUsage) {
 		t.Fatalf("expected usage error, got %v", err)
 	}
 	cm.Concurrency = 1
 	cm.HeaderTimeoutStr = "until the cat wakes up"
 	err = cm.Run(context.Background())
-	if !errors.Is(err, UsageError) {
+	if !errors.Is(err, ErrUsage) {
 		t.Fatalf("expected usage error, got %v", err)
 	}
 }

ctl/chksum.go

@@ -7,7 +7,6 @@ import (
 	"crypto/tls"
 	"fmt"
 	"io"
-	"os"
 
 	"github.com/cespare/xxhash"
 	pilosa "github.com/featurebasedb/featurebase/v3"
@@ -38,9 +37,9 @@ func (cmd *ChkSumCommand) Logger() logger.Logger {
 }
 
 // NewChkSumCommand returns a new instance of BackupCommand.
-func NewChkSumCommand(logdest logger.Logger) *ChkSumCommand {
+func NewChkSumCommand(logdest logger.Logger, stdout io.Writer) *ChkSumCommand {
 	return &ChkSumCommand{
-		stdout:  os.Stdout,
+		stdout:  stdout,
 		logDest: logdest,
 	}
 }

ctl/chksum_test.go

@@ -13,9 +13,8 @@ import (
 
 func TestChkSumCommand_Run(t *testing.T) {
 	cmLog := logger.NewStandardLogger(os.Stderr)
-	cm := NewChkSumCommand(cmLog)
 	buf := &bytes.Buffer{}
-	cm.stdout = buf
+	cm := NewChkSumCommand(cmLog, buf)
 
 	cluster := test.MustRunCluster(t, 1)
 	defer cluster.Close()

ctl/dataframe-csv-loader.go

@@ -36,7 +36,7 @@ func init() {
 
 // TODO(rdp): add refresh token to this as well
 
-// NewDataframeCsvLoaderCommand
+// DataframeCsvLoaderCommand is used to load a dataframe from CSV.
 type DataframeCsvLoaderCommand struct {
 	tlsConfig *tls.Config
 
@@ -50,6 +50,9 @@ type DataframeCsvLoaderCommand struct {
 	// max line length of csv file
 	MaxCapacity int
 
+	// Batch Size
+	BatchSize int
+
 	// Host:port on which to listen for pprof.
 	Pprof string `json:"pprof"`
 
@@ -146,7 +149,7 @@ func (cmd *DataframeCsvLoaderCommand) Run(ctx context.Context) (err error) {
 
 	// Validate arguments.
 	if cmd.Path == "" {
-		return fmt.Errorf("%w: --csv flag required", UsageError)
+		return fmt.Errorf("%w: --csv flag required", ErrUsage)
 	}
 
 	readFile, err := os.Open(cmd.Path)
@@ -234,7 +237,9 @@ func (cmd *DataframeCsvLoaderCommand) Run(ctx context.Context) (err error) {
 	fileScanner.Scan() // skip the header
 	cmd.Logger().Infof("Build the dataframe input package in memory")
 	id := uint64(0)
+	recordCounter := 0
 	for fileScanner.Scan() {
+		recordCounter++
 		records := strings.Split(fileScanner.Text(), ",")
 		if cmd.needTranslation {
 			id = lookup[records[0]]
@@ -278,13 +283,23 @@ func (cmd *DataframeCsvLoaderCommand) Run(ctx context.Context) (err error) {
 				}
 			}
 		}
+		if recordCounter > cmd.BatchSize {
+			cmd.Logger().Infof("sending package to featurebase")
+			err = sharder.Store(arrowSchema, cmd.client)
+			if err != nil {
+				return err
+			}
+			sharder.Reset()
+			recordCounter = 0
+		}
 	}
-	cmd.Logger().Infof("sending package to featurebase")
-	err = sharder.Store(arrowSchema, cmd.client)
-	if err != nil {
-		return err
+	if recordCounter > 0 {
+		err = sharder.Store(arrowSchema, cmd.client)
+		if err != nil {
+			return err
+		}
 	}
-	return err
+	return nil
 }
 
 type pair struct {
@@ -378,6 +393,10 @@ type Sharder struct {
 	log logger.Logger
 }
 
+func (s *Sharder) Reset() {
+	s.shards = make(map[uint64]*ShardDiff)
+}
+
 func (s *Sharder) GetShard(shard uint64) (*ShardDiff, error) {
 	f, ok := s.shards[shard]
 	if ok {

ctl/dax.go

@@ -17,9 +17,21 @@ func BuildDAXFlags(cmd *cobra.Command, srv *server.Command) {
 	// Controller
 	flags.BoolVar(&srv.Config.Controller.Run, "controller.run", srv.Config.Controller.Run, "Run the Controller service in process.")
 	flags.DurationVar(&srv.Config.Controller.Config.RegistrationBatchTimeout, "controller.config.registration-batch-timeout", srv.Config.Controller.Config.RegistrationBatchTimeout, "Timeout for node registration batches.")
-	flags.StringVar(&srv.Config.Controller.Config.DataDir, "controller.config.data-dir", srv.Config.Controller.Config.DataDir, "Controller directory to use in process.")
+	flags.StringVar(&srv.Config.Controller.Config.StorageMethod, "controller.config.storage-method", srv.Config.Controller.Config.StorageMethod, "Backing store. boltdb or sqldb.")
 	flags.DurationVar(&srv.Config.Controller.Config.SnappingTurtleTimeout, "controller.config.snapping-turtle-timeout", srv.Config.Controller.Config.SnappingTurtleTimeout, "Period for running automatic snapshotting routine.")
 
+	// Controller.SQLDB
+	flags.StringVar(&srv.Config.Controller.Config.SQLDB.Database, "controller.config.sqldb.database", srv.Config.Controller.Config.SQLDB.Database, "Database name.")
+	flags.StringVar(&srv.Config.Controller.Config.SQLDB.Host, "controller.config.sqldb.host", srv.Config.Controller.Config.SQLDB.Host, "Hostname of SQL Database")
+	flags.StringVar(&srv.Config.Controller.Config.SQLDB.Port, "controller.config.sqldb.port", srv.Config.Controller.Config.SQLDB.Port, "Port of SQL Database")
+	flags.StringVar(&srv.Config.Controller.Config.SQLDB.User, "controller.config.sqldb.user", srv.Config.Controller.Config.SQLDB.User, "Username for connection to SQL Database")
+	flags.StringVar(&srv.Config.Controller.Config.SQLDB.Password, "controller.config.sqldb.password", srv.Config.Controller.Config.SQLDB.Password, "Password for connection to SQL Database")
+	flags.StringVar(&srv.Config.Controller.Config.SQLDB.URL, "controller.config.sqldb.url", srv.Config.Controller.Config.SQLDB.URL, "URL for connection to SQL Database (supersedes host/port/username etc)")
+	flags.IntVar(&srv.Config.Controller.Config.SQLDB.Pool, "controller.config.sqldb.pool", srv.Config.Controller.Config.SQLDB.Pool, "Max number of open connections to database. 0=unlimited")
+	flags.IntVar(&srv.Config.Controller.Config.SQLDB.IdlePool, "controller.config.sqldb.idle-pool", srv.Config.Controller.Config.SQLDB.IdlePool, "Maximum number of idle connections to database.")
+	flags.DurationVar(&srv.Config.Controller.Config.SQLDB.ConnMaxLifetime, "controller.config.sqldb.conn-max-lifetime", srv.Config.Controller.Config.SQLDB.ConnMaxLifetime, "See https://golang.org/pkg/database/sql/#DB.SetConnMaxLifetime")
+	flags.DurationVar(&srv.Config.Controller.Config.SQLDB.ConnMaxIdleTime, "controller.config.sqldb.conn-max-idle-time", srv.Config.Controller.Config.SQLDB.ConnMaxIdleTime, "See https://golang.org/pkg/database/sql/#DB.SetConnMaxIdletime")
+
 	// Queryer
 	flags.BoolVar(&srv.Config.Queryer.Run, "queryer.run", srv.Config.Queryer.Run, "Run the Queryer service in process.")
 	flags.StringVar(&srv.Config.Queryer.Config.ControllerAddress, "queryer.config.controller-address", srv.Config.Queryer.Config.ControllerAddress, "Address of remote Controller process.")

ctl/export.go

@@ -50,9 +50,9 @@ func (cmd *ExportCommand) Run(ctx context.Context) error {
 
 	// Validate arguments.
 	if cmd.Index == "" {
-		return fmt.Errorf("%w: %v", UsageError, pilosa.ErrIndexRequired)
+		return fmt.Errorf("%w: %v", ErrUsage, pilosa.ErrIndexRequired)
 	} else if cmd.Field == "" {
-		return fmt.Errorf("%w: %v", UsageError, pilosa.ErrFieldRequired)
+		return fmt.Errorf("%w: %v", ErrUsage, pilosa.ErrFieldRequired)
 	}
 
 	// Use output file, if specified.

ctl/import.go

@@ -86,11 +86,11 @@ func (cmd *ImportCommand) Run(ctx context.Context) error {
 	// Validate arguments.
 	// Index and field are validated early before the files are parsed.
 	if cmd.Index == "" {
-		return fmt.Errorf("%w: %v", UsageError, pilosa.ErrIndexRequired)
+		return fmt.Errorf("%w: %v", ErrUsage, pilosa.ErrIndexRequired)
 	} else if cmd.Field == "" {
-		return fmt.Errorf("%w: %v", UsageError, pilosa.ErrFieldRequired)
+		return fmt.Errorf("%w: %v", ErrUsage, pilosa.ErrFieldRequired)
 	} else if len(cmd.Paths) == 0 {
-		return fmt.Errorf("%w: path required", UsageError)
+		return fmt.Errorf("%w: path required", ErrUsage)
 	}
 	// Create a client to the server.
 	client, err := commandClient(cmd)

ctl/keygen.go

@@ -12,7 +12,7 @@ import (
 	"github.com/gorilla/securecookie"
 )
 
-// Keygen represents a command for generating a cryptographic key.
+// KeygenCommand represents a command for generating a cryptographic key.
 type KeygenCommand struct {
 	stdout    io.Writer
 	logDest   logger.Logger

ctl/parquet-info.go

@@ -4,7 +4,6 @@ package ctl
 
 import (
 	"context"
-	"errors"
 	"fmt"
 	"io"
 	"net/http"
@@ -47,23 +46,23 @@ func (cmd *ParquetInfoCommand) Run(ctx context.Context) error {
 			return err
 		}
 		if response.StatusCode != 200 {
-			return errors.New(fmt.Sprintf("unexpected response %d", response.StatusCode))
+			return fmt.Errorf("unexpected response %d", response.StatusCode)
 		}
 		defer response.Body.Close()
 		// download to temp file first
 		f, err = os.CreateTemp("", "BulkParquetFile.parquet")
 		if err != nil {
-			return errors.New(fmt.Sprintf("error creating tempfile %v", err))
+			return fmt.Errorf("error creating tempfile %v", err)
 		}
 		_, err = io.Copy(f, response.Body)
 		if err != nil {
-			return errors.New(fmt.Sprintf("error downloading url %v %v", cmd.Path, err))
+			return fmt.Errorf("error downloading url %v %v", cmd.Path, err)
 		}
 		defer os.Remove(f.Name())
 
 		_, err = f.Seek(0, io.SeekStart)
 		if err != nil {
-			return errors.New(fmt.Sprintf("error reseting file for reading %v ", err))
+			return fmt.Errorf("error reseting file for reading %v ", err)
 		}
 	} else {
 		f, err = os.Open(cmd.Path)

ctl/restore.go

@@ -83,9 +83,9 @@ func (cmd *RestoreCommand) Run(ctx context.Context) (err error) {
 
 	// Validate arguments.
 	if cmd.Path == "" {
-		return fmt.Errorf("%w: -s flag required", UsageError)
+		return fmt.Errorf("%w: -s flag required", ErrUsage)
 	} else if cmd.Concurrency <= 0 {
-		return fmt.Errorf("%w: concurrency must be at least one", UsageError)
+		return fmt.Errorf("%w: concurrency must be at least one", ErrUsage)
 	}
 
 	// Parse TLS configuration for node-specific clients.

ctl/restore_tar.go

@@ -3,13 +3,13 @@ package ctl
 
 import (
 	"archive/tar"
-	"bytes"
 	"compress/gzip"
 	"context"
 	"crypto/tls"
 	"fmt"
 	"io"
-	gohttp "net/http"
+	"io/ioutil"
+	"net/http"
 	"os"
 	"strconv"
 	"strings"
@@ -17,6 +17,7 @@ import (
 
 	pilosa "github.com/featurebasedb/featurebase/v3"
 	"github.com/featurebasedb/featurebase/v3/authn"
+	"github.com/featurebasedb/featurebase/v3/buffer"
 	"github.com/featurebasedb/featurebase/v3/disco"
 	"github.com/featurebasedb/featurebase/v3/logger"
 	"github.com/featurebasedb/featurebase/v3/server"
@@ -48,6 +49,9 @@ type RestoreTarCommand struct {
 	TLS server.TLSConfig
 
 	AuthToken string
+
+	// TempDir location of scratch files
+	TempDir string
 }
 
 // Logger returns the command's associated Logger to maintain CommandWithTLSSupport interface compatibility
@@ -75,7 +79,7 @@ func (cmd *RestoreTarCommand) Run(ctx context.Context) (err error) {
 
 	// Validate arguments.
 	if cmd.Path == "" {
-		return fmt.Errorf("%w: -s flag required", UsageError)
+		return fmt.Errorf("%w: -s flag required", ErrUsage)
 	}
 	useStdin := cmd.Path == "-"
 
@@ -133,8 +137,13 @@ func (cmd *RestoreTarCommand) Run(ctx context.Context) (err error) {
 	if primary == nil {
 		return errors.New("no primary")
 	}
-	c := &gohttp.Client{}
+	c := &http.Client{}
+	// buf := new(bytes.Buffer)
+	mb512 := 2 << 29
+	buf := buffer.NewFileBuffer(mb512, cmd.TempDir)
+	defer buf.Reset()
 	for {
+		buf.Reset()
 		header, err := tarReader.Next()
 		if err == io.EOF {
 			break
@@ -175,21 +184,21 @@ func (cmd *RestoreTarCommand) Run(ctx context.Context) (err error) {
 			} else if len(fragmentNodes) == 0 {
 				return fmt.Errorf("no fragmentNodes available")
 			}
-
-			shardBytes, err := io.ReadAll(tarReader) // this feels wrong but works for now
+			_, err = io.Copy(buf, tarReader)
 			if err != nil {
-				return err
+				return errors.Wrap(err, "copying")
 			}
 			g, _ := errgroup.WithContext(ctx)
 			for _, node := range fragmentNodes {
 				node := node
+				rd, err := buf.NewReader()
+				if err != nil {
+					return err
+				}
 				g.Go(func() error {
-					client := &gohttp.Client{}
-					rd := bytes.NewReader(shardBytes)
 					logger.Printf("shard %v %v", shard, indexName)
 					url := node.URI.Path(fmt.Sprintf("/internal/restore/%v/%v", indexName, shard))
-					_, err = client.Post(url, "application/octet-stream", rd)
-					return err
+					return Post(ctx, url, "application/octet-stream", rd, nil)
 				})
 			}
 			if err := g.Wait(); err != nil {
@@ -207,20 +216,22 @@ func (cmd *RestoreTarCommand) Run(ctx context.Context) (err error) {
 				return fmt.Errorf("no fragmentNodes available")
 			}
 
-			shardBytes, err := io.ReadAll(tarReader) // this feels wrong but works for now
+			_, err = io.Copy(buf, tarReader)
 			if err != nil {
-				return err
+				return errors.Wrap(err, "copying")
 			}
+
 			g, _ := errgroup.WithContext(ctx)
 			for _, node := range fragmentNodes {
 				node := node
+				rd, err := buf.NewReader()
+				if err != nil {
+					return err
+				}
 				g.Go(func() error {
-					client := &gohttp.Client{}
-					rd := bytes.NewReader(shardBytes)
 					logger.Printf("dataframe shard %v %v", shard, indexName)
 					url := node.URI.Path(fmt.Sprintf("/internal/dataframe/restore/%v/%v", indexName, shard))
-					_, err = client.Post(url, "application/octet-stream", rd)
-					return err
+					return Post(ctx, url, "application/octet-stream", rd, nil)
 				})
 			}
 			if err := g.Wait(); err != nil {
@@ -236,20 +247,22 @@ func (cmd *RestoreTarCommand) Run(ctx context.Context) (err error) {
 			if err != nil {
 				return err
 			}
-			shardBytes, err := io.ReadAll(tarReader) // this feels wrong but works for now
+
+			_, err = io.Copy(buf, tarReader)
 			if err != nil {
-				return err
+				return errors.Wrap(err, "copying")
 			}
+
 			g, _ := errgroup.WithContext(ctx)
 			for _, node := range partitionNodes {
 				node := node
+				rd, err := buf.NewReader()
+				if err != nil {
+					return err
+				}
 				g.Go(func() error {
-					// rd := bytes.NewReader(shardBytes)
-					rd := func() (io.Reader, error) {
-						return bytes.NewReader(shardBytes), nil
-					}
-
-					return cmd.client.ImportIndexKeys(ctx, &node.URI, indexName, partitionID, false, rd)
+					url := node.URI.Path(fmt.Sprintf("/internal/translate/index/%s/%d", indexName, partitionID))
+					return Post(ctx, url, "application/octet-stream", rd, nil)
 				})
 			}
 			if err := g.Wait(); err != nil {
@@ -263,21 +276,20 @@ func (cmd *RestoreTarCommand) Run(ctx context.Context) (err error) {
 			switch action := record[4]; action {
 			case "translate":
 				logger.Printf("field keys %v %v", indexName, fieldName)
-				// needs to go to all nodes
-				shardBytes, err := io.ReadAll(tarReader) // this feels wrong but works for now
+				_, err = io.Copy(buf, tarReader)
 				if err != nil {
-					return err
+					return errors.Wrap(err, "copying")
 				}
 				g, _ := errgroup.WithContext(ctx)
 				for _, node := range nodes {
 					node := node
+					rd, err := buf.NewReader()
+					if err != nil {
+						return err
+					}
 					g.Go(func() error {
-						// rd := bytes.NewReader(shardBytes)
-						rd := func() (io.Reader, error) {
-							return bytes.NewReader(shardBytes), nil
-						}
-
-						return cmd.client.ImportFieldKeys(ctx, &node.URI, indexName, fieldName, false, rd)
+						url := node.URI.Path(fmt.Sprintf("/internal/translate/field/%s/%s", indexName, fieldName))
+						return Post(ctx, url, "application/octet-stream", rd, nil)
 					})
 				}
 				if err := g.Wait(); err != nil {
@@ -304,3 +316,33 @@ func (cmd *RestoreTarCommand) Run(ctx context.Context) (err error) {
 func (cmd *RestoreTarCommand) TLSHost() string { return cmd.Host }
 
 func (cmd *RestoreTarCommand) TLSConfiguration() server.TLSConfig { return cmd.TLS }
+
+func Post(ctx context.Context, url, contentType string, rd io.Reader, query map[string]string) error {
+	client := &http.Client{}
+	req, err := http.NewRequest(http.MethodPost, url, rd)
+	if err != nil {
+		return err
+	}
+	req.Header.Set("User-Agent", "pilosa/"+pilosa.Version)
+	req.Header.Set("Content-Type", contentType)
+	pilosa.AddAuthToken(ctx, &req.Header)
+
+	// appending to existing query args
+
+	q := req.URL.Query()
+	for k, v := range query {
+		q.Add(k, v)
+	}
+
+	// assign encoded query string to http request
+	req.URL.RawQuery = q.Encode()
+
+	resp, err := client.Do(req)
+	if err != nil {
+		fmt.Println("Errored when sending request to the server")
+		return err
+	}
+	defer resp.Body.Close()
+	_, err = ioutil.ReadAll(resp.Body) // drain the response
+	return err
+}

ctl/restore_tar_test.go

@@ -24,7 +24,7 @@ func TestRestoreTarCommand_Run(t *testing.T) {
 	cm.Host = hostport
 	cm.Path = ""
 	err := cm.Run(context.Background())
-	if !errors.Is(err, UsageError) {
+	if !errors.Is(err, ErrUsage) {
 		t.Fatalf("expected usage error with empty path, got %v", err)
 	}
 	cm.Path = "nonexistent-file"

ctl/restore_test.go

@@ -15,13 +15,13 @@ func TestRestoreCommand_Run(t *testing.T) {
 	cm := NewRestoreCommand(cmLog)
 	cm.Path = ""
 	err := cm.Run(context.Background())
-	if !errors.Is(err, UsageError) {
+	if !errors.Is(err, ErrUsage) {
 		t.Fatalf("expected usage error, got %v", err)
 	}
 	cm.Path = "foo"
 	cm.Concurrency = 0
 	err = cm.Run(context.Background())
-	if !errors.Is(err, UsageError) {
+	if !errors.Is(err, ErrUsage) {
 		t.Fatalf("expected usage error, got %v", err)
 	}
 }

ctl/server.go

@@ -49,6 +49,8 @@ func serverFlagSet(srv *server.Config, prefix string) *pflag.FlagSet {
 	flags.DurationVar((*time.Duration)(&srv.LongQueryTime), pre("long-query-time"), time.Duration(srv.LongQueryTime), "Duration that will trigger log and stat messages for slow queries. Zero to disable.")
 	flags.IntVar(&srv.QueryHistoryLength, pre("query-history-length"), srv.QueryHistoryLength, "Number of queries to remember in history.")
 	flags.Int64Var(&srv.MaxQueryMemory, pre("max-query-memory"), srv.MaxQueryMemory, "Maximum memory allowed per Extract() or SELECT query.")
+	flags.StringVar(&srv.VerChkAddress, pre("verchk-address"), srv.VerChkAddress, "Address to contact to check for latest version.")
+	flags.StringVar(&srv.UUIDFile, pre("uuid-file"), srv.UUIDFile, "File to store UUID used in checking latest version. If this is a relative path, the file will be stored in the server's data directory.")
 
 	// TLS
 	SetTLSConfig(flags, pre(""), &srv.TLS.CertificatePath, &srv.TLS.CertificateKeyPath, &srv.TLS.CACertPath, &srv.TLS.SkipVerify, &srv.TLS.EnableClientVerification)
@@ -110,7 +112,9 @@ func serverFlagSet(srv *server.Config, prefix string) *pflag.FlagSet {
 	// RBF specific flags. See pilosa/rbf/cfg/cfg.go for definitions.
 	srv.RBFConfig.DefineFlags(flags, prefix)
 
-	flags.BoolVar(&srv.SQL.EndpointEnabled, pre("sql.endpoint-enabled"), srv.SQL.EndpointEnabled, "Enable FeatureBase SQL /sql endpoint (default false)")
+	pfalse := false
+	flags.BoolVar(&pfalse, pre("sql.endpoint-enabled"), false, "Enable FeatureBase SQL /sql endpoint (default false)")
+	flags.MarkDeprecated("sql.endpoint-enabled", "sql.endpoint-enabled is deprecated")
 
 	flags.DurationVar(&srv.CheckInInterval, pre("check-in-interval"), srv.CheckInInterval, "Interval between check-ins to the Controller")
 

ctl/server_test.go

@@ -20,4 +20,10 @@ func TestBuildServerFlags(t *testing.T) {
 	if cm.Flags().Lookup("log-path").Name == "" {
 		t.Fatal("log-path flag is required")
 	}
+	if cm.Flags().Lookup("verchk-address").Name == "" {
+		t.Fatal("verchk-address flag is required")
+	}
+	if cm.Flags().Lookup("uuid-file").Name == "" {
+		t.Fatal("uuid-file flag is required")
+	}
 }

ctl/util.go

@@ -19,7 +19,7 @@ func (c ctlUsageError) Error() string {
 	return "usage error"
 }
 
-var UsageError ctlUsageError
+var ErrUsage ctlUsageError
 
 // startProfilingServer starts a server which handles /debug/pprof and
 // /debug/fgprof for use in utilities we might want to profile but

dax/Makefile

@@ -20,6 +20,11 @@ test-integration:
 testv-integration:
 	$(GO) test -v ./test/dax -count 1 -timeout 20m -run TestDAXIntegration/$(RUN)
 
+######################## Postgres Backend Stuff #########################
+
+run-postgres:
+	mkdir -p $(HOME)/pgdata
+	docker run -p 5432:5432 -v $(HOME)/pgdata:/var/lib/postgresql/data:Z  -e POSTGRES_PASSWORD=testpass --user=$(shell id -u) postgres:14.7 -c log_min_duration_statement=0
 
 ############################### AWS STUFF ###############################
 

dax/boltdb/boltdb.go

@@ -1,158 +0,0 @@
-// Package boltdb contains the boltdb implementations of the DAX interfaces.
-package boltdb
-
-import (
-	"context"
-	"os"
-	"path/filepath"
-	"strings"
-	"time"
-
-	"github.com/featurebasedb/featurebase/v3/errors"
-	bolt "go.etcd.io/bbolt"
-)
-
-const (
-	ErrFmtBucketNotFound = "boltdb: bucket '%s' not found"
-)
-
-type Bucket []byte
-
-// DB represents the database connection.
-type DB struct {
-	db     *bolt.DB
-	ctx    context.Context // background context
-	cancel func()          // cancel background context
-
-	// Datasource name.
-	DSN string
-
-	// Destination for events to be published.
-	// EventService wtf.EventService
-
-	// Returns the current time. Defaults to time.Now().
-	// Can be mocked for tests.
-	Now func() time.Time
-
-	filePath string
-
-	// bucketQueue contains a list of buckets to create upon Open.
-	bucketQueue []Bucket
-}
-
-// NewDB returns a new instance of DB associated with the given datasource name.
-func NewDB(dsn string) *DB {
-	db := &DB{
-		DSN: dsn,
-		Now: time.Now,
-
-		//EventService: wtf.NopEventService(),
-	}
-	db.ctx, db.cancel = context.WithCancel(context.Background())
-	return db
-}
-
-// NewSvcBolt gets, opens, and creates buckets for a boltDB for a
-// particular named service (the data file will be named after the
-// service).
-func NewSvcBolt(dir, svc string, buckets ...Bucket) (*DB, error) {
-	dir = strings.TrimPrefix(dir, "file:")
-	filename := filepath.Join(dir, svc+".boltdb")
-	db := NewDB("file:" + filename)
-	db.RegisterBuckets(buckets...)
-	err := db.Open()
-	return db, errors.Wrap(err, "opening")
-}
-
-// path returns the file path to the boltdb database file.
-func (db *DB) path() (string, error) {
-	if !strings.HasPrefix(db.DSN, "file:") {
-		return "", errors.New(errors.ErrUncoded, "boltdb package only supports a DSN beginning with `file:`")
-	}
-
-	return db.DSN[5:], nil
-}
-
-// RegisterBuckets queues up the buckets to be created when the database is
-// first opened.
-func (db *DB) RegisterBuckets(buckets ...Bucket) {
-	db.bucketQueue = append(db.bucketQueue, buckets...)
-}
-
-// InitializeBuckets creates the given buckets if they do not already exist.
-func (db *DB) InitializeBuckets(buckets ...Bucket) (err error) {
-	return db.db.Update(func(tx *bolt.Tx) error {
-		for _, bucket := range buckets {
-			if _, err := tx.CreateBucketIfNotExists(bucket); err != nil {
-				return errors.Wrapf(err, "creating bucket: %s", bucket)
-			}
-		}
-		return nil
-	})
-}
-
-// Open opens the database connection.
-func (db *DB) Open() (err error) {
-	path, err := db.path()
-	if err != nil {
-		return errors.Wrap(err, "getting path from DSN")
-	}
-
-	if err := os.MkdirAll(filepath.Dir(path), 0777); err != nil {
-		return errors.Wrapf(err, "mkdir %s", filepath.Dir(path))
-	} else if db.db, err = bolt.Open(path, 0666, &bolt.Options{Timeout: 1 * time.Second}); err != nil {
-		return errors.Wrapf(err, "open file: %s", err)
-	}
-
-	// cache the path in db.filePath.
-	db.filePath = path
-
-	if err := db.InitializeBuckets(db.bucketQueue...); err != nil {
-		return errors.Wrap(err, "initializing buckets")
-	}
-
-	// Reset the bucketQueue.
-	db.bucketQueue = make([]Bucket, 0)
-
-	return nil
-}
-
-// Close closes the database connection.
-func (db *DB) Close() (err error) {
-	return db.db.Close()
-}
-
-// BeginTx starts a transaction and returns a wrapper Tx type. This type
-// provides a reference to the database and a fixed timestamp at the start of
-// the transaction. The timestamp allows us to mock time during tests as well.
-// The wrapper also contains the context.
-func (db *DB) BeginTx(ctx context.Context, writable bool) (*Tx, error) {
-	tx, err := db.db.Begin(writable)
-	if err != nil {
-		return nil, err
-	}
-
-	// Return wrapper Tx that includes the transaction start time.
-	return &Tx{
-		Tx:  tx,
-		ctx: ctx,
-		db:  db,
-		now: db.Now().UTC().Truncate(time.Second),
-	}, nil
-}
-
-// Tx wraps the SQL Tx object to provide a timestamp at the start of the transaction.
-type Tx struct {
-	*bolt.Tx
-	ctx context.Context
-	db  *DB
-	now time.Time
-}
-
-func (tx *Tx) Context() context.Context {
-	return tx.ctx
-}
-
-func (db *DB) Path() string {
-	return db.filePath
-}

dax/boltdb/boltdb_test.go

@@ -1,17 +0,0 @@
-package boltdb_test
-
-import (
-	"testing"
-
-	"github.com/featurebasedb/featurebase/v3/dax/test/boltdb"
-)
-
-// Ensure the test database can open & close.
-func TestDB(t *testing.T) {
-	db := boltdb.MustOpenDB(t)
-	defer boltdb.MustCloseDB(t, db)
-
-	t.Cleanup(func() {
-		boltdb.CleanupDB(t, db.Path())
-	})
-}