Update README.md

changed links to the community help repo

.github/workflows/ci.yml

@@ -36,7 +36,7 @@ jobs:
         with:
           # Optional: version of golangci-lint to use in form of v1.2 or v1.2.3 or `latest` to use the latest version
           # version: v1.29
-          args: --timeout=5m
+          args: --timeout=8m
 
   validate:
     name: Code Checks

.gitignore

@@ -84,3 +84,4 @@ dax/dax-data
 
 coverage-from-docker
 *.client_id.txt
+

.gitlab/.gitlab-ci.yml

@@ -12,10 +12,22 @@ include:
   - template: Security/License-Scanning.gitlab-ci.yml
   - template: Security/Dependency-Scanning.gitlab-ci.yml
 
+default:
+  retry:
+    max: 2 # This is confusing but this means "3 runs at max".
+    when:
+      - unknown_failure
+      - api_failure
+      - runner_system_failure
+      - job_execution_timeout
+      - stuck_or_timeout_failure
+
 variables:
   GOVERSION: "1.19.3"
   GOFUTURE: "latest"
   CI_IMAGE: "${CI_REGISTRY_IMAGE}/ci-builder:0.0.1"
+  CI_PRE_CLONE_SCRIPT: |
+    set -x
 
 stages:
   - ci_image_build
@@ -161,16 +173,48 @@ build featurebase:
       for goos in "darwin" "linux"; do
         for goarch in "amd64" "arm64"; do
           GOOS="${goos}" GOARCH="${goarch}" make build FLAGS="-o featurebase_${goos}_${goarch}"
-          GOOS="${goos}" GOARCH="${goarch}" make build-fbsql FLAGS="-o fbsql_${goos}_${goarch}"
         done
       done
   artifacts:
     paths:
       - featurebase_*
-      - fbsql_*
   needs:
     - job: build lattice
 
+build fbsql amd64:
+  stage: test
+  variables:
+    BUILD_NAME: build_${CI_COMMIT_SHA}_${CI_CONCURRENT_ID}
+  tags:
+    - shell
+  rules:
+    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
+  script:
+    - export SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct)
+    - date
+    - GOOS="linux" GOARCH="amd64" make docker-build-fbsql BUILD_CGO=1
+    - GOOS="darwin" GOARCH="amd64" make docker-build-fbsql
+  artifacts:
+    paths:
+      - ./build/fbsql_*
+
+build fbsql arm64:
+  stage: test
+  variables:
+    BUILD_NAME: build_${CI_COMMIT_SHA}_${CI_CONCURRENT_ID}
+  tags:
+    - shell-arm64
+  rules:
+    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
+  script:
+    - export SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct)
+    - date
+    - GOOS="linux" GOARCH="arm64" make docker-build-fbsql BUILD_CGO=1
+    - GOOS="darwin" GOARCH="arm64" make docker-build-fbsql
+  artifacts:
+    paths:
+      - ./build/fbsql_*
+
 build amd container fb:
   stage: test
   tags:
@@ -211,6 +255,16 @@ run go tests race:
   stage: nonblocking # don't let this job block any other jobs because it takes much longer than the other tests.
   image: golang:$GOVERSION
   extends: .go-cache
+  variables:
+    FEATUREBASE_CONTROLLER_CONFIG_SQLDB_DATABASE: run_go_tests_race
+    POSTGRES_DB: run_go_tests_race
+    FEATUREBASE_CONTROLLER_CONFIG_SQLDB_USER: postgres
+    POSTGRES_USER: postgres
+    FEATUREBASE_CONTROLLER_CONFIG_SQLDB_PASSWORD: $POSTGRES_PASSWORD
+    POSTGRES_PASSWORD: $POSTGRES_PASSWORD
+    FEATUREBASE_CONTROLLER_CONFIG_SQLDB_HOST: postgres
+  services:
+    - postgres:14.7
   rules:
     - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
   needs: ["smoke build"] # we do block on smoke build though bc it's pretty dumb to test stuff if it doesn't build
@@ -232,6 +286,16 @@ run go tests:
   stage: test
   image: golang:$GOVERSION
   extends: .go-cache
+  variables:
+    FEATUREBASE_CONTROLLER_CONFIG_SQLDB_DATABASE: run_go_tests
+    POSTGRES_DB: run_go_tests
+    FEATUREBASE_CONTROLLER_CONFIG_SQLDB_USER: postgres
+    POSTGRES_USER: postgres
+    FEATUREBASE_CONTROLLER_CONFIG_SQLDB_PASSWORD: $POSTGRES_PASSWORD
+    POSTGRES_PASSWORD: $POSTGRES_PASSWORD
+    FEATUREBASE_CONTROLLER_CONFIG_SQLDB_HOST: postgres
+  services:
+    - postgres:14.7
   rules:
     - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
   script:
@@ -251,42 +315,45 @@ run go tests:
 run go tests dax/test/dax:
   stage: test
   image: golang:$GOVERSION
+  extends: .go-cache
   tags:
-    - aws
+    - docker
+  variables:
+    FEATUREBASE_CONTROLLER_CONFIG_SQLDB_DATABASE: run_go_tests_dax
+    POSTGRES_DB: run_go_tests_dax
+    FEATUREBASE_CONTROLLER_CONFIG_SQLDB_USER: postgres
+    POSTGRES_USER: postgres
+    FEATUREBASE_CONTROLLER_CONFIG_SQLDB_PASSWORD: $POSTGRES_PASSWORD
+    POSTGRES_PASSWORD: $POSTGRES_PASSWORD
+    FEATUREBASE_CONTROLLER_CONFIG_SQLDB_HOST: postgres
+  services:
+    - postgres:14.7
   rules:
     - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
   script:
     - echo "Building FB and Datagen docker images for DAX tests"
     - PKG_LIST=$(go list ./... | grep -Ev 'internal/clustertests|simulacraData' | paste -s -d, -)
+    - export TMPDIR=/mnt/ramdisk/test-$CI_JOB_ID
+    - mkdir -p $TMPDIR
     - go test -coverprofile=coverage-dax-integration.out -covermode=atomic -coverpkg=${PKG_LIST} -timeout=20m ./dax/test/dax
+  after_script:
+    - rm -rf /mnt/ramdisk/test-$CI_JOB_ID
   artifacts:
     paths:
       - coverage-dax-integration.out
 
-.idk_changed:
-  rules:
-    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
-      changes:
-        compare_to: refs/heads/master
-        paths:
-          - idk/**
-          - client/**
-          - batch/**
-
 # idk tests
 run go tests idk race:
   variables:
     PROJECT: race_${CI_CONCURRENT_ID}
   stage: nonblocking
   retry: 1
-  rules:
-    - !reference [.idk_changed, rules]
   script:
     - echo "Running test-all-race"
     - cd ./idk/
     - echo $PROJECT
     - echo $CI_COMMIT_REF_SLUG
-    - BRANCH_NAME=${CI_COMMIT_REF_SLUG} make test-all-race
+    - BRANCH_NAME=${CI_COMMIT_REF_SLUG} IDK_FEATUREBASE_TAG=${CI_COMMIT_TAG} IDK_FEATUREBASE_HASH=${CI_COMMIT_SHA} make test-all-race
   after_script:
     - cd ./idk/
     - make save-pilosa-logs
@@ -306,14 +373,12 @@ run go tests idk shard transactional:
     PROJECT: shardttrans_${CI_CONCURRENT_ID}
   stage: nonblocking
   retry: 1
-  rules:
-    - !reference [.idk_changed, rules]
   script:
     - echo "Running shard transactional tests"
     - cd ./idk/
     - echo $PROJECT
     - echo $CI_COMMIT_REF_SLUG
-    - BRANCH_NAME=${CI_COMMIT_REF_SLUG} make test-all
+    - BRANCH_NAME=${CI_COMMIT_REF_SLUG} IDK_FEATUREBASE_TAG=${CI_COMMIT_TAG} IDK_FEATUREBASE_HASH=${CI_COMMIT_SHA} make test-all
   after_script:
     - cd ./idk/
     - make save-pilosa-logs
@@ -339,13 +404,11 @@ run go tests idk 533:
     - cd ./idk/
     - echo $PROJECT
     - echo $CI_COMMIT_REF_SLUG
-    - CONFLUENT_VERSION=5.3.3 BRANCH_NAME=${CI_COMMIT_REF_SLUG} make test-all
+    - CONFLUENT_VERSION=5.3.3 BRANCH_NAME=${CI_COMMIT_REF_SLUG} IDK_FEATUREBASE_TAG=${CI_COMMIT_TAG} IDK_FEATUREBASE_HASH=${CI_COMMIT_SHA} make test-all
   after_script:
     - cd ./idk/
     - make save-pilosa-logs
     - make shutdown
-  rules:
-    - !reference [.idk_changed, rules]
   tags:
     - shell
     - aws
@@ -365,13 +428,11 @@ run go tests idk sasl:
     - cd ./idk/
     - echo $PROJECT
     - echo $CI_COMMIT_REF_SLUG
-    - BRANCH_NAME=${CI_COMMIT_REF_SLUG} make test-all-kafka-sasl
+    - BRANCH_NAME=${CI_COMMIT_REF_SLUG} IDK_FEATUREBASE_TAG=${CI_COMMIT_TAG} IDK_FEATUREBASE_HASH=${CI_COMMIT_SHA} make test-all-kafka-sasl
   after_script:
     - cd ./idk/
     - make save-pilosa-logs
     - make shutdown
-  rules:
-    - !reference [.idk_changed, rules]
   tags:
     - shell
     - aws
@@ -637,14 +698,16 @@ s3 dump:
     - |
       for goos in "darwin" "linux"; do
         for goarch in "amd64" "arm64"; do
-          for binary in "featurebase" "fbsql"; do
-            aws s3 cp ${binary}_${goos}_${goarch} s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/${CI_COMMIT_SHORT_SHA}/${binary}_${goos}_${goarch}
-            aws s3 cp ${binary}_${goos}_${goarch} s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/_latest/${binary}_${goos}_${goarch}
-          done
+          aws s3 cp featurebase_${goos}_${goarch} s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/${CI_COMMIT_SHORT_SHA}/featurebase_${goos}_${goarch}
+          aws s3 cp featurebase_${goos}_${goarch} s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/_latest/featurebase_${goos}_${goarch}
+          aws s3 cp ./build/fbsql_${goos}_${goarch} s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/${CI_COMMIT_SHORT_SHA}/fbsql_${goos}_${goarch}
+          aws s3 cp ./build/fbsql_${goos}_${goarch} s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/_latest/fbsql_${goos}_${goarch}
         done
       done
   needs:
     - job: build featurebase
+    - job: build fbsql amd64
+    - job: build fbsql arm64
 
 s3 dump tag:
   stage: post build
@@ -670,6 +733,7 @@ s3 dump tag:
           echo "Directory ${dir}"
           mkdir $dir
           mv featurebase_${goos}_${goarch} ${dir}/featurebase
+          mv ./build/fbsql_${goos}_${goarch} ${dir}/fbsql
           cp NOTICE install/featurebase.conf install/featurebase.*.service ${dir}/
           tar cvzf ${dir}.tar.gz ${dir}
           aws s3 cp ${dir} s3://${LOCATION}/${CI_COMMIT_TAG}/${dir}/ --recursive
@@ -679,3 +743,5 @@ s3 dump tag:
 
   needs:
     - job: build featurebase
+    - job: build fbsql amd64
+    - job: build fbsql arm64

.golangci.yml

@@ -10,9 +10,26 @@ run:
     - pql/pql.peg.go
 linters:
   enable:
+    # Recommended to be enabled by default (https://golangci-lint.run).
+    # - errcheck (lots to fix)
+    - gosimple
     - govet
-    - gofmt
+    - ineffassign
     - staticcheck
+    - typecheck
+    # - unused (about 20 to fix)
+
+    # Additional linters we choose to enable.
+    # - bodyclose (lots to fix, but we should)
+    - errchkjson
+    - errname
+    - gofmt
+    # - misspell (lots to fix, but we should)
+    - prealloc
+    # - predeclared (20 to fix)
+    # - stylecheck (quite a lot to fix, but we should definitely work on this)
+    - stylecheck
+    # - unconvert (not at all critical, but makes for cleaner code)
   enable-all: false
   disable-all: true
 
@@ -50,6 +67,12 @@ linters-settings:
       - shadow
     disable-all: false
 
+  stylecheck:
+    # ST1000: at least one file in a package should have a package comment
+    # ST1003: golang naming standards
+    # ST1016: methods on the same type should have the same receiver name
+    # ST1020: comment on exported function
+    checks: ["all", "-ST1000", "-ST1003", "-ST1016", "-ST1020"]
 
 issues:
   exclude-use-default: false

Dockerfile-fbsql

@@ -0,0 +1,48 @@
+ARG GO_VERSION=1.19
+
+FROM golang:1.19-buster as builder
+
+WORKDIR /
+RUN apt-get update -y -qq && apt-get install -y -qq \
+  build-essential \
+  git \
+  musl-tools \
+  netcat \
+  unixodbc \
+  unixodbc-dev \
+  && rm -rf /var/lib/apt/lists/*
+RUN ["git", "clone", "https://github.com/edenhill/librdkafka.git"]
+WORKDIR /librdkafka
+RUN ./configure --prefix /usr && \
+  make && \
+  make install
+
+WORKDIR /featurebase
+
+COPY . .
+
+ARG MAKE_FLAGS
+ARG GO_BUILD_FLAGS
+ARG SOURCE_DATE_EPOCH
+
+WORKDIR /featurebase/
+
+ENV SOURCE_DATE_EPOCH=${SOURCE_DATE_EPOCH}
+RUN make build-fbsql GO_BUILD_FLAGS="-mod=vendor ${GO_BUILD_FLAGS}" ${MAKE_FLAGS}
+
+FROM ubuntu:20.04 as runner
+
+RUN apt-get update -y -qq && apt-get install -y -qq \
+  ca-certificates \
+  musl-tools \
+  netcat \
+  unixodbc-dev \
+  && rm -rf /var/lib/apt/lists/*
+
+COPY --from=builder /featurebase/fbsql /usr/local/bin/
+
+# Verify that the linker can find everything.
+FROM runner as linkcheck
+RUN if [ -e /usr/local/bin/fbsql ] ; then ldd /usr/local/bin/fbsql; fi
+
+FROM runner

Makefile

@@ -1,4 +1,4 @@
-.PHONY: build clean build-lattice cover cover-viz default docker docker-build docker-tag-push generate generate-protoc generate-pql generate-statik generate-stringer install install-protoc-gen-gofast install-protoc install-statik install-peg test docker-login
+.PHONY: build clean build-lattice cover cover-viz default docker docker-build docker-build-fbsql docker-tag-push generate generate-protoc generate-pql generate-statik generate-stringer install install-protoc-gen-gofast install-protoc install-statik install-peg test docker-login
 
 SHELL := /bin/bash
 VERSION := $(shell git describe --tags 2> /dev/null || echo unknown)
@@ -19,6 +19,7 @@ SHARD_WIDTH = 20
 COMMIT := $(shell git describe --exact-match >/dev/null 2>&1 || git rev-parse --short HEAD)
 LDFLAGS="-X github.com/featurebasedb/featurebase/v3.Version=$(VERSION) -X github.com/featurebasedb/featurebase/v3.BuildTime=$(BUILD_TIME) -X github.com/featurebasedb/featurebase/v3.Variant=$(VARIANT) -X github.com/featurebasedb/featurebase/v3.Commit=$(COMMIT) -X github.com/featurebasedb/featurebase/v3.TrialDeadline=$(TRIAL_DEADLINE)"
 GO_VERSION=1.19
+GO_BUILD_FLAGS=
 DOCKER_BUILD= # set to 1 to use `docker-build` instead of `build` when creating a release
 BUILD_TAGS += 
 TEST_TAGS = roaringparanoia
@@ -50,7 +51,7 @@ version:
 
 # We build a list of packages that omits the IDK and batch packages because
 # those packages require fancy environment setup.
-GOPACKAGES := $(shell $(GO) list ./... | grep -v "/idk" | grep -v "/batch")
+GOPACKAGES := $(shell $(GO) list ./... | grep -v "/v3/idk" | grep -v "/v3/batch")
 
 # Run test suite
 test:
@@ -120,11 +121,6 @@ cover-viz: cover
 build:
 	$(GO) build -tags='$(BUILD_TAGS)' -ldflags $(LDFLAGS) $(FLAGS) ./cmd/featurebase
 
-# Build fbsql
-build-fbsql:
-	$(GO) build -tags='$(BUILD_TAGS)' -ldflags $(LDFLAGS) $(FLAGS) ./cmd/fbsql
-
-
 package:
 	GOOS=$(GOOS) GOARCH=$(GOARCH) $(MAKE) build
 	GOOS=$(GOOS) GOARCH=$(GOARCH) $(MAKE) build-fbsql
@@ -167,7 +163,7 @@ install-idk:
 	$(MAKE) -C ./idk install
 
 install-fbsql:
-	$(GO) install ./cmd/fbsql
+	CGO_ENABLED=1 $(GO) install ./cmd/fbsql
 
 # Build the lattice assets
 build-lattice:
@@ -237,6 +233,13 @@ docker-image-featurebase: vendor
 		--file Dockerfile-dax \
 		--tag dax/featurebase .
 
+docker-image-featurebase-linux-amd64: vendor
+	docker build \
+	    --build-arg GO_VERSION=$(GO_VERSION) \
+	    --platform linux/amd64 \
+	    --file Dockerfile-dax \
+	    --tag dax/featurebase .
+
 docker-image-featurebase-test: vendor
 	docker build \
 	    --build-arg GO_VERSION=$(GO_VERSION) \
@@ -263,7 +266,12 @@ docker-image-featurebase-quick: build-for-quick
 docker-image-datagen: vendor
 	docker build --tag dax/datagen --file Dockerfile-datagen .
 
+get-account-id:
+	$(eval AWS_ACCOUNTID := $(shell aws sts get-caller-identity --output=json | jq -r .Account))
+
+
 ecr-push-featurebase: docker-login
+	echo "Pushing to account $(AWS_ACCOUNTID), profile $(AWS_PROFILE)"
 	docker tag dax/featurebase:latest $(AWS_ACCOUNTID).dkr.ecr.us-east-2.amazonaws.com/dax/featurebase:latest
 	docker push $(AWS_ACCOUNTID).dkr.ecr.us-east-2.amazonaws.com/dax/featurebase:latest
 
@@ -271,7 +279,7 @@ ecr-push-datagen: docker-login
 	docker tag dax/datagen:latest $(AWS_ACCOUNTID).dkr.ecr.us-east-2.amazonaws.com/dax/datagen:latest
 	docker push $(AWS_ACCOUNTID).dkr.ecr.us-east-2.amazonaws.com/dax/datagen:latest
 
-docker-login:
+docker-login: get-account-id
 	aws ecr get-login-password --region us-east-2 | docker login --username AWS --password-stdin $(AWS_ACCOUNTID).dkr.ecr.us-east-2.amazonaws.com
 
 # Create docker image (alias)
@@ -345,3 +353,53 @@ test-external-lookup:
 
 bnf:
 	ebnf2railroad --no-overview-diagram  --no-optimizations ./sql3/sql3.ebnf
+
+#################################
+# fbsql builds in docker
+#################################
+
+# This allows multiple concurrent builds to happen in CI without
+# creating container name conflicts and such. (different BUILD_NAMEs
+# are passed in from gitlab-ci.yml)
+BUILD_NAME ?= fbsql-build
+
+LDFLAGS_STATIC="-linkmode external -extldflags \"-static\" -X 'github.com/featurebasedb/featurebase/v3/fbsql.Version=$(VERSION)' -X 'github.com/featurebasedb/featurebase/v3/fbsql.BuildTime=$(BUILD_TIME)' "
+
+UNAME_P := $(shell uname -p)
+BUILD_CGO ?= 0
+
+# Build fbsql
+build-fbsql:
+	@echo GOOS=$(GOOS) GOARCH=$(GOARCH) uname -p=$(UNAME_P) build_cgo=$(BUILD_CGO)
+ifeq ($(BUILD_CGO), 0)
+	make build-fbsql-non-cgo
+endif
+ifeq ($(BUILD_CGO), 1)
+	make build-fbsql-cgo
+endif
+
+build-fbsql-non-cgo:
+	CGO_ENABLED=0 $(GO) build -ldflags $(LDFLAGS) $(GO_BUILD_FLAGS) -o fbsql ./cmd/fbsql
+
+build-fbsql-cgo:
+ifeq ($(GOARCH), arm64)
+	CGO_ENABLED=1 $(GO) build -tags dynamic $(GO_BUILD_FLAGS) -o fbsql ./cmd/fbsql
+endif
+ifeq ($(GOARCH), amd64)
+	CC=/usr/bin/musl-gcc CGO_ENABLED=1 $(GO) build -tags "musl static" -ldflags $(LDFLAGS_STATIC) $(GO_BUILD_FLAGS) -o fbsql ./cmd/fbsql
+endif
+
+docker-build-fbsql: vendor
+	DOCKER_BUILDKIT=0 docker build \
+		--file Dockerfile-fbsql \
+	    --build-arg GO_VERSION=$(GO_VERSION) \
+	    --build-arg MAKE_FLAGS="GOOS=$(GOOS) GOARCH=$(GOARCH) BUILD_CGO=$(BUILD_CGO)" \
+	    --build-arg GO_BUILD_FLAGS=$(GO_BUILD_FLAGS) \
+		--build-arg SOURCE_DATE_EPOCH=$(SOURCE_DATE_EPOCH) \
+	    --target builder \
+	    --tag fbsql:$(BUILD_NAME) .
+	mkdir -p build
+	docker create --name $(BUILD_NAME) fbsql:$(BUILD_NAME)
+	docker cp $(BUILD_NAME):/featurebase/fbsql ./build/fbsql_$(GOOS)_$(GOARCH)
+	docker rm $(BUILD_NAME)
+

OPENSOURCE.md

@@ -0,0 +1,45 @@
+## User Contribution Guidelines for FeatureBase
+
+Thank you for your interest in contributing to FeatureBase! We appreciate your support in making this open-source project even better. Here are some guidelines to help you get started with contributing to FeatureBase:
+
+1. Familiarize Yourself with the Project:
+   - Visit the FeatureBase website at www.featurebase.com to understand the project's goals, capabilities, and features.
+   - Read the documentation available on the website, including the installation guide, configuration options, and data modeling concepts.
+   - Explore the codebase by cloning the repository and reviewing the source code.
+
+2. Join the Community:
+   - Visit the FeatureBase community page at https://www.featurebase.com/community to learn more about the project's community and how to get involved.
+   - Join the Discord server at https://discord.gg/FBn2vEp7Na to chat with other contributors and users, ask questions, and share your ideas.
+
+3. Set Up Your Development Environment:
+   - Ensure you have Go installed on your machine. Make sure your shell's search path includes the go/bin directory.
+   - Clone the FeatureBase repository or download it as a zip file from the repository's page.
+   - Follow the "Build FeatureBase Server from source" instructions in the README file to compile the server binary and the ingester binaries.
+
+4. Choose a Contribution Area:
+   - Identify the area you'd like to contribute to, such as bug fixes, new features, performance improvements, documentation updates, or community support.
+   - Check the issue tracker on the repository or the FeatureBase community for open issues or feature requests that align with your interests and skills. Alternatively, propose your own idea by creating a new issue.
+
+5. Create a New Branch:
+   - Before making any changes, create a new branch in the repository's Git repository. This branch will contain your contributions.
+   - Give your branch a descriptive name that reflects the nature of your contribution.
+
+6. Make Your Changes:
+   - Follow the coding style and conventions used in the existing codebase.
+   - Write clear and concise commit messages for each logical change.
+   - If you're introducing new features or modifying existing behavior, make sure to update the documentation to reflect the changes.
+
+7. Test Your Changes:
+   - Run the existing test suite to ensure that your modifications do not introduce any regressions.
+   - If applicable, write additional tests to cover the changes you made.
+   - Document any new testing procedures required for your contribution.
+
+8. Submitting Your Contribution:
+   - Push your branch to the main repository or create a fork and submit a pull request to the main repository.
+   - Provide a detailed description of your changes, including the problem you solved and the approach you took.
+   - Be responsive to any feedback or suggestions provided by the project maintainers or other contributors.
+   - Once your contribution is approved, it will be reviewed and merged into the main codebase.
+
+Please note that by contributing to FeatureBase, you agree that your contributions will be licensed under the Apache 2.0 license, which governs the project.
+
+Thank you for considering contributing to FeatureBase! Your contributions are valuable and help improve the project for everyone.

README.md

@@ -1,4 +1,10 @@
-# FeatureBase
+# FeatureBase Community
+
+FeatureBase Community is now archived and no longer maintained.
+
+* [FeatureBase Community Help](https://github.com/FeatureBaseDB/FB-community-help)
+
+
 
 ## Pilosa is now FeatureBase
 
@@ -10,6 +16,8 @@ For more information about FeatureBase, please visit [www.featurebase.com][HomeP
 
 ## Getting Started
 
+* [Learn how to install FeatureBase Community](https://github.com/FeatureBaseDB/FB-community-help/blob/main/docs/community/com-getstart/com-getstart-home.md)
+
 ### Build FeatureBase Server from source
 
 0. Install go. Ensure that your shell's search path includes the go/bin directory.
@@ -19,42 +27,20 @@ For more information about FeatureBase, please visit [www.featurebase.com][HomeP
 4. Run `featurebase server --handler.allowed-origins=http://localhost:3000` to run FeatureBase server with default settings (learn more about configuring FeatureBase at the link below). The `--handler.allowed-origins` parameter allows the standalone web UI to talk to the server; this can be omitted if the web UI is not needed.
 5. Run `curl localhost:10101/status` to verify the server is running and accessible.
 
-### Ingest Data and Query
-
-1. Run 
-```
-molecula-consumer-csv \
-    --index repository \
-    --header "language__ID_F,project_id__ID_F" \
-    --id-field project_id \
-    --batch-size 1000 \
-    --files example.csv
-```
-
-This will ingest the `example.csv` file into a FeatureBase table called `repository`. If the table does not exist, it will be automatically created. Learn more about [ingesting data into FeatureBase][Ingest]
-
-2. Query your data. 
-```
-curl localhost:10101/index/repository/query \
-     -X POST \
-     -d 'Row(example=5)'
-```
-Learn about supported [SQL][SQL], native [Pilosa Query Language (PQL)][PQL].
-
 ### Data Model
 
 Because FeatureBase is built on bitmaps, there is bit of a learning curve to grasp how your data is represented. 
-[Learn about Data Modeling][DataModel].
 
-### More Information
+* [Learn about Data Modeling](https://github.com/FeatureBaseDB/FB-community-help/blob/main/docs/concepts/concepts-home.md)
 
-[Installation][Install]
 
-[Configuration][Config]
+### Ingest Data and Query
+
+* [Learn how to ingest data from multiple data sources](https://github.com/FeatureBaseDB/FB-community-help/blob/main/docs/community/com-ingest/com-ingest-manage.md)
 
 ## Community
 
-You can email us at community@featurebase.com or learn more about contributing at [https://www.featurebase.com/community][Community].
+You can email us at community@featurebase.com and [learn more about contributing](https://github.com/FeatureBaseDB/featurebase/blob/master/OPENSOURCE.md).
 
 Chat with us: [https://discord.gg/FBn2vEp7Na][Discord]
 
@@ -73,13 +59,14 @@ A lot has changed since the days of Pilosa. This list highlights some new capabi
 
 FeatureBase is licensed under the [Apache License, Version 2.0][License]
 
-[Community]: http://www.featurebase.com/community?utm_campaign=Open%20Source&utm_source=GitHub
-[Config]: https://docs.featurebase.com/docs/community/com-config/old-config-flags/?utm_campaign=Open%20Source&utm_source=GitHub
-[DataModel]: https://docs.featurebase.com/docs/concepts/overview-data-modeling/?utm_campaign=Open%20Source&utm_source=GitHub
+[Community]: https://github.com/FeatureBaseDB/FB-community-help/tree/main
+[Install]:https://github.com/FeatureBaseDB/FB-community-help/blob/main/docs/community/com-getstart/com-getstart-home.md
+[Config]: https://github.com/FeatureBaseDB/FB-community-help/tree/main/docs/community/com-config
+[DataModel]: https://github.com/FeatureBaseDB/FB-community-help/blob/main/docs/concepts/concepts-home.md
 [Discord]: https://discord.gg/FBn2vEp7Na
 [HomePage]: http://featurebase.com?utm_campaign=Open%20Source&utm_source=GitHub
-[Ingest]: https://docs.featurebase.com/docs/community/com-ingest/old-ingesters/?utm_campaign=Open%20Source&utm_source=GitHub
-[Install]: https://docs.featurebase.com/docs/community/com-home/#install-featurebase-community?utm_campaign=Open%20Source&utm_source=GitHub
+[Ingest]: https://github.com/FeatureBaseDB/FB-community-help/blob/main/docs/community/com-ingest/com-ingest-manage.md
+ 
 [License]: http://www.apache.org/licenses/LICENSE-2.0
 [PQL]: https://docs.featurebase.com/docs/pql-guide/pql-home/?utm_campaign=Open%20Source&utm_source=GitHub
 [SQL]: https://docs.featurebase.com/docs/sql-guide/sql-guide-home/?utm_campaign=Open%20Source&utm_source=GitHub

api.go

@@ -367,8 +367,8 @@ func (api *API) DeleteIndex(ctx context.Context, indexName string) error {
 }
 
 // CreateField makes the named field in the named index with the given options.
-// This method currently only takes a single functional option, but that may be
-// changed in the future to support multiple options.
+//
+// The resulting field will always have TrackExistence set.
 func (api *API) CreateField(ctx context.Context, indexName string, fieldName string, opts ...FieldOption) (*Field, error) {
 	span, _ := tracing.StartSpanFromContext(ctx, "API.CreateField")
 	defer span.Finish()
@@ -381,6 +381,11 @@ func (api *API) CreateField(ctx context.Context, indexName string, fieldName str
 	// authN/Z info
 	requestUserID, _ := fbcontext.UserID(ctx) // requestUserID is "" if not in ctx
 
+	// newFieldOptions is also used in the path through the index creating
+	// a field from an update from DAX, so it can't assume it can always
+	// override this. But we're the call path for creating new fields, and
+	// new fields should always have TrackExistence on.
+	opts = append(opts, OptFieldTrackExistence())
 	// Apply and validate functional options.
 	fo, err := newFieldOptions(opts...)
 	if err != nil {
@@ -494,16 +499,9 @@ func importWorker(importWork chan importJob) {
 	for j := range importWork {
 		err := func() (err0 error) {
 			for viewName, viewData := range j.req.Views {
-				// The logic here corresponds to the logic in fragment.cleanViewName().
-				// Unfortunately, the logic in that method is not completely exclusive
-				// (i.e. an "other" view named with format YYYYMMDD would be handled
-				// incorrectly). One way to address this would be to change the logic
-				// overall so there weren't conflicts. For now, we just
-				// rely on the field type to inform the intended view name.
-				if viewName == "" {
-					viewName = viewStandard
-				} else if j.field.Type() == FieldTypeTime {
-					viewName = fmt.Sprintf("%s_%s", viewStandard, viewName)
+				viewName, err0 = j.field.cleanupViewName(viewName)
+				if err0 != nil {
+					return err0
 				}
 				if len(viewData) == 0 {
 					return fmt.Errorf("no data to import for view: %s", viewName)
@@ -1316,7 +1314,6 @@ type ImportOptions struct {
 	Clear          bool
 	IgnoreKeyCheck bool
 	Presorted      bool
-	fullySorted    bool // format-aware sorting, internal use only please.
 	suppressLog    bool
 
 	// test Tx atomicity if > 0
@@ -1523,7 +1520,6 @@ func (api *API) ImportWithTx(ctx context.Context, qcx *Qcx, req *ImportRequest,
 		return errors.Wrap(err, "validating api method")
 	}
 
-	api.server.logger.Debugf("ImportWithTx: %v %v %v", req.Index, req.Field, req.Shard)
 	idx, field, err := api.indexField(req.Index, req.Field, req.Shard)
 	if err != nil {
 		return errors.Wrap(err, "getting index and field")
@@ -1642,6 +1638,12 @@ func (api *API) ImportWithTx(ctx context.Context, qcx *Qcx, req *ImportRequest,
 // across many fields in a single shard. It can both set and clear
 // bits and updates caches/bitDepth as appropriate, although only the
 // bitmap parts happen truly transactionally.
+//
+// This function does not attempt to do existence tracking, because
+// it can't; there's no way to distinguish empty sets from not setting
+// bits. As a result, users of this endpoint are responsible for
+// providing corrected existence views for fields with existence
+// tracking. Our batch API does that.
 func (api *API) ImportRoaringShard(ctx context.Context, indexName string, shard uint64, req *ImportRoaringShardRequest) error {
 	index, err := api.Index(ctx, indexName)
 	if err != nil {
@@ -1672,7 +1674,7 @@ func (api *API) ImportRoaringShard(ctx context.Context, indexName string, shard
 		}
 
 		fieldType := field.Options().Type
-		if err1 = cleanupView(fieldType, &viewUpdate); err1 != nil {
+		if viewUpdate.View, err1 = field.cleanupViewName(viewUpdate.View); err1 != nil {
 			return err1
 		}
 
@@ -1764,27 +1766,6 @@ func (api *API) ImportRoaringShard(ctx context.Context, indexName string, shard
 	return nil
 }
 
-func cleanupView(fieldType string, viewUpdate *RoaringUpdate) error {
-	// TODO wouldn't hurt to have consolidated logic somewhere for validating view names.
-	switch fieldType {
-	case FieldTypeSet, FieldTypeTime:
-		if viewUpdate.View == "" {
-			viewUpdate.View = "standard"
-		}
-		// add 'standard_' if we just have a time... this is how IDK works by default
-		if fieldType == FieldTypeTime && !strings.HasPrefix(viewUpdate.View, viewStandard) {
-			viewUpdate.View = fmt.Sprintf("%s_%s", viewStandard, viewUpdate.View)
-		}
-	case FieldTypeInt, FieldTypeDecimal, FieldTypeTimestamp:
-		if viewUpdate.View == "" {
-			viewUpdate.View = "bsig_" + viewUpdate.Field
-		} else if viewUpdate.View != "bsig_"+viewUpdate.Field {
-			return NewBadRequestError(errors.Errorf("invalid view name (%s) for field %s of type %s", viewUpdate.View, viewUpdate.Field, fieldType))
-		}
-	}
-	return nil
-}
-
 // ImportValue is a wrapper around the common code in ImportValueWithTx, which
 // currently just translates req.Clear into a clear ImportOption.
 func (api *API) ImportValue(ctx context.Context, qcx *Qcx, req *ImportValueRequest, opts ...ImportOption) error {
@@ -2038,21 +2019,20 @@ func (api *API) ImportValueWithTx(ctx context.Context, qcx *Qcx, req *ImportValu
 	return nil
 }
 
-func importExistenceColumns(qcx *Qcx, index *Index, columnIDs []uint64, shard uint64) error {
+func importExistenceColumns(qcx *Qcx, index *Index, columnIDs []uint64, shard uint64) (err0 error) {
 	ef := index.existenceField()
 	if ef == nil {
 		return nil
 	}
-
-	existenceRowIDs := make([]uint64, len(columnIDs))
-	// If we don't gratuitously hand-duplicate things in field.Import,
-	// the fact that fragment.bulkImport rewrites its row and column
-	// lists can burn us if we don't make a copy before doing the
-	// existence field write.
-	columnCopy := make([]uint64, len(columnIDs))
-	copy(columnCopy, columnIDs)
-	options := ImportOptions{}
-	return ef.Import(qcx, existenceRowIDs, columnCopy, nil, shard, &options)
+	tx, finisher, err := qcx.GetTx(Txo{Write: true, Index: index, Shard: shard})
+	if err != nil {
+		return err
+	}
+	defer finisher(&err0)
+	// markExistingInView is simpler/faster than Import, but unusually, we use the
+	// standard view of the existence field, instead of the existence view of
+	// a specific field, when doing the index-wide update.
+	return ef.markExistingInView(tx, columnIDs, viewStandard, shard)
 }
 
 // ShardDistribution returns an object representing the distribution of shards
@@ -3377,6 +3357,7 @@ func (n *NopSchemaAPI) DeleteField(ctx context.Context, tname dax.TableName, fna
 
 type ClusterNode struct {
 	ID        string
+	Type      string
 	State     string
 	URI       string
 	GRPCURI   string

api_directive.go

@@ -35,6 +35,22 @@ func (api *API) ApplyDirective(ctx context.Context, d *dax.Directive) error {
 	// Handle the operations based on the directive method.
 	switch d.Method {
 	case dax.DirectiveMethodDiff:
+		// In order to prevent adding too much code specific to handling a diff
+		// directive (e.g. adding something like an `enactDirectiveDiff()`
+		// method), we are instead going to build a full Directive based on the
+		// diff, and then proceed normally as if we had received a full
+		// Directive. We do that by copying the previous Directive and then
+		// applying the diffs to the copy.
+		newD := previousDirective.Copy()
+
+		// Apply the diffs from the incoming Directive to the new, copied
+		// Directive.
+		newD.ApplyDiff(d)
+
+		// Now proceed with the new diff as if we had received it as a full diff.
+		d = newD
+
+	case dax.DirectiveMethodFull:
 		// pass: normal operation
 
 	case dax.DirectiveMethodReset:
@@ -964,7 +980,7 @@ func createField(idx *Index, fld *dax.Field) error {
 		return errors.Wrapf(err, "creating field options from field: %s", fld.Name)
 	}
 
-	if _, err := idx.CreateField(string(fld.Name), "", opts...); err != nil {
+	if _, err := idx.createNullableField(string(fld.Name), "", opts...); err != nil {
 		return errors.Wrapf(err, "creating field on index: %s", fld.Name)
 	}
 	return nil

api_directive_test.go

@@ -30,7 +30,7 @@ func TestAPI_Directive(t *testing.T) {
 		// Empty directive (and empty holder).
 		{
 			d := &dax.Directive{
-				Method:  dax.DirectiveMethodDiff,
+				Method:  dax.DirectiveMethodFull,
 				Version: 1,
 			}
 			err := api.ApplyDirective(ctx, d)
@@ -41,7 +41,7 @@ func TestAPI_Directive(t *testing.T) {
 		// Add a new table.
 		{
 			d := &dax.Directive{
-				Method: dax.DirectiveMethodDiff,
+				Method: dax.DirectiveMethodFull,
 				Tables: []*dax.QualifiedTable{
 					tbl1,
 				},
@@ -55,7 +55,7 @@ func TestAPI_Directive(t *testing.T) {
 		// Add a new table, and keep the existing table.
 		{
 			d := &dax.Directive{
-				Method: dax.DirectiveMethodDiff,
+				Method: dax.DirectiveMethodFull,
 				Tables: []*dax.QualifiedTable{
 					tbl1,
 					tbl2,
@@ -70,7 +70,7 @@ func TestAPI_Directive(t *testing.T) {
 		// Add a new table and remove one of the existing tables.
 		{
 			d := &dax.Directive{
-				Method: dax.DirectiveMethodDiff,
+				Method: dax.DirectiveMethodFull,
 				Tables: []*dax.QualifiedTable{
 					tbl2,
 					tbl3,

api_test.go

@@ -837,7 +837,7 @@ func TestAPI_IDAlloc(t *testing.T) {
 			t.Fatalf("obtaining random bytes: %v", err)
 		}
 		ids3, err := primary.ReserveIDs(key, session, 0, 2)
-		var esync pilosa.ErrIDOffsetDesync
+		var esync pilosa.IDOffsetDesyncError
 		if errors.As(err, &esync) {
 			if esync.Requested != 0 {
 				t.Errorf("incorrect requested offset in error: provided %d but got %d", 0, esync.Requested)

arrow.go

@@ -478,7 +478,7 @@ func readTableArrow(filename string, mem memory.Allocator) (arrow.Table, error)
 		return nil, err
 	}
 	defer rr.Close()
-	records := make([]arrow.Record, rr.NumRecords(), rr.NumRecords())
+	records := make([]arrow.Record, rr.NumRecords())
 	i := 0
 	for {
 		rec, err := rr.Read()

batch/batch.go

@@ -23,6 +23,7 @@ import (
 const (
 	DefaultKeyTranslateBatchSize = 100000
 	existenceFieldName           = "_exists"
+	existenceViewName            = "existence" // this should match top level featurebase viewExistence
 )
 
 // TODO if using column translation, column ids might get way out of
@@ -573,7 +574,11 @@ func (b *Batch) Add(rec Row) error {
 		case int64:
 			b.values[field.Name] = append(b.values[field.Name], val)
 		case []string:
-			if len(val) == 0 {
+			// note that a length of 0 can be valid, and represents an
+			// empty set. an empty set counts as a non-NULL value for
+			// SQL purposes -- it means the existence view bit should
+			// get set.
+			if val == nil {
 				continue
 			}
 			rowIDSets, ok := b.rowIDSets[field.Name]
@@ -608,7 +613,11 @@ func (b *Batch) Add(rec Row) error {
 			}
 			b.rowIDSets[field.Name] = append(rowIDSets, rowIDs)
 		case []uint64:
-			if len(val) == 0 {
+			// note that a length of 0 can be valid, and represents an
+			// empty set. an empty set counts as a non-NULL value for
+			// SQL purposes -- it means the existence view bit should
+			// get set.
+			if val == nil {
 				continue
 			}
 			rowIDSets, ok := b.rowIDSets[field.Name]
@@ -663,6 +672,9 @@ func (b *Batch) Add(rec Row) error {
 
 	for i, uval := range rec.Clears {
 		field := b.header[i]
+		if field.Options.Type == featurebase.FieldTypeMutex && uval != nil {
+			return errors.Errorf("individual-bit clears not allowed on mutex fields; use nil to clear a mutex")
+		}
 		if _, ok := b.clearRowIDs[i]; !ok {
 			b.clearRowIDs[i] = make(map[int]uint64)
 		}
@@ -1245,7 +1257,7 @@ func (b *Batch) doImport(frags, clearFrags fragments) error {
 			}
 
 			ferr := b.importer.ImportRoaringBitmap(ctx, b.tbl.ID, fld, shard, viewMap, false)
-			b.log.Debugf("imp-roar    field: %s, shard:%d, views:%d %v", field, shard, len(clearViewMap), time.Since(starty))
+			b.log.Debugf("imp-roar    field: %s, shard:%d, views:%d %v", field, shard, len(viewMap), time.Since(starty))
 			return errors.Wrapf(ferr, "importing data for %s", field)
 		})
 	}
@@ -1343,6 +1355,7 @@ func (b *Batch) makeFragments(frags, clearFrags fragments) (fragments, fragments
 		curShard := ^uint64(0) // impossible sentinel value for shard.
 		var curBM *roaring.Bitmap
 		var clearBM *roaring.Bitmap
+		var existCurBM *roaring.Bitmap
 		for j := range b.ids {
 			col := b.ids[j]
 			row := nilSentinel
@@ -1355,8 +1368,12 @@ func (b *Batch) makeFragments(frags, clearFrags fragments) (fragments, fragments
 
 			if col/shardWidth != curShard {
 				curShard = col / shardWidth
+				// the API treats "" as standard
 				curBM = frags.GetOrCreate(curShard, field.Name, "")
 				clearBM = clearFrags.GetOrCreate(curShard, field.Name, "")
+				if opts.ActuallyTrackingExistence() {
+					existCurBM = frags.GetOrCreate(curShard, field.Name, existenceViewName)
+				}
 			}
 			if row != nilSentinel {
 				// TODO this is super ugly, but we want to avoid setting
@@ -1366,6 +1383,9 @@ func (b *Batch) makeFragments(frags, clearFrags fragments) (fragments, fragments
 				// the NoStandardView case would be great.
 				if !(opts.Type == featurebase.FieldTypeTime && opts.NoStandardView) {
 					curBM.DirectAdd(row*shardWidth + (col % shardWidth))
+					if opts.ActuallyTrackingExistence() {
+						existCurBM.DirectAdd(col % shardWidth)
+					}
 				}
 				if opts.Type == featurebase.FieldTypeTime {
 					views, err := b.times[j].views(opts.TimeQuantum)
@@ -1386,6 +1406,16 @@ func (b *Batch) makeFragments(frags, clearFrags fragments) (fragments, fragments
 				// we want to make sure that at this point, the "set"
 				// fragments don't contain the bit that we're clearing
 				curBM.DirectRemoveN(clearRow*shardWidth + (col % shardWidth))
+				// Because this is RowIDs, not RowIDSets, there's only one
+				// bit. We should not be setting the existence bit based on
+				// this value, if we're actually clearing it. This doesn't
+				// mean we will clear an existing existence bit, though.
+				// The case where we would clear an existence bit is the
+				// case where someone specified row[mutexField].Clears = nil,
+				// which is far from here.
+				if opts.ActuallyTrackingExistence() {
+					existCurBM.DirectRemoveN(col % shardWidth)
+				}
 			}
 		}
 	}
@@ -1404,14 +1434,23 @@ func (b *Batch) makeFragments(frags, clearFrags fragments) (fragments, fragments
 		opts := field.Options
 		curShard := ^uint64(0) // impossible sentinel value for shard.
 		var curBM *roaring.Bitmap
+		var existCurBM *roaring.Bitmap
 		for j := range b.ids {
 			col, rowIDs := b.ids[j], rowIDSets[j]
-			if len(rowIDs) == 0 {
-				continue
-			}
 			if col/shardWidth != curShard {
 				curShard = col / shardWidth
 				curBM = frags.GetOrCreate(curShard, fname, "")
+				if opts.ActuallyTrackingExistence() {
+					existCurBM = frags.GetOrCreate(curShard, fname, existenceViewName)
+				}
+			}
+			if len(rowIDs) == 0 {
+				// you can validly specify an empty set, which is not the same as a null,
+				// but which still ought to set the existence bit if we're tracking that.
+				if opts.ActuallyTrackingExistence() && rowIDs != nil {
+					existCurBM.DirectAdd(col % shardWidth)
+				}
+				continue
 			}
 			// TODO this is super ugly, but we want to avoid setting
 			// bits on the standard view in the specific case when
@@ -1422,6 +1461,9 @@ func (b *Batch) makeFragments(frags, clearFrags fragments) (fragments, fragments
 				for _, row := range rowIDs {
 					curBM.DirectAdd(row*shardWidth + (col % shardWidth))
 				}
+				if opts.ActuallyTrackingExistence() {
+					existCurBM.DirectAdd(col % shardWidth)
+				}
 			}
 			if opts.Type == featurebase.FieldTypeTime {
 				views, err := b.times[j].views(opts.TimeQuantum)
@@ -1549,6 +1591,11 @@ func (b *Batch) makeSingleValFragments(frags, clearFrags fragments) (fragments,
 		shard := ids[0] / shardWidth
 		bitmap := frags.GetOrCreate(shard, field.Name, "standard")
 		clearBM := clearFrags.GetOrCreate(shard, field.Name, "standard")
+		var existBM, existClearBM *roaring.Bitmap
+		if field.Options.ActuallyTrackingExistence() {
+			existBM = frags.GetOrCreate(shard, field.Name, existenceViewName)
+			existClearBM = clearFrags.GetOrCreate(shard, field.Name, existenceViewName)
+		}
 		for i, id := range ids {
 			if i+1 < len(ids) {
 				// we only want the last value set for each id
@@ -1561,6 +1608,10 @@ func (b *Batch) makeSingleValFragments(frags, clearFrags fragments) (fragments,
 				shard = id / shardWidth
 				bitmap = frags.GetOrCreate(shard, field.Name, "standard")
 				clearBM = clearFrags.GetOrCreate(shard, field.Name, "standard")
+				if field.Options.ActuallyTrackingExistence() {
+					existBM = frags.GetOrCreate(shard, field.Name, existenceViewName)
+					existClearBM = clearFrags.GetOrCreate(shard, field.Name, existenceViewName)
+				}
 			}
 			fragmentColumn := id % shardWidth
 			clearBM.Add(fragmentColumn) // Will use this to clear columns.
@@ -1568,6 +1619,11 @@ func (b *Batch) makeSingleValFragments(frags, clearFrags fragments) (fragments,
 				// clearSentinel is used for deletion
 				// so this value should only be added if its not clearSentinel
 				bitmap.Add(row*shardWidth + fragmentColumn)
+				if field.Options.ActuallyTrackingExistence() {
+					existBM.Add(fragmentColumn)
+				}
+			} else if field.Options.ActuallyTrackingExistence() {
+				existClearBM.Add(fragmentColumn)
 			}
 		}
 	}
@@ -1596,6 +1652,11 @@ func (b *Batch) makeSingleValFragments(frags, clearFrags fragments) (fragments,
 			fragmentColumn := recID % shardWidth
 
 			clearBM.Add(fragmentColumn)
+			if field.Options.ActuallyTrackingExistence() {
+				existClearBM := clearFrags.GetOrCreate(shard, field.Name, existenceViewName)
+
+				existClearBM.Add(fragmentColumn)
+			}
 		}
 	}
 
@@ -1618,6 +1679,10 @@ func (b *Batch) makeSingleValFragments(frags, clearFrags fragments) (fragments,
 
 			fragmentColumn := recID % shardWidth
 			clearBM.Add(fragmentColumn)
+			if field.Options.ActuallyTrackingExistence() {
+				exist := frags.GetOrCreate(shard, field.Name, existenceViewName)
+				exist.Add(fragmentColumn)
+			}
 
 			if boolVal {
 				bitmap.Add(trueRowOffset + fragmentColumn)

batch/batch_test.go

@@ -103,6 +103,12 @@ func testStringSliceCombos(t *testing.T, importer featurebase.Importer, sapi fea
 		Index: idx.Name,
 		Query: "TopN(a1, n=10)",
 	})
+	if resp.Err != nil {
+		t.Fatalf("unexpected error from TopN query: %v", resp.Err)
+	}
+	if len(resp.Results) < 1 {
+		t.Fatalf("expected non-empty result set, got empty results")
+	}
 	pairsField, ok := resp.Results[0].(*featurebase.PairsField)
 	assert.True(t, ok, "wrong return type: %T", resp.Results[0])
 
@@ -508,10 +514,11 @@ func testStringSliceEmptyAndNil(t *testing.T, importer featurebase.Importer, sap
 			{
 				Name: "strslice",
 				Options: featurebase.FieldOptions{
-					Type:      featurebase.FieldTypeSet,
-					Keys:      true,
-					CacheType: featurebase.CacheTypeRanked,
-					CacheSize: 100,
+					Type:           featurebase.FieldTypeSet,
+					Keys:           true,
+					CacheType:      featurebase.CacheTypeRanked,
+					CacheSize:      100,
+					TrackExistence: true,
 				},
 			},
 		},
@@ -611,6 +618,14 @@ func testStringSliceEmptyAndNil(t *testing.T, importer featurebase.Importer, sap
 			pql: "Row(strslice='z')",
 			exp: []uint64{2},
 		},
+		{
+			pql: "Row(strslice==null)",
+			exp: []uint64{1},
+		},
+		{
+			pql: "Row(strslice!=null)",
+			exp: []uint64{0, 2, 3, 4},
+		},
 	}
 	for i, test := range tests {
 		t.Run(fmt.Sprintf("test-%d", i), func(t *testing.T) {
@@ -2045,7 +2060,7 @@ func mutexClearRegression(t *testing.T, importer featurebase.Importer, sapi feat
 	}
 
 	col := uint64(0)
-	row := uint64(1)
+	row := uint64(0)
 	for i := uint64(0); i <= 21; i++ {
 		col = (i%2+1)*featurebase.ShardWidth + i%5
 		row = i % 3
@@ -2126,7 +2141,7 @@ func mutexNilClearID(t *testing.T, importer featurebase.Importer, sapi featureba
 	}
 
 	col := uint64(0)
-	row := uint64(1)
+	row := uint64(0)
 	// populate mutex with some data
 	for i := uint64(0); i < 11; i++ {
 		col = (i%2+1)*featurebase.ShardWidth + i%5

batch/batcher.go

@@ -0,0 +1,20 @@
+package batch
+
+import (
+	"time"
+
+	"github.com/featurebasedb/featurebase/v3/dax"
+)
+
+// Batcher is an interface implemented by anything which can allocate new
+// batches.
+type Batcher interface {
+	NewBatch(cfg Config, tbl *dax.Table, fields []*dax.Field) (RecordBatch, error)
+}
+
+// Config is the configuration options passed to NewBatch for any implementation
+// of the Batcher interface.
+type Config struct {
+	Size         int
+	MaxStaleness time.Duration
+}

batch/egpool/egpool.go

@@ -58,11 +58,11 @@ func (eg *Group) err(err error) {
 	eg.errs = append(eg.errs, err)
 }
 
-type ErrPanic struct {
+type PanicError struct {
 	Value interface{}
 }
 
-func (p ErrPanic) Error() string {
+func (p PanicError) Error() string {
 	return fmt.Sprintf("panic: %v", p.Value)
 }
 
@@ -77,7 +77,7 @@ func (eg *Group) processJobs() {
 	defer func() {
 		if !finished {
 			if p := recover(); p != nil {
-				eg.err(ErrPanic{p})
+				eg.err(PanicError{p})
 			} else {
 				eg.err(ErrGoexit)
 			}

bufferpool/page.go

@@ -53,10 +53,10 @@ const PAGE_PREV_POINTER_OFFSET = 12 // offset 12, length 4, end 16
 const PAGE_NEXT_POINTER_OFFSET = 16 // offset 16, length 4, end 20
 const PAGE_SLOTS_START_OFFSET = 20  // offset 20
 
-// page slots
+// PAGE_SLOT_LENGTH is the size of the page slot key/value.
 //
-//	key offset int16    	//offset 0, length 2, end 2
-//	value offset int16	    //offset 2, length 2, end 4
+// key offset int16    	//offset 0, length 2, end 2
+// value offset int16	    //offset 2, length 2, end 4
 const PAGE_SLOT_LENGTH = 4
 
 // Page represents a page on disk

cache_test.go

@@ -6,7 +6,7 @@ import (
 	"reflect"
 	"testing"
 
-	"github.com/featurebasedb/featurebase/v3"
+	pilosa "github.com/featurebasedb/featurebase/v3"
 )
 
 // Ensure cache stays constrained to its configured size.
@@ -62,7 +62,7 @@ func TestCache_Rank_Dirty(t *testing.T) {
 		cache.Add(v.ID, v.Count)
 	}
 
-	var got []pair
+	var got []pair //nolint:prealloc
 	for _, p := range cache.Top() {
 		got = append(got, pair(p))
 	}

catcher.go

@@ -124,6 +124,17 @@ func (c *catcherTx) Remove(index, field, view string, shard uint64, a ...uint64)
 	return c.b.Remove(index, field, view, shard, a...)
 }
 
+func (c *catcherTx) Removed(index, field, view string, shard uint64, a ...uint64) (changed []uint64, err error) {
+
+	defer func() {
+		if r := recover(); r != nil {
+			vprint.AlwaysPrintf("see Removed() PanicOn '%v' at '%v'", r, vprint.Stack())
+			vprint.PanicOn(r)
+		}
+	}()
+	return c.b.Removed(index, field, view, shard, a...)
+}
+
 func (c *catcherTx) Contains(index, field, view string, shard uint64, key uint64) (exists bool, err error) {
 
 	defer func() {

cli/Makefile

@@ -0,0 +1,15 @@
+.PHONY: test testv test-integration testv-integration
+
+GO=go
+
+test:
+	$(GO) test ./... -short
+
+testv:
+	$(GO) test -v ./... -short
+
+test-integration:
+	$(GO) test . -count 1 -timeout 20m -run TestCLIIntegration/$(RUN)
+
+testv-integration:
+	$(GO) test -v . -count 1 -timeout 20m -run TestCLIIntegration/$(RUN)

cli/batch/inserter.go

@@ -0,0 +1,8 @@
+package batch
+
+// Inserter can be implemented by anything which can handle a SQL statement
+// representing a write operation. An example is `BULK INSERT`. The Insert()
+// method on this interface does not return any results other than an error.
+type Inserter interface {
+	Insert(sql string) error
+}

cli/batch/sql.go

@@ -0,0 +1,215 @@
+package batch
+
+import (
+	"encoding/json"
+	"fmt"
+	"strings"
+	"time"
+
+	fbbatch "github.com/featurebasedb/featurebase/v3/batch"
+	"github.com/featurebasedb/featurebase/v3/dax"
+	"github.com/featurebasedb/featurebase/v3/errors"
+	"github.com/featurebasedb/featurebase/v3/pql"
+)
+
+// Ensure type implements interface.
+var _ fbbatch.Batcher = (*sqlBatcher)(nil)
+
+type sqlBatcher struct {
+	inserter Inserter
+	fields   []*dax.Field
+}
+
+func NewSQLBatcher(i Inserter, flds []*dax.Field) *sqlBatcher {
+	return &sqlBatcher{
+		inserter: i,
+		fields:   flds,
+	}
+}
+
+func (b *sqlBatcher) NewBatch(cfg fbbatch.Config, tbl *dax.Table, flds []*dax.Field) (fbbatch.RecordBatch, error) {
+	fields := flds
+	if b.fields != nil {
+		fields = b.fields
+	}
+	return &sqlBatch{
+		table:        tbl,
+		fields:       fields,
+		size:         cfg.Size,
+		maxStaleness: cfg.MaxStaleness,
+		ids:          make([]interface{}, 0, cfg.Size),
+		rows:         make([][]interface{}, 0, cfg.Size),
+		inserter:     b.inserter,
+	}, nil
+}
+
+// Ensure type implements interface.
+var _ fbbatch.RecordBatch = (*sqlBatch)(nil)
+
+type sqlBatch struct {
+	table  *dax.Table
+	fields []*dax.Field
+	size   int
+
+	ids  []interface{}
+	rows [][]interface{}
+
+	// staleTime tracks the time the first record of the batch was inserted
+	// plus the maxStaleness, in order to raise ErrBatchNowStale if the
+	// maxStaleness has elapsed
+	staleTime    time.Time
+	maxStaleness time.Duration
+
+	// inserter handles SQL INSERT statements generated for each batch.
+	inserter Inserter
+}
+
+func (b *sqlBatch) Add(rec fbbatch.Row) error {
+	// Clear rec.Values and rec.Clears upon return.
+	defer func() {
+		for i := range rec.Values {
+			rec.Values[i] = nil
+		}
+		for k := range rec.Clears {
+			delete(rec.Clears, k)
+		}
+	}()
+
+	if len(b.ids) == cap(b.ids) {
+		return fbbatch.ErrBatchAlreadyFull
+	}
+	if len(rec.Values) != len(b.fields) {
+		return errors.Errorf("record needs to match up with batch fields, got %d fields and %d record", len(b.fields), len(rec.Values))
+	}
+
+	// Append the ID to b.ids.
+	b.ids = append(b.ids, rec.ID)
+
+	// Convert decimal fields (which come in as int64, along with the scale in
+	// field) to pql.Decimal.
+	for i, fld := range b.fields {
+		switch b.fields[i].Type {
+		case dax.BaseTypeDecimal:
+			if val, ok := rec.Values[i].(int64); ok {
+				rec.Values[i] = pql.NewDecimal(val, fld.Options.Scale)
+			}
+		case dax.BaseTypeTimestamp:
+			if val, ok := rec.Values[i].(int64); ok {
+				ts := time.Unix(val, 0)
+				rec.Values[i] = ts.Format(time.RFC3339)
+			}
+		}
+	}
+
+	// Append the record to b.rows.
+	vals := make([]interface{}, 0, len(rec.Values))
+	vals = append(vals, rec.Values...)
+	b.rows = append(b.rows, vals)
+
+	// Check for batch full or stale.
+	if len(b.ids) == cap(b.ids) {
+		return fbbatch.ErrBatchNowFull
+	}
+	if b.maxStaleness != time.Duration(0) { // set maxStaleness to 0 to disable staleness checking
+		if len(b.ids) == 1 {
+			b.staleTime = time.Now().Add(b.maxStaleness)
+		} else if time.Now().After(b.staleTime) {
+			return fbbatch.ErrBatchNowStale
+		}
+	}
+	return nil
+}
+
+func (b *sqlBatch) Import() error {
+	if len(b.rows) == 0 {
+		return nil
+	}
+
+	// Construct the BULK INSERT statement based on the table and fields.
+	sql, err := buildBulkInsert(b.table, b.fields, b.ids, b.rows)
+	if err != nil {
+		return errors.Wrap(err, "building bulk insert statement")
+	}
+
+	// Reset batch data.
+	b.reset()
+
+	// Submit the SQL statement.
+	return b.inserter.Insert(sql)
+}
+
+func (b *sqlBatch) reset() {
+	b.ids = b.ids[:0]
+	b.rows = b.rows[:0]
+}
+
+func (b *sqlBatch) Len() int {
+	return len(b.rows)
+}
+
+func (b *sqlBatch) Flush() error {
+	return nil
+}
+
+func buildBulkInsert(tbl *dax.Table, fields []*dax.Field, ids []interface{}, rows [][]interface{}) (string, error) {
+	// Validation.
+	if tbl.Name == "" {
+		return "", errors.New(errors.ErrUncoded, "table name is required")
+	} else if len(fields) == 0 {
+		return "", errors.New(errors.ErrUncoded, "at least one field is required")
+	}
+
+	var sb strings.Builder
+
+	sb.WriteString(`BULK INSERT INTO `)
+	sb.WriteString(string(tbl.Name))
+	sb.WriteString(` (_id,`)
+
+	flds := make([]string, 0, len(fields))
+	maps := make([]string, 0, len(fields))
+	for i := range fields {
+		flds = append(flds, string(fields[i].Name))
+		maps = append(maps, fmt.Sprintf("'$.col_%d' %s", i, fields[i].FullType()))
+	}
+	// Fields
+	sb.WriteString(strings.Join(flds, ","))
+
+	// MAP
+	keyType := dax.BaseTypeID
+	if tbl.StringKeys() {
+		keyType = dax.BaseTypeString
+	}
+	sb.WriteString(`) MAP ('$._id' `)
+	sb.WriteString(keyType)
+	sb.WriteString(`,`)
+	sb.WriteString(strings.Join(maps, ","))
+	sb.WriteString(`) FROM x'`)
+
+	// Row values.
+
+	// m is a map representing a single row to be marshalled and added to the
+	// bulk insert as one line in the NDJSON payload. We re-use the map for each
+	// row.
+	m := make(map[string]interface{})
+	for i := range rows {
+		// Write the ID value.
+		m[string(dax.PrimaryKeyFieldName)] = ids[i]
+		// Write the rest of the data values.
+		for col := range rows[i] {
+			m[fmt.Sprintf("col_%d", col)] = rows[i][col]
+		}
+
+		// Marshal the map to json and add to the sql statement.
+		if j, err := json.Marshal(m); err != nil {
+			return "", errors.Wrap(err, "marshalling row to json")
+		} else {
+			sb.Write(j)
+			sb.WriteString("\n")
+		}
+	}
+
+	// WITH
+	sb.WriteString(fmt.Sprintf(`' WITH BATCHSIZE %d FORMAT 'NDJSON' INPUT 'STREAM'`, len(rows)))
+
+	return sb.String(), nil
+}

cli/batch/sql_test.go

@@ -0,0 +1,47 @@
+package batch
+
+import (
+	"testing"
+
+	"github.com/featurebasedb/featurebase/v3/dax"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestBatchSQL(t *testing.T) {
+	tbl := &dax.Table{
+		Name: "foo",
+	}
+	fields := []*dax.Field{
+		{
+			Name: "name",
+			Type: dax.BaseTypeString,
+		},
+		{
+			Name: "age",
+			Type: dax.BaseTypeInt,
+		},
+	}
+	ids := []interface{}{
+		0, 1, 2,
+	}
+	rows := [][]interface{}{
+		{
+			[]interface{}{"Alice", int64(11)},
+		},
+		{
+			[]interface{}{"Bob", int64(22)},
+		},
+		{
+			[]interface{}{"Carl,Comma", int64(33)},
+		},
+	}
+
+	s, err := buildBulkInsert(tbl, fields, ids, rows)
+	assert.NoError(t, err)
+
+	exp := `BULK INSERT INTO foo (_id,name,age) MAP ('$._id' id,'$.col_0' string,'$.col_1' int) FROM x'{"_id":0,"col_0":["Alice",11]}
+{"_id":1,"col_0":["Bob",22]}
+{"_id":2,"col_0":["Carl,Comma",33]}
+' WITH BATCHSIZE 3 FORMAT 'NDJSON' INPUT 'STREAM'`
+	assert.Equal(t, exp, s)
+}

cli/cli.go

@@ -13,6 +13,7 @@ import (
 
 	"github.com/chzyer/readline"
 	featurebase "github.com/featurebasedb/featurebase/v3"
+	"github.com/featurebasedb/featurebase/v3/cli/batch"
 	"github.com/featurebasedb/featurebase/v3/cli/fbcloud"
 	"github.com/featurebasedb/featurebase/v3/errors"
 	"github.com/featurebasedb/featurebase/v3/logger"
@@ -20,8 +21,8 @@ import (
 
 const (
 	defaultHost     string = "localhost"
-	promptBegin     string = "fbsql> "
-	promptMid       string = "    -> "
+	defaultClientID string = "6i2gs7mu215ab23cnvmshdoq6t" // production Cognito client ID
+	defaultRegion   string = "us-east-2"
 	terminationChar string = ";"
 	nullValue       string = "NULL"
 )
@@ -32,14 +33,13 @@ var (
 	Stderr io.Writer     = os.Stderr
 )
 
-var (
-	splash string = fmt.Sprintf(`FeatureBase CLI (%s)
+var splash string = fmt.Sprintf(`FeatureBase CLI (%s)
 Type "\q" to quit.
 `, featurebase.Version)
-)
 
 // Ensure type implments interfaces.
 var _ printer = (*Command)(nil)
+var _ batch.Inserter = (*Command)(nil)
 
 type Command struct {
 	host string
@@ -56,9 +56,9 @@ type Command struct {
 
 	Queryer Queryer `json:"-"`
 
-	Stdin  io.ReadCloser `json:"-"`
-	Stdout io.Writer     `json:"-"`
-	Stderr io.Writer     `json:"-"`
+	stdin  io.ReadCloser `json:"-"`
+	stdout io.Writer     `json:"-"`
+	stderr io.Writer     `json:"-"`
 
 	// output is where actual results are written. This might point to stdout,
 	// or to a file, based on the current configuration.
@@ -103,22 +103,24 @@ func NewCommand(logdest logger.Logger) *Command {
 			Database:       "",
 
 			CloudAuth: CloudAuthConfig{
-				ClientID: "",
-				Region:   "",
+				ClientID: defaultClientID,
+				Region:   defaultRegion,
 				Email:    "",
 				Password: "",
 			},
 
 			HistoryPath: "",
+
+			CSV: false,
 		},
 
 		buffer:     newBuffer(),
 		splitter:   newSplitter(newReplacer(variables)),
 		workingDir: newWorkingDir(),
 
-		Stdin:  Stdin,
-		Stdout: Stdout,
-		Stderr: Stderr,
+		stdin:  Stdin,
+		stdout: Stdout,
+		stderr: Stderr,
 
 		output:       Stdout,
 		writeOptions: defaultWriteOptions(),
@@ -129,60 +131,112 @@ func NewCommand(logdest logger.Logger) *Command {
 	}
 }
 
+// SetStdin sets stdin. This is useful for initial configuration in tests.
+func (cmd *Command) SetStdin(rc io.ReadCloser) {
+	cmd.stdin = rc
+}
+
+// SetStdout sets both stdout and output to the value provided. This is useful
+// for initial configuration in tests.
+func (cmd *Command) SetStdout(w io.Writer) {
+	cmd.stdout = w
+	cmd.output = w
+}
+
+// SetStderr sets stderr. This is useful for initial configuration in tests.
+func (cmd *Command) SetStderr(w io.Writer) {
+	cmd.stderr = w
+}
+
 // Run is the main entry-point to the CLI.
 func (cmd *Command) Run(ctx context.Context) error {
-	cmd.setupConfig()
+	if err := cmd.run(ctx); err != nil {
+		cmd.Errorf(err.Error() + "\n")
+		return err
+	}
+	return nil
+}
+
+// run is effectively wrapped by the Run() method, but it's split out this way
+// so that run() can simply return errors, rather than worrying about how errors
+// should be printed; printing errors returned by run() is left up to the Run()
+// method.
+func (cmd *Command) run(ctx context.Context) error {
+	if err := cmd.setupConfig(); err != nil {
+		return errors.Wrap(err, "setting up config")
+	}
 
 	// Check to see if Command needs to run in non-interactive mode.
-	if len(cmd.Commands) > 0 || len(cmd.Files) > 0 {
+	if len(cmd.Commands) > 0 ||
+		len(cmd.Files) > 0 ||
+		cmd.Config.KafkaConfig != "" ||
+		cmd.Config.CSV {
 		cmd.nonInteractiveMode = true
+	}
 
-		if err := cmd.setupClient(); err != nil {
-			return errors.Wrap(err, "setting up client")
-		}
-		if err := cmd.connectToDatabase(cmd.database); err != nil {
-			cmd.Errorf(errors.Wrap(err, "connecting to database").Error() + "\n")
-		}
+	// Print the splash message.
+	if !cmd.nonInteractiveMode {
+		cmd.Printf(splash)
+	}
 
+	if err := cmd.setupClient(); err != nil {
+		return errors.Wrap(err, "setting up client")
+	}
+
+	// Print the connection info.
+	if !cmd.nonInteractiveMode {
+		cmd.printConnInfo()
+	}
+
+	if err := cmd.connectToDatabase(cmd.database); err != nil {
+		cmd.Errorf(errors.Wrap(err, "connecting to database").Error() + "\n")
+		// We intentionally do not return err here.
+	}
+
+	// Run in non-interactive mode based on flags and configuration.
+	// This includes either handling `-c` and/or `-f` flags, or handling a
+	// `--kafka-config` flag.
+	if len(cmd.Commands) > 0 || len(cmd.Files) > 0 {
 		// Run Commands.
 		for _, line := range cmd.Commands {
 			if err := cmd.handleLine(line); err != nil {
-				cmd.Errorf(err.Error())
-				return nil
+				return errors.Wrapf(err, "handling line: %s", line)
 			}
 		}
 
 		// Run Files.
 		for _, fname := range cmd.Files {
 			if _, err := executeFile(cmd, fname); err != nil {
-				cmd.Errorf(err.Error())
-				return nil
+				return errors.Wrapf(err, "executing file: %s", fname)
 			}
 		}
 
+		return nil
+	} else if cmd.Config.KafkaConfig != "" {
+		runner, err := cmd.newKafkaRunner(cmd.Config.KafkaConfig)
+		if err != nil {
+			return errors.Wrap(err, "getting new kafka runner")
+		}
+		if err := runner.Main.Run(); err != nil {
+			return errors.Wrap(err, "running kafka")
+		}
 		return nil
 	}
 
-	// Print the splash message.
-	cmd.Printf(splash)
+	// From this point on, we should be in interactive mode.
+
+	// Set up history for saving user input.
 	cmd.setupHistory()
-	if err := cmd.setupClient(); err != nil {
-		return errors.Wrap(err, "setting up client")
-	}
-	cmd.printConnInfo()
-	if err := cmd.connectToDatabase(cmd.database); err != nil {
-		cmd.Errorf(errors.Wrap(err, "connecting to database").Error() + "\n")
-	}
 
 	rl, err := readline.NewEx(&readline.Config{
-		Prompt:                 promptBegin,
+		Prompt:                 cmd.prompt(false),
 		HistoryFile:            cmd.historyPath,
 		HistoryLimit:           100000,
 		DisableAutoSaveHistory: true,
 
-		Stdin:  cmd.Stdin,
-		Stdout: cmd.Stdout,
-		Stderr: cmd.Stderr,
+		Stdin:  cmd.stdin,
+		Stdout: cmd.stdout,
+		Stderr: cmd.stderr,
 	})
 	if err != nil {
 		return errors.Wrap(err, "getting readline")
@@ -194,11 +248,7 @@ func (cmd *Command) Run(ctx context.Context) error {
 	var inMidCommand bool
 
 	for {
-		if inMidCommand {
-			rl.SetPrompt(promptMid)
-		} else {
-			rl.SetPrompt(promptBegin)
-		}
+		rl.SetPrompt(cmd.prompt(inMidCommand))
 
 		// Read user provided input.
 		line, err := rl.Readline()
@@ -297,11 +347,25 @@ func (cmd *Command) Run(ctx context.Context) error {
 			}
 			return nil
 		default:
-			//pass
+			// pass
 		}
 	}
 }
 
+// prompt constructs the prompt that the user sees based on the currently
+// connected database and whether the user is in the middle of a sql statement.
+func (cmd *Command) prompt(mid bool) string {
+	db := "fbsql" // default prompt when a database is not set.
+	if cmd.databaseName != "" {
+		db = cmd.databaseName
+	}
+
+	if mid {
+		return strings.Repeat(" ", len(db)) + "-# "
+	}
+	return db + "=# "
+}
+
 // close is called upon quitting. It should close any remaining open file
 // handles used by the CLICommand.
 func (cmd *Command) close() error {
@@ -310,9 +374,9 @@ func (cmd *Command) close() error {
 
 // setupConfig sets up private struct members based on values provided via the
 // configuration flags.
-func (cmd *Command) setupConfig() {
+func (cmd *Command) setupConfig() error {
 	if cmd.Config == nil {
-		return
+		return nil
 	}
 
 	cmd.host = cmd.Config.Host
@@ -322,15 +386,58 @@ func (cmd *Command) setupConfig() {
 	cmd.database = cmd.Config.Database
 
 	cmd.historyPath = cmd.Config.HistoryPath
+
+	// Apply any pset flag arguments.
+	for _, pset := range cmd.Config.PSets {
+		if err := cmd.applyPSet(pset); err != nil {
+			return errors.Wrapf(err, "applying pset: %s", pset)
+		}
+	}
+
+	// If running with the `--csv` flag, configure things to ensure the output
+	// is correct (i.e. that it's just the csv).
+	if cmd.Config.CSV {
+		cmd.writeOptions.format = formatCSV
+	}
+
+	return nil
+}
+
+// applyPSet takes a pset string of the form `arg` or `arg=val` and applies it
+// as if the user had run `\pset arg val`. The only difference is that applying
+// pset here suppresses any output to stdout.
+func (cmd *Command) applyPSet(pset string) error {
+	// We expect arg to be one of the folowing formats:
+	// arg
+	// arg=val
+	args := strings.SplitN(pset, "=", 2)
+
+	// This is kind of hacky, but until we re-think the metaCommand interface to
+	// take a printer interface somewhere (so we can pass in the nopPrinter
+	// here), we're just going to discard stdout for the duration of this apply,
+	// and then set stdout back to its previous writer after the apply.
+	hold := cmd.stdout
+	cmd.stdout = io.Discard
+	defer func() {
+		cmd.stdout = hold
+	}()
+
+	_, err := newMetaPSet(args).execute(cmd)
+	return err
 }
 
 func (cmd *Command) executeAndWriteQuery(qry query) error {
 	queryResponse, err := cmd.executeQuery(qry)
 	if err != nil {
+		if errors.Is(err, ErrOrganizationRequired) {
+			// Print an error message and return nil, effectively aborting any
+			// further writes for this query.
+			cmd.Errorf("Organization required. Use \\org to set an organization.\n")
+			return nil
+		}
 		return errors.Wrap(err, "making query")
 	}
-
-	if err := writeTable(queryResponse, cmd.writeOptions, cmd.output, cmd.Stdout, cmd.Stderr); err != nil {
+	if err := writeOutput(queryResponse, cmd.writeOptions, cmd.output, cmd.stdout, cmd.stderr); err != nil {
 		return errors.Wrap(err, "writing out response")
 	}
 
@@ -375,7 +482,7 @@ func (n *nopPrinter) Errorf(format string, a ...any)  {}
 // Printf is a helper method which sends the given payload to stdout.
 func (cmd *Command) Printf(format string, a ...any) {
 	out := fmt.Sprintf(format, a...)
-	cmd.Stdout.Write([]byte(out))
+	cmd.stdout.Write([]byte(out))
 }
 
 // Outputf is a helper method which sends the given payload to output.
@@ -387,7 +494,7 @@ func (cmd *Command) Outputf(format string, a ...any) {
 // Errorf is a helper method which sends the given payload to stderr.
 func (cmd *Command) Errorf(format string, a ...any) {
 	out := fmt.Sprintf(format, a...)
-	cmd.Stderr.Write([]byte(out))
+	cmd.stderr.Write([]byte(out))
 }
 
 func (cmd *Command) setupHistory() {
@@ -406,7 +513,7 @@ func (cmd *Command) setupHistory() {
 		if err != nil {
 			cmd.Errorf("Creating directory for history: %v\n", err)
 		} else {
-			historyPath = filepath.Join(historyDir, "cli_history")
+			historyPath = filepath.Join(historyDir, "fbsql_history")
 		}
 	}
 	cmd.historyPath = historyPath
@@ -424,7 +531,12 @@ func (cmd *Command) connectToDatabase(dbName string) error {
 		p = newNopPrinter()
 	}
 
-	if dbName == "" {
+	// Providing a blank ("") or hyphen ("-") dbName is the equivalent of
+	// disconnecting from the current database. We support the hyphen option
+	// because calling the `\c` meta-command without an argument is how you
+	// print the current connection.
+	switch dbName {
+	case "-", "":
 		cmd.databaseID = ""
 		cmd.databaseName = ""
 		p.Printf(cmd.connectionMessage())
@@ -432,16 +544,12 @@ func (cmd *Command) connectToDatabase(dbName string) error {
 	}
 
 	// Look up dbID based on dbName.
-	qry := []queryPart{
-		newPartRaw("SHOW DATABASES"),
-	}
-
-	qr, err := cmd.executeQuery(qry)
+	wqr, err := cmd.executeQuery(newRawQuery("SHOW DATABASES"))
 	if err != nil {
 		return errors.Wrap(err, "executing query")
 	}
 
-	for _, db := range qr.Data {
+	for _, db := range wqr.Data {
 		// 0: _id
 		// 1: name
 		if db[1] == dbName {
@@ -465,7 +573,7 @@ func (cmd *Command) connectionMessage() string {
 	if cmd.databaseName == "" {
 		return "You are not connected to a database.\n"
 	}
-	return fmt.Sprintf("You are now connected to database \"%s\" (%s) as user \"???\".\n", cmd.databaseName, cmd.databaseID)
+	return fmt.Sprintf("You are now connected to database \"%s\" (%s).\n", cmd.databaseName, cmd.databaseID)
 }
 
 func (cmd *Command) setupClient() error {
@@ -698,3 +806,11 @@ func (cmd *Command) handleLineAsQueryParts(line string) error {
 	}
 	return nil
 }
+
+func (cmd *Command) Insert(sql string) error {
+	wqr, err := cmd.executeQuery(newRawQuery(sql))
+	if wqr.Error != "" {
+		return errors.Errorf(wqr.Error)
+	}
+	return err
+}

cli/cli_integration_test.go

@@ -0,0 +1,257 @@
+package cli_test
+
+import (
+	"bufio"
+	"context"
+	"fmt"
+	"os"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/featurebasedb/featurebase/v3/cli"
+	"github.com/featurebasedb/featurebase/v3/dax/server/test"
+	"github.com/featurebasedb/featurebase/v3/logger"
+	"github.com/stretchr/testify/require"
+)
+
+func TestCLIIntegration(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test")
+	}
+
+	ctx := context.Background()
+
+	t.Run("Stubbed Framework", func(t *testing.T) {
+		mc := test.MustRunManagedCommand(t)
+		defer mc.Close()
+
+		addr := mc.Address()
+
+		capture := newCapture(t)
+
+		comparer := newComparer(t)
+		comparer.run()
+
+		fbsql := cli.NewCommand(logger.StderrLogger)
+		fbsql.SetStdin(capture)
+		fbsql.SetStdout(comparer)
+		fbsql.SetStderr(comparer)
+
+		fbsql.Config = &cli.Config{
+			Host: addr.Host(),
+			Port: fmt.Sprintf("%d", addr.Port()),
+		}
+
+		// Run fbsql in a goroutine so we can continue to send it commands
+		// below.
+		didQuit := make(chan struct{})
+		go func() {
+			require.NoError(t, fbsql.Run(ctx))
+			close(didQuit)
+		}()
+
+		// testFiles reference files located in the cli/testdata directory. All
+		// tests should be placed there; other than adding another test file to
+		// this list, you probably shouldn't be editing this file unless you are
+		// trying to modify the way the test framework itself works.
+		testFiles := []string{
+			"setup",
+			"database",
+			"table",
+			// the tests below may be dependent on the previous tests, which do
+			// setup and some shared database and table creation.
+			"query_buffer",
+			// meta commands
+			"meta_bang",
+			"meta_cd",
+			"meta_echo",
+			"meta_describe",
+			"meta_file",
+			"meta_pset_border",
+			"meta_pset_expanded",
+			"meta_pset_format_csv",
+			"meta_pset_tuples_only",
+			"meta_include",
+			"meta_output",
+			"meta_set",
+			"meta_timing",
+			"meta_write",
+		}
+
+		for _, testFile := range testFiles {
+			t.Run(testFile, func(t *testing.T) {
+				f, err := os.Open("testdata/" + testFile)
+				require.NoError(t, err)
+
+				scanner := bufio.NewScanner(f)
+				var lineNo int
+				for scanner.Scan() {
+					line := scanner.Text()
+					lineNo++
+
+					// Empty lines and comments (//) are ignored.
+					if line == "" {
+						continue
+					} else if strings.HasPrefix(line, "//") {
+						continue
+					}
+
+					parts := strings.SplitN(line, ":", 2)
+
+					switch parts[0] {
+					case "SEND":
+						v := ""
+						if len(parts) == 2 {
+							v = parts[1]
+						}
+						capture.sendLine(v)
+					case "EXPECT":
+						v := ""
+						if len(parts) == 2 {
+							v = parts[1]
+						}
+						comparer.expectLine(v, testFile, lineNo)
+					case "EXPECTCOMP":
+						if len(parts) == 2 {
+							comps := strings.SplitN(parts[1], ":", 2)
+							v := ""
+							if len(comps) == 2 {
+								v = comps[1]
+							}
+							comparer.expectLineComp(comparator(comps[0]), v, testFile, lineNo)
+						} else {
+							t.Errorf("unexpected line: %s[%d]:%s", testFile, lineNo, line)
+						}
+					default:
+						t.Errorf("unexpected line: %s[%d]:%s", testFile, lineNo, line)
+					}
+				}
+				require.NoError(t, scanner.Err())
+			})
+		}
+
+		// End with quit to ensure that fbsql closes without error.
+		capture.sendLine(`\q`)
+
+		// Ensure fbsql quits cleanly.
+		select {
+		case <-didQuit:
+		case <-time.After(time.Second):
+			t.Fatalf("expected fbsql to quit")
+		}
+	})
+}
+
+// compare is used to compare fbsql output written to its Stdout with expected
+// lines.
+type comparer struct {
+	t       *testing.T
+	out     chan byte
+	outline chan []byte
+	exp     chan []byte
+}
+
+func newComparer(t *testing.T) *comparer {
+	return &comparer{
+		t:       t,
+		out:     make(chan byte, 1024),
+		outline: make(chan []byte, 128),
+		exp:     make(chan []byte, 1024),
+	}
+}
+
+func (c *comparer) run() {
+	// Read bytes off output, and for every line (designated by a line feed "\n"),
+	// push the line onto the outline channel.
+	go func() {
+		var line []byte
+		for {
+			b := <-c.out
+			if b == byte('\n') {
+				c.outline <- line
+				line = []byte{}
+				continue
+			}
+			line = append(line, b)
+		}
+	}()
+}
+
+type comparator string
+
+const (
+	compEquals     = "Equals"
+	compHasPrefix  = "HasPrefix"
+	compWithFormat = "WithFormat"
+)
+
+// expectLine is a convenience method which calls expectLineComp with the compEq
+// comparator and the given line.
+func (c *comparer) expectLine(line string, fileName string, lineNo int) {
+	c.expectLineComp(compEquals, line, fileName, lineNo)
+}
+
+// expectLineComp reads the next line from the outline channel and compares it
+// with the given `line`. A comparator can be provided to inform how the lines
+// should be compared (for example, the compHasPrefix comparator will just
+// compare the beginning part of the outline).
+func (c *comparer) expectLineComp(comp comparator, line string, fileName string, lineNo int) {
+	var outline []byte
+	select {
+	case outline = <-c.outline:
+	case <-time.After(10 * time.Second):
+		// TODO(tlt): this is 10 seconds to account for the fb_views creation on
+		// a local mac. This should really be something like 2 seconds. Put this
+		// back to 2 once fb_views issue is addressed.
+		c.t.Fatalf("expected output line %s[%d]: >%s<", fileName, lineNo, line)
+	}
+
+	// msg is included in any require which fails.
+	msg := []interface{}{"exp: %s[%d], got: >%s<", fileName, lineNo, outline}
+
+	switch comp {
+	case compEquals:
+		require.Equal(c.t, []byte(line), outline, msg...)
+	case compHasPrefix:
+		require.True(c.t, strings.HasPrefix(string(outline), line), msg...)
+	case compWithFormat:
+		require.True(c.t, compareByteSlices(outline, []byte(line)), msg...)
+	default:
+		c.t.Fatalf("invalid comparator: %s", comp)
+	}
+}
+
+func (c *comparer) Write(b []byte) (n int, err error) {
+	for i := range b {
+		c.out <- b[i]
+	}
+	return len(b), err
+}
+
+// compareByteSlices compares a byte slice s with another byte slice format and
+// returns true if they are the same. It will accept underscore as a
+// single-character wildcard anywhere in slice format.
+func compareByteSlices(s, format []byte) bool {
+	// Replace some helpers in format before comparing.
+	f := string(format)
+	f = strings.ReplaceAll(f, `{uuid}`, `________-____-____-____-____________`)
+	f = strings.ReplaceAll(f, `{timestamp}`, `____-__-__T__:__:__Z`)
+	format = []byte(f)
+
+	if len(s) != len(format) {
+		return false
+	}
+
+	for i := range s {
+		if format[i] == '_' {
+			continue
+		}
+		if s[i] != format[i] {
+			// log.Printf("DEBUG: characters differ: (%d): '%v' != '%v'", i, s[i], format[i])
+			return false
+		}
+	}
+
+	return true
+}

cli/cli_test.go

@@ -22,8 +22,8 @@ func TestCLI(t *testing.T) {
 		capture := newCapture(t)
 
 		cli := cli.NewCommand(logger.StderrLogger)
-		cli.Stdin = capture
-		cli.Stdout = capture
+		cli.SetStdin(capture)
+		cli.SetStdout(capture)
 		cli.Queryer = capture
 
 		go func() {
@@ -74,7 +74,7 @@ var _ cli.Queryer = (*capture)(nil)
 
 // capture implements the various CLI interfaces in order to capture test input
 // and submit it as though that input were being read from the command line. It
-// also captures calls made to the Queryer.Query method and ensures the sql the
+// also captures calls made to the Queryer.Query method and ensures the sql they
 // contain is expected.
 type capture struct {
 	t *testing.T

cli/config.go

@@ -11,7 +11,16 @@ type Config struct {
 	// CloudAuth
 	CloudAuth CloudAuthConfig `json:"cloud-auth"`
 
+	// Kafka
+	KafkaConfig string `json:"kafka-config"`
+
 	HistoryPath string `json:"history-path"`
+
+	// CSV (Comma-Separated Values) table output mode.
+	CSV bool `json:"csv"`
+
+	// PSet takes one or more pset arguments of the form: `--pset=VAR[=ARG]`.
+	PSets []string `json:"pset"`
 }
 
 type CloudAuthConfig struct {

cli/errors.go

@@ -0,0 +1,16 @@
+package cli
+
+import (
+	"github.com/featurebasedb/featurebase/v3/errors"
+)
+
+const (
+	ErrOrganizationRequired errors.Code = "OrganizationRequired"
+)
+
+func NewErrOrganizationRequired() error {
+	return errors.New(
+		ErrOrganizationRequired,
+		"organization required",
+	)
+}

cli/kafka.go

@@ -0,0 +1,70 @@
+package cli
+
+import (
+	"fmt"
+
+	"github.com/featurebasedb/featurebase/v3/cli/batch"
+	"github.com/featurebasedb/featurebase/v3/cli/kafka"
+	"github.com/featurebasedb/featurebase/v3/errors"
+	"github.com/spf13/viper"
+)
+
+func (cmd *Command) newKafkaRunner(cfgFile string) (*kafka.Runner, error) {
+	// Read the kafka config file.
+	v := viper.New()
+	v.SetConfigFile(cfgFile)
+	v.SetConfigType("toml")
+	err := v.ReadInConfig()
+	if err != nil {
+		return nil, fmt.Errorf("error reading configuration file '%s': %v", cfgFile, err)
+	}
+
+	cfg := kafka.Config{}
+	if err := v.Unmarshal(&cfg); err != nil {
+		return nil, errors.Wrap(err, "unmarshalling config")
+	}
+
+	if err := kafka.ValidateConfig(cfg); err != nil {
+		return nil, errors.Wrap(err, "validating config")
+	}
+
+	// Create a new config with defaults.
+
+	// Look up fields based on table provided in the config.
+	wqr, err := cmd.executeQuery(newRawQuery("SHOW COLUMNS FROM " + cfg.Table))
+	if err != nil {
+		return nil, errors.Wrap(err, "executing query")
+	}
+
+	scr, err := wqr.ShowColumnsResponse()
+	if err != nil {
+		return nil, errors.Wrap(err, "getting show columns from wire query response")
+	}
+
+	// If no fields were provided in the config, use the fields defined on the
+	// table and assume a 1-to-1 mapping of source to destination.
+	if len(cfg.Fields) == 0 {
+		cfg.Fields = kafka.FieldsToConfig(scr.Fields)
+	} else {
+		cfg.Fields, err = kafka.CheckFieldCompatibility(cfg.Fields, scr)
+		if err != nil {
+			return nil, errors.Wrap(err, "validating config fields")
+		}
+	}
+
+	idkCfg, err := kafka.ConvertConfig(cfg)
+	if err != nil {
+		return nil, errors.Wrap(err, "cleaning config")
+	}
+
+	flds, err := kafka.ConfigToFields(cfg)
+	if err != nil {
+		return nil, errors.Wrap(err, "getting fields from config")
+	}
+
+	return kafka.NewRunner(
+		idkCfg,
+		batch.NewSQLBatcher(cmd, flds),
+		cmd.stderr,
+	), nil
+}

cli/kafka/config.go

@@ -0,0 +1,251 @@
+package kafka
+
+import (
+	"fmt"
+	"time"
+
+	featurebase "github.com/featurebasedb/featurebase/v3"
+	"github.com/featurebasedb/featurebase/v3/dax"
+	"github.com/featurebasedb/featurebase/v3/idk"
+	"github.com/pkg/errors"
+)
+
+// Config is the user-facing configuration for kafka support in the CLI. This is
+// unmarshalled from the the toml config file supplied by the user.
+type Config struct {
+	Hosts  []string `mapstructure:"hosts" help:"Kafka hosts."`
+	Group  string   `mapstructure:"group" help:"Kafka group."`
+	Topics []string `mapstructure:"topics" help:"Kafka topics to read from."`
+
+	BatchSize         int           `mapstructure:"batch-size" help:"Batch size."`
+	BatchMaxStaleness time.Duration `mapstructure:"batch-max-staleness" help:"Maximum length of time that the oldest record in a batch can exist before flushing the batch. Note that this can potentially stack with timeouts waiting for the source."`
+	Timeout           time.Duration `mapstructure:"timeout" help:"Time to wait for more records from Kafka before flushing a batch. 0 to disable."`
+
+	Table  string  `mapstructure:"table" help:"Destination table name."`
+	Fields []Field `mapstructure:"fields"`
+}
+
+// Field is a user-facing configuration field.
+type Field struct {
+	Name       string   `mapstructure:"name"`
+	SourceType string   `mapstructure:"source-type"`
+	SourcePath []string `mapstructure:"source-path"`
+	PrimaryKey bool     `mapstructure:"primary-key"`
+}
+
+// ConfigForIDK represents Config converted to values suitable for IDK. In
+// particular, the idk.RawField is used in parsing the schema in IDK.
+type ConfigForIDK struct {
+	Hosts  []string
+	Group  string
+	Topics []string
+
+	BatchSize         int
+	BatchMaxStaleness time.Duration
+	Timeout           time.Duration
+
+	Table   string
+	IDField string
+	Fields  []idk.RawField
+}
+
+// ValidateConfig validates the config is usable.
+func ValidateConfig(c Config) error {
+	if c.Table == "" {
+		return errors.Errorf("table is required")
+	} else if len(c.Topics) == 0 {
+		return errors.Errorf("at least one topic is required")
+	} else if len(c.Fields) > 0 {
+		// We only need to do these checks if any fields are specified at all.
+		// If no fields are specified, that's ok because then we default to
+		// using fields based off the existing table.
+		if len(c.Fields) < 2 {
+			return errors.Errorf("at least two fields are required (one should be a primary key)")
+		} else {
+			var found int
+			for i := range c.Fields {
+				if c.Fields[i].PrimaryKey {
+					found++
+				}
+				if c.Fields[i].Name == "" {
+					return errors.Errorf("a name attribute (which isn't equal to \"\") should exist for all fields")
+				}
+			}
+			if found != 1 {
+				return errors.Errorf("exactly one primary key field is required")
+			}
+		}
+	}
+	return nil
+}
+
+// ConvertConfig converts a Config to one that suitable for IDK.
+func ConvertConfig(c Config) (ConfigForIDK, error) {
+	// Set a default kafka host in case one isn't provided.
+	hosts := []string{"localhost:9092"}
+	if len(c.Hosts) > 0 {
+		hosts = c.Hosts
+	}
+
+	// Copy all the shared members from Config to ConfigForIDK.
+	out := ConfigForIDK{
+		Hosts:             hosts,
+		Group:             c.Group,
+		Topics:            c.Topics,
+		BatchSize:         c.BatchSize,
+		BatchMaxStaleness: c.BatchMaxStaleness,
+		Timeout:           c.Timeout,
+		Table:             c.Table,
+	}
+
+	if len(c.Fields) == 0 {
+		return out, errors.New("fields cannot be empty")
+	}
+
+	// rawFields wil be the same as c.Fields, but possibly enhanced.
+	rawFields := make([]idk.RawField, 0, len(c.Fields))
+
+	var foundPK bool
+	for _, fld := range c.Fields {
+		if fld.PrimaryKey {
+			out.IDField = fld.Name
+			foundPK = true
+		}
+
+		typ, quals, err := dax.SplitFieldType(fld.SourceType)
+		if err != nil {
+			return out, errors.Wrap(err, "getting base type")
+		}
+
+		rawFld := idk.RawField{
+			Name: fld.Name,
+			Type: string(typ),
+			Path: fld.SourcePath,
+		}
+		// If a SourcePath wasn't provided, default to using the field name.
+		if len(rawFld.Path) == 0 {
+			rawFld.Path = []string{fld.Name}
+		}
+
+		switch typ {
+		case dax.BaseTypeInt:
+			// We don't have to handle min/max because we don't create the table.
+		case dax.BaseTypeDecimal:
+			if len(quals) != 1 {
+				return out, errors.Errorf("expected decimal scale")
+			}
+			rawFld.Config = []byte(fmt.Sprintf(`{"scale":%d}`, quals[0]))
+		case dax.BaseTypeID:
+			rawFld.Config = []byte("{\"mutex\":true}")
+		case dax.BaseTypeIDSet:
+			rawFld.Type = "ids"
+		case dax.BaseTypeString:
+			rawFld.Config = []byte("{\"mutex\":true}")
+		case dax.BaseTypeStringSet:
+			rawFld.Type = "strings"
+		case dax.BaseTypeTimestamp:
+			// No timestamp options are handled for now.
+		}
+
+		rawFields = append(rawFields, rawFld)
+	}
+	if !foundPK {
+		return out, errors.New("primary-key not found in fields")
+	}
+
+	out.Fields = rawFields
+
+	return out, nil
+}
+
+// ConfigToFields returns a list of *dax.Field based on the IDField and Fields
+// in the Config.
+func ConfigToFields(c Config) ([]*dax.Field, error) {
+	// We don't know if a primary key will be found, so we can't set the
+	// capacity to `len(c.Fields)-1`.
+	out := make([]*dax.Field, 0, len(c.Fields))
+
+	for _, fld := range c.Fields {
+		if fld.PrimaryKey {
+			continue
+		}
+		typ, quals, err := dax.SplitFieldType(fld.SourceType)
+		if err != nil {
+			return nil, errors.Wrap(err, "splitting field type")
+		}
+		dfld := &dax.Field{
+			Name: dax.FieldName(fld.Name),
+			Type: typ,
+		}
+		switch typ {
+		case dax.BaseTypeDecimal:
+			if len(quals) != 1 {
+				return nil, errors.Errorf("expected decimal scale")
+			}
+			scale, ok := quals[0].(int64)
+			if !ok {
+				return nil, errors.Errorf("invalid decimal scale: %v", quals[0])
+			}
+			dfld.Options.Scale = scale
+		}
+		out = append(out, dfld)
+	}
+
+	return out, nil
+}
+
+// FieldsToConfig returns a Config.Fields based on a list of *dax.Field.
+func FieldsToConfig(flds []*dax.Field) []Field {
+	out := make([]Field, 0, len(flds))
+	for _, fld := range flds {
+		out = append(out, Field{
+			Name:       string(fld.Name),
+			SourceType: fld.FullType(),
+			PrimaryKey: fld.IsPrimaryKey(),
+		})
+	}
+	return out
+}
+
+// CheckFieldCompatibility ensures that the fields provided in the kafka config
+// are compatible with the fields in the existing table. It returns a copy of
+// the kafka config fields with empty values defaulted to the table field
+// configuration.
+func CheckFieldCompatibility(cflds []Field, scr *featurebase.ShowColumnsResponse) ([]Field, error) {
+	out := make([]Field, len(cflds))
+	for i, cfld := range cflds {
+		out[i] = cfld
+		cfldName := dax.FieldName(cfld.Name)
+
+		// Primary key field.
+		if cfld.PrimaryKey {
+			f := scr.Field(dax.PrimaryKeyFieldName)
+			if f == nil {
+				return nil, dax.NewErrFieldDoesNotExist(dax.PrimaryKeyFieldName) // It should be impossible to hit this.
+			}
+			if out[i].SourceType == "" {
+				if f.StringKeys() {
+					out[i].SourceType = dax.BaseTypeString
+				} else {
+					out[i].SourceType = dax.BaseTypeID
+				}
+			}
+			continue
+		}
+
+		// Non primary key fields.
+		if cfldName == dax.PrimaryKeyFieldName {
+			return nil, errors.Errorf("field named '%s' must be a primary key", dax.PrimaryKeyFieldName)
+		}
+
+		f := scr.Field(cfldName)
+		if f == nil {
+			return nil, dax.NewErrFieldDoesNotExist(cfldName)
+		}
+
+		if out[i].SourceType == "" {
+			out[i].SourceType = f.FullType()
+		}
+	}
+	return out, nil
+}

cli/kafka/runner.go

@@ -0,0 +1,67 @@
+package kafka
+
+import (
+	"io"
+	"time"
+
+	fbbatch "github.com/featurebasedb/featurebase/v3/batch"
+	"github.com/featurebasedb/featurebase/v3/errors"
+	"github.com/featurebasedb/featurebase/v3/idk"
+	"github.com/featurebasedb/featurebase/v3/idk/kafka_static"
+	"github.com/featurebasedb/featurebase/v3/logger"
+)
+
+// Runner is a CLI-specific kafka consumer. It's similar to
+// idk.kafka_static.Main in that it embeds idk.Main and contains additional
+// functionality specific to its use case.
+type Runner struct {
+	idk.Main   `flag:"!embed"`
+	KafkaHosts []string       `help:"Comma separated list of host:port pairs for Kafka."`
+	Group      string         `help:"Kafka group."`
+	Topics     []string       `help:"Kafka topics to read from."`
+	Timeout    time.Duration  `help:"Time to wait for more records from Kafka before flushing a batch. 0 to disable."`
+	Header     []idk.RawField `help:"Header configuration."`
+}
+
+func NewRunner(cfg ConfigForIDK, batcher fbbatch.Batcher, logWriter io.Writer) *Runner {
+	idkMain := idk.NewMain()
+	idkMain.IDField = cfg.IDField
+	idkMain.Index = cfg.Table
+	idkMain.Batcher = batcher
+	idkMain.BatchSize = cfg.BatchSize
+	idkMain.BatchMaxStaleness = cfg.BatchMaxStaleness
+	idkMain.SetBasic()
+	idkMain.SetLog(logger.NewStandardLogger(logWriter))
+
+	kr := &Runner{
+		Main:       *idkMain,
+		KafkaHosts: cfg.Hosts,
+		Group:      cfg.Group,
+		Topics:     cfg.Topics,
+		Header:     cfg.Fields,
+		Timeout:    cfg.Timeout,
+	}
+	kr.OffsetMode = true
+	kr.Main.Namespace = "cli_kafka_runner"
+	kr.Main.Pprof = "" // don't initialize pprof until we actually use it in tests
+	kr.NewSource = func() (idk.Source, error) {
+		source := kafka_static.NewSource()
+		source.Hosts = kr.KafkaHosts
+		source.Group = kr.Group
+		source.Topics = kr.Topics
+		source.Log = kr.Main.Log()
+		// source.TLS = m.KafkaTLS
+		source.Timeout = kr.Timeout
+		// source.SkipOld = m.SkipOld
+		source.HeaderFields = kr.Header
+		// source.S3Region = m.S3Region
+		// source.AllowMissingFields = m.AllowMissingFields
+
+		err := source.Open()
+		if err != nil {
+			return nil, errors.Wrap(err, "opening source")
+		}
+		return source, nil
+	}
+	return kr
+}

cli/meta.go

@@ -2,6 +2,7 @@ package cli
 
 import (
 	"bufio"
+	"fmt"
 	"io"
 	"os"
 	"os/exec"
@@ -17,10 +18,11 @@ import (
 	"github.com/featurebasedb/featurebase/v3/errors"
 )
 
-// action is used to indicate how CLICommand should respond after execution a
-// given metaCommand. For example, an action of type "reset" tells CLICommand
-// that the buffer has been reset and it needs to change its user prompt.
-type action string
+// responseAction is used to indicate how CLICommand should respond after
+// executing a given metaCommand. For example, a responseAction of type "reset"
+// tells CLICommand that the buffer has been reset and it needs to change its
+// user prompt.
+type responseAction string
 
 const (
 	actionNone  = ""
@@ -30,7 +32,7 @@ const (
 
 // metaCommand is the interface for any type responding to a "\" meta-command.
 type metaCommand interface {
-	execute(cmd *Command) (action, error)
+	execute(cmd *Command) (responseAction, error)
 }
 
 // Ensure type implements interface.
@@ -38,6 +40,7 @@ var _ metaCommand = (*metaBang)(nil)
 var _ metaCommand = (*metaBorder)(nil)
 var _ metaCommand = (*metaChangeDirectory)(nil)
 var _ metaCommand = (*metaConnect)(nil)
+var _ metaCommand = (*metaDescribe)(nil)
 var _ metaCommand = (*metaEcho)(nil)
 var _ metaCommand = (*metaExpanded)(nil)
 var _ metaCommand = (*metaFile)(nil)
@@ -45,6 +48,7 @@ var _ metaCommand = (*metaHelp)(nil)
 var _ metaCommand = (*metaInclude)(nil)
 var _ metaCommand = (*metaListDatabases)(nil)
 var _ metaCommand = (*metaListTables)(nil)
+var _ metaCommand = (*metaListViews)(nil)
 var _ metaCommand = (*metaOrg)(nil)
 var _ metaCommand = (*metaOutput)(nil)
 var _ metaCommand = (*metaPrint)(nil)
@@ -73,13 +77,13 @@ func newMetaBang(args []string) *metaBang {
 	}
 }
 
-func (m *metaBang) execute(cmd *Command) (action, error) {
+func (m *metaBang) execute(cmd *Command) (responseAction, error) {
 	if len(m.args) == 0 {
 		return actionNone, errors.Errorf("meta command '!' requires at least one argument")
 	}
 	c := exec.Command(m.args[0])
 	c.Args = m.args
-	c.Stdout = cmd.Stdout
+	c.Stdout = cmd.stdout
 	err := c.Run()
 	return actionNone, errors.Wrap(err, "running bang command")
 }
@@ -97,7 +101,7 @@ func newMetaBorder(args []string) *metaBorder {
 	}
 }
 
-func (m *metaBorder) execute(cmd *Command) (action, error) {
+func (m *metaBorder) execute(cmd *Command) (responseAction, error) {
 	switch len(m.args) {
 	case 0:
 		// pass
@@ -131,12 +135,23 @@ func newMetaChangeDirectory(args []string) *metaChangeDirectory {
 	}
 }
 
-func (m *metaChangeDirectory) execute(cmd *Command) (action, error) {
-	if len(m.args) != 1 {
-		return actionNone, errors.Errorf("meta command 'cd' requires exactly one argument")
+func (m *metaChangeDirectory) execute(cmd *Command) (responseAction, error) {
+	var dir string
+	switch len(m.args) {
+	case 0:
+		if d, err := os.UserHomeDir(); err != nil {
+			return actionNone, errors.Wrapf(err, "getting home directory")
+		} else {
+			dir = d
+		}
+	case 1:
+		dir = m.args[0]
+	default:
+		return actionNone, errors.Errorf("meta command 'cd' takes zero or one argument")
 	}
-	err := cmd.workingDir.cd(m.args[0])
-	return actionNone, errors.Wrap(err, "running cd command")
+
+	err := cmd.workingDir.cd(dir)
+	return actionNone, errors.Wrapf(err, "changing directory to: %s", dir)
 }
 
 // ////////////////////////////////////////////////////////////////////////////
@@ -152,7 +167,7 @@ func newMetaConnect(args []string) *metaConnect {
 	}
 }
 
-func (m *metaConnect) execute(cmd *Command) (action, error) {
+func (m *metaConnect) execute(cmd *Command) (responseAction, error) {
 	switch len(m.args) {
 	case 0:
 		cmd.Printf(cmd.connectionMessage())
@@ -160,7 +175,6 @@ func (m *metaConnect) execute(cmd *Command) (action, error) {
 	case 1:
 		err := cmd.connectToDatabase(m.args[0])
 		return actionNone, err
-
 	default:
 		return actionNone, errors.Errorf("meta command 'connect' takes zero or one argument")
 	}
@@ -179,11 +193,11 @@ func newMetaEcho(args []string) *metaEcho {
 	}
 }
 
-func (m *metaEcho) execute(cmd *Command) (action, error) {
-	return echo(m.args, cmd.Stdout)
+func (m *metaEcho) execute(cmd *Command) (responseAction, error) {
+	return echo(m.args, cmd.stdout)
 }
 
-func echo(args []string, w io.Writer) (action, error) {
+func echo(args []string, w io.Writer) (responseAction, error) {
 	switch len(args) {
 	case 0:
 		w.Write([]byte("\n"))
@@ -220,7 +234,7 @@ func newMetaExpanded(args []string) *metaExpanded {
 	}
 }
 
-func (m *metaExpanded) execute(cmd *Command) (action, error) {
+func (m *metaExpanded) execute(cmd *Command) (responseAction, error) {
 	switch len(m.args) {
 	case 0:
 		cmd.writeOptions.expanded = !cmd.writeOptions.expanded
@@ -259,7 +273,7 @@ func newMetaFile(args []string) *metaFile {
 	}
 }
 
-func (m *metaFile) execute(cmd *Command) (action, error) {
+func (m *metaFile) execute(cmd *Command) (responseAction, error) {
 	if len(m.args) != 1 {
 		return actionNone, errors.Errorf("meta command 'file' requires exactly one argument")
 	}
@@ -281,6 +295,40 @@ func (m *metaFile) execute(cmd *Command) (action, error) {
 	return actionNone, errors.Wrap(err, "adding part file")
 }
 
+// ////////////////////////////////////////////////////////////////////////////
+// format (sub-command of pset)
+// ////////////////////////////////////////////////////////////////////////////
+type metaFormat struct {
+	args []string
+}
+
+func newMetaFormat(args []string) *metaFormat {
+	return &metaFormat{
+		args: args,
+	}
+}
+
+func (m *metaFormat) execute(cmd *Command) (responseAction, error) {
+	switch len(m.args) {
+	case 0:
+		// pass
+	case 1:
+		switch m.args[0] {
+		case formatAligned:
+			cmd.writeOptions.format = formatAligned
+		case formatCSV:
+			cmd.writeOptions.format = formatCSV
+		default:
+			return actionNone, errors.Errorf("\\pset: allowed formats are aligned, csv")
+		}
+	default:
+		return actionNone, errors.Errorf("meta command 'format' takes zero or one argument")
+	}
+
+	cmd.Printf("Output format is %s.\n", cmd.writeOptions.format)
+	return actionNone, nil
+}
+
 // ////////////////////////////////////////////////////////////////////////////
 // help (?)
 // ////////////////////////////////////////////////////////////////////////////
@@ -294,7 +342,7 @@ func newMetaHelp(args []string) *metaHelp {
 	}
 }
 
-func (m *metaHelp) execute(cmd *Command) (action, error) {
+func (m *metaHelp) execute(cmd *Command) (responseAction, error) {
 	helpText := `General
   \q[uit]                quit psql
   \watch [SEC]           execute query every SEC seconds
@@ -305,30 +353,32 @@ Help
 Query Buffer
   \p[rint]               show the contents of the query buffer
   \r[eset]               reset (clear) the query buffer
-  \w FILE                write query buffer to file
+  \w[rite] FILE          write query buffer to file
 
 Input/Output
   \echo [-n] [STRING]    write string to standard output (-n for no newline)
   \file ...              reference a local file to stream to the server
   \i[nclude] FILE        execute commands from file
-  \o [FILE]              send all query results to file
+  \o[ut] [FILE]          send all query results to file
   \qecho [-n] [STRING]   write string to \o output stream (-n for no newline)
   \warn [-n] [STRING]    write string to standard error (-n for no newline)
 
 Informational
-  \d                     list tables and views
+  \d                     list tables, including system tables
+  \d NAME                describe table
   \dt                    list tables
   \dv                    list views
-  \l                     list databases
+  \l[ist]                list databases
 
 Formatting
   \pset [NAME [VALUE]]   set table output option
-                         (border|expanded|tuples_only)
+                         (border|expanded|format|location|tuples_only)
   \t [on|off]            show only rows
   \x [on|off]            toggle expanded output
 
 Connection
   \c[onnect] [DBNAME]    connect to new database
+                         disconnect by sending DBNAME "-"
   \org [ORGNAME]         set organization id
 
 Operating System
@@ -358,7 +408,7 @@ func newMetaInclude(args []string) *metaInclude {
 	}
 }
 
-func (m *metaInclude) execute(cmd *Command) (action, error) {
+func (m *metaInclude) execute(cmd *Command) (responseAction, error) {
 	if len(m.args) != 1 {
 		return actionNone, errors.Errorf("meta command 'include' requires exactly one argument")
 	}
@@ -366,7 +416,7 @@ func (m *metaInclude) execute(cmd *Command) (action, error) {
 	return executeFile(cmd, m.args[0])
 }
 
-func executeFile(cmd *Command, fileName string) (action, error) {
+func executeFile(cmd *Command, fileName string) (responseAction, error) {
 	file, err := os.Open(fileName)
 	if err != nil {
 		return actionNone, errors.Wrapf(err, "opening file: %s", fileName)
@@ -385,8 +435,6 @@ func executeFile(cmd *Command, fileName string) (action, error) {
 		qps, mcs, err := splitter.split(line)
 		if err != nil {
 			return actionNone, errors.Wrapf(err, "splitting lines")
-		} else if len(mcs) > 0 {
-			return actionNone, errors.Errorf("include does not support meta-commands")
 		}
 
 		for i := range qps {
@@ -398,6 +446,18 @@ func executeFile(cmd *Command, fileName string) (action, error) {
 				}
 			}
 		}
+
+		for i := range mcs {
+			action, err := mcs[i].execute(cmd)
+			if err != nil {
+				return actionNone, errors.Wrap(err, "executing meta command")
+			}
+			switch action {
+			case actionQuit:
+				close(cmd.quit)
+				return action, nil
+			}
+		}
 	}
 	if err := sc.Err(); err != nil {
 		return actionNone, errors.Wrapf(err, "scanning file: %s", fileName)
@@ -415,7 +475,7 @@ func newMetaListDatabases() *metaListDatabases {
 	return &metaListDatabases{}
 }
 
-func (m *metaListDatabases) execute(cmd *Command) (action, error) {
+func (m *metaListDatabases) execute(cmd *Command) (responseAction, error) {
 	qry := []queryPart{
 		newPartRaw("SHOW DATABASES"),
 	}
@@ -428,7 +488,53 @@ func (m *metaListDatabases) execute(cmd *Command) (action, error) {
 }
 
 // ////////////////////////////////////////////////////////////////////////////
-// list tables (d or dt)
+// describe (d)
+// ////////////////////////////////////////////////////////////////////////////
+type metaDescribe struct {
+	args []string
+}
+
+func newMetaDescribe(args []string) *metaDescribe {
+	return &metaDescribe{
+		args: args,
+	}
+}
+
+func (m *metaDescribe) execute(cmd *Command) (responseAction, error) {
+	switch len(m.args) {
+	case 0:
+		// Describe with no args should list all relations (tables, views,
+		// etc.). For now, we're just going to list the tables, including system
+		// tables.
+		qry := []queryPart{
+			newPartRaw("SHOW TABLES WITH SYSTEM"),
+		}
+
+		if err := cmd.executeAndWriteQuery(qry); err != nil {
+			return actionNone, errors.Wrap(err, "executing query")
+		}
+
+		return actionReset, nil
+
+	case 1:
+		// Describe with a single arg will assume the arg is a table name, so it
+		// runs a `SHOW COLUMNS` for that table.
+		qry := []queryPart{
+			newPartRaw(fmt.Sprintf(`SHOW COLUMNS FROM "%s"`, m.args[0])),
+		}
+
+		if err := cmd.executeAndWriteQuery(qry); err != nil {
+			return actionNone, errors.Wrap(err, "executing query")
+		}
+
+		return actionReset, nil
+	default:
+		return actionNone, errors.Errorf("meta command 'describe' takes zero or one argument")
+	}
+}
+
+// ////////////////////////////////////////////////////////////////////////////
+// describe (dt)
 // ////////////////////////////////////////////////////////////////////////////
 type metaListTables struct{}
 
@@ -436,7 +542,7 @@ func newMetaListTables() *metaListTables {
 	return &metaListTables{}
 }
 
-func (m *metaListTables) execute(cmd *Command) (action, error) {
+func (m *metaListTables) execute(cmd *Command) (responseAction, error) {
 	qry := []queryPart{
 		newPartRaw("SHOW TABLES"),
 	}
@@ -448,6 +554,58 @@ func (m *metaListTables) execute(cmd *Command) (action, error) {
 	return actionReset, nil
 }
 
+// ////////////////////////////////////////////////////////////////////////////
+// describe views (dv)
+// ////////////////////////////////////////////////////////////////////////////
+type metaListViews struct{}
+
+func newMetaListViews() *metaListViews {
+	return &metaListViews{}
+}
+
+func (m *metaListViews) execute(cmd *Command) (responseAction, error) {
+	qry := []queryPart{
+		newPartRaw("SELECT * FROM fb_views"),
+	}
+
+	if err := cmd.executeAndWriteQuery(qry); err != nil {
+		return actionNone, errors.Wrap(err, "executing query")
+	}
+
+	return actionReset, nil
+}
+
+// ////////////////////////////////////////////////////////////////////////////
+// location (sub-command of pset)
+// ////////////////////////////////////////////////////////////////////////////
+type metaLocation struct {
+	args []string
+}
+
+func newMetaLocation(args []string) *metaLocation {
+	return &metaLocation{
+		args: args,
+	}
+}
+
+func (m *metaLocation) execute(cmd *Command) (responseAction, error) {
+	switch len(m.args) {
+	case 0:
+		// pass
+	case 1:
+		loc, err := time.LoadLocation(m.args[0])
+		if err != nil {
+			return actionNone, errors.Wrapf(err, "loading location: %s", m.args[0])
+		}
+		cmd.writeOptions.location = loc
+	default:
+		return actionNone, errors.Errorf("meta command 'location' takes zero or one argument")
+	}
+
+	cmd.Printf("Location is %s.\n", cmd.writeOptions.location)
+	return actionNone, nil
+}
+
 // ////////////////////////////////////////////////////////////////////////////
 // org
 // ////////////////////////////////////////////////////////////////////////////
@@ -461,7 +619,7 @@ func newMetaOrg(args []string) *metaOrg {
 	}
 }
 
-func (m *metaOrg) execute(cmd *Command) (action, error) {
+func (m *metaOrg) execute(cmd *Command) (responseAction, error) {
 	switch len(m.args) {
 
 	case 0:
@@ -490,7 +648,7 @@ func newMetaOutput(args []string) *metaOutput {
 	}
 }
 
-func (m *metaOutput) execute(cmd *Command) (action, error) {
+func (m *metaOutput) execute(cmd *Command) (responseAction, error) {
 	switch len(m.args) {
 	case 0:
 		// Close cmd.output (if closable).
@@ -499,7 +657,7 @@ func (m *metaOutput) execute(cmd *Command) (action, error) {
 		}
 
 		// Set cmd.output to cmd.Stdout.
-		cmd.output = cmd.Stdout
+		cmd.output = cmd.stdout
 
 		return actionNone, nil
 
@@ -532,7 +690,7 @@ func newMetaPrint() *metaPrint {
 	return &metaPrint{}
 }
 
-func (m *metaPrint) execute(cmd *Command) (action, error) {
+func (m *metaPrint) execute(cmd *Command) (responseAction, error) {
 	cmd.Printf(cmd.buffer.print() + "\n")
 	return actionNone, nil
 }
@@ -560,18 +718,22 @@ func (m *metaPSet) print(cmd *Command) {
 
 	fmt := `border      %d
 expanded    %s
+format      %s
+location    %s
 tuples_only %s
 `
 
 	cmd.Printf(fmt,
 		cmd.writeOptions.border,
 		onOff(cmd.writeOptions.expanded),
+		cmd.writeOptions.format,
+		cmd.writeOptions.location,
 		onOff(cmd.writeOptions.tuplesOnly),
 	)
 
 }
 
-func (m *metaPSet) execute(cmd *Command) (action, error) {
+func (m *metaPSet) execute(cmd *Command) (responseAction, error) {
 	switch len(m.args) {
 	case 0:
 		m.print(cmd)
@@ -581,10 +743,16 @@ func (m *metaPSet) execute(cmd *Command) (action, error) {
 		case "border":
 			sub := newMetaBorder(m.args[1:])
 			return sub.execute(cmd)
-		case "expanded":
+		case "expanded", "x":
 			sub := newMetaExpanded(m.args[1:])
 			return sub.execute(cmd)
-		case "tuples_only":
+		case "format":
+			sub := newMetaFormat(m.args[1:])
+			return sub.execute(cmd)
+		case "location":
+			sub := newMetaLocation(m.args[1:])
+			return sub.execute(cmd)
+		case "tuples_only", "t":
 			sub := newMetaTuplesOnly(m.args[1:])
 			return sub.execute(cmd)
 		default:
@@ -608,7 +776,7 @@ func newMetaQEcho(args []string) *metaQEcho {
 	}
 }
 
-func (m *metaQEcho) execute(cmd *Command) (action, error) {
+func (m *metaQEcho) execute(cmd *Command) (responseAction, error) {
 	return echo(m.args, cmd.output)
 }
 
@@ -621,7 +789,7 @@ func newMetaQuit() *metaQuit {
 	return &metaQuit{}
 }
 
-func (m *metaQuit) execute(cmd *Command) (action, error) {
+func (m *metaQuit) execute(cmd *Command) (responseAction, error) {
 	return actionQuit, nil
 }
 
@@ -634,8 +802,8 @@ func newMetaReset() *metaReset {
 	return &metaReset{}
 }
 
-func (m *metaReset) execute(cmd *Command) (action, error) {
-	cmd.Printf(cmd.buffer.reset())
+func (m *metaReset) execute(cmd *Command) (responseAction, error) {
+	cmd.Printf(cmd.buffer.reset() + "\n")
 	return actionReset, nil
 }
 
@@ -652,7 +820,7 @@ func newMetaSet(args []string) *metaSet {
 	}
 }
 
-func (m *metaSet) execute(cmd *Command) (action, error) {
+func (m *metaSet) execute(cmd *Command) (responseAction, error) {
 	switch len(m.args) {
 	case 0:
 		// Sort the variables before printing them.
@@ -692,7 +860,7 @@ func newMetaTiming(args []string) *metaTiming {
 	}
 }
 
-func (m *metaTiming) execute(cmd *Command) (action, error) {
+func (m *metaTiming) execute(cmd *Command) (responseAction, error) {
 	switch len(m.args) {
 	case 0:
 		cmd.writeOptions.timing = !cmd.writeOptions.timing
@@ -703,7 +871,7 @@ func (m *metaTiming) execute(cmd *Command) (action, error) {
 		case "off":
 			cmd.writeOptions.timing = false
 		default:
-			return actionNone, errors.Errorf("unrecognized value \"%s\" for \"\timing\": Boolean expected", m.args[0])
+			return actionNone, errors.Errorf("unrecognized value \"%s\" for \"\\timing\": Boolean expected", m.args[0])
 		}
 	default:
 		return actionNone, errors.Errorf("meta command 'timing' takes zero or one argument")
@@ -731,7 +899,7 @@ func newMetaTuplesOnly(args []string) *metaTuplesOnly {
 	}
 }
 
-func (m *metaTuplesOnly) execute(cmd *Command) (action, error) {
+func (m *metaTuplesOnly) execute(cmd *Command) (responseAction, error) {
 	switch len(m.args) {
 	case 0:
 		cmd.writeOptions.tuplesOnly = !cmd.writeOptions.tuplesOnly
@@ -770,7 +938,7 @@ func newMetaUnset(args []string) *metaUnset {
 	}
 }
 
-func (m *metaUnset) execute(cmd *Command) (action, error) {
+func (m *metaUnset) execute(cmd *Command) (responseAction, error) {
 	switch len(m.args) {
 	case 0:
 		cmd.Printf("\\unset: missing required argument\n")
@@ -799,8 +967,8 @@ func newMetaWarn(args []string) *metaWarn {
 	}
 }
 
-func (m *metaWarn) execute(cmd *Command) (action, error) {
-	return echo(m.args, cmd.Stderr)
+func (m *metaWarn) execute(cmd *Command) (responseAction, error) {
+	return echo(m.args, cmd.stderr)
 }
 
 // ////////////////////////////////////////////////////////////////////////////
@@ -816,7 +984,7 @@ func newMetaWatch(args []string) *metaWatch {
 	}
 }
 
-func (m *metaWatch) execute(cmd *Command) (action, error) {
+func (m *metaWatch) execute(cmd *Command) (responseAction, error) {
 	period := 2 * time.Second
 
 	qry := cmd.buffer.lastQuery
@@ -870,7 +1038,7 @@ func newMetaWrite(args []string) *metaWrite {
 	}
 }
 
-func (m *metaWrite) execute(cmd *Command) (action, error) {
+func (m *metaWrite) execute(cmd *Command) (responseAction, error) {
 	switch len(m.args) {
 	case 0:
 		cmd.Errorf(`\w: missing required argument` + "\n")
@@ -946,8 +1114,12 @@ func splitMetaCommand(in string, replacer *replacer) (metaCommand, error) {
 		return newMetaChangeDirectory(args), nil
 	case "c", "connect":
 		return newMetaConnect(args), nil
-	case "d", "dt":
+	case "d":
+		return newMetaDescribe(args), nil
+	case "dt":
 		return newMetaListTables(), nil
+	case "dv":
+		return newMetaListViews(), nil
 	case "echo":
 		return newMetaEcho(args), nil
 	case "file":
@@ -956,9 +1128,9 @@ func splitMetaCommand(in string, replacer *replacer) (metaCommand, error) {
 		return newMetaHelp(args), nil
 	case "i", "include":
 		return newMetaInclude(args), nil
-	case "l":
+	case "l", "list":
 		return newMetaListDatabases(), nil
-	case "o":
+	case "o", "out":
 		return newMetaOutput(args), nil
 	case "org":
 		return newMetaOrg(args), nil
@@ -984,7 +1156,7 @@ func splitMetaCommand(in string, replacer *replacer) (metaCommand, error) {
 		return newMetaWarn(args), nil
 	case "watch":
 		return newMetaWatch(args), nil
-	case "w":
+	case "w", "write":
 		return newMetaWrite(args), nil
 	case "x":
 		return newMetaExpanded(args), nil

cli/parts.go

@@ -39,6 +39,12 @@ type queryPart interface {
 	Reader() io.Reader
 }
 
+func newRawQuery(s string) query {
+	return []queryPart{
+		newPartRaw(s),
+	}
+}
+
 // ////////////////////////////////////////////////////////////////////////////
 // raw
 // ////////////////////////////////////////////////////////////////////////////

cli/queryer.go

@@ -45,8 +45,10 @@ func (qryr *standardQueryer) Query(org string, db string, sql io.Reader) (*featu
 	if err != nil {
 		return nil, errors.Wrap(err, "reading response")
 	}
+
 	sqlResponse := &featurebase.WireQueryResponse{}
 	// TODO(tlt): switch this back once all responses are typed
+	// TODO(twg) 2023/03/01 using json.Number to decode large ints so care must be made
 	// if err := json.Unmarshal(fullbod, sqlResponse); err != nil {
 	if err := sqlResponse.UnmarshalJSONTyped(fullbod, true); err != nil {
 		return nil, errors.Wrapf(err, "unmarshaling query response, body:\n'%s'\n", fullbod)
@@ -66,7 +68,10 @@ type serverlessQueryer struct {
 }
 
 func (qryr *serverlessQueryer) Query(org string, db string, sql io.Reader) (*featurebase.WireQueryResponse, error) {
-	// buf := bytes.Buffer{}
+	if org == "" {
+		return nil, NewErrOrganizationRequired()
+	}
+
 	url := fmt.Sprintf("%s/queryer/databases/%s/sql", hostPort(qryr.Host, qryr.Port), db)
 	if db == "" {
 		url = fmt.Sprintf("%s/queryer/sql", hostPort(qryr.Host, qryr.Port))

cli/splitter.go

@@ -30,7 +30,12 @@ func newSplitter(r *replacer) *splitter {
 // 2- [metaCommands...]: "\! pwd \q"
 // 3- [queryParts...][metaCommands...]: "select * from \i file.sql"
 func (s *splitter) split(line string) ([]queryPart, []metaCommand, error) {
-	// Look for a meta command
+	// Look for a comment line.
+	if strings.HasPrefix(line, "--") {
+		return nil, nil, nil
+	}
+
+	// Look for a meta command.
 	parts := strings.SplitN(line, `\`, 2)
 
 	switch len(parts) {

cli/testdata/database

@@ -0,0 +1,53 @@
+// Show databases now that we have set org.
+SEND:SHOW DATABASES;
+EXPECT:+-----+------+-------+------------+------------+------------+-------+-------------+
+EXPECT:| _id | name | owner | updated_by | created_at | updated_at | units | description |
+EXPECT:+-----+------+-------+------------+------------+------------+-------+-------------+
+EXPECT:+-----+------+-------+------------+------------+------------+-------+-------------+
+EXPECT:
+
+// Create db1.
+SEND:CREATE DATABASE db1 WITH UNITS 1;
+EXPECT:
+
+// List databases via SHOW DATABASES.
+SEND:SHOW DATABASES;
+EXPECT:+--------------------------------------+------+-------+------------+----------------------+----------------------+-------+-------------+
+EXPECT:| _id                                  | name | owner | updated_by | created_at           | updated_at           | units | description |
+EXPECT:+--------------------------------------+------+-------+------------+----------------------+----------------------+-------+-------------+
+EXPECTCOMP:WithFormat:| {uuid} | db1  |       |            | {timestamp} | {timestamp} |     1 |             |
+EXPECT:+--------------------------------------+------+-------+------------+----------------------+----------------------+-------+-------------+
+EXPECT:
+
+// List databases via SHOW DATABASES.
+SEND:\l
+EXPECT:+--------------------------------------+------+-------+------------+----------------------+----------------------+-------+-------------+
+EXPECT:| _id                                  | name | owner | updated_by | created_at           | updated_at           | units | description |
+EXPECT:+--------------------------------------+------+-------+------------+----------------------+----------------------+-------+-------------+
+EXPECTCOMP:WithFormat:| {uuid} | db1  |       |            | {timestamp} | {timestamp} |     1 |             |
+EXPECT:+--------------------------------------+------+-------+------------+----------------------+----------------------+-------+-------------+
+EXPECT:
+
+// Check database connection.
+SEND:\c
+EXPECT:You are not connected to a database.
+
+// Try connecting to an invalid database.
+SEND:\c invalid
+EXPECT:executing meta command: invalid database: invalid
+
+// Try connecting with too many arguments.
+SEND:\c db1 extra
+EXPECT:executing meta command: meta command 'connect' takes zero or one argument
+
+// Connect to a database.
+SEND:\c db1
+EXPECTCOMP:WithFormat:You are now connected to database "db1" ({uuid}).
+
+// Disconnect from the current database.
+SEND:\c -
+EXPECT:You are not connected to a database.
+
+// Connect to a database again.
+SEND:\c db1
+EXPECTCOMP:WithFormat:You are now connected to database "db1" ({uuid}).

cli/testdata/famous.csv

@@ -0,0 +1,10 @@
+
+"Id", "Name", "Short description", "Gender", "Country", "Occupation", "Birth year", "Death year", "Manner of death", "Age of death"
+1, "George Washington", "1st president of the United States (1732–1799)", "Male", "United States of America; Kingdom of Great Britain", "Politician", "1732", "1799", "natural causes", "67"
+2, "Douglas Adams", "English writer and humorist", "Male", "United Kingdom", "Artist", "1952", "2001", "natural causes", "49"
+3, "Abraham Lincoln", "16th president of the United States (1809-1865)", "Male", "United States of America", "Politician", "1809", "1865", "homicide", "56"
+4, "Wolfgang Amadeus Mozart", "Austrian composer of the Classical period", "Male", "Archduchy of Austria; Archbishopric of Salzburg", "Artist", "1756", "1791", "0", "35"
+5, "Ludwig van Beethoven", "German classical and romantic composer", "Male", "Holy Roman Empire; Austrian Empire", "Artist", "1770", "1827", "0", "57"
+6, "Jean-François Champollion", "French classical scholar", "Male", "Kingdom of France; First French Empire", "Egyptologist", "1790", "1832", "natural causes", "42"
+7, "Paul Morand", "French writer", "Male", "France", "Artist", "1888", "1976", "0", "88"
+8, "Claude Monet", "French impressionist painter (1840-1926)", "Male", "France", "Artist", "1840", "1926", "natural causes", "86"

cli/testdata/meta_bang

@@ -0,0 +1,8 @@
+SEND:\! echo 'foo'
+EXPECT:foo
+
+SEND:\! echo "foo"
+EXPECT:"foo"
+
+SEND:\!
+EXPECT:executing meta command: meta command '!' requires at least one argument

cli/testdata/meta_cd

@@ -0,0 +1,17 @@
+// Make a directory so we can test \cd'ing into it.
+SEND:\! mkdir cli-test-dir
+SEND:\cd cli-test-dir
+SEND:\cd ..
+SEND:\! rmdir cli-test-dir
+
+// TODO(tlt): before we do this, we should implement the ability to execute
+// commands in a \set like:
+// \set homedir `pwd`
+// then we can store what directory we're in so we can move back to it 
+// at the end of the test
+// Switch to home directory.
+// SEND:\cd
+
+// Expect error on extra argument to \cd.
+SEND:\cd dir extra
+EXPECT:executing meta command: meta command 'cd' takes zero or one argument

cli/testdata/meta_describe

@@ -0,0 +1,33 @@
+// TODO(tlt): we can't run this test until we get the system tables under control (i.e. sorted). Currently, fb_views is in a map with users, so the following can fail 50% of the time.
+// Show tables for database by calling describe with no args.
+// SEND:\d
+// EXPECT:+-------------------------+-------------------------+-------+------------+----------------------+----------------------+-------+------------+------------------------+
+// EXPECT:| _id                     | name                    | owner | updated_by | created_at           | updated_at           | keys  | space_used | description            |
+// EXPECT:+-------------------------+-------------------------+-------+------------+----------------------+----------------------+-------+------------+------------------------+
+// EXPECTCOMP:WithFormat:| fb_veiws                | fb_views                |       |            | {timestamp} | {timestamp} | true  |          0 | system table for views |
+// EXPECTCOMP:WithFormat:| users                   | users                   |       |            | {timestamp} | {timestamp} | false |          0 |                        |
+// EXPECTCOMP:WithFormat:| fb_____________________ | fb_____________________ |       |            | {timestamp} | {timestamp} | false |          0 |                        |
+// EXPECTCOMP:WithFormat:| fb_____________________ | fb_____________________ |       |            | {timestamp} | {timestamp} | false |          0 |                        |
+// EXPECTCOMP:WithFormat:| fb_____________________ | fb_____________________ |       |            | {timestamp} | {timestamp} | false |          0 |                        |
+// EXPECTCOMP:WithFormat:| fb_____________________ | fb_____________________ |       |            | {timestamp} | {timestamp} | false |          0 |                        |
+// EXPECTCOMP:WithFormat:| fb_____________________ | fb_____________________ |       |            | {timestamp} | {timestamp} | false |          0 |                        |
+// EXPECT:+-------------------------+-------------------------+-------+------------+----------------------+----------------------+-------+------------+------------------------+
+// EXPECT:
+
+// Show columns for table.
+SEND:\d users
+EXPECT:+------+------+--------+----------------------+-------+------------+------------+-------+----------------------+---------------------+----------+-------+-------------+-----+
+EXPECT:| _id  | name | type   | created_at           | keys  | cache_type | cache_size | scale |                  min |                 max | timeunit | epoch | timequantum | ttl |
+EXPECT:+------+------+--------+----------------------+-------+------------+------------+-------+----------------------+---------------------+----------+-------+-------------+-----+
+EXPECTCOMP:WithFormat:| _id  | _id  | id     | {timestamp} | false |            |          0 |     0 |                    0 |                   0 |          |     0 |             | 0s  |
+EXPECTCOMP:WithFormat:| name | name | string | {timestamp} | true  | ranked     |      50000 |     0 |                    0 |                   0 |          |     0 |             | 0s  |
+EXPECTCOMP:WithFormat:| age  | age  | int    | {timestamp} | false |            |          0 |     0 | -9223372036854775808 | 9223372036854775807 |          |     0 |             | 0s  |
+EXPECT:+------+------+--------+----------------------+-------+------------+------------+-------+----------------------+---------------------+----------+-------+-------------+-----+
+EXPECT:
+
+// Show columns for an invalid table.
+SEND:\d invalid
+EXPECT:Error: compiling plan: [1:19] table 'invalid' not found
+
+SEND:\d users extra
+EXPECT:executing meta command: meta command 'describe' takes zero or one argument

cli/testdata/meta_echo

@@ -0,0 +1,6 @@
+SEND:\echo
+EXPECT:
+
+// Simple \echo.
+SEND:\echo foo bar
+EXPECT:foo bar

cli/testdata/meta_file

@@ -0,0 +1,70 @@
+// Create a table.
+SEND:CREATE TABLE famous (
+SEND:    _id ID,
+SEND:    name STRING,
+SEND:    description STRING,
+SEND:    gender STRING,
+SEND:    country STRING,
+SEND:    occupation STRING,
+SEND:    birth_year INT min -32767 max 32767,
+SEND:    death_year INT min -32767 max 32767,
+SEND:    death_manner STRING,
+SEND:    birth_age INT min -32767 max 32767
+SEND:);
+EXPECT:
+
+// Open bulk insert.
+SEND:BULK INSERT
+SEND:INTO famous (_id, name, description, gender, country, occupation,
+SEND:    birth_year, death_year, death_manner, birth_age )
+SEND:MAP(0 INT,
+SEND:1 STRING,
+SEND:2 STRING,
+SEND:3 STRING,
+SEND:4 STRING,
+SEND:5 STRING,
+SEND:6 INT,
+SEND:7 INT,
+SEND:8 STRING,
+SEND:9 INT )
+SEND:FROM
+SEND:    x'
+
+// Call \file
+SEND:\file testdata/famous.csv
+
+// Close bulk insert.
+SEND:'
+SEND:WITH
+SEND:    BATCHSIZE 100000
+SEND:    FORMAT 'CSV'
+SEND:    INPUT 'STREAM'
+SEND:    HEADER_ROW;
+EXPECT:
+
+// Query table to ensure we have data.
+SEND:SELECT * FROM famous;
+EXPECT:+-----+---------------------------+-------------------------------------------------+--------+----------------------------------------------------+--------------+------------+------------+----------------+-----------+
+EXPECT:| _id | name                      | description                                     | gender | country                                            | occupation   | birth_year | death_year | death_manner   | birth_age |
+EXPECT:+-----+---------------------------+-------------------------------------------------+--------+----------------------------------------------------+--------------+------------+------------+----------------+-----------+
+EXPECT:|   1 | George Washington         | 1st president of the United States (1732–1799)  | Male   | United States of America; Kingdom of Great Britain | Politician   |       1732 |       1799 | natural causes |        67 |
+EXPECT:|   2 | Douglas Adams             | English writer and humorist                     | Male   | United Kingdom                                     | Artist       |       1952 |       2001 | natural causes |        49 |
+EXPECT:|   3 | Abraham Lincoln           | 16th president of the United States (1809-1865) | Male   | United States of America                           | Politician   |       1809 |       1865 | homicide       |        56 |
+EXPECT:|   4 | Wolfgang Amadeus Mozart   | Austrian composer of the Classical period       | Male   | Archduchy of Austria; Archbishopric of Salzburg    | Artist       |       1756 |       1791 | 0              |        35 |
+EXPECT:|   5 | Ludwig van Beethoven      | German classical and romantic composer          | Male   | Holy Roman Empire; Austrian Empire                 | Artist       |       1770 |       1827 | 0              |        57 |
+EXPECT:|   6 | Jean-François Champollion | French classical scholar                        | Male   | Kingdom of France; First French Empire             | Egyptologist |       1790 |       1832 | natural causes |        42 |
+EXPECT:|   7 | Paul Morand               | French writer                                   | Male   | France                                             | Artist       |       1888 |       1976 | 0              |        88 |
+EXPECT:|   8 | Claude Monet              | French impressionist painter (1840-1926)        | Male   | France                                             | Artist       |       1840 |       1926 | natural causes |        86 |
+EXPECT:+-----+---------------------------+-------------------------------------------------+--------+----------------------------------------------------+--------------+------------+------------+----------------+-----------+
+EXPECT:
+
+// TODO(tlt): dropping the table seems to cause problems.
+// Drop the table.
+//SEND:DROP TABLE famous;
+
+// Ensure that invalid aruments (none or too many) return an error.
+SEND:\file
+EXPECT:executing meta command: meta command 'file' requires exactly one argument
+SEND:\file filename extra
+EXPECT:executing meta command: meta command 'file' requires exactly one argument
+

cli/testdata/meta_include

@@ -0,0 +1,24 @@
+// Include with no argument should error.
+SEND:\i
+EXPECT:executing meta command: meta command 'include' requires exactly one argument
+
+// Include with too many arguments should error.
+SEND:\include testdata/people.sql extra
+EXPECT:executing meta command: meta command 'include' requires exactly one argument
+
+// Invalid file should error.
+SEND:\include invalid.file
+EXPECT:executing meta command: opening file: invalid.file: open invalid.file: no such file or directory
+
+SEND:\include testdata/people.sql
+EXPECT:
+EXPECT:
+EXPECT:+-----+------+-----+
+EXPECT:| _id | name | age |
+EXPECT:+-----+------+-----+
+EXPECT:|   1 | Amy  |  42 |
+EXPECT:|   2 | Bob  |  27 |
+EXPECT:|   3 | Carl |  33 |
+EXPECT:+-----+------+-----+
+EXPECT:
+EXPECT:mix in a meta command

cli/testdata/meta_output

@@ -0,0 +1,67 @@
+SEND:SELECT * FROM users;
+EXPECT:+-----+-------+-----+
+EXPECT:| _id | name  | age |
+EXPECT:+-----+-------+-----+
+EXPECT:|   1 | Anne  |  38 |
+EXPECT:|   2 | Bill  |  23 |
+EXPECT:|   3 | Cindy |  64 |
+EXPECT:+-----+-------+-----+
+EXPECT:
+
+// Redirect output to a file.
+SEND:\o test-output-file
+SEND:SELECT * FROM users;
+
+// Ensure the output went to the file.
+SEND:\! cat test-output-file
+EXPECT:+-----+-------+-----+
+EXPECT:| _id | name  | age |
+EXPECT:+-----+-------+-----+
+EXPECT:|   1 | Anne  |  38 |
+EXPECT:|   2 | Bill  |  23 |
+EXPECT:|   3 | Cindy |  64 |
+EXPECT:+-----+-------+-----+
+EXPECT:
+
+// Let's test some qecho stuff here while we're at it.
+SEND:\qecho string with "double quotes"
+SEND:\qecho -n one
+SEND:\qecho -n two
+SEND:\qecho three
+SEND:\qecho four
+
+SEND:\! cat test-output-file
+EXPECT:+-----+-------+-----+
+EXPECT:| _id | name  | age |
+EXPECT:+-----+-------+-----+
+EXPECT:|   1 | Anne  |  38 |
+EXPECT:|   2 | Bill  |  23 |
+EXPECT:|   3 | Cindy |  64 |
+EXPECT:+-----+-------+-----+
+EXPECT:
+EXPECT:string with "double quotes"
+EXPECT:onetwothree
+EXPECT:four
+
+// And \warn messages should still go to stderr, not the file.
+SEND:\warn a warning string
+EXPECT:a warning string
+
+// Remove the file.
+SEND:\! rm test-output-file
+
+// Set the output back to stdout.
+SEND:\o
+SEND:SELECT * FROM users;
+EXPECT:+-----+-------+-----+
+EXPECT:| _id | name  | age |
+EXPECT:+-----+-------+-----+
+EXPECT:|   1 | Anne  |  38 |
+EXPECT:|   2 | Bill  |  23 |
+EXPECT:|   3 | Cindy |  64 |
+EXPECT:+-----+-------+-----+
+EXPECT:
+
+// Ensure extra arguments to \output causes an error.
+SEND:\o filename extra
+EXPECT:executing meta command: meta command 'output' takes zero or one argument

cli/testdata/meta_pset_border

@@ -0,0 +1,52 @@
+SEND:\pset border 2
+EXPECT:Border style is 2.
+
+SEND:SELECT 1 as foo, 'baz' as bar;
+EXPECT:+-----+-----+
+EXPECT:| foo | bar |
+EXPECT:+-----+-----+
+EXPECT:|   1 | baz |
+EXPECT:+-----+-----+
+EXPECT:
+
+SEND:\pset border
+EXPECT:Border style is 2.
+
+SEND:\pset border 999
+EXPECT:Border style is 0.
+
+SEND:\pset border 1
+EXPECT:Border style is 1.
+
+SEND:SELECT 1 as foo, 'baz' as bar;
+EXPECT: foo | bar 
+EXPECT:-----+-----
+EXPECT:   1 | baz 
+EXPECT:
+
+SEND:\pset border 2
+EXPECT:Border style is 2.
+
+SEND:SELECT 1 as foo, 'baz' as bar;
+EXPECT:+-----+-----+
+EXPECT:| foo | bar |
+EXPECT:+-----+-----+
+EXPECT:|   1 | baz |
+EXPECT:+-----+-----+
+EXPECT:
+
+SEND:\pset border 0
+EXPECT:Border style is 0.
+
+SEND:SELECT 1 as foo, 'baz' as bar;
+EXPECT:foo bar
+EXPECT:--- ---
+EXPECT:  1 baz
+EXPECT:
+
+SEND:\pset border 1 extra
+EXPECT:executing meta command: meta command 'pset' takes zero, one, or two arguments
+
+// Set border back to the testing default.
+SEND:\pset border 2
+EXPECT:Border style is 2.

cli/testdata/meta_pset_expanded

@@ -0,0 +1,35 @@
+// Set to off.
+SEND:\pset expanded off
+EXPECT:Expanded display is off.
+
+// Set to on.
+SEND:\pset expanded on
+EXPECT:Expanded display is on.
+
+// Toggle to off.
+SEND:\pset expanded
+EXPECT:Expanded display is off.
+
+// Toggle to on.
+SEND:\pset expanded
+EXPECT:Expanded display is on.
+
+// Set to something invalid.
+SEND:\pset expanded invalid
+EXPECT:executing meta command: unrecognized value "invalid" for "expanded": Boolean expected
+
+// Ensure expanded shows results vertically.
+SEND:SELECT 1 as foo, 'baz' as bar;
+EXPECT:+-----+-----+
+EXPECT:| foo | 1   |
+EXPECT:| bar | baz |
+EXPECT:+-----+-----+
+EXPECT:
+
+// Set back to off as we started.
+SEND:\pset expanded off
+EXPECT:Expanded display is off.
+
+// make sure the \x meta-command returns expected errors
+SEND:\x on extra
+EXPECT:executing meta command: meta command 'expanded' takes zero or one argument

cli/testdata/meta_pset_format_csv

@@ -0,0 +1,47 @@
+SEND:\pset format csv
+EXPECT:Output format is csv.
+
+SEND:SELECT * FROM users;
+EXPECT:_id,name,age
+EXPECT:1,Anne,38
+EXPECT:2,Bill,23
+EXPECT:3,Cindy,64
+
+// Exclude headers.
+SEND:\t on
+EXPECT:Tuples only is on.
+
+SEND:SELECT * FROM users;
+EXPECT:1,Anne,38
+EXPECT:2,Bill,23
+EXPECT:3,Cindy,64
+
+// Reset headers.
+SEND:\t off
+EXPECT:Tuples only is off.
+
+// Set expanded to on.
+SEND:\x on
+EXPECT:Expanded display is on.
+
+SEND:SELECT * FROM users;
+EXPECT:_id,1
+EXPECT:name,Anne
+EXPECT:age,38
+EXPECT:_id,2
+EXPECT:name,Bill
+EXPECT:age,23
+EXPECT:_id,3
+EXPECT:name,Cindy
+EXPECT:age,64
+
+// Set expanded back to off.
+SEND:\x off
+EXPECT:Expanded display is off.
+
+// Set format back to aligned as we started.
+SEND:\pset format aligned
+EXPECT:Output format is aligned.
+
+SEND:\pset format invalid
+EXPECT:executing meta command: \pset: allowed formats are aligned, csv

cli/testdata/meta_pset_tuples_only

@@ -0,0 +1,34 @@
+// Set to off.
+SEND:\pset tuples_only off
+EXPECT:Tuples only is off.
+
+// Set to on.
+SEND:\pset tuples_only on
+EXPECT:Tuples only is on.
+
+// Toggle to off.
+SEND:\pset tuples_only
+EXPECT:Tuples only is off.
+
+// Toggle to on.
+SEND:\pset tuples_only
+EXPECT:Tuples only is on.
+
+// Set to something invalid.
+SEND:\pset tuples_only invalid
+EXPECT:executing meta command: unrecognized value "invalid" for "tuples_only": Boolean expected
+
+// Ensure tuples_only shows only tuples.
+SEND:SELECT 1 as foo, 'baz' as bar;
+EXPECT:+---+-----+
+EXPECT:| 1 | baz |
+EXPECT:+---+-----+
+EXPECT:
+
+// Set back to off as we started.
+SEND:\pset tuples_only off
+EXPECT:Tuples only is off.
+
+// make sure the \t meta-command returns expected errors
+SEND:\t off extra
+EXPECT:executing meta command: meta command 'tuples_only' takes zero or one argument

cli/testdata/meta_set

@@ -0,0 +1,31 @@
+SEND:\set
+
+SEND:\set var1 foo
+SEND:\set
+EXPECT:var1 = 'foo'
+
+SEND:\set var2 bar
+SEND:\set
+EXPECT:var1 = 'foo'
+EXPECT:var2 = 'bar'
+
+SEND:\set var3 zoo
+SEND:\set
+EXPECT:var1 = 'foo'
+EXPECT:var2 = 'bar'
+EXPECT:var3 = 'zoo'
+
+SEND:\unset
+EXPECT:\unset: missing required argument
+
+SEND:\unset non-existent-key
+
+SEND:\unset var1
+SEND:\set
+EXPECT:var2 = 'bar'
+EXPECT:var3 = 'zoo'
+
+SEND:\unset var2 extra
+EXPECT:\unset: extra argument "extra" ignored
+SEND:\set
+EXPECT:var3 = 'zoo'

cli/testdata/meta_timing

@@ -0,0 +1,50 @@
+// Start by ensuring timing is off.
+SEND:\timing off
+EXPECT:Timing is off.
+
+// Set timing on.
+SEND:\timing on
+EXPECT:Timing is on.
+
+// Toggle timing.
+SEND:\timing
+EXPECT:Timing is off.
+
+// Toggle timing again.
+SEND:\timing
+EXPECT:Timing is on.
+
+// Send extra argument to \timing.
+SEND:\timing on extra
+EXPECT:executing meta command: meta command 'timing' takes zero or one argument
+
+
+SEND:SELECT * FROM users;
+EXPECT:+-----+-------+-----+
+EXPECT:| _id | name  | age |
+EXPECT:+-----+-------+-----+
+EXPECT:|   1 | Anne  |  38 |
+EXPECT:|   2 | Bill  |  23 |
+EXPECT:|   3 | Cindy |  64 |
+EXPECT:+-----+-------+-----+
+EXPECT:
+EXPECTCOMP:HasPrefix:Execution time:
+
+// Turn timing back off.
+SEND:\timing off
+EXPECT:Timing is off.
+
+// Ensure we don't get timing.
+SEND:SELECT * FROM users;
+EXPECT:+-----+-------+-----+
+EXPECT:| _id | name  | age |
+EXPECT:+-----+-------+-----+
+EXPECT:|   1 | Anne  |  38 |
+EXPECT:|   2 | Bill  |  23 |
+EXPECT:|   3 | Cindy |  64 |
+EXPECT:+-----+-------+-----+
+EXPECT:
+
+// Ensure an invalid timing value returns an error.
+SEND:\timing invalid
+EXPECT:executing meta command: unrecognized value "invalid" for "\timing": Boolean expected

cli/testdata/meta_write

@@ -0,0 +1,24 @@
+// Make sure there's something in the query buffer.
+// This is left unterminated because we don't need to execute the query;
+// we just need there to be something in the buffer.
+SEND:SELECT * FROM invalid-table
+SEND:\write query-buffer-contents
+
+// Reset the buffer.
+SEND:\r
+EXPECT:Query buffer reset (cleared).
+
+// Read from the file.
+SEND:\! cat query-buffer-contents
+EXPECT:SELECT * FROM invalid-table
+
+// Remove the file.
+SEND:\! rm query-buffer-contents
+
+// Send \write with no arguments.
+SEND:\write
+EXPECT:\w: missing required argument
+
+// Send \write with extra arguments.
+SEND:\write filename extra
+EXPECT:executing meta command: meta command 'w' exactly one argument

cli/testdata/people.sql

@@ -0,0 +1,12 @@
+-- Create a table.
+create table people (_id id, name string, age int);
+
+-- Insert some values.
+insert into people values (1, 'Amy', 42), (2, 'Bob', 27), (3, 'Carl', 33);
+
+-- Get all rows from the table.
+select * from people;
+
+-- Mix in a meta-command to show that both are supported
+-- in the include file.
+\echo mix in a meta command

cli/testdata/query_buffer

@@ -0,0 +1,40 @@
+SEND:select 1 as foo;
+EXPECT:+-----+
+EXPECT:| foo |
+EXPECT:+-----+
+EXPECT:|   1 |
+EXPECT:+-----+
+EXPECT:
+
+SEND:\p
+EXPECT:select 1 as foo;
+
+SEND:select 2
+SEND:\p
+EXPECT:select 2
+SEND:\r
+EXPECT:Query buffer reset (cleared).
+
+SEND:\p
+EXPECT:select 1 as foo;
+
+SEND:select 3
+SEND:\p
+EXPECT:select 3
+
+SEND:as foo
+SEND:\p
+EXPECT:select 3
+EXPECT:as foo
+
+SEND:;
+EXPECT:+-----+
+EXPECT:| foo |
+EXPECT:+-----+
+EXPECT:|   3 |
+EXPECT:+-----+
+EXPECT:
+
+SEND:\p
+EXPECT:select 3
+EXPECT:as foo;

cli/testdata/setup

@@ -0,0 +1,51 @@
+// Startup splash.
+EXPECT:FeatureBase CLI ()
+EXPECT:Type "\q" to quit.
+EXPECT:Detected on-prem, serverless deployment.
+EXPECTCOMP:HasPrefix:Host: http://localhost:
+EXPECT:You are not connected to a database.
+
+// Show databases.
+SEND:SHOW DATABASES;
+EXPECT:Organization required. Use \org to set an organization.
+
+// Get current org.
+SEND:\org
+EXPECT:You have not set an organization.
+
+// Set org.
+SEND:\org acme
+EXPECT:You have set organization "acme".
+
+// Try to set org with too many arguments.
+SEND:\org acme extra
+EXPECT:executing meta command: meta command 'org' takes zero or one argument
+
+// Set location to UTC so that expected timestamp size is consistent.
+// Without this, a test running locally in may have a timestamp that
+// ends in a timezone offset such as `-06:00`, while one running as UTC
+// will have `Z`. Since these string lengths differ, our generic
+// {timestamp} comparison will fail.
+SEND:\pset location UTC
+EXPECT:Location is UTC.
+
+// Set an invalid location.
+SEND:\pset location invalid
+EXPECT:executing meta command: loading location: invalid: unknown time zone invalid
+
+// Try to set location with too many arguments.
+SEND:\pset location UTC extra
+EXPECT:executing meta command: meta command 'pset' takes zero, one, or two arguments
+
+// Set border to 2 for testing because it makes it easier to visually see
+// what the tests are expecting (because lines don't end in spaces).
+SEND:\pset border 2
+EXPECT:Border style is 2.
+
+// Check the state of pset.
+SEND:\pset
+EXPECT:border      2
+EXPECT:expanded    off
+EXPECT:format      aligned
+EXPECT:location    UTC
+EXPECT:tuples_only off

cli/testdata/table

@@ -0,0 +1,72 @@
+// Show tables for database using SHOW TABLES WITH SYSTEM.
+SEND:SHOW TABLES WITH SYSTEM;
+EXPECT:+-------------------------+-------------------------+-------+------------+----------------------+----------------------+-------+------------+-------------+
+EXPECT:| _id                     | name                    | owner | updated_by | created_at           | updated_at           | keys  | space_used | description |
+EXPECT:+-------------------------+-------------------------+-------+------------+----------------------+----------------------+-------+------------+-------------+
+EXPECTCOMP:WithFormat:| fb_____________________ | fb_____________________ |       |            | {timestamp} | {timestamp} | false |          0 |             |
+EXPECTCOMP:WithFormat:| fb_____________________ | fb_____________________ |       |            | {timestamp} | {timestamp} | false |          0 |             |
+EXPECTCOMP:WithFormat:| fb_____________________ | fb_____________________ |       |            | {timestamp} | {timestamp} | false |          0 |             |
+EXPECTCOMP:WithFormat:| fb_____________________ | fb_____________________ |       |            | {timestamp} | {timestamp} | false |          0 |             |
+EXPECTCOMP:WithFormat:| fb_____________________ | fb_____________________ |       |            | {timestamp} | {timestamp} | false |          0 |             |
+EXPECT:+-------------------------+-------------------------+-------+------------+----------------------+----------------------+-------+------------+-------------+
+EXPECT:
+
+// Show tables for database using \d.
+SEND:\d
+EXPECT:+-------------------------+-------------------------+-------+------------+----------------------+----------------------+-------+------------+-------------+
+EXPECT:| _id                     | name                    | owner | updated_by | created_at           | updated_at           | keys  | space_used | description |
+EXPECT:+-------------------------+-------------------------+-------+------------+----------------------+----------------------+-------+------------+-------------+
+EXPECTCOMP:WithFormat:| fb_____________________ | fb_____________________ |       |            | {timestamp} | {timestamp} | false |          0 |             |
+EXPECTCOMP:WithFormat:| fb_____________________ | fb_____________________ |       |            | {timestamp} | {timestamp} | false |          0 |             |
+EXPECTCOMP:WithFormat:| fb_____________________ | fb_____________________ |       |            | {timestamp} | {timestamp} | false |          0 |             |
+EXPECTCOMP:WithFormat:| fb_____________________ | fb_____________________ |       |            | {timestamp} | {timestamp} | false |          0 |             |
+EXPECTCOMP:WithFormat:| fb_____________________ | fb_____________________ |       |            | {timestamp} | {timestamp} | false |          0 |             |
+EXPECT:+-------------------------+-------------------------+-------+------------+----------------------+----------------------+-------+------------+-------------+
+EXPECT:
+
+// Show tables for database using SHOW TABLES.
+SEND:SHOW TABLES;
+EXPECT:+-----+------+-------+------------+------------+------------+------+------------+-------------+
+EXPECT:| _id | name | owner | updated_by | created_at | updated_at | keys | space_used | description |
+EXPECT:+-----+------+-------+------------+------------+------------+------+------------+-------------+
+EXPECT:+-----+------+-------+------------+------------+------------+------+------------+-------------+
+EXPECT:
+
+
+// Show tables for database using \dt.
+SEND:\dt
+EXPECT:+-----+------+-------+------------+------------+------------+------+------------+-------------+
+EXPECT:| _id | name | owner | updated_by | created_at | updated_at | keys | space_used | description |
+EXPECT:+-----+------+-------+------------+------------+------------+------+------------+-------------+
+EXPECT:+-----+------+-------+------------+------------+------------+------+------------+-------------+
+EXPECT:
+
+// Create a table. That can be used for general testing.
+SEND:CREATE TABLE users (_id id, name string, age int);
+EXPECT:
+SEND:INSERT INTO users VALUES (1, 'Anne', 38), (2, 'Bill', 23), (3, 'Cindy', 64);
+EXPECT:
+
+
+// Show tables for database to get the newly created table.
+SEND:\dt
+EXPECT:+-------+-------+-------+------------+----------------------+----------------------+-------+------------+-------------+
+EXPECT:| _id   | name  | owner | updated_by | created_at           | updated_at           | keys  | space_used | description |
+EXPECT:+-------+-------+-------+------------+----------------------+----------------------+-------+------------+-------------+
+EXPECTCOMP:WithFormat:| users | users |       |            | {timestamp} | {timestamp} | false |          0 |             |
+EXPECT:+-------+-------+-------+------------+----------------------+----------------------+-------+------------+-------------+
+EXPECT:
+
+// We don't select from users until AFTER we check SHOW TABLES above because
+// running this creates the fb_views sytem table which has a description.
+// And it's annoying to mask out all of the description fields because we
+// don't know which row fb_views will fall into.
+SEND:SELECT * FROM users;
+EXPECT:+-----+-------+-----+
+EXPECT:| _id | name  | age |
+EXPECT:+-----+-------+-----+
+EXPECT:|   1 | Anne  |  38 |
+EXPECT:|   2 | Bill  |  23 |
+EXPECT:|   3 | Cindy |  64 |
+EXPECT:+-----+-------+-----+
+EXPECT:

cli/writer.go

@@ -1,8 +1,11 @@
 package cli
 
 import (
+	"encoding/csv"
 	"fmt"
 	"io"
+	"log"
+	"time"
 
 	featurebase "github.com/featurebasedb/featurebase/v3"
 	"github.com/jedib0t/go-pretty/table"
@@ -15,23 +18,32 @@ import (
 type writeOptions struct {
 	border     int
 	expanded   bool
+	format     string
+	location   *time.Location
 	timing     bool
 	tuplesOnly bool
 }
 
+const (
+	formatAligned = "aligned"
+	formatCSV     = "csv"
+)
+
 func defaultWriteOptions() *writeOptions {
 	return &writeOptions{
 		border:     1,
 		expanded:   false,
+		format:     formatAligned,
+		location:   time.Local,
 		timing:     false,
 		tuplesOnly: false,
 	}
 }
 
-// writeTable writes the query response, taking the format into consideration.
+// writeOutput writes the query response, taking the format into consideration.
 // It sends query output to qOut, non-error informational output (such as query
 // timing) to wOut, and errors to wErr.
-func writeTable(r *featurebase.WireQueryResponse, format *writeOptions, qOut io.Writer, wOut io.Writer, wErr io.Writer) error {
+func writeOutput(r *featurebase.WireQueryResponse, opts *writeOptions, qOut io.Writer, wOut io.Writer, wErr io.Writer) error {
 	if r == nil {
 		return errors.New("attempt to write out nil response")
 	}
@@ -42,9 +54,104 @@ func writeTable(r *featurebase.WireQueryResponse, format *writeOptions, qOut io.
 		return writeWarnings(r, wErr)
 	}
 
+	switch opts.format {
+	case formatAligned:
+		if err := writeTable(r, opts, qOut); err != nil {
+			return errors.Wrap(err, "writing table")
+		}
+
+		// Add some white space after query results.
+		qOut.Write([]byte("\n"))
+
+	case formatCSV:
+		if err := writeCSV(r, opts, qOut); err != nil {
+			return errors.Wrap(err, "writing csv")
+		}
+
+	default:
+		return errors.Errorf("invalid format: %s", opts.format)
+	}
+
+	if err := writeWarnings(r, wErr); err != nil {
+		return err
+	}
+
+	// Timing.
+	if opts.timing {
+		if _, err := wOut.Write([]byte(fmt.Sprintf("Execution time: %dμs\n", r.ExecutionTime))); err != nil {
+			return errors.Wrapf(err, "writing execution time: %s", r.Error)
+		}
+	}
+
+	return nil
+}
+
+// writeCSV writes the WireQueryResponse to qOut as csv.
+func writeCSV(r *featurebase.WireQueryResponse, opts *writeOptions, qOut io.Writer) error {
+	w := csv.NewWriter(qOut)
+
+	if opts.expanded {
+		// Expanded csv
+
+		// rec is used to write the row as a slice of strings. It is reused to
+		// avoid unnecessary memory allocation.
+		rec := make([]string, 2)
+
+		for _, row := range r.Data {
+			cleanRow(row, opts)
+			for i, col := range r.Schema.Fields {
+				rec[0] = string(col.Name)
+				rec[1] = fmt.Sprintf("%v", row[i])
+
+				// Write the record.
+				if err := w.Write(rec); err != nil {
+					log.Fatalln("error writing expanded record to csv:", err)
+				}
+			}
+		}
+
+	} else {
+		// Normal csv (i.e. NOT expanded)
+
+		// Write the schema.
+		if !opts.tuplesOnly {
+			header := make([]string, 0, len(r.Schema.Fields))
+			for i := range r.Schema.Fields {
+				header = append(header, string(r.Schema.Fields[i].Name))
+			}
+			if err := w.Write(header); err != nil {
+				return errors.Wrapf(err, "error writing header to csv")
+			}
+		}
+
+		// Write the records.
+
+		// rec is used to write the row as a slice of strings. It is reused to
+		// avoid unnecessary memory allocation.
+		rec := make([]string, len(r.Schema.Fields))
+
+		for _, row := range r.Data {
+			cleanRow(row, opts)
+			for i := range row {
+				rec[i] = fmt.Sprintf("%v", row[i])
+			}
+			if err := w.Write(rec); err != nil {
+				log.Fatalln("error writing record to csv:", err)
+			}
+		}
+	}
+
+	// Write any buffered data to the underlying writer (standard output).
+	w.Flush()
+
+	return w.Error()
+}
+
+// writeTable writes the WireQueryResponse to qOut in a tabular format.
+func writeTable(r *featurebase.WireQueryResponse, opts *writeOptions, qOut io.Writer) error {
 	t := table.NewWriter()
 	t.SetOutputMirror(qOut)
-	switch format.border {
+	switch opts.border {
 	case 0:
 		t.SetStyle(styleBorder0)
 	case 1:
@@ -52,7 +159,7 @@ func writeTable(r *featurebase.WireQueryResponse, format *writeOptions, qOut io.
 	default:
 		t.SetStyle(styleBorder2)
 		// In expanded mode with a border, we need borders between each record.
-		if format.expanded {
+		if opts.expanded {
 			t.Style().Options.SeparateRows = true
 		}
 	}
@@ -60,9 +167,10 @@ func writeTable(r *featurebase.WireQueryResponse, format *writeOptions, qOut io.
 	// Don't uppercase the header values.
 	t.Style().Format.Header = text.FormatDefault
 
-	if format.expanded {
+	if opts.expanded {
 		// Expanded table
 		for _, row := range r.Data {
+			cleanRow(row, opts)
 			colRow := make([]interface{}, 2)
 			scolRow := make([]string, 2)
 			div := "\n"
@@ -71,11 +179,7 @@ func writeTable(r *featurebase.WireQueryResponse, format *writeOptions, qOut io.
 					div = ""
 				}
 				scolRow[0] += fmt.Sprintf("%s%s", col.Name, div)
-				if row[i] == nil {
-					scolRow[1] += fmt.Sprintf("%s%s", nullValue, div)
-				} else {
-					scolRow[1] += fmt.Sprintf("%v%s", row[i], div)
-				}
+				scolRow[1] += fmt.Sprintf("%v%s", row[i], div)
 			}
 			colRow[0] = scolRow[0]
 			colRow[1] = scolRow[1]
@@ -83,37 +187,36 @@ func writeTable(r *featurebase.WireQueryResponse, format *writeOptions, qOut io.
 		}
 	} else {
 		// Normal table (i.e. NOT expanded)
-		if !format.tuplesOnly {
+		if !opts.tuplesOnly {
 			t.AppendHeader(schemaToRow(r.Schema))
 		}
 		for _, row := range r.Data {
-			// If the value is nil, replace it with a null string; go-pretty doesn't
-			// expect nil pointers in the data values.
-			for i := range row {
-				if row[i] == nil {
-					row[i] = nullValue
-				}
-			}
+			cleanRow(row, opts)
 			t.AppendRow(table.Row(row))
 		}
 	}
 	t.Render()
 
-	if err := writeWarnings(r, wErr); err != nil {
-		return err
-	}
+	return nil
+}
 
-	// Add some white space after query results.
-	qOut.Write([]byte("\n"))
-
-	// Timing.
-	if format.timing {
-		if _, err := wOut.Write([]byte(fmt.Sprintf("Execution time: %dμs\n", r.ExecutionTime))); err != nil {
-			return errors.Wrapf(err, "writing execution time: %s", r.Error)
+// cleanRow loops through all the columns of row and modifies its value based on
+// type.
+//
+// If the value is nil, replace it with a null string; go-pretty doesn't expect
+// nil pointers in the data values.
+//
+// If the value is a time.Time, we want to print it using RFC3339Nano to be
+// consistent with everything else.
+func cleanRow(row []interface{}, opts *writeOptions) {
+	for i := range row {
+		switch v := row[i].(type) {
+		case nil:
+			row[i] = nullValue
+		case time.Time:
+			row[i] = v.In(opts.location).Format(time.RFC3339Nano)
 		}
 	}
-
-	return nil
 }
 
 func schemaToRow(schema featurebase.WireQuerySchema) []interface{} {

cli/writer_test.go

@@ -58,6 +58,7 @@ func TestWriter(t *testing.T) {
 				format: &writeOptions{
 					border:     1,
 					expanded:   false,
+					format:     formatAligned,
 					timing:     true,
 					tuplesOnly: false,
 				},
@@ -77,6 +78,7 @@ func TestWriter(t *testing.T) {
 				format: &writeOptions{
 					border:     2,
 					expanded:   false,
+					format:     formatAligned,
 					timing:     false,
 					tuplesOnly: false,
 				},
@@ -98,6 +100,7 @@ func TestWriter(t *testing.T) {
 				format: &writeOptions{
 					border:     0,
 					expanded:   false,
+					format:     formatAligned,
 					timing:     false,
 					tuplesOnly: false,
 				},
@@ -117,6 +120,7 @@ func TestWriter(t *testing.T) {
 				format: &writeOptions{
 					border:     1,
 					expanded:   false,
+					format:     formatAligned,
 					timing:     false,
 					tuplesOnly: true,
 				},
@@ -134,6 +138,7 @@ func TestWriter(t *testing.T) {
 				format: &writeOptions{
 					border:     2,
 					expanded:   true,
+					format:     formatAligned,
 					timing:     false,
 					tuplesOnly: false,
 				},
@@ -164,7 +169,7 @@ func TestWriter(t *testing.T) {
 				wOut := bytes.NewBuffer(make([]byte, 0, 100000))
 				wErr := bytes.NewBuffer(make([]byte, 0, 100000))
 
-				assert.NoError(t, writeTable(wqr, test.format, qOut, wOut, wErr))
+				assert.NoError(t, writeOutput(wqr, test.format, qOut, wOut, wErr))
 
 				assert.Equal(t, test.expQOut, qOut.String())
 				assert.Equal(t, test.expOut, wOut.String())

cmd/auth_token.go

@@ -16,7 +16,7 @@ func newAuthTokenCommand(logdest logger.Logger) *cobra.Command {
 		Long: `
 Retrieves an auth-token for use in authenticating with FeatureBase from the configured identity provider.
 `,
-		RunE: usageErrorWrapper(cmd),
+		RunE: UsageErrorWrapper(cmd),
 	}
 
 	flags := ccmd.Flags()

cmd/backup.go

@@ -16,7 +16,7 @@ func newBackupCommand(logdest logger.Logger) *cobra.Command {
 		Long: `
 Backs up a FeatureBase server to a local, tar-formatted snapshot file.
 `,
-		RunE: usageErrorWrapper(cmd),
+		RunE: UsageErrorWrapper(cmd),
 	}
 
 	flags := ccmd.Flags()

cmd/backup_tar.go

@@ -16,7 +16,7 @@ func newBackupTarCommand(logdest io.Writer) *cobra.Command {
 		Long: `
 Backs up a FeatureBase server to a local, tar-formatted snapshot file.
 `,
-		RunE: usageErrorWrapper(cmd),
+		RunE: UsageErrorWrapper(cmd),
 	}
 
 	flags := ccmd.Flags()

cmd/chksum.go

@@ -19,7 +19,7 @@ func newChkSumCommand(logdest logger.Logger) *cobra.Command {
 			Generates a digital signature of all the data associated with a provided FeatureBase server
 			WARNING: could be slow if high cardinality fields exist
 `,
-		RunE: usageErrorWrapper(cmd),
+		RunE: UsageErrorWrapper(cmd),
 	}
 
 	flags := ccmd.Flags()

cmd/cli.go

@@ -1,35 +0,0 @@
-// Copyright 2021 Molecula Corp. All rights reserved.
-package cmd
-
-import (
-	"io"
-
-	"github.com/featurebasedb/featurebase/v3/cli"
-	"github.com/featurebasedb/featurebase/v3/ctl"
-	"github.com/featurebasedb/featurebase/v3/logger"
-	"github.com/spf13/cobra"
-	"github.com/spf13/viper"
-)
-
-var cliCmd *cli.Command
-
-// NewCLICommand runs the FeatureBase CLI subcommand.
-func NewCLICommand(stderr io.Writer) *cobra.Command {
-	logdest := logger.NewStandardLogger(stderr)
-	cliCmd = cli.NewCommand(logdest)
-	cobraCmd := &cobra.Command{
-		Use:   "fbsql",
-		Short: "Query FeatureBase with SQL from the command line",
-		Long:  ``,
-		RunE:  usageErrorWrapper(cliCmd),
-		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
-			v := viper.New()
-			return setAllConfig(v, cmd.Flags(), "FBSQL")
-		},
-		SilenceErrors: true,
-	}
-
-	// Attach flags to the command.
-	ctl.BuildCLIFlags(cobraCmd, cliCmd)
-	return cobraCmd
-}

cmd/dataframe-csv-loader.go

@@ -15,7 +15,7 @@ func newDataframeCsvLoaderCommand(logdest logger.Logger) *cobra.Command {
 		Short: "load dataframe integer and floating point values into featurebase",
 		Long: `
 `,
-		RunE: usageErrorWrapper(cmd),
+		RunE: UsageErrorWrapper(cmd),
 	}
 	flags := loaderCmd.Flags()
 	flags.StringVar(&cmd.Path, "csv", "", "path to csv input file")

cmd/export.go

@@ -26,7 +26,7 @@ The format of the CSV file is:
 
 The file does not contain any headers.
 `,
-		RunE: usageErrorWrapper(Exporter),
+		RunE: UsageErrorWrapper(Exporter),
 	}
 	flags := exportCmd.Flags()
 

cmd/fbsql/main.go

@@ -2,12 +2,65 @@
 package main
 
 import (
+	"io"
 	"os"
 
+	"github.com/featurebasedb/featurebase/v3/cli"
 	"github.com/featurebasedb/featurebase/v3/cmd"
+	"github.com/featurebasedb/featurebase/v3/logger"
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
 )
 
 func main() {
-	command := cmd.NewCLICommand(os.Stderr)
+	command := newCLICommand(os.Stderr)
 	command.Execute()
 }
+
+// newCLICommand runs the FeatureBase CLI subcommand.
+func newCLICommand(stderr io.Writer) *cobra.Command {
+	logdest := logger.NewStandardLogger(stderr)
+	cliCmd := cli.NewCommand(logdest)
+	cobraCmd := &cobra.Command{
+		Use:   "fbsql",
+		Short: "Query FeatureBase with SQL from the command line",
+		Long:  ``,
+		RunE:  cmd.UsageErrorWrapper(cliCmd),
+		PersistentPreRunE: func(cobraCmd *cobra.Command, args []string) error {
+			v := viper.New()
+			return cmd.SetAllConfig(v, cobraCmd.Flags(), "FBSQL")
+		},
+		SilenceErrors: true,
+	}
+
+	// Attach flags to the command.
+	buildFlags(cobraCmd, cliCmd)
+	return cobraCmd
+}
+
+// buildFlags attaches a set of flags to the command for a cli instance.
+func buildFlags(cmd *cobra.Command, cliCmd *cli.Command) {
+	flags := cmd.Flags()
+
+	// Base struct flags.
+	flags.StringSliceVarP(&cliCmd.Commands, "command", "c", cliCmd.Commands, "Command to run in non-interactive mode. Provide multiple flags to execute more than one command. All `--command` flags run before all `--file` flags.")
+	flags.StringSliceVarP(&cliCmd.Files, "file", "f", cliCmd.Files, "File to run in non-interactive mode. Provide multiple flags to execute more than one file. All `--command` flags run before all `--file` flags.")
+
+	// Config flags.
+	flags.StringVarP(&cliCmd.Config.Host, "host", "", cliCmd.Config.Host, "hostname of FeatureBase.")
+	flags.StringVarP(&cliCmd.Config.Port, "port", "p", cliCmd.Config.Port, "port of FeatureBase.")
+	flags.StringVar(&cliCmd.Config.HistoryPath, "history-path", cliCmd.Config.HistoryPath, "path for history files.")
+	flags.StringVar(&cliCmd.Config.OrganizationID, "org-id", cliCmd.Config.OrganizationID, "OrganizationID.")
+	flags.StringVarP(&cliCmd.Config.Database, "dbname", "d", cliCmd.Config.Database, "Name of the database to connect to.")
+
+	flags.StringVar(&cliCmd.Config.CloudAuth.ClientID, "client-id", cliCmd.Config.CloudAuth.ClientID, "Cognito Client ID for FeatureBase Cloud access.")
+	flags.StringVar(&cliCmd.Config.CloudAuth.Region, "region", cliCmd.Config.CloudAuth.Region, "Cloud region for FeatureBase Cloud access (e.g. us-east-2).")
+	flags.StringVar(&cliCmd.Config.CloudAuth.Email, "email", cliCmd.Config.CloudAuth.Email, "Email address for FeatureBase Cloud access.")
+	flags.StringVar(&cliCmd.Config.CloudAuth.Password, "password", cliCmd.Config.CloudAuth.Password, "Password for FeatureBase Cloud access.")
+
+	flags.StringVar(&cliCmd.Config.KafkaConfig, "kafka-config", cliCmd.Config.KafkaConfig, "Kafka configuration file to read from.")
+	flags.BoolVar(&cliCmd.Config.CSV, "csv", cliCmd.Config.CSV, "CSV (Comma-Separated Values) table output mode.")
+	flags.StringSliceVar(&cliCmd.Config.PSets, "pset", cliCmd.Config.PSets, "Set printing option VAR to ARG (see \\pset command). Use form: --pset=VAR[=ARG]")
+
+	flags.String("config", "", "Configuration file to read from.")
+}

cmd/generate_config.go

@@ -18,7 +18,7 @@ func newGenerateConfigCommand(logdest logger.Logger) *cobra.Command {
 		Short: "Print the default configuration.",
 		Long: `generate-config prints the default configuration to stdout
 `,
-		RunE: usageErrorWrapper(generateConf),
+		RunE: UsageErrorWrapper(generateConf),
 	}
 
 	return confCmd

cmd/keygen.go

@@ -16,7 +16,7 @@ func newKeygenCommand(logdest logger.Logger) *cobra.Command {
 		Long: `
 Generate secret key to configure FeatureBase for Authentication.
 `,
-		RunE: usageErrorWrapper(cmd),
+		RunE: UsageErrorWrapper(cmd),
 	}
 
 	flags := ccmd.Flags()

cmd/parquet-info.go

@@ -27,7 +27,7 @@ Displays schema and sample data from the specified file
 			c.Path = args[0]
 			return nil
 		},
-		RunE: usageErrorWrapper(c),
+		RunE: UsageErrorWrapper(c),
 	}
 	return cmd
 }

cmd/presort.go

@@ -15,7 +15,7 @@ func newPreSortCommand(logdest logger.Logger) *cobra.Command {
 		Long: `
 Takes all input files and writes PartitionN numbered files to a directory, where each file contains only records that will go into the partition it is named for.
 `,
-		RunE: usageErrorWrapper(cmd),
+		RunE: UsageErrorWrapper(cmd),
 	}
 
 	flags := ccmd.Flags()

cmd/rbf.go

@@ -44,7 +44,7 @@ Executes a consistency check on an RBF data directory.
 			c.Path = args[0]
 			return nil
 		},
-		RunE: usageErrorWrapper(c),
+		RunE: UsageErrorWrapper(c),
 	}
 	return cmd
 }
@@ -76,7 +76,7 @@ Dumps the raw hex data for one or more RBF pages.
 
 			return nil
 		},
-		RunE: usageErrorWrapper(c),
+		RunE: UsageErrorWrapper(c),
 	}
 	return cmd
 }
@@ -98,7 +98,7 @@ Prints a line for every page in the database with its type/status.
 			c.Path = args[0]
 			return nil
 		},
-		RunE: usageErrorWrapper(c),
+		RunE: UsageErrorWrapper(c),
 	}
 
 	flags := cmd.Flags()
@@ -133,7 +133,7 @@ Prints the header & cell data for one or more pages.
 
 			return nil
 		},
-		RunE: usageErrorWrapper(c),
+		RunE: UsageErrorWrapper(c),
 	}
 	return cmd
 }

cmd/restore.go

@@ -16,7 +16,7 @@ func newRestoreCommand(logdest logger.Logger) *cobra.Command {
 		Long: `
 The Restore command will take a backup archive and restore it to a new, clean cluster.
 `,
-		RunE: usageErrorWrapper(cmd),
+		RunE: UsageErrorWrapper(cmd),
 	}
 	flags := restoreCmd.Flags()
 	flags.StringVarP(&cmd.Path, "source", "s", "", "backup file; specify '-' to restore from stdin tar stream")

cmd/restore_tar.go

@@ -15,7 +15,7 @@ func newRestoreTarCommand(logdest logger.Logger) *cobra.Command {
 		Long: `
 The Restore command will take a tar-formatted backup archive and restore it to a new, clean cluster.
 `,
-		RunE: usageErrorWrapper(cmd),
+		RunE: UsageErrorWrapper(cmd),
 	}
 	flags := restoreCmd.Flags()
 	flags.StringVarP(&cmd.Path, "source", "s", "", "backup file; specify '-' to restore from stdin tar stream")

cmd/root.go

@@ -25,11 +25,11 @@ type runner interface {
 	Run(context.Context) error
 }
 
-// usageErrorWrapper takes a thing with a Run(context) error, and produces
+// UsageErrorWrapper takes a thing with a Run(context) error, and produces
 // a func(*cobra.Command, []string) error from it which will run that
 // command, and then set Cobra's SilenceUsage flag unless the returned
 // error errors.Is() a ctl.UsageError.
-func usageErrorWrapper(inner runner) func(*cobra.Command, []string) error {
+func UsageErrorWrapper(inner runner) func(*cobra.Command, []string) error {
 	return func(c *cobra.Command, args []string) error {
 		return considerUsageError(c, inner.Run(context.Background()))
 	}
@@ -41,7 +41,7 @@ func usageErrorWrapper(inner runner) func(*cobra.Command, []string) error {
 // wrappers.
 func considerUsageError(cmd *cobra.Command, err error) error {
 	cmd.SilenceErrors = true
-	if !errors.Is(err, ctl.UsageError) {
+	if !errors.Is(err, ctl.ErrUsage) {
 		cmd.SilenceUsage = true
 	}
 	return err
@@ -68,7 +68,7 @@ at https://docs.featurebase.com/.
 			case "dax":
 				v.Set("future.rename", true) // always use FEATUREBASE env for dax
 			}
-			if err := setAllConfig(v, cmd.Flags(), ""); err != nil {
+			if err := SetAllConfig(v, cmd.Flags(), ""); err != nil {
 				return err
 			}
 
@@ -114,17 +114,17 @@ at https://docs.featurebase.com/.
 	return rc
 }
 
-// setAllConfig takes a FlagSet to be the definition of all configuration
+// SetAllConfig takes a FlagSet to be the definition of all configuration
 // options, as well as their defaults. It then reads from the command line, the
 // environment, and a config file (if specified), and applies the configuration
 // in that priority order. Since each flag in the set contains a pointer to
-// where its value should be stored, setAllConfig can directly modify the value
+// where its value should be stored, SetAllConfig can directly modify the value
 // of each config variable.
 //
-// setAllConfig looks for environment variables which are capitalized versions
+// SetAllConfig looks for environment variables which are capitalized versions
 // of the flag names with dashes replaced by underscores, and prefixed with
 // envPrefix plus an underscore.
-func setAllConfig(v *viper.Viper, flags *pflag.FlagSet, envPrefix string) error { // nolint: unparam
+func SetAllConfig(v *viper.Viper, flags *pflag.FlagSet, envPrefix string) error { // nolint: unparam
 	// add cmd line flag def to viper
 	err := v.BindPFlags(flags)
 	if err != nil {

ctl/backup.go

@@ -10,7 +10,6 @@ import (
 	"fmt"
 	"io"
 	"io/fs"
-	"log"
 	"os"
 	"path/filepath"
 	"time"
@@ -95,13 +94,13 @@ func (cmd *BackupCommand) Run(ctx context.Context) (err error) {
 
 	// Validate arguments.
 	if cmd.OutputDir == "" {
-		return fmt.Errorf("%w: -o flag required", UsageError)
+		return fmt.Errorf("%w: -o flag required", ErrUsage)
 	} else if cmd.Concurrency <= 0 {
-		return fmt.Errorf("%w: concurrency must be at least one", UsageError)
+		return fmt.Errorf("%w: concurrency must be at least one", ErrUsage)
 	}
 	if cmd.HeaderTimeoutStr != "" {
 		if dur, err := time.ParseDuration(cmd.HeaderTimeoutStr); err != nil {
-			return fmt.Errorf("%w: could not parse '%s' as a duration: %v", UsageError, cmd.HeaderTimeoutStr, err)
+			return fmt.Errorf("%w: could not parse '%s' as a duration: %v", ErrUsage, cmd.HeaderTimeoutStr, err)
 		} else {
 			cmd.HeaderTimeout = dur
 		}
@@ -549,7 +548,7 @@ func (cmd *BackupCommand) backupShardDataframe(ctx context.Context, indexName st
 	resp, err := client.GetDataframeShard(ctx, indexName, shard)
 	// no error if doesn't exist
 	if err != nil {
-		log.Fatal(err)
+		return fmt.Errorf("getting dataframe: %w", err)
 	}
 	defer resp.Body.Close()
 	if resp.StatusCode == 404 {

ctl/backup_tar.go

@@ -8,7 +8,6 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
-	"log"
 	"os"
 	"path"
 	"path/filepath"
@@ -83,7 +82,7 @@ func (cmd *BackupTarCommand) Run(ctx context.Context) (err error) {
 	logdest := cmd.Logger()
 	// Validate arguments.
 	if cmd.OutputPath == "" {
-		return fmt.Errorf("%w: -o flag required", UsageError)
+		return fmt.Errorf("%w: -o flag required", ErrUsage)
 	}
 	useStdout := cmd.OutputPath == "-"
 	if useStdout && cmd.logwriter == os.Stdout {
@@ -101,7 +100,7 @@ func (cmd *BackupTarCommand) Run(ctx context.Context) (err error) {
 
 	if cmd.HeaderTimeoutStr != "" {
 		if dur, err := time.ParseDuration(cmd.HeaderTimeoutStr); err != nil {
-			return fmt.Errorf("%w: could not parse '%s' as a duration: %v", UsageError, cmd.HeaderTimeoutStr, err)
+			return fmt.Errorf("%w: could not parse '%s' as a duration: %v", ErrUsage, cmd.HeaderTimeoutStr, err)
 		} else {
 			cmd.HeaderTimeout = dur
 		}
@@ -329,7 +328,7 @@ func (cmd *BackupTarCommand) backupTarShardDataframe(ctx context.Context, tw *ta
 	resp, err := client.GetDataframeShard(ctx, indexName, shard)
 	// no error if doesn't exist
 	if err != nil {
-		log.Fatal(err)
+		return fmt.Errorf("getting dataframe: %w", err)
 	}
 	defer resp.Body.Close()
 	if resp.StatusCode == 404 {

ctl/backup_test.go

@@ -15,19 +15,19 @@ func TestBackupCommand_Run(t *testing.T) {
 	cm := NewBackupCommand(cmLog)
 	cm.OutputDir = ""
 	err := cm.Run(context.Background())
-	if !errors.Is(err, UsageError) {
+	if !errors.Is(err, ErrUsage) {
 		t.Fatalf("expected usage error, got %v", err)
 	}
 	cm.OutputDir = "foo"
 	cm.Concurrency = 0
 	err = cm.Run(context.Background())
-	if !errors.Is(err, UsageError) {
+	if !errors.Is(err, ErrUsage) {
 		t.Fatalf("expected usage error, got %v", err)
 	}
 	cm.Concurrency = 1
 	cm.HeaderTimeoutStr = "until the cat wakes up"
 	err = cm.Run(context.Background())
-	if !errors.Is(err, UsageError) {
+	if !errors.Is(err, ErrUsage) {
 		t.Fatalf("expected usage error, got %v", err)
 	}
 }

ctl/cli.go

@@ -1,39 +0,0 @@
-package ctl
-
-import (
-	"github.com/featurebasedb/featurebase/v3/cli"
-	"github.com/spf13/cobra"
-	"github.com/spf13/pflag"
-)
-
-// BuildCLIFlags attaches a set of flags to the command for a cli instance.
-func BuildCLIFlags(cmd *cobra.Command, cliCmd *cli.Command) {
-	flags := cmd.Flags()
-
-	// Base struct flags.
-	flags.StringSliceVarP(&cliCmd.Commands, "command", "c", cliCmd.Commands, "Command to run in non-interactive mode. Provide multiple flags to execute more than one command. All `--command` flags run before all `--file` flags.")
-	flags.StringSliceVarP(&cliCmd.Files, "file", "f", cliCmd.Files, "File to run in non-interactive mode. Provide multiple flags to execute more than one file. All `--command` flags run before all `--file` flags.")
-
-	// Config flags.
-	flags.AddFlagSet(cliConfigFlagSet(cliCmd.Config))
-}
-
-// cliConfigFlagSet returns a pflag.FlagSet for the CLI Config struct.
-func cliConfigFlagSet(cfg *cli.Config) *pflag.FlagSet {
-	flags := pflag.NewFlagSet("cli", pflag.ExitOnError)
-
-	flags.StringVarP(&cfg.Host, "host", "", cfg.Host, "hostname of FeatureBase.")
-	flags.StringVarP(&cfg.Port, "port", "", cfg.Port, "port of FeatureBase.")
-	flags.StringVar(&cfg.HistoryPath, "history-path", cfg.HistoryPath, "path for history files.")
-	flags.StringVar(&cfg.OrganizationID, "org-id", cfg.OrganizationID, "OrganizationID.")
-	flags.StringVar(&cfg.Database, "db", cfg.Database, "Name of the database to connect to.")
-
-	flags.StringVar(&cfg.CloudAuth.ClientID, "client-id", cfg.CloudAuth.ClientID, "Cognito Client ID for FeatureBase Cloud access.")
-	flags.StringVar(&cfg.CloudAuth.Region, "region", cfg.CloudAuth.Region, "Cloud region for FeatureBase Cloud access (e.g. us-east-2).")
-	flags.StringVar(&cfg.CloudAuth.Email, "email", cfg.CloudAuth.Email, "Email address for FeatureBase Cloud access.")
-	flags.StringVar(&cfg.CloudAuth.Password, "password", cfg.CloudAuth.Password, "Password for FeatureBase Cloud access.")
-
-	flags.String("config", "", "Configuration file to read from.")
-
-	return flags
-}

ctl/dataframe-csv-loader.go

@@ -36,7 +36,7 @@ func init() {
 
 // TODO(rdp): add refresh token to this as well
 
-// NewDataframeCsvLoaderCommand
+// DataframeCsvLoaderCommand is used to load a dataframe from CSV.
 type DataframeCsvLoaderCommand struct {
 	tlsConfig *tls.Config
 
@@ -149,7 +149,7 @@ func (cmd *DataframeCsvLoaderCommand) Run(ctx context.Context) (err error) {
 
 	// Validate arguments.
 	if cmd.Path == "" {
-		return fmt.Errorf("%w: --csv flag required", UsageError)
+		return fmt.Errorf("%w: --csv flag required", ErrUsage)
 	}
 
 	readFile, err := os.Open(cmd.Path)

ctl/dax.go

@@ -17,9 +17,21 @@ func BuildDAXFlags(cmd *cobra.Command, srv *server.Command) {
 	// Controller
 	flags.BoolVar(&srv.Config.Controller.Run, "controller.run", srv.Config.Controller.Run, "Run the Controller service in process.")
 	flags.DurationVar(&srv.Config.Controller.Config.RegistrationBatchTimeout, "controller.config.registration-batch-timeout", srv.Config.Controller.Config.RegistrationBatchTimeout, "Timeout for node registration batches.")
-	flags.StringVar(&srv.Config.Controller.Config.DataDir, "controller.config.data-dir", srv.Config.Controller.Config.DataDir, "Controller directory to use in process.")
+	flags.StringVar(&srv.Config.Controller.Config.StorageMethod, "controller.config.storage-method", srv.Config.Controller.Config.StorageMethod, "Backing store. boltdb or sqldb.")
 	flags.DurationVar(&srv.Config.Controller.Config.SnappingTurtleTimeout, "controller.config.snapping-turtle-timeout", srv.Config.Controller.Config.SnappingTurtleTimeout, "Period for running automatic snapshotting routine.")
 
+	// Controller.SQLDB
+	flags.StringVar(&srv.Config.Controller.Config.SQLDB.Database, "controller.config.sqldb.database", srv.Config.Controller.Config.SQLDB.Database, "Database name.")
+	flags.StringVar(&srv.Config.Controller.Config.SQLDB.Host, "controller.config.sqldb.host", srv.Config.Controller.Config.SQLDB.Host, "Hostname of SQL Database")
+	flags.StringVar(&srv.Config.Controller.Config.SQLDB.Port, "controller.config.sqldb.port", srv.Config.Controller.Config.SQLDB.Port, "Port of SQL Database")
+	flags.StringVar(&srv.Config.Controller.Config.SQLDB.User, "controller.config.sqldb.user", srv.Config.Controller.Config.SQLDB.User, "Username for connection to SQL Database")
+	flags.StringVar(&srv.Config.Controller.Config.SQLDB.Password, "controller.config.sqldb.password", srv.Config.Controller.Config.SQLDB.Password, "Password for connection to SQL Database")
+	flags.StringVar(&srv.Config.Controller.Config.SQLDB.URL, "controller.config.sqldb.url", srv.Config.Controller.Config.SQLDB.URL, "URL for connection to SQL Database (supersedes host/port/username etc)")
+	flags.IntVar(&srv.Config.Controller.Config.SQLDB.Pool, "controller.config.sqldb.pool", srv.Config.Controller.Config.SQLDB.Pool, "Max number of open connections to database. 0=unlimited")
+	flags.IntVar(&srv.Config.Controller.Config.SQLDB.IdlePool, "controller.config.sqldb.idle-pool", srv.Config.Controller.Config.SQLDB.IdlePool, "Maximum number of idle connections to database.")
+	flags.DurationVar(&srv.Config.Controller.Config.SQLDB.ConnMaxLifetime, "controller.config.sqldb.conn-max-lifetime", srv.Config.Controller.Config.SQLDB.ConnMaxLifetime, "See https://golang.org/pkg/database/sql/#DB.SetConnMaxLifetime")
+	flags.DurationVar(&srv.Config.Controller.Config.SQLDB.ConnMaxIdleTime, "controller.config.sqldb.conn-max-idle-time", srv.Config.Controller.Config.SQLDB.ConnMaxIdleTime, "See https://golang.org/pkg/database/sql/#DB.SetConnMaxIdletime")
+
 	// Queryer
 	flags.BoolVar(&srv.Config.Queryer.Run, "queryer.run", srv.Config.Queryer.Run, "Run the Queryer service in process.")
 	flags.StringVar(&srv.Config.Queryer.Config.ControllerAddress, "queryer.config.controller-address", srv.Config.Queryer.Config.ControllerAddress, "Address of remote Controller process.")

ctl/export.go

@@ -50,9 +50,9 @@ func (cmd *ExportCommand) Run(ctx context.Context) error {
 
 	// Validate arguments.
 	if cmd.Index == "" {
-		return fmt.Errorf("%w: %v", UsageError, pilosa.ErrIndexRequired)
+		return fmt.Errorf("%w: %v", ErrUsage, pilosa.ErrIndexRequired)
 	} else if cmd.Field == "" {
-		return fmt.Errorf("%w: %v", UsageError, pilosa.ErrFieldRequired)
+		return fmt.Errorf("%w: %v", ErrUsage, pilosa.ErrFieldRequired)
 	}
 
 	// Use output file, if specified.

ctl/import.go

@@ -86,11 +86,11 @@ func (cmd *ImportCommand) Run(ctx context.Context) error {
 	// Validate arguments.
 	// Index and field are validated early before the files are parsed.
 	if cmd.Index == "" {
-		return fmt.Errorf("%w: %v", UsageError, pilosa.ErrIndexRequired)
+		return fmt.Errorf("%w: %v", ErrUsage, pilosa.ErrIndexRequired)
 	} else if cmd.Field == "" {
-		return fmt.Errorf("%w: %v", UsageError, pilosa.ErrFieldRequired)
+		return fmt.Errorf("%w: %v", ErrUsage, pilosa.ErrFieldRequired)
 	} else if len(cmd.Paths) == 0 {
-		return fmt.Errorf("%w: path required", UsageError)
+		return fmt.Errorf("%w: path required", ErrUsage)
 	}
 	// Create a client to the server.
 	client, err := commandClient(cmd)

ctl/keygen.go

@@ -12,7 +12,7 @@ import (
 	"github.com/gorilla/securecookie"
 )
 
-// Keygen represents a command for generating a cryptographic key.
+// KeygenCommand represents a command for generating a cryptographic key.
 type KeygenCommand struct {
 	stdout    io.Writer
 	logDest   logger.Logger

ctl/parquet-info.go

@@ -4,7 +4,6 @@ package ctl
 
 import (
 	"context"
-	"errors"
 	"fmt"
 	"io"
 	"net/http"
@@ -47,23 +46,23 @@ func (cmd *ParquetInfoCommand) Run(ctx context.Context) error {
 			return err
 		}
 		if response.StatusCode != 200 {
-			return errors.New(fmt.Sprintf("unexpected response %d", response.StatusCode))
+			return fmt.Errorf("unexpected response %d", response.StatusCode)
 		}
 		defer response.Body.Close()
 		// download to temp file first
 		f, err = os.CreateTemp("", "BulkParquetFile.parquet")
 		if err != nil {
-			return errors.New(fmt.Sprintf("error creating tempfile %v", err))
+			return fmt.Errorf("error creating tempfile %v", err)
 		}
 		_, err = io.Copy(f, response.Body)
 		if err != nil {
-			return errors.New(fmt.Sprintf("error downloading url %v %v", cmd.Path, err))
+			return fmt.Errorf("error downloading url %v %v", cmd.Path, err)
 		}
 		defer os.Remove(f.Name())
 
 		_, err = f.Seek(0, io.SeekStart)
 		if err != nil {
-			return errors.New(fmt.Sprintf("error reseting file for reading %v ", err))
+			return fmt.Errorf("error reseting file for reading %v ", err)
 		}
 	} else {
 		f, err = os.Open(cmd.Path)

ctl/restore.go

@@ -83,9 +83,9 @@ func (cmd *RestoreCommand) Run(ctx context.Context) (err error) {
 
 	// Validate arguments.
 	if cmd.Path == "" {
-		return fmt.Errorf("%w: -s flag required", UsageError)
+		return fmt.Errorf("%w: -s flag required", ErrUsage)
 	} else if cmd.Concurrency <= 0 {
-		return fmt.Errorf("%w: concurrency must be at least one", UsageError)
+		return fmt.Errorf("%w: concurrency must be at least one", ErrUsage)
 	}
 
 	// Parse TLS configuration for node-specific clients.

ctl/restore_tar.go

@@ -10,7 +10,6 @@ import (
 	"io"
 	"io/ioutil"
 	"net/http"
-	gohttp "net/http"
 	"os"
 	"strconv"
 	"strings"
@@ -80,7 +79,7 @@ func (cmd *RestoreTarCommand) Run(ctx context.Context) (err error) {
 
 	// Validate arguments.
 	if cmd.Path == "" {
-		return fmt.Errorf("%w: -s flag required", UsageError)
+		return fmt.Errorf("%w: -s flag required", ErrUsage)
 	}
 	useStdin := cmd.Path == "-"
 
@@ -138,7 +137,7 @@ func (cmd *RestoreTarCommand) Run(ctx context.Context) (err error) {
 	if primary == nil {
 		return errors.New("no primary")
 	}
-	c := &gohttp.Client{}
+	c := &http.Client{}
 	// buf := new(bytes.Buffer)
 	mb512 := 2 << 29
 	buf := buffer.NewFileBuffer(mb512, cmd.TempDir)
@@ -319,7 +318,7 @@ func (cmd *RestoreTarCommand) TLSHost() string { return cmd.Host }
 func (cmd *RestoreTarCommand) TLSConfiguration() server.TLSConfig { return cmd.TLS }
 
 func Post(ctx context.Context, url, contentType string, rd io.Reader, query map[string]string) error {
-	client := &gohttp.Client{}
+	client := &http.Client{}
 	req, err := http.NewRequest(http.MethodPost, url, rd)
 	if err != nil {
 		return err

ctl/restore_tar_test.go

@@ -24,7 +24,7 @@ func TestRestoreTarCommand_Run(t *testing.T) {
 	cm.Host = hostport
 	cm.Path = ""
 	err := cm.Run(context.Background())
-	if !errors.Is(err, UsageError) {
+	if !errors.Is(err, ErrUsage) {
 		t.Fatalf("expected usage error with empty path, got %v", err)
 	}
 	cm.Path = "nonexistent-file"

ctl/restore_test.go

@@ -15,13 +15,13 @@ func TestRestoreCommand_Run(t *testing.T) {
 	cm := NewRestoreCommand(cmLog)
 	cm.Path = ""
 	err := cm.Run(context.Background())
-	if !errors.Is(err, UsageError) {
+	if !errors.Is(err, ErrUsage) {
 		t.Fatalf("expected usage error, got %v", err)
 	}
 	cm.Path = "foo"
 	cm.Concurrency = 0
 	err = cm.Run(context.Background())
-	if !errors.Is(err, UsageError) {
+	if !errors.Is(err, ErrUsage) {
 		t.Fatalf("expected usage error, got %v", err)
 	}
 }

ctl/util.go

@@ -19,7 +19,7 @@ func (c ctlUsageError) Error() string {
 	return "usage error"
 }
 
-var UsageError ctlUsageError
+var ErrUsage ctlUsageError
 
 // startProfilingServer starts a server which handles /debug/pprof and
 // /debug/fgprof for use in utilities we might want to profile but

dax/Makefile

@@ -20,6 +20,11 @@ test-integration:
 testv-integration:
 	$(GO) test -v ./test/dax -count 1 -timeout 20m -run TestDAXIntegration/$(RUN)
 
+######################## Postgres Backend Stuff #########################
+
+run-postgres:
+	mkdir -p $(HOME)/pgdata
+	docker run -p 5432:5432 -v $(HOME)/pgdata:/var/lib/postgresql/data:Z  -e POSTGRES_PASSWORD=testpass --user=$(shell id -u) postgres:14.7 -c log_min_duration_statement=0
 
 ############################### AWS STUFF ###############################
 

dax/boltdb/boltdb.go

@@ -1,158 +0,0 @@
-// Package boltdb contains the boltdb implementations of the DAX interfaces.
-package boltdb
-
-import (
-	"context"
-	"os"
-	"path/filepath"
-	"strings"
-	"time"
-
-	"github.com/featurebasedb/featurebase/v3/errors"
-	bolt "go.etcd.io/bbolt"
-)
-
-const (
-	ErrFmtBucketNotFound = "boltdb: bucket '%s' not found"
-)
-
-type Bucket []byte
-
-// DB represents the database connection.
-type DB struct {
-	db     *bolt.DB
-	ctx    context.Context // background context
-	cancel func()          // cancel background context
-
-	// Datasource name.
-	DSN string
-
-	// Destination for events to be published.
-	// EventService wtf.EventService
-
-	// Returns the current time. Defaults to time.Now().
-	// Can be mocked for tests.
-	Now func() time.Time
-
-	filePath string
-
-	// bucketQueue contains a list of buckets to create upon Open.
-	bucketQueue []Bucket
-}
-
-// NewDB returns a new instance of DB associated with the given datasource name.
-func NewDB(dsn string) *DB {
-	db := &DB{
-		DSN: dsn,
-		Now: time.Now,
-
-		//EventService: wtf.NopEventService(),
-	}
-	db.ctx, db.cancel = context.WithCancel(context.Background())
-	return db
-}
-
-// NewSvcBolt gets, opens, and creates buckets for a boltDB for a
-// particular named service (the data file will be named after the
-// service).
-func NewSvcBolt(dir, svc string, buckets ...Bucket) (*DB, error) {
-	dir = strings.TrimPrefix(dir, "file:")
-	filename := filepath.Join(dir, svc+".boltdb")
-	db := NewDB("file:" + filename)
-	db.RegisterBuckets(buckets...)
-	err := db.Open()
-	return db, errors.Wrap(err, "opening")
-}
-
-// path returns the file path to the boltdb database file.
-func (db *DB) path() (string, error) {
-	if !strings.HasPrefix(db.DSN, "file:") {
-		return "", errors.New(errors.ErrUncoded, "boltdb package only supports a DSN beginning with `file:`")
-	}
-
-	return db.DSN[5:], nil
-}
-
-// RegisterBuckets queues up the buckets to be created when the database is
-// first opened.
-func (db *DB) RegisterBuckets(buckets ...Bucket) {
-	db.bucketQueue = append(db.bucketQueue, buckets...)
-}
-
-// InitializeBuckets creates the given buckets if they do not already exist.
-func (db *DB) InitializeBuckets(buckets ...Bucket) (err error) {
-	return db.db.Update(func(tx *bolt.Tx) error {
-		for _, bucket := range buckets {
-			if _, err := tx.CreateBucketIfNotExists(bucket); err != nil {
-				return errors.Wrapf(err, "creating bucket: %s", bucket)
-			}
-		}
-		return nil
-	})
-}
-
-// Open opens the database connection.
-func (db *DB) Open() (err error) {
-	path, err := db.path()
-	if err != nil {
-		return errors.Wrap(err, "getting path from DSN")
-	}
-
-	if err := os.MkdirAll(filepath.Dir(path), 0777); err != nil {
-		return errors.Wrapf(err, "mkdir %s", filepath.Dir(path))
-	} else if db.db, err = bolt.Open(path, 0666, &bolt.Options{Timeout: 1 * time.Second}); err != nil {
-		return errors.Wrapf(err, "open file: %s", err)
-	}
-
-	// cache the path in db.filePath.
-	db.filePath = path
-
-	if err := db.InitializeBuckets(db.bucketQueue...); err != nil {
-		return errors.Wrap(err, "initializing buckets")
-	}
-
-	// Reset the bucketQueue.
-	db.bucketQueue = make([]Bucket, 0)
-
-	return nil
-}
-
-// Close closes the database connection.
-func (db *DB) Close() (err error) {
-	return db.db.Close()
-}
-
-// BeginTx starts a transaction and returns a wrapper Tx type. This type
-// provides a reference to the database and a fixed timestamp at the start of
-// the transaction. The timestamp allows us to mock time during tests as well.
-// The wrapper also contains the context.
-func (db *DB) BeginTx(ctx context.Context, writable bool) (*Tx, error) {
-	tx, err := db.db.Begin(writable)
-	if err != nil {
-		return nil, err
-	}
-
-	// Return wrapper Tx that includes the transaction start time.
-	return &Tx{
-		Tx:  tx,
-		ctx: ctx,
-		db:  db,
-		now: db.Now().UTC().Truncate(time.Second),
-	}, nil
-}
-
-// Tx wraps the SQL Tx object to provide a timestamp at the start of the transaction.
-type Tx struct {
-	*bolt.Tx
-	ctx context.Context
-	db  *DB
-	now time.Time
-}
-
-func (tx *Tx) Context() context.Context {
-	return tx.ctx
-}
-
-func (db *DB) Path() string {
-	return db.filePath
-}

dax/boltdb/boltdb_test.go

@@ -1,17 +0,0 @@
-package boltdb_test
-
-import (
-	"testing"
-
-	"github.com/featurebasedb/featurebase/v3/dax/test/boltdb"
-)
-
-// Ensure the test database can open & close.
-func TestDB(t *testing.T) {
-	db := boltdb.MustOpenDB(t)
-	defer boltdb.MustCloseDB(t, db)
-
-	t.Cleanup(func() {
-		boltdb.CleanupDB(t, db.Path())
-	})
-}

dax/boltdb/directiveversion.go

@@ -1,60 +0,0 @@
-package boltdb
-
-import (
-	"encoding/binary"
-
-	"github.com/featurebasedb/featurebase/v3/dax"
-	"github.com/featurebasedb/featurebase/v3/errors"
-)
-
-var (
-	bucketDirective     = Bucket("nodeDirective")
-	keyDirectiveVersion = []byte("directiveVersion")
-)
-
-// DirectiveBuckets defines the buckets used by this package. It can be called
-// during setup to create the buckets ahead of time.
-var DirectiveBuckets []Bucket = []Bucket{
-	bucketDirective,
-}
-
-// Ensure type implements interface.
-var _ dax.DirectiveVersion = (*DirectiveVersion)(nil)
-
-type DirectiveVersion struct {
-	db *DB
-}
-
-func NewDirectiveVersion(db *DB) *DirectiveVersion {
-	return &DirectiveVersion{
-		db: db,
-	}
-}
-
-func (d *DirectiveVersion) Increment(tx dax.Transaction, delta uint64) (uint64, error) {
-	txx, ok := tx.(*Tx)
-	if !ok {
-		return 0, dax.NewErrInvalidTransaction()
-	}
-
-	bkt := txx.Bucket(bucketDirective)
-	if bkt == nil {
-		return 0, errors.Errorf(ErrFmtBucketNotFound, bucketDirective)
-	}
-
-	var nextVersion uint64 = 1 // Start at 1; 0 is an invalid version.
-
-	b := bkt.Get(keyDirectiveVersion)
-	if b != nil {
-		nextVersion = binary.LittleEndian.Uint64(b) + delta
-	}
-
-	vsn := make([]byte, 8)
-	binary.LittleEndian.PutUint64(vsn, nextVersion)
-
-	if err := bkt.Put(keyDirectiveVersion, vsn); err != nil {
-		return 0, errors.Wrap(err, "putting next directive version")
-	}
-
-	return nextVersion, nil
-}

dax/computer/service/computer.go

@@ -74,6 +74,7 @@ func (c *computerService) Start() error {
 				dax.RoleTypeCompute,
 				dax.RoleTypeTranslate,
 			},
+			HasDirective: false,
 		}
 
 		if err := c.computer.Registrar.RegisterNode(context.TODO(), node); err != nil {

dax/controller/balancer.go

@@ -13,6 +13,9 @@ type Balancer interface {
 	// be either transferred to other workers or placed on the free job list.
 	RemoveWorker(tx dax.Transaction, addr dax.Address) ([]dax.WorkerDiff, error)
 
+	// ReleaseWorkers dissociates the given workers from a database.
+	ReleaseWorkers(tx dax.Transaction, addrs ...dax.Address) error
+
 	// AddJobs adds new jobs for the given database.
 	AddJobs(tx dax.Transaction, roleType dax.RoleType, qtid dax.QualifiedTableID, jobs ...dax.Job) ([]dax.WorkerDiff, error)
 
@@ -61,6 +64,9 @@ func (b *NopBalancer) AddWorker(tx dax.Transaction, node *dax.Node) ([]dax.Worke
 func (b *NopBalancer) RemoveWorker(tx dax.Transaction, addr dax.Address) ([]dax.WorkerDiff, error) {
 	return []dax.WorkerDiff{}, nil
 }
+func (b *NopBalancer) ReleaseWorkers(tx dax.Transaction, addrs ...dax.Address) error {
+	return nil
+}
 func (b *NopBalancer) AddJobs(tx dax.Transaction, roleType dax.RoleType, qtid dax.QualifiedTableID, jobs ...dax.Job) ([]dax.WorkerDiff, error) {
 	return []dax.WorkerDiff{}, nil
 }

dax/controller/balancer/balancer.go

@@ -28,7 +28,7 @@ type Balancer struct {
 	// current represents the current state of worker/job assigments.
 	current WorkerJobService
 
-	nodeService controller.NodeService
+	workerRegistry controller.WorkerRegistry
 
 	// freeJobs is the set of jobs which have yet to be assigned to a worker.
 	// This could be because there are no available workers, or because a worker
@@ -44,47 +44,37 @@ type Balancer struct {
 }
 
 // New returns a new instance of Balancer.
-func New(ns controller.NodeService, fjs FreeJobService, wjs WorkerJobService, fws FreeWorkerService, schemar schemar.Schemar, logger logger.Logger) *Balancer {
+func New(wr controller.WorkerRegistry, fjs FreeJobService, wjs WorkerJobService, fws FreeWorkerService, schemar schemar.Schemar, logger logger.Logger) *Balancer {
 	return &Balancer{
-		current:     wjs,
-		nodeService: ns,
-		freeJobs:    fjs,
-		freeWorkers: fws,
-		schemar:     schemar,
-		logger:      logger,
+		current:        wjs,
+		workerRegistry: wr,
+		freeJobs:       fjs,
+		freeWorkers:    fws,
+		schemar:        schemar,
+		logger:         logger,
 	}
 }
 
-// AddWorker adds the given Node to the Balancer's available worker pool.
-// TODO(tlt): this method takes a Node (as opposed to a Worker) because in the
-// future we may want to maintain separate worker pools based on RoleType
-// (compute, translate, etc.).
+// AddWorker adds the given Node to the Balancer's available worker pool. Note
+// that a node is used for ALL of the role types specified. In other words,
+// specifying roleTypes = {compute, translate}, does not mean that the node can
+// be used as either a compute worker or a translate worker. It means that it
+// will be used as both.
 func (b *Balancer) AddWorker(tx dax.Transaction, node *dax.Node) ([]dax.WorkerDiff, error) {
-	addr := node.Address
-	b.logger.Debugf("AddWorker(%s)", addr)
+	b.logger.Debugf("AddWorker(%s)", node.Address)
 
-	if err := b.nodeService.CreateNode(tx, addr, node); err != nil {
-		return nil, errors.Wrapf(err, "creating node on node service: %s", addr)
+	if err := b.workerRegistry.AddWorker(tx, node); err != nil {
+		return nil, errors.Wrapf(err, "creating node on node service: %s", node.Address)
 	}
 
 	diffs := NewInternalDiffs()
 
-	// This logic means that a node is used for ALL of the role types specified.
-	// In other words, specifying roleTypes = {compute, translate}, does not
-	// mean that the node can be used as either a compute worker or a translate
-	// worker. It means that it will be used as both.
-	for _, rt := range node.RoleTypes {
-		if err := b.addWorker(tx, rt, addr); err != nil {
-			return nil, errors.Wrapf(err, "adding worker: (%s) %s", rt, addr)
-		}
-	}
-
-	// Process the freeWorkers.
+	// Process the newly added workers.
 	// TODO(tlt): this is a little heavy-handed. I'm sure we'll need to be more
-	// intentional about knowing which databases needs workers, as opposed to
+	// intentional about knowing which databases need workers, as opposed to
 	// this brute force loop over all databases every time.
 	if diff, err := b.balance(tx); err != nil {
-		return nil, errors.Wrapf(err, "balancing new worker: %s", addr)
+		return nil, errors.Wrapf(err, "balancing new worker: %s", node.Address)
 	} else {
 		diffs.Merge(diff)
 	}
@@ -92,22 +82,9 @@ func (b *Balancer) AddWorker(tx dax.Transaction, node *dax.Node) ([]dax.WorkerDi
 	return diffs.Output(), nil
 }
 
-// addWorker adds a worker to the free worker list. From there, it can be used
-// by any database which needs a worker.
-func (b *Balancer) addWorker(tx dax.Transaction, roleType dax.RoleType, addr dax.Address) error {
-	// If this worker already exists, don't do anything.
-	if dbkey := b.current.DatabaseForWorker(tx, addr); dbkey != "" {
-		return nil
-	}
+func (b *Balancer) assignMinWorkers(tx dax.Transaction, roleType dax.RoleType, qdbid dax.QualifiedDatabaseID) (InternalDiffs, error) {
+	b.logger.Debugf("assigning min workers for '%s', '%s'", roleType, qdbid)
 
-	if err := b.freeWorkers.AddWorkers(tx, roleType, addr); err != nil {
-		return errors.Wrap(err, "adding free worker")
-	}
-
-	return nil
-}
-
-func (b *Balancer) assignMinWorkers(tx dax.Transaction, roleType dax.RoleType) (InternalDiffs, error) {
 	// Find out how many free workers we have.
 	freeWorkers, err := b.freeWorkers.ListWorkers(tx, roleType)
 	if err != nil {
@@ -117,13 +94,16 @@ func (b *Balancer) assignMinWorkers(tx dax.Transaction, roleType dax.RoleType) (
 
 	// If there are no free workers, return early.
 	if freeWorkerCount == 0 {
+		b.logger.Debugf("No free workers for '%s'", roleType)
 		return InternalDiffs{}, nil
 	}
 
-	// Get all database and their minWorkerCount (Database.Options.WorkersMin).
-	qdbs, err := b.schemar.Databases(tx, "")
+	// Get database and its minWorkerCount (Database.Options.WorkersMin). This
+	// used to get all databases, but now this method is specific to a single
+	// database. That's why we just get the one here.
+	qdbs, err := b.schemar.Databases(tx, qdbid.OrganizationID, qdbid.DatabaseID)
 	if err != nil {
-		return nil, errors.Wrap(err, "getting all database")
+		return nil, errors.Wrap(err, "getting database")
 	}
 
 	// Create a map[database]int where int is the number of workers required to
@@ -167,7 +147,7 @@ func (b *Balancer) assignMinWorkers(tx dax.Transaction, roleType dax.RoleType) (
 
 	diffs := NewInternalDiffs()
 
-	// Create an ordered slice of map keys so that tests are predicatable.
+	// Create an ordered slice of map keys so that tests are predictable.
 	qdbids := make([]dax.QualifiedDatabaseID, 0, len(m))
 	for qdbid := range m {
 		qdbids = append(qdbids, qdbid)
@@ -176,7 +156,7 @@ func (b *Balancer) assignMinWorkers(tx dax.Transaction, roleType dax.RoleType) (
 
 	// For each database, if there are enough free workers to
 	// satisfy its min, then pop that number of workers from the free list. If
-	// not, contine to the next database until either reaching the end of the
+	// not, continue to the next database until either reaching the end of the
 	// database list or until there are no more free workers in the list,
 	// whichever comes first.
 	for _, qdbid := range qdbids {
@@ -242,46 +222,40 @@ func (b *Balancer) databaseHasJobs(tx dax.Transaction, roleType dax.RoleType, qd
 func (b *Balancer) RemoveWorker(tx dax.Transaction, addr dax.Address) ([]dax.WorkerDiff, error) {
 	diffs := NewInternalDiffs()
 
-	// Remove the worker from the free worker list (if it's there).
-	for _, rt := range []dax.RoleType{dax.RoleTypeCompute, dax.RoleTypeTranslate} {
-		if err := b.freeWorkers.RemoveWorker(tx, rt, addr); err != nil {
-			return nil, errors.Wrapf(err, "removing worker from free list: (%s) %s", rt, addr)
+	// See if the worker is assigned to a database. If it is, disassociate the
+	// worker from all of its jobs for the database.
+	dbkey := b.current.DatabaseForWorker(tx, addr)
+	if dbkey != "" {
+		qdbid := dbkey.QualifiedDatabaseID()
+		for _, rt := range dax.AllRoleTypes {
+			if diff, err := b.removeDatabaseWorker(tx, rt, qdbid, addr); err != nil {
+				return nil, errors.Wrapf(err, "removing worker: (%s) %s", rt, addr)
+			} else {
+				diffs.Merge(diff)
+			}
 		}
 	}
 
-	// Remove the worker (i.e. Node) from the node service.
-	if err := b.nodeService.DeleteNode(tx, addr); err != nil {
+	// Remove the worker from the worker registry.
+	if err := b.workerRegistry.RemoveWorker(tx, addr); err != nil {
 		return nil, errors.Wrapf(err, "deleting node from node service: %s", addr)
 	}
 
-	////// The rest is database specific. ////////////
-
-	// See if the worker is assigned to a database. If it's not, return early.
-	dbkey := b.current.DatabaseForWorker(tx, addr)
-	if dbkey == "" {
-		return diffs.Output(), nil
-	}
-
-	qdbid := dbkey.QualifiedDatabaseID()
-
-	for _, rt := range []dax.RoleType{dax.RoleTypeCompute, dax.RoleTypeTranslate} {
-		if diff, err := b.removeDatabaseWorker(tx, rt, qdbid, addr); err != nil {
-			return nil, errors.Wrapf(err, "removing worker: (%s) %s", rt, addr)
+	if dbkey != "" {
+		qdbid := dbkey.QualifiedDatabaseID()
+		// Balance the affected database.
+		if diff, err := b.balanceDatabase(tx, qdbid); err != nil {
+			return nil, errors.Wrapf(err, "balancing database: %s", qdbid)
 		} else {
 			diffs.Merge(diff)
 		}
 	}
 
-	// Balance the affected database.
-	if diff, err := b.balanceDatabase(tx, qdbid); err != nil {
-		return nil, errors.Wrapf(err, "balancing database: %s", qdbid)
-	} else {
-		diffs.Merge(diff)
-	}
-
 	return diffs.Output(), nil
 }
 
+// removeDatabaseWorker is used to remove a worker that has been associated with
+// a database. The worker here is determined by address.
 func (b *Balancer) removeDatabaseWorker(tx dax.Transaction, roleType dax.RoleType, qdbid dax.QualifiedDatabaseID, addr dax.Address) (InternalDiffs, error) {
 	jobs, err := b.current.ListJobs(tx, roleType, qdbid, addr)
 	if err != nil {
@@ -289,13 +263,8 @@ func (b *Balancer) removeDatabaseWorker(tx dax.Transaction, roleType dax.RoleTyp
 	}
 
 	// Before removing the worker, mark its jobs as free.
-	if err := b.freeJobs.MergeJobs(tx, roleType, qdbid, jobs); err != nil {
-		return nil, errors.Wrap(err, "merging free jobs")
-	}
-
-	// Remove the worker.
-	if err := b.current.DeleteWorker(tx, roleType, qdbid, addr); err != nil {
-		return nil, errors.Wrap(err, "deleting worker")
+	if err := b.freeJobs.MarkJobsAsFree(tx, roleType, qdbid, jobs); err != nil {
+		return nil, errors.Wrap(err, "marking jobs as free")
 	}
 
 	// Even though this may not be useful to the caller (for example, in the
@@ -309,6 +278,10 @@ func (b *Balancer) removeDatabaseWorker(tx dax.Transaction, roleType dax.RoleTyp
 	return diff, nil
 }
 
+func (b *Balancer) ReleaseWorkers(tx dax.Transaction, addrs ...dax.Address) error {
+	return errors.Wrap(b.current.ReleaseWorkers(tx, addrs...), "freeing workers")
+}
+
 func (b *Balancer) AddJobs(tx dax.Transaction, roleType dax.RoleType, qtid dax.QualifiedTableID, jobs ...dax.Job) ([]dax.WorkerDiff, error) {
 	start := time.Now()
 	defer func() {
@@ -360,7 +333,7 @@ func (b *Balancer) addJobs(tx dax.Transaction, roleType dax.RoleType, qdbid dax.
 		// assigned workers until it has at least one job (which this database
 		// now has).
 		if diff, err := b.balanceDatabaseForRole(tx, roleType, qdbid); err != nil {
-			return nil, errors.Wrapf(err, "assigning min workers: (%s)", roleType)
+			return nil, errors.Wrapf(err, "balancing database for role: (%s)", roleType)
 		} else {
 			diffs.Merge(diff)
 		}
@@ -390,6 +363,7 @@ func (b *Balancer) addDatabaseJobs(tx dax.Transaction, roleType dax.RoleType, qd
 	if err != nil {
 		return nil, errors.Wrapf(err, "getting workers jobs: %s", roleType)
 	}
+
 	jset := dax.NewSet[dax.Job]()
 	for _, workerInfo := range workerJobs {
 		jset.Merge(dax.NewSet(workerInfo.Jobs...))
@@ -431,12 +405,12 @@ func (b *Balancer) addDatabaseJobs(tx dax.Transaction, roleType dax.RoleType, qd
 		jobCounts[lowWorker]++
 	}
 
-	for worker, jobs := range jobsToCreate {
-		if err := b.current.CreateJobs(tx, roleType, qdbid, worker, jobs...); err != nil {
+	for addr, jobs := range jobsToCreate {
+		if err := b.current.AssignWorkerToJobs(tx, roleType, qdbid, addr, jobs...); err != nil {
 			return nil, errors.Wrap(err, "creating job")
 		}
 		for _, job := range jobs {
-			diffs.Added(worker, job)
+			diffs.Added(addr, job)
 		}
 	}
 
@@ -521,7 +495,7 @@ func (b *Balancer) BalanceDatabase(tx dax.Transaction, qdbid dax.QualifiedDataba
 func (b *Balancer) balanceDatabase(tx dax.Transaction, qdbid dax.QualifiedDatabaseID) (InternalDiffs, error) {
 	diffs := NewInternalDiffs()
 
-	for _, role := range []dax.RoleType{dax.RoleTypeCompute, dax.RoleTypeTranslate} {
+	for _, role := range dax.AllRoleTypes {
 		diff, err := b.balanceDatabaseForRole(tx, role, qdbid)
 		if err != nil {
 			return nil, errors.Wrapf(err, "getting worker count: (%s) %s", role, qdbid)
@@ -533,12 +507,12 @@ func (b *Balancer) balanceDatabase(tx dax.Transaction, qdbid dax.QualifiedDataba
 }
 
 func (b *Balancer) balanceDatabaseForRole(tx dax.Transaction, roleType dax.RoleType, qdbid dax.QualifiedDatabaseID) (InternalDiffs, error) {
+	b.logger.Debugf("balancing database %s for role: %s\n", qdbid, roleType)
 	diffs := NewInternalDiffs()
 
 	// Before balancing, make sure the database has its minimum number of
 	// workers satisfied.
-	// TODO(tlt): make assignMinWorkers database specific.
-	if diff, err := b.assignMinWorkers(tx, roleType); err != nil {
+	if diff, err := b.assignMinWorkers(tx, roleType, qdbid); err != nil {
 		return nil, errors.Wrapf(err, "assigning min workers: (%s) %s", roleType, qdbid)
 	} else {
 		diffs.Merge(diff)
@@ -776,16 +750,13 @@ func (b *Balancer) processFreeJobs(tx dax.Transaction, roleType dax.RoleType, qd
 	if err != nil {
 		return nil, errors.Wrapf(err, "listing free jobs: %s", roleType)
 	}
-	for _, job := range jobs {
-		if aj, err := b.addDatabaseJobs(tx, roleType, qdbid, job); err != nil {
-			return nil, errors.Wrapf(err, "adding job: %s", job)
-		} else {
-			diffs.Merge(aj)
-		}
-		if err := b.freeJobs.DeleteJob(tx, roleType, qdbid, job); err != nil {
-			return nil, errors.Wrapf(err, "deleting free job: %s", job)
-		}
+
+	if aj, err := b.addDatabaseJobs(tx, roleType, qdbid, jobs...); err != nil {
+		return nil, errors.Wrapf(err, "adding jobs: %s", jobs)
+	} else {
+		diffs.Merge(aj)
 	}
+
 	return diffs, nil
 }
 
@@ -805,11 +776,11 @@ func (b *Balancer) workerForJob(tx dax.Transaction, roleType dax.RoleType, qdbid
 }
 
 func (b *Balancer) ReadNode(tx dax.Transaction, addr dax.Address) (*dax.Node, error) {
-	return b.nodeService.ReadNode(tx, addr)
+	return b.workerRegistry.Worker(tx, addr)
 }
 
 func (b *Balancer) Nodes(tx dax.Transaction) ([]*dax.Node, error) {
-	return b.nodeService.Nodes(tx)
+	return b.workerRegistry.Workers(tx)
 }
 
 type WorkerJobService interface {
@@ -819,9 +790,9 @@ type WorkerJobService interface {
 	ListWorkers(tx dax.Transaction, roleType dax.RoleType, qdbid dax.QualifiedDatabaseID) (dax.Addresses, error)
 
 	CreateWorker(tx dax.Transaction, roleType dax.RoleType, qdbid dax.QualifiedDatabaseID, addr dax.Address) error
-	DeleteWorker(tx dax.Transaction, roleType dax.RoleType, qdbid dax.QualifiedDatabaseID, addr dax.Address) error
+	ReleaseWorkers(tx dax.Transaction, addrs ...dax.Address) error
 
-	CreateJobs(tx dax.Transaction, roleType dax.RoleType, qdbid dax.QualifiedDatabaseID, addr dax.Address, job ...dax.Job) error
+	AssignWorkerToJobs(tx dax.Transaction, roleType dax.RoleType, qdbid dax.QualifiedDatabaseID, addr dax.Address, job ...dax.Job) error
 	DeleteJob(tx dax.Transaction, roleType dax.RoleType, qdbid dax.QualifiedDatabaseID, addr dax.Address, job dax.Job) error
 	DeleteJobsForTable(tx dax.Transaction, roleType dax.RoleType, qtid dax.QualifiedTableID) (InternalDiffs, error)
 	JobCounts(tx dax.Transaction, roleType dax.RoleType, qdbid dax.QualifiedDatabaseID, addr ...dax.Address) (map[dax.Address]int, error)
@@ -835,12 +806,10 @@ type FreeJobService interface {
 	DeleteJob(tx dax.Transaction, roleType dax.RoleType, qdbid dax.QualifiedDatabaseID, job dax.Job) error
 	DeleteJobsForTable(tx dax.Transaction, roleType dax.RoleType, qtid dax.QualifiedTableID) error
 	ListJobs(tx dax.Transaction, roleType dax.RoleType, qdbid dax.QualifiedDatabaseID) (dax.Jobs, error)
-	MergeJobs(tx dax.Transaction, roleType dax.RoleType, qdbid dax.QualifiedDatabaseID, jobs dax.Jobs) error
+	MarkJobsAsFree(tx dax.Transaction, roleType dax.RoleType, qdbid dax.QualifiedDatabaseID, jobs dax.Jobs) error
 }
 
 type FreeWorkerService interface {
-	AddWorkers(tx dax.Transaction, roleType dax.RoleType, addrs ...dax.Address) error
-	RemoveWorker(tx dax.Transaction, roleType dax.RoleType, addr dax.Address) error
 	PopWorkers(tx dax.Transaction, roleType dax.RoleType, num int) ([]dax.Address, error)
 	ListWorkers(tx dax.Transaction, roleType dax.RoleType) (dax.Addresses, error)
 }

dax/controller/balancer/boltdb/balancer.go

@@ -1,819 +0,0 @@
-// Package boltdb contains the boltdb implementation of the Balancer interface.
-package boltdb
-
-import (
-	"bytes"
-	"encoding/json"
-	"fmt"
-	"strings"
-
-	"github.com/featurebasedb/featurebase/v3/dax"
-	"github.com/featurebasedb/featurebase/v3/dax/boltdb"
-	balancer "github.com/featurebasedb/featurebase/v3/dax/controller/balancer"
-	"github.com/featurebasedb/featurebase/v3/dax/controller/schemar"
-	"github.com/featurebasedb/featurebase/v3/errors"
-	"github.com/featurebasedb/featurebase/v3/logger"
-)
-
-var (
-	bucketBalancer = boltdb.Bucket("balancer")
-)
-
-// BalancerBuckets defines the buckets used by this package. It can be
-// called during setup to create the buckets ahead of time.
-var BalancerBuckets []boltdb.Bucket = []boltdb.Bucket{
-	bucketBalancer,
-}
-
-// NewBalancer returns a new instance of controller.Balancer.
-func NewBalancer(db *boltdb.DB, schemar schemar.Schemar, logger logger.Logger) *balancer.Balancer {
-	fjs := newFreeJobService(db)
-	wjs := newWorkerJobService(db, logger)
-	fws := newFreeWorkerService(db)
-	ns := NewNodeService(db, logger)
-
-	return balancer.New(ns, fjs, wjs, fws, schemar, logger)
-}
-
-// Ensure type implements interface.
-var _ balancer.WorkerJobService = (*workerJobService)(nil)
-
-type workerJobService struct {
-	db     *boltdb.DB
-	logger logger.Logger
-}
-
-func newWorkerJobService(db *boltdb.DB, logger logger.Logger) *workerJobService {
-	return &workerJobService{
-		db:     db,
-		logger: logger,
-	}
-}
-
-func (w *workerJobService) WorkersJobs(tx dax.Transaction, roleType dax.RoleType, qdbid dax.QualifiedDatabaseID) ([]dax.WorkerInfo, error) {
-	workerInfos, err := w.getWorkerInfos(tx, roleType, qdbid)
-	if err != nil {
-		return nil, errors.Wrapf(err, "getting worker infos: %s", roleType)
-	}
-
-	return workerInfos, nil
-}
-
-func (w *workerJobService) WorkerCount(tx dax.Transaction, roleType dax.RoleType, qdbid dax.QualifiedDatabaseID) (int, error) {
-	workers, err := w.getWorkers(tx, roleType, qdbid)
-	if err != nil {
-		return 0, errors.Wrapf(err, "getting workers: %s", roleType)
-	}
-
-	return len(workers), nil
-}
-
-func (w *workerJobService) ListWorkers(tx dax.Transaction, roleType dax.RoleType, qdbid dax.QualifiedDatabaseID) (dax.Addresses, error) {
-	return w.getWorkers(tx, roleType, qdbid)
-}
-
-func (w *workerJobService) getWorkers(tx dax.Transaction, roleType dax.RoleType, qdbid dax.QualifiedDatabaseID) (dax.Addresses, error) {
-	txx, ok := tx.(*boltdb.Tx)
-	if !ok {
-		return nil, dax.NewErrInvalidTransaction()
-	}
-
-	c := txx.Bucket(bucketBalancer).Cursor()
-
-	// Deserialize rows into Worker objects.
-	addrs := make(dax.Addresses, 0)
-
-	prefix := []byte(fmt.Sprintf(prefixFmtWorkersDB, roleType, qdbid.Key()))
-	for k, v := c.Seek(prefix); k != nil && bytes.HasPrefix(k, prefix); k, v = c.Next() {
-		if v == nil {
-			w.logger.Printf("nil value for key: %s", k)
-			continue
-		}
-
-		addr, err := keyWorker(k)
-		if err != nil {
-			return nil, errors.Wrapf(err, "getting worker from key: %s", k)
-		}
-
-		addrs = append(addrs, addr)
-	}
-
-	return addrs, nil
-}
-
-func (w *workerJobService) getWorkerInfos(tx dax.Transaction, roleType dax.RoleType, qdbid dax.QualifiedDatabaseID) (dax.WorkerInfos, error) {
-	txx, ok := tx.(*boltdb.Tx)
-	if !ok {
-		return nil, dax.NewErrInvalidTransaction()
-	}
-
-	c := txx.Bucket(bucketBalancer).Cursor()
-
-	// Deserialize rows into WorkerInfo objects.
-	workerInfos := make(dax.WorkerInfos, 0)
-
-	var prefix []byte
-	empty := dax.QualifiedDatabaseID{}
-	if roleType == "" && qdbid == empty {
-		prefix = []byte("workers/role/")
-	} else {
-		prefix = []byte(fmt.Sprintf(prefixFmtWorkersDB, roleType, qdbid.Key()))
-	}
-	for k, v := c.Seek(prefix); k != nil && bytes.HasPrefix(k, prefix); k, v = c.Next() {
-		addr, err := keyWorker(k)
-		if err != nil {
-			return nil, errors.Wrapf(err, "getting worker from key: %s", k)
-		}
-
-		jobs := dax.NewSet[dax.Job]()
-		if v != nil {
-			jobs, err = decodeJobSet(v)
-			if err != nil {
-				return nil, errors.Wrap(err, "decoding job set")
-			}
-		}
-
-		workerInfo := dax.WorkerInfo{
-			Address: addr,
-			Jobs:    jobs.Sorted(),
-		}
-
-		workerInfos = append(workerInfos, workerInfo)
-	}
-
-	return workerInfos, nil
-}
-
-func (w *workerJobService) CreateWorker(tx dax.Transaction, roleType dax.RoleType, qdbid dax.QualifiedDatabaseID, addr dax.Address) error {
-	txx, ok := tx.(*boltdb.Tx)
-	if !ok {
-		return dax.NewErrInvalidTransaction()
-	}
-
-	bkt := txx.Bucket(bucketBalancer)
-	if bkt == nil {
-		return errors.Errorf(boltdb.ErrFmtBucketNotFound, bucketBalancer)
-	}
-
-	// If this worker already exists, don't do anything.
-	wrkr := bkt.Get(workerDBKey(roleType, qdbid, addr))
-	if wrkr != nil {
-		return nil
-	}
-
-	val := []byte("[]")
-	if err := bkt.Put(workerDBKey(roleType, qdbid, addr), val); err != nil {
-		return errors.Wrapf(err, "putting db worker: %s, %s", qdbid, addr)
-	}
-
-	if err := bkt.Put(workerAssignedKey(addr), []byte(qdbid.Key())); err != nil {
-		return errors.Wrapf(err, "putting assigned worker: %s, %s", qdbid, addr)
-	}
-
-	return nil
-}
-
-func (w *workerJobService) DeleteWorker(tx dax.Transaction, roleType dax.RoleType, qdbid dax.QualifiedDatabaseID, addr dax.Address) error {
-	txx, ok := tx.(*boltdb.Tx)
-	if !ok {
-		return dax.NewErrInvalidTransaction()
-	}
-
-	bkt := txx.Bucket(bucketBalancer)
-	if bkt == nil {
-		return errors.Errorf(boltdb.ErrFmtBucketNotFound, bucketBalancer)
-	}
-
-	if err := bkt.Delete(workerDBKey(roleType, qdbid, addr)); err != nil {
-		return errors.Wrapf(err, "deleting node key: %s", workerDBKey(roleType, qdbid, addr))
-	}
-
-	if err := bkt.Delete(workerAssignedKey(addr)); err != nil {
-		return errors.Wrapf(err, "deleting assigned worker: %s", workerAssignedKey(addr))
-	}
-
-	return nil
-}
-
-func (w *workerJobService) CreateJobs(tx dax.Transaction, roleType dax.RoleType, qdbid dax.QualifiedDatabaseID, addr dax.Address, jobs ...dax.Job) error {
-	txx, ok := tx.(*boltdb.Tx)
-	if !ok {
-		return dax.NewErrInvalidTransaction()
-	}
-
-	bkt := txx.Bucket(bucketBalancer)
-	if bkt == nil {
-		return errors.Errorf(boltdb.ErrFmtBucketNotFound, bucketBalancer)
-	}
-
-	jobset := dax.NewSet[dax.Job]()
-	var err error
-
-	// get worker
-	wrkr := bkt.Get(workerDBKey(roleType, qdbid, addr))
-	if wrkr != nil {
-		jobset, err = decodeJobSet(wrkr)
-		if err != nil {
-			return errors.Wrap(err, "decoding job set")
-		}
-	}
-
-	for _, job := range jobs {
-		jobset.Add(job)
-	}
-	val, err := encodeJobSet(jobset)
-	if err != nil {
-		return errors.Wrap(err, "encoding job set")
-	}
-
-	if err := bkt.Put(workerDBKey(roleType, qdbid, addr), val); err != nil {
-		return errors.Wrap(err, "putting worker")
-	}
-
-	return nil
-}
-
-func (w *workerJobService) DeleteJob(tx dax.Transaction, roleType dax.RoleType, qdbid dax.QualifiedDatabaseID, addr dax.Address, job dax.Job) error {
-	txx, ok := tx.(*boltdb.Tx)
-	if !ok {
-		return dax.NewErrInvalidTransaction()
-	}
-
-	bkt := txx.Bucket(bucketBalancer)
-	if bkt == nil {
-		return errors.Errorf(boltdb.ErrFmtBucketNotFound, bucketBalancer)
-	}
-
-	// get worker
-	wrkr := bkt.Get(workerDBKey(roleType, qdbid, addr))
-	if wrkr == nil {
-		return nil
-	}
-
-	jobset, err := decodeJobSet(wrkr)
-	if err != nil {
-		return errors.Wrap(err, "decoding job set")
-	}
-	if !jobset.Contains(job) {
-		return nil
-	}
-
-	jobset.Remove(job)
-	val, err := encodeJobSet(jobset)
-	if err != nil {
-		return errors.Wrap(err, "encoding job set")
-	}
-
-	if err := bkt.Put(workerDBKey(roleType, qdbid, addr), val); err != nil {
-		return errors.Wrap(err, "putting worker")
-	}
-
-	return nil
-}
-
-func (w *workerJobService) DeleteJobsForTable(tx dax.Transaction, roleType dax.RoleType, qtid dax.QualifiedTableID) (balancer.InternalDiffs, error) {
-	txx, ok := tx.(*boltdb.Tx)
-	if !ok {
-		return nil, dax.NewErrInvalidTransaction()
-	}
-
-	bkt := txx.Bucket(bucketBalancer)
-	if bkt == nil {
-		return nil, errors.Errorf(boltdb.ErrFmtBucketNotFound, bucketBalancer)
-	}
-
-	qdbid := qtid.QualifiedDatabaseID
-	prefix := string(qtid.Key())
-
-	workers, err := w.getWorkers(tx, roleType, qdbid)
-	if err != nil {
-		return nil, errors.Wrap(err, "getting workers")
-	}
-
-	idiffs := balancer.NewInternalDiffs()
-	for _, worker := range workers {
-		// get worker
-		wrkr := bkt.Get(workerDBKey(roleType, qdbid, worker))
-		if wrkr == nil {
-			panic("didn't find worker that should... definitely exist")
-		}
-		jobset, err := decodeJobSet(wrkr)
-		if err != nil {
-			return nil, errors.Wrap(err, "decoding job set")
-		}
-
-		jobs := jobset.RemoveByPrefix(prefix)
-		for _, job := range jobs {
-			idiffs.Removed(worker, job)
-		}
-		val, err := encodeJobSet(jobset)
-		if err != nil {
-			return nil, errors.Wrap(err, "encoding job set")
-		}
-
-		if err := bkt.Put(workerDBKey(roleType, qdbid, worker), val); err != nil {
-			return nil, errors.Wrap(err, "putting worker")
-		}
-
-	}
-
-	return idiffs, nil
-}
-
-func (w *workerJobService) ListJobs(tx dax.Transaction, roleType dax.RoleType, qdbid dax.QualifiedDatabaseID, addr dax.Address) (dax.Jobs, error) {
-	txx, ok := tx.(*boltdb.Tx)
-	if !ok {
-		return nil, dax.NewErrInvalidTransaction()
-	}
-
-	bkt := txx.Bucket(bucketBalancer)
-	if bkt == nil {
-		return nil, errors.Errorf(boltdb.ErrFmtBucketNotFound, bucketBalancer)
-	}
-
-	jobset := dax.NewSet[dax.Job]()
-	var err error
-
-	// get worker
-	wrkr := bkt.Get(workerDBKey(roleType, qdbid, addr))
-	if wrkr != nil {
-		jobset, err = decodeJobSet(wrkr)
-		if err != nil {
-			return nil, errors.Wrap(err, "decoding job set")
-		}
-	}
-
-	return jobset.Sorted(), nil
-}
-
-func (w *workerJobService) JobCounts(tx dax.Transaction, roleType dax.RoleType, qdbid dax.QualifiedDatabaseID, addrs ...dax.Address) (map[dax.Address]int, error) {
-	txx, ok := tx.(*boltdb.Tx)
-	if !ok {
-		return nil, dax.NewErrInvalidTransaction()
-	}
-
-	bkt := txx.Bucket(bucketBalancer)
-	if bkt == nil {
-		return nil, errors.Errorf(boltdb.ErrFmtBucketNotFound, bucketBalancer)
-	}
-
-	m := make(map[dax.Address]int)
-
-	for _, addr := range addrs {
-		jobset := dax.NewSet[dax.Job]()
-		var err error
-
-		// get worker
-		wrkr := bkt.Get(workerDBKey(roleType, qdbid, addr))
-		if wrkr != nil {
-			jobset, err = decodeJobSet(wrkr)
-			if err != nil {
-				return nil, errors.Wrap(err, "decoding job set")
-			}
-		}
-
-		m[addr] = len(jobset)
-	}
-
-	return m, nil
-}
-
-func (w *workerJobService) DatabaseForWorker(tx dax.Transaction, addr dax.Address) dax.DatabaseKey {
-	txx, ok := tx.(*boltdb.Tx)
-	if !ok {
-		return "" // TODO(tlt): return error here?
-	}
-
-	bkt := txx.Bucket(bucketBalancer)
-	if bkt == nil {
-		return ""
-	}
-
-	wrkr := bkt.Get(workerAssignedKey(addr))
-
-	return dax.DatabaseKey(wrkr)
-}
-
-// encodeJobSet encode the jobSet into a JSON array of strings.
-func encodeJobSet(jobSet dax.Set[dax.Job]) ([]byte, error) {
-	arr := jobSet.Sorted()
-	b, err := json.Marshal(arr)
-	if err != nil {
-		return nil, errors.Wrap(err, "marshalling json")
-	}
-	return b, nil
-}
-
-// decodeJobSet decode the string (a JSON array of strings) into jobSet.
-func decodeJobSet(v []byte) (dax.Set[dax.Job], error) {
-	var arr []string
-	err := json.Unmarshal(v, &arr)
-	if err != nil {
-		return nil, errors.Wrap(err, "unmarshalling json")
-	}
-
-	js := dax.NewSet[dax.Job]()
-	for _, s := range arr {
-		js.Add(dax.Job(s))
-	}
-
-	return js, nil
-}
-
-// encodeWorkerSet encode the workerSet into a JSON array of strings.
-func encodeWorkerSet(workerSet dax.Set[dax.Address]) ([]byte, error) {
-	arr := workerSet.Sorted()
-	b, err := json.Marshal(arr)
-	if err != nil {
-		return nil, errors.Wrap(err, "marshalling json")
-	}
-	return b, nil
-}
-
-// decodeWorkerSet decode the string (a JSON array of strings) into workerSet.
-func decodeWorkerSet(v []byte) (dax.Set[dax.Address], error) {
-	var arr []string
-	err := json.Unmarshal(v, &arr)
-	if err != nil {
-		return nil, errors.Wrap(err, "unmarshalling json")
-	}
-
-	ws := dax.NewSet[dax.Address]()
-	for _, s := range arr {
-		ws.Add(dax.Address(s))
-	}
-
-	return ws, nil
-}
-
-// Ensure type implements interface.
-var _ balancer.FreeJobService = (*freeJobService)(nil)
-
-type freeJobService struct {
-	db *boltdb.DB
-}
-
-func newFreeJobService(db *boltdb.DB) *freeJobService {
-	return &freeJobService{
-		db: db,
-	}
-}
-
-func (f *freeJobService) CreateJobs(tx dax.Transaction, roleType dax.RoleType, qdbid dax.QualifiedDatabaseID, jobs ...dax.Job) error {
-	return f.MergeJobs(tx, roleType, qdbid, jobs)
-}
-
-func (f *freeJobService) DeleteJob(tx dax.Transaction, roleType dax.RoleType, qdbid dax.QualifiedDatabaseID, job dax.Job) error {
-	txx, ok := tx.(*boltdb.Tx)
-	if !ok {
-		return dax.NewErrInvalidTransaction()
-	}
-
-	bkt := txx.Bucket(bucketBalancer)
-	if bkt == nil {
-		return errors.Errorf(boltdb.ErrFmtBucketNotFound, bucketBalancer)
-	}
-
-	// get free jobs
-	fjs := bkt.Get(freeJobKey(roleType, qdbid))
-	if fjs == nil {
-		return nil
-	}
-
-	jobset, err := decodeJobSet(fjs)
-	if err != nil {
-		return errors.Wrap(err, "decoding job set")
-	}
-	if !jobset.Contains(job) {
-		return nil
-	}
-
-	jobset.Remove(job)
-	val, err := encodeJobSet(jobset)
-	if err != nil {
-		return errors.Wrap(err, "encoding job set")
-	}
-
-	if err := bkt.Put(freeJobKey(roleType, qdbid), val); err != nil {
-		return errors.Wrap(err, "putting free job")
-	}
-
-	return nil
-}
-
-func (f *freeJobService) DeleteJobsForTable(tx dax.Transaction, roleType dax.RoleType, qtid dax.QualifiedTableID) error {
-	txx, ok := tx.(*boltdb.Tx)
-	if !ok {
-		return dax.NewErrInvalidTransaction()
-	}
-
-	bkt := txx.Bucket(bucketBalancer)
-	if bkt == nil {
-		return errors.Errorf(boltdb.ErrFmtBucketNotFound, bucketBalancer)
-	}
-
-	qdbid := qtid.QualifiedDatabaseID
-	prefix := string(qtid.Key())
-
-	// get free jobs
-	fjs := bkt.Get(freeJobKey(roleType, qdbid))
-	if fjs == nil {
-		return nil
-	}
-
-	jobset, err := decodeJobSet(fjs)
-	if err != nil {
-		return errors.Wrap(err, "decoding job set")
-	}
-
-	jobset.RemoveByPrefix(prefix)
-	val, err := encodeJobSet(jobset)
-	if err != nil {
-		return errors.Wrap(err, "encoding job set")
-	}
-
-	if err := bkt.Put(freeJobKey(roleType, qdbid), val); err != nil {
-		return errors.Wrap(err, "putting free job")
-	}
-
-	return nil
-}
-
-func (f *freeJobService) ListJobs(tx dax.Transaction, roleType dax.RoleType, qdbid dax.QualifiedDatabaseID) (dax.Jobs, error) {
-	txx, ok := tx.(*boltdb.Tx)
-	if !ok {
-		return nil, dax.NewErrInvalidTransaction()
-	}
-
-	bkt := txx.Bucket(bucketBalancer)
-	if bkt == nil {
-		return nil, errors.Errorf(boltdb.ErrFmtBucketNotFound, bucketBalancer)
-	}
-
-	jobset := dax.NewSet[dax.Job]()
-	var err error
-
-	// get free jobs
-	fjs := bkt.Get(freeJobKey(roleType, qdbid))
-	if fjs != nil {
-		jobset, err = decodeJobSet(fjs)
-		if err != nil {
-			return nil, errors.Wrap(err, "decoding job set")
-		}
-	}
-
-	return jobset.Sorted(), nil
-}
-
-func (f *freeJobService) MergeJobs(tx dax.Transaction, roleType dax.RoleType, qdbid dax.QualifiedDatabaseID, jobs dax.Jobs) error {
-	txx, ok := tx.(*boltdb.Tx)
-	if !ok {
-		return dax.NewErrInvalidTransaction()
-	}
-
-	bkt := txx.Bucket(bucketBalancer)
-	if bkt == nil {
-		return errors.Errorf(boltdb.ErrFmtBucketNotFound, bucketBalancer)
-	}
-
-	jobset := dax.NewSet[dax.Job]()
-	var err error
-
-	// get free jobs
-	fjs := bkt.Get(freeJobKey(roleType, qdbid))
-	if fjs != nil {
-		jobset, err = decodeJobSet(fjs)
-		if err != nil {
-			return errors.Wrap(err, "decoding job set")
-		}
-	}
-
-	for _, j := range jobs {
-		jobset.Add(j)
-	}
-	val, err := encodeJobSet(jobset)
-	if err != nil {
-		return errors.Wrap(err, "encoding job set")
-	}
-
-	if err := bkt.Put(freeJobKey(roleType, qdbid), val); err != nil {
-		return errors.Wrap(err, "putting free job")
-	}
-
-	return nil
-}
-
-//////////////////////////////////////////////////////
-
-// Ensure type implements interface.
-var _ balancer.FreeWorkerService = (*freeWorkerService)(nil)
-
-type freeWorkerService struct {
-	db *boltdb.DB
-}
-
-func newFreeWorkerService(db *boltdb.DB) *freeWorkerService {
-	return &freeWorkerService{
-		db: db,
-	}
-}
-
-func (f *freeWorkerService) AddWorkers(tx dax.Transaction, roleType dax.RoleType, addres ...dax.Address) error {
-	txx, ok := tx.(*boltdb.Tx)
-	if !ok {
-		return dax.NewErrInvalidTransaction()
-	}
-
-	bkt := txx.Bucket(bucketBalancer)
-	if bkt == nil {
-		return errors.Errorf(boltdb.ErrFmtBucketNotFound, bucketBalancer)
-	}
-
-	workerset := dax.NewSet[dax.Address]()
-	var err error
-
-	// get free workers
-	fws := bkt.Get(freeWorkerKey(roleType))
-	if fws != nil {
-		workerset, err = decodeWorkerSet(fws)
-		if err != nil {
-			return errors.Wrap(err, "decoding worker set")
-		}
-	}
-
-	for _, w := range addres {
-		workerset.Add(w)
-	}
-	val, err := encodeWorkerSet(workerset)
-	if err != nil {
-		return errors.Wrap(err, "encoding worker set")
-	}
-
-	if err := bkt.Put(freeWorkerKey(roleType), val); err != nil {
-		return errors.Wrap(err, "putting free worker")
-	}
-
-	return nil
-}
-
-func (f *freeWorkerService) RemoveWorker(tx dax.Transaction, roleType dax.RoleType, addr dax.Address) error {
-	txx, ok := tx.(*boltdb.Tx)
-	if !ok {
-		return dax.NewErrInvalidTransaction()
-	}
-
-	workers, err := f.ListWorkers(tx, roleType)
-	if err != nil {
-		return errors.Wrap(err, "listing free workers")
-	}
-
-	// Create a workerset containing the free workers which remain after
-	// removing num workers.
-	workerset := dax.NewSet[dax.Address]()
-	for _, w := range workers {
-		workerset.Add(w)
-	}
-
-	if !workerset.Contains(addr) {
-		return nil
-	}
-
-	// Remove the worker.
-	workerset.Remove(addr)
-
-	bkt := txx.Bucket(bucketBalancer)
-	if bkt == nil {
-		return errors.Errorf(boltdb.ErrFmtBucketNotFound, bucketBalancer)
-	}
-
-	val, err := encodeWorkerSet(workerset)
-	if err != nil {
-		return errors.Wrap(err, "encoding worker set")
-	}
-
-	if err := bkt.Put(freeWorkerKey(roleType), val); err != nil {
-		return errors.Wrap(err, "putting free worker")
-	}
-
-	return nil
-}
-
-func (f *freeWorkerService) PopWorkers(tx dax.Transaction, roleType dax.RoleType, num int) ([]dax.Address, error) {
-	txx, ok := tx.(*boltdb.Tx)
-	if !ok {
-		return nil, dax.NewErrInvalidTransaction()
-	}
-
-	workers, err := f.ListWorkers(tx, roleType)
-	if err != nil {
-		return nil, errors.Wrap(err, "listing free workers")
-	}
-
-	if len(workers) < num {
-		return nil, errors.Errorf("not enough free workers to pop: wanted %d, have: %d", num, len(workers))
-	}
-
-	// Get num workers from the list.
-	workersToAssign := workers[0:num]
-
-	// Create a workerset containing the free workers which remain after
-	// removing num workers.
-	workerset := dax.NewSet[dax.Address]()
-	for _, worker := range workers[num:] {
-		workerset.Add(worker)
-	}
-
-	bkt := txx.Bucket(bucketBalancer)
-	if bkt == nil {
-		return nil, errors.Errorf(boltdb.ErrFmtBucketNotFound, bucketBalancer)
-	}
-
-	val, err := encodeWorkerSet(workerset)
-	if err != nil {
-		return nil, errors.Wrap(err, "encoding worker set")
-	}
-
-	if err := bkt.Put(freeWorkerKey(roleType), val); err != nil {
-		return nil, errors.Wrap(err, "putting free worker")
-	}
-
-	return workersToAssign, nil
-}
-
-func (f *freeWorkerService) ListWorkers(tx dax.Transaction, roleType dax.RoleType) (dax.Addresses, error) {
-	txx, ok := tx.(*boltdb.Tx)
-	if !ok {
-		return nil, dax.NewErrInvalidTransaction()
-	}
-
-	bkt := txx.Bucket(bucketBalancer)
-	if bkt == nil {
-		return nil, errors.Errorf(boltdb.ErrFmtBucketNotFound, bucketBalancer)
-	}
-
-	workerset := dax.NewSet[dax.Address]()
-	var err error
-
-	// get free workers
-	fws := bkt.Get(freeWorkerKey(roleType))
-	if fws != nil {
-		workerset, err = decodeWorkerSet(fws)
-		if err != nil {
-			return nil, errors.Wrap(err, "decoding worker set")
-		}
-	}
-
-	return workerset.Sorted(), nil
-}
-
-//////////////////////////////////////////////////////
-
-const (
-	prefixFmtWorkersDB       = "workers/role/%s/db/%s/" // %s - role, dbKey
-	prefixFmtWorkersAssigned = "workers/assigned/"
-
-	prefixFmtFreeJobs    = "freejobs/role/%s/db/%s" // %s - role, dbKey
-	prefixFmtFreeWorkers = "freeworkers/role/%s"    // %s - role
-)
-
-// workerDBKey returns a key based on worker.
-//
-// Format: workers/role/[role]/db/[dbKey]/[worker] = [job1, job2, ...]
-func workerDBKey(roleType dax.RoleType, qdbid dax.QualifiedDatabaseID, addr dax.Address) []byte {
-	key := fmt.Sprintf(prefixFmtWorkersDB+"%s", roleType, qdbid.Key(), addr)
-	return []byte(key)
-}
-
-// workerAssignedKey returns a key based on worker.
-//
-// Format: workers/assigned/[worker] = dbKey
-func workerAssignedKey(addr dax.Address) []byte {
-	key := fmt.Sprintf(prefixFmtWorkersAssigned+"%s", addr)
-	return []byte(key)
-}
-
-// keyWorker gets the worker out of the key.
-func keyWorker(key []byte) (dax.Address, error) {
-	parts := strings.SplitN(string(key), "/", 6)
-	if len(parts) != 6 {
-		return "", errors.New(errors.ErrUncoded, "worker key format expected: `workers/role/[role]/db/[db]/worker`")
-	}
-
-	return dax.Address(parts[5]), nil
-}
-
-// freeJobKey returns a key for all freeJobs.
-//
-// Format: freejobs/role/[role]/db/[dbKey] = [job1, job2, ...]
-func freeJobKey(roleType dax.RoleType, qdbid dax.QualifiedDatabaseID) []byte {
-	key := fmt.Sprintf(prefixFmtFreeJobs, roleType, qdbid.Key())
-	return []byte(key)
-}
-
-// freeWorkerKey returns a key for all freeWorkers.
-//
-// Format: freeworkers/role/[role] = [worker1, worker2, ...]
-func freeWorkerKey(roleType dax.RoleType) []byte {
-	key := fmt.Sprintf(prefixFmtFreeWorkers, roleType)
-	return []byte(key)
-}

dax/controller/balancer/boltdb/node.go

@@ -1,144 +0,0 @@
-package boltdb
-
-import (
-	"bytes"
-	"encoding/json"
-	"fmt"
-
-	"github.com/featurebasedb/featurebase/v3/dax"
-	"github.com/featurebasedb/featurebase/v3/dax/boltdb"
-	"github.com/featurebasedb/featurebase/v3/dax/controller"
-	"github.com/featurebasedb/featurebase/v3/errors"
-	"github.com/featurebasedb/featurebase/v3/logger"
-)
-
-// Ensure type implements interface.
-var _ controller.NodeService = (*NodeService)(nil)
-
-// NodeService represents a service for managing nodes.
-type NodeService struct {
-	db *boltdb.DB
-
-	logger logger.Logger
-}
-
-// NewNodeService returns a new instance of NodeService with default values.
-func NewNodeService(db *boltdb.DB, logger logger.Logger) *NodeService {
-	return &NodeService{
-		db:     db,
-		logger: logger,
-	}
-}
-
-func (s *NodeService) CreateNode(tx dax.Transaction, addr dax.Address, node *dax.Node) error {
-	txx, ok := tx.(*boltdb.Tx)
-	if !ok {
-		return dax.NewErrInvalidTransaction()
-	}
-
-	bkt := txx.Bucket(bucketBalancer)
-	if bkt == nil {
-		return errors.Errorf(boltdb.ErrFmtBucketNotFound, bucketBalancer)
-	}
-
-	val, err := json.Marshal(node)
-	if err != nil {
-		return errors.Wrap(err, "marshalling node to json")
-	}
-
-	if err := bkt.Put(addressKey(addr), val); err != nil {
-		return errors.Wrap(err, "putting node")
-	}
-
-	return nil
-}
-
-func (s *NodeService) ReadNode(tx dax.Transaction, addr dax.Address) (*dax.Node, error) {
-	txx, ok := tx.(*boltdb.Tx)
-	if !ok {
-		return nil, dax.NewErrInvalidTransaction()
-	}
-
-	bkt := txx.Bucket(bucketBalancer)
-	if bkt == nil {
-		return nil, errors.Errorf(boltdb.ErrFmtBucketNotFound, bucketBalancer)
-	}
-
-	b := bkt.Get(addressKey(addr))
-	if b == nil {
-		return nil, dax.NewErrNodeDoesNotExist(addr)
-	}
-
-	node := &dax.Node{}
-	if err := json.Unmarshal(b, node); err != nil {
-		return nil, errors.Wrap(err, "unmarshalling node json")
-	}
-
-	return node, nil
-}
-
-func (s *NodeService) DeleteNode(tx dax.Transaction, addr dax.Address) error {
-	txx, ok := tx.(*boltdb.Tx)
-	if !ok {
-		return dax.NewErrInvalidTransaction()
-	}
-
-	bkt := txx.Bucket(bucketBalancer)
-	if bkt == nil {
-		return errors.Errorf(boltdb.ErrFmtBucketNotFound, bucketBalancer)
-	}
-
-	if err := bkt.Delete(addressKey(addr)); err != nil {
-		return errors.Wrapf(err, "deleting node key: %s", addressKey(addr))
-	}
-
-	return nil
-}
-
-func (s *NodeService) Nodes(tx dax.Transaction) ([]*dax.Node, error) {
-	txx, ok := tx.(*boltdb.Tx)
-	if !ok {
-		return nil, dax.NewErrInvalidTransaction()
-	}
-
-	nodes, err := s.getNodes(txx)
-	if err != nil {
-		return nil, errors.Wrap(err, "getting nodes")
-	}
-
-	return nodes, nil
-}
-
-func (s *NodeService) getNodes(tx *boltdb.Tx) ([]*dax.Node, error) {
-	c := tx.Bucket(bucketBalancer).Cursor()
-
-	// Deserialize rows into Node objects.
-	nodes := make([]*dax.Node, 0)
-
-	prefix := []byte(prefixFmtNodes)
-	for k, v := c.Seek(prefix); k != nil && bytes.HasPrefix(k, prefix); k, v = c.Next() {
-		if v == nil {
-			s.logger.Printf("nil value for key: %s", k)
-			continue
-		}
-
-		node := &dax.Node{}
-		if err := json.Unmarshal(v, node); err != nil {
-			return nil, errors.Wrap(err, "unmarshalling node json")
-		}
-
-		nodes = append(nodes, node)
-	}
-
-	return nodes, nil
-}
-
-const (
-	prefixFmtNodes = "nodes/"
-)
-
-// addressKey returns a key based on address.
-func addressKey(addr dax.Address) []byte {
-	key := fmt.Sprintf(prefixFmtNodes+"%s", addr)
-	return []byte(key)
-}

dax/controller/balancer/boltdb/node_test.go

@@ -1,61 +0,0 @@
-package boltdb_test
-
-import (
-	"context"
-	"testing"
-
-	"github.com/featurebasedb/featurebase/v3/dax"
-	"github.com/featurebasedb/featurebase/v3/dax/controller/balancer/boltdb"
-	testbolt "github.com/featurebasedb/featurebase/v3/dax/test/boltdb"
-	"github.com/featurebasedb/featurebase/v3/errors"
-	"github.com/featurebasedb/featurebase/v3/logger"
-	"github.com/stretchr/testify/assert"
-)
-
-func TestNodeService(t *testing.T) {
-	db := testbolt.MustOpenDB(t)
-	defer testbolt.MustCloseDB(t, db)
-
-	t.Cleanup(func() {
-		testbolt.CleanupDB(t, db.Path())
-	})
-
-	ctx := context.Background()
-
-	// Initialize the buckets.
-	assert.NoError(t, db.InitializeBuckets(boltdb.BalancerBuckets...))
-
-	t.Run("Nodes", func(t *testing.T) {
-		ns := boltdb.NewNodeService(db, logger.NopLogger)
-
-		node1 := &dax.Node{
-			Address: "localhost:10101",
-			RoleTypes: []dax.RoleType{
-				"compute",
-			},
-		}
-
-		tx, err := db.BeginTx(ctx, true)
-		assert.NoError(t, err)
-		defer tx.Rollback()
-
-		// Create node.
-		assert.NoError(t, ns.CreateNode(tx, node1.Address, node1))
-
-		// Read node.
-		n, err := ns.ReadNode(tx, node1.Address)
-		assert.NoError(t, err)
-		assert.Equal(t, node1, n)
-
-		// Delete node.
-		assert.NoError(t, ns.DeleteNode(tx, node1.Address))
-
-		// Read node.
-		_, err = ns.ReadNode(tx, node1.Address)
-		if assert.Error(t, err) {
-			assert.True(t, errors.Is(err, dax.ErrNodeDoesNotExist))
-		}
-
-		assert.NoError(t, tx.Commit())
-	})
-}