.github/workflows/ci.yml

@@ -22,22 +22,6 @@ jobs:
           github-token: ${{ secrets.GITHUB_TOKEN }}
           dry: true
 
-  # golangci-lint must be run separately from "validate" as there are go mod issues if you run it after the vet
-  golangci:
-    name: lint
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/setup-go@v3
-        with:
-          go-version: 1.19
-      - uses: actions/checkout@v3
-      - name: golangci-lint
-        uses: golangci/golangci-lint-action@v3
-        with:
-          # Optional: version of golangci-lint to use in form of v1.2 or v1.2.3 or `latest` to use the latest version
-          # version: v1.29
-          args: --timeout=8m
-
   validate:
     name: Code Checks
     runs-on: ubuntu-latest
@@ -55,3 +39,11 @@ jobs:
 
       - name: go vet
         run: go vet ./...
+
+      - name: golangci-lint
+        uses: golangci/golangci-lint-action@v3
+        with:
+          args: --timeout=5m
+
+      - name: test
+        run: go test ./...

.gitignore

@@ -83,5 +83,3 @@ staticcheck.conf
 dax/dax-data
 
 coverage-from-docker
-*.client_id.txt
-

.gitlab/.gitlab-ci.yml

@@ -1,747 +0,0 @@
-# You will see a couple of instances of:
-# PKG_LIST=$(go list ./... | grep -Ev 'internal/clustertests|simulacraData|batch|idk|v3/dax/test/dax' | paste -s -d, -)
-# This gets us a package list, comma-separated, which excludes the batch
-# and IDK tests, and a couple of subdirs with specialized stuff. We can
-# then use this with -coverpkg, or we can use ${PKG_LIST//,/ } to get a
-# space-separated list for use with `go test` to run the tests for those
-# directories only.
-
-include:
-  - local: /.gitlab/batch-ci.yml
-  - template: Security/SAST.gitlab-ci.yml
-  - template: Security/License-Scanning.gitlab-ci.yml
-  - template: Security/Dependency-Scanning.gitlab-ci.yml
-
-default:
-  retry:
-    max: 2 # This is confusing but this means "3 runs at max".
-    when:
-      - unknown_failure
-      - api_failure
-      - runner_system_failure
-      - job_execution_timeout
-      - stuck_or_timeout_failure
-
-variables:
-  GOVERSION: "1.19.3"
-  GOFUTURE: "latest"
-  CI_IMAGE: "${CI_REGISTRY_IMAGE}/ci-builder:0.0.1"
-  CI_PRE_CLONE_SCRIPT: |
-    set -x
-
-stages:
-  - ci_image_build
-  - lint
-  - test
-  - build
-  - post build
-  - integration
-  - gauntlet
-  - performance
-  - nonblocking
-  - cleanup_build
-
-gosec-sast:
-  allow_failure: false
-  before_script:
-    - export GOPRIVATE=github.com/molecula/*
-    - apk add openssh-client
-    - eval $(ssh-agent -s)
-    - echo "$FB_SSH_PRIVATE_KEY" | tr -d '\r' | ssh-add -
-    - mkdir -p ~/.ssh
-    - chmod 700 ~/.ssh
-    - git config --global --add url."ssh://git@github.com/".insteadOf "https://github.com/"
-    - ssh-keygen -F github.com || echo "$SSH_KNOWN_HOSTS_HASHED" >> ~/.ssh/known_hosts
-    - chmod 644 ~/.ssh/known_hosts
-  script:
-    ## securego/gosec works for scanning, but not converting to the gitlab report format.
-    - go install github.com/securego/gosec/v2/cmd/gosec@v2.12.0
-    - gosec -fmt=json -out=gosec.json -tests ./... || true
-    ## gitlab's wrapper for gosec works for converting, but not for scanning.
-    - go install 'gitlab.com/gitlab-org/security-products/analyzers/gosec@v1.4.0'
-    - gosec convert gosec.json > gl-sast-report.json
-
-.go-cache:
-  variables:
-    GOPATH: $CI_PROJECT_DIR/.go
-  before_script:
-    - mkdir -p .go
-  cache:
-    # this caching strategy makes it so each branch uses the same cache
-    key: "$CI_COMMIT_REF_SLUG"
-    paths:
-      - .go/pkg/mod/
-
-smoke build:
-  image: golang:$GOVERSION
-  extends: .go-cache
-  stage: lint
-  allow_failure: false
-  rules:
-    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
-  script:
-    - echo "Let's just see if it compiles... (sometimes the linter gives unclear errors if it doesn't)"
-    - go build ./...
-
-golangci-lint:
-  image: golangci/golangci-lint:v1.46.2
-  extends: .go-cache
-  stage: lint
-  allow_failure: true
-  rules:
-    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
-  script:
-    - echo "Checking for issues in new code"
-    - golangci-lint run -v --timeout=8m
-
-go mod tidy:
-  stage: lint
-  image: golang:$GOVERSION
-  extends: .go-cache
-  rules:
-    - if: '$CI_COMMIT_TAG == null && ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web")'
-  script:
-    - go mod tidy
-    - git diff --exit-code -- go.mod go.sum
-
-build lattice:
-  stage: test
-  image: node:14
-  variables:
-    AWS_PROFILE: "service-fb-ci"
-    AWS_ACCESS_KEY_ID: $AWS_FBCI_ACCESS_KEY_ID
-    AWS_SECRET_ACCESS_KEY: $AWS_FBCI_SECRET_ACCESS_KEY
-    CI: "false"
-  rules:
-    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
-  before_script:
-    - curl -sS "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
-    - unzip -qq awscliv2.zip
-    - ./aws/install
-    - aws --version
-    - aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID
-    - aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY
-    - aws configure set region "us-east-2"
-    - aws configure set aws_profile $AWS_PROFILE
-    - aws sts get-caller-identity # ensure we have a valid AWS login
-  script:
-    - cd lattice
-    - cache=$(find . -type f -print0 | sort -z | xargs -0 sha1sum | sha1sum | cut -d ' ' -f 1)
-    - echo "'$cache'"
-    - echo "looking for s3://molecula-artifact-storage/lattice/$cache/build.tar.gz"
-    # if is for if we had a cache object in S3
-    # else is for if we didn't have a cache object (and have to build).
-    - |
-      if aws s3api head-object --bucket molecula-artifact-storage --key "lattice/$cache/build.tar.gz"; then
-        # download object, extract, name the folder `build`
-        aws s3 cp "s3://molecula-artifact-storage/lattice/$cache/build.tar.gz" build.tar.gz
-        tar -xf build.tar.gz
-      else # cache file not found
-        yarn install --frozen-lockfile # CI needs to enforce that the lockfile doesn't need to be updated
-        yarn build
-        tar -czvf "$cache.tar.gz" build/
-        aws s3 mv "$cache.tar.gz" "s3://molecula-artifact-storage/lattice/$cache/build.tar.gz"
-        touch "$CI_COMMIT_SHA"
-        aws s3 mv "$CI_COMMIT_SHA" "s3://molecula-artifact-storage/lattice/$cache/$CI_COMMIT_SHA"
-      fi
-    - | # Ensure that we have build directory after the caching step
-      if [ ! -d build ]; then
-        echo "no build directory, erroring out" || exit 1
-      fi
-    - mv build ../
-    - cd ../
-    - rm -r lattice
-    - mv build lattice
-    - tar -czvf lattice.tar.gz lattice
-  artifacts:
-    paths:
-      - lattice.tar.gz
-
-build featurebase:
-  stage: test
-  image: golang:$GOVERSION
-  extends: .go-cache
-  rules:
-    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
-  script:
-    - rm -r lattice
-    - tar -xvf lattice.tar.gz
-    - go install github.com/rakyll/statik@v0.1.7
-    - $GOPATH/bin/statik -src=lattice
-    - export SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct)
-    - |
-      for goos in "darwin" "linux"; do
-        for goarch in "amd64" "arm64"; do
-          GOOS="${goos}" GOARCH="${goarch}" make build FLAGS="-o featurebase_${goos}_${goarch}"
-        done
-      done
-  artifacts:
-    paths:
-      - featurebase_*
-  needs:
-    - job: build lattice
-
-build fbsql amd64:
-  stage: test
-  variables:
-    BUILD_NAME: build_${CI_COMMIT_SHA}_${CI_CONCURRENT_ID}
-  tags:
-    - shell
-  rules:
-    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
-  script:
-    - export SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct)
-    - date
-    - GOOS="linux" GOARCH="amd64" make docker-build-fbsql BUILD_CGO=1
-    - GOOS="darwin" GOARCH="amd64" make docker-build-fbsql
-  artifacts:
-    paths:
-      - ./build/fbsql_*
-
-build fbsql arm64:
-  stage: test
-  variables:
-    BUILD_NAME: build_${CI_COMMIT_SHA}_${CI_CONCURRENT_ID}
-  tags:
-    - shell-arm64
-  rules:
-    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
-  script:
-    - export SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct)
-    - date
-    - GOOS="linux" GOARCH="arm64" make docker-build-fbsql BUILD_CGO=1
-    - GOOS="darwin" GOARCH="arm64" make docker-build-fbsql
-  artifacts:
-    paths:
-      - ./build/fbsql_*
-
-build amd container fb:
-  stage: test
-  tags:
-    - shell
-  rules:
-    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
-  before_script:
-    - echo "${DOCKER_DEPLOY_TOKEN}" | docker login -u ${DOCKER_DEPLOY_USER} --password-stdin ${CI_REGISTRY}
-  script:
-    - tag=${CI_REGISTRY_IMAGE}/featurebase:linux-amd64-${CI_COMMIT_REF_SLUG}
-    - export SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct)
-    - docker build --build-arg GO_VERSION=$GOVERSION --build-arg ARCH=amd64 --build-arg SOURCE_DATE_EPOCH=$SOURCE_DATE_EPOCH -t $tag -f .gitlab/Dockerfile .
-    - docker push $tag
-    - echo Created docker featurebase image with tag "$tag"
-  needs:
-    - job: build featurebase
-
-run jest tests:
-  stage: test
-  image: node:14
-  variables:
-    CI: "true"
-  rules:
-    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
-  script:
-    - echo "Testing lattice..."
-    - cd lattice
-    - npm install --force
-    - npm test -- --coverage --testResultsProcessor=jest-sonar-reporter
-  artifacts:
-    paths:
-      - lattice/coverage/lcov.info
-
-# We run go test -race on all the standard packages, skipping the ones that have their
-# own separate tests. we spin this off as nonblocking because it used to take a really
-# long time and even now it's pretty slow.
-run go tests race:
-  stage: nonblocking # don't let this job block any other jobs because it takes much longer than the other tests.
-  image: golang:$GOVERSION
-  extends: .go-cache
-  variables:
-    FEATUREBASE_CONTROLLER_CONFIG_SQLDB_DATABASE: run_go_tests_race
-    POSTGRES_DB: run_go_tests_race
-    FEATUREBASE_CONTROLLER_CONFIG_SQLDB_USER: postgres
-    POSTGRES_USER: postgres
-    FEATUREBASE_CONTROLLER_CONFIG_SQLDB_PASSWORD: $POSTGRES_PASSWORD
-    POSTGRES_PASSWORD: $POSTGRES_PASSWORD
-    FEATUREBASE_CONTROLLER_CONFIG_SQLDB_HOST: postgres
-  services:
-    - postgres:14.7
-  rules:
-    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
-  needs: ["smoke build"] # we do block on smoke build though bc it's pretty dumb to test stuff if it doesn't build
-  script:
-    - echo "Running featurebase race tests..."
-    - PKG_LIST=$(go list ./... | grep -Ev 'internal/clustertests|simulacraData|batch|idk|v3/dax/test/dax' | paste -s -d, -)
-    - export TMPDIR=/mnt/ramdisk/test-$CI_JOB_ID
-    - mkdir -p $TMPDIR
-    - go test -race -v -timeout=10m ${PKG_LIST//,/ }
-  after_script:
-    - rm -rf /mnt/ramdisk/test-$CI_JOB_ID
-  tags:
-    - docker
-
-# We run our base tests against $GOVERSION (a reasonably current version that we trust)
-# and use shardwidth22 for them. This gives us a canary for things breaking for
-# unusual shard widths.
-run go tests:
-  stage: test
-  image: golang:$GOVERSION
-  extends: .go-cache
-  variables:
-    FEATUREBASE_CONTROLLER_CONFIG_SQLDB_DATABASE: run_go_tests
-    POSTGRES_DB: run_go_tests
-    FEATUREBASE_CONTROLLER_CONFIG_SQLDB_USER: postgres
-    POSTGRES_USER: postgres
-    FEATUREBASE_CONTROLLER_CONFIG_SQLDB_PASSWORD: $POSTGRES_PASSWORD
-    POSTGRES_PASSWORD: $POSTGRES_PASSWORD
-    FEATUREBASE_CONTROLLER_CONFIG_SQLDB_HOST: postgres
-  services:
-    - postgres:14.7
-  rules:
-    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
-  script:
-    - echo "Running featurebase unit tests..."
-    - PKG_LIST=$(go list ./... | grep -Ev 'internal/clustertests|simulacraData|batch|idk|v3/dax/test/dax' | paste -s -d, -)
-    - export TMPDIR=/mnt/ramdisk/test-$CI_JOB_ID
-    - mkdir -p $TMPDIR
-    - go test -tags=shardwidth22 -timeout=10m -coverprofile=coverage.out -covermode=atomic -coverpkg=${PKG_LIST} ${PKG_LIST//,/ }
-  after_script:
-    - rm -rf /mnt/ramdisk/test-$CI_JOB_ID
-  artifacts:
-    paths:
-      - coverage.out
-  tags:
-    - docker
-
-run go tests dax/test/dax:
-  stage: test
-  image: golang:$GOVERSION
-  extends: .go-cache
-  tags:
-    - docker
-  variables:
-    FEATUREBASE_CONTROLLER_CONFIG_SQLDB_DATABASE: run_go_tests_dax
-    POSTGRES_DB: run_go_tests_dax
-    FEATUREBASE_CONTROLLER_CONFIG_SQLDB_USER: postgres
-    POSTGRES_USER: postgres
-    FEATUREBASE_CONTROLLER_CONFIG_SQLDB_PASSWORD: $POSTGRES_PASSWORD
-    POSTGRES_PASSWORD: $POSTGRES_PASSWORD
-    FEATUREBASE_CONTROLLER_CONFIG_SQLDB_HOST: postgres
-  services:
-    - postgres:14.7
-  rules:
-    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
-  script:
-    - echo "Building FB and Datagen docker images for DAX tests"
-    - PKG_LIST=$(go list ./... | grep -Ev 'internal/clustertests|simulacraData' | paste -s -d, -)
-    - export TMPDIR=/mnt/ramdisk/test-$CI_JOB_ID
-    - mkdir -p $TMPDIR
-    - go test -coverprofile=coverage-dax-integration.out -covermode=atomic -coverpkg=${PKG_LIST} -timeout=20m ./dax/test/dax
-  after_script:
-    - rm -rf /mnt/ramdisk/test-$CI_JOB_ID
-  artifacts:
-    paths:
-      - coverage-dax-integration.out
-
-# idk tests
-run go tests idk race:
-  variables:
-    PROJECT: race_${CI_CONCURRENT_ID}
-  stage: nonblocking
-  retry: 1
-  script:
-    - echo "Running test-all-race"
-    - cd ./idk/
-    - echo $PROJECT
-    - echo $CI_COMMIT_REF_SLUG
-    - BRANCH_NAME=${CI_COMMIT_REF_SLUG} IDK_FEATUREBASE_TAG=${CI_COMMIT_TAG} IDK_FEATUREBASE_HASH=${CI_COMMIT_SHA} make test-all-race
-  after_script:
-    - cd ./idk/
-    - make save-pilosa-logs
-    - make shutdown
-  artifacts:
-    paths:
-      - ./idk/testdata/*_coverage.out
-  tags:
-    - shell
-    - aws
-  needs:
-    - job: build amd container fb
-
-run go tests idk shard transactional:
-  variables:
-    IDK_DEFAULT_SHARD_TRANSACTIONAL: 1
-    PROJECT: shardttrans_${CI_CONCURRENT_ID}
-  stage: nonblocking
-  retry: 1
-  script:
-    - echo "Running shard transactional tests"
-    - cd ./idk/
-    - echo $PROJECT
-    - echo $CI_COMMIT_REF_SLUG
-    - BRANCH_NAME=${CI_COMMIT_REF_SLUG} IDK_FEATUREBASE_TAG=${CI_COMMIT_TAG} IDK_FEATUREBASE_HASH=${CI_COMMIT_SHA} make test-all
-  after_script:
-    - cd ./idk/
-    - make save-pilosa-logs
-    - make shutdown
-  artifacts:
-    paths:
-      - ./idk/testdata/*_coverage.out
-      - ./idk/testdata/*_logs.txt
-  tags:
-    - shell
-    - aws
-  needs:
-    - job: build amd container fb
-
-run go tests idk 533:
-  variables:
-    USERNAME: fb-idk-access
-    PROJECT: test533_${CI_CONCURRENT_ID}
-  stage: nonblocking
-  retry: 1
-  script:
-    - echo "Running confluent 5.3.3 test-all"
-    - cd ./idk/
-    - echo $PROJECT
-    - echo $CI_COMMIT_REF_SLUG
-    - CONFLUENT_VERSION=5.3.3 BRANCH_NAME=${CI_COMMIT_REF_SLUG} IDK_FEATUREBASE_TAG=${CI_COMMIT_TAG} IDK_FEATUREBASE_HASH=${CI_COMMIT_SHA} make test-all
-  after_script:
-    - cd ./idk/
-    - make save-pilosa-logs
-    - make shutdown
-  tags:
-    - shell
-    - aws
-  artifacts:
-    paths:
-      - ./idk/testdata/*_coverage.out
-  needs:
-    - job: build amd container fb
-
-run go tests idk sasl:
-  variables:
-    PROJECT: sasl_${CI_CONCURRENT_ID}
-  stage: nonblocking
-  retry: 1
-  script:
-    - echo "Running test-all-kafka-sasl"
-    - cd ./idk/
-    - echo $PROJECT
-    - echo $CI_COMMIT_REF_SLUG
-    - BRANCH_NAME=${CI_COMMIT_REF_SLUG} IDK_FEATUREBASE_TAG=${CI_COMMIT_TAG} IDK_FEATUREBASE_HASH=${CI_COMMIT_SHA} make test-all-kafka-sasl
-  after_script:
-    - cd ./idk/
-    - make save-pilosa-logs
-    - make shutdown
-  tags:
-    - shell
-    - aws
-  artifacts:
-    paths:
-      - ./idk/testdata/*_coverage.out
-  needs:
-    - job: build amd container fb
-
-upload to sonarcloud:
-  stage: nonblocking
-  image: sonarsource/sonar-scanner-cli:4.7
-  variables:
-    SONAR_TOKEN: $SONAR_TOKEN
-  rules:
-    - if: '$CI_COMMIT_TAG == null && ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web")'
-  script:
-    - sonar-scanner -Dsonar.projectKey=molecula_featurebase -Dsonar.organization=molecula -Dsonar.sources=. -Dsonar.host.url=https://sonarcloud.io -Dsonar.go.coverage.reportPaths=coverage*.out,results/coverage*out,idk/testdata/*coverage.out,batch/testdata/*coverage.out,coverage-from-docker/*.out -Dsonar.javascript.lcov.reportPaths=lattice/coverage/lcov.info
-  needs:
-    - job: run go tests
-    - job: run jest tests
-    - job: external lookup tests
-    - job: run go tests idk race
-      optional: true
-    - job: run go tests idk shard transactional
-      optional: true
-    - job: run go tests idk sasl
-      optional: true
-    - job: run go tests idk 533
-      optional: true
-    - job: run go tests batch
-      optional: true
-    - job: run go tests dax/test/dax
-      optional: true
-
-package for linux amd64:
-  stage: build
-  image: golang:$GOVERSION
-  extends: .go-cache
-  rules:
-    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
-  variables:
-    GOOS: "linux"
-    GOARCH: "amd64"
-  script:
-    - echo 'deb [trusted=yes] https://repo.goreleaser.com/apt/ /' | tee /etc/apt/sources.list.d/goreleaser.list
-    - apt update && apt install nfpm=2.11.3
-    - make package
-  artifacts:
-    paths:
-      - "*.deb"
-      - "*.rpm"
-
-upload_artifacts_to_nexus:
-  stage: post build
-  image: golang:$GOVERSION
-  rules:
-    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
-  script:
-    - curl -v --user "$NEXUS_YUM_CREDS" --upload-file *.arm64.rpm https://nexus.molecula.com/repository/molecula-yum/release/
-    - curl -v --user "$NEXUS_YUM_CREDS" --upload-file *.amd64.rpm https://nexus.molecula.com/repository/molecula-yum/release/
-  dependencies:
-    - package for linux amd64
-    - package for linux arm64
-
-trigger_m-cloud-images:
-  stage: post build
-  rules:
-    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
-  needs:
-    - upload_artifacts_to_nexus
-  trigger: molecula/m-cloud-images
-
-package for linux arm64:
-  stage: build
-  image: golang:$GOVERSION
-  extends: .go-cache
-  rules:
-    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
-  variables:
-    GOOS: "linux"
-    GOARCH: "arm64"
-  script:
-    - echo 'deb [trusted=yes] https://repo.goreleaser.com/apt/ /' | tee /etc/apt/sources.list.d/goreleaser.list
-    - apt update && apt install nfpm=2.11.3
-    - make package
-  artifacts:
-    paths:
-      - "*.deb"
-      - "*.rpm"
-
-build arm container fb:
-  stage: build
-  needs:
-    - "build featurebase"
-  tags:
-    - shell
-  rules:
-    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
-  before_script:
-    - echo "${DOCKER_DEPLOY_TOKEN}" | docker login -u ${DOCKER_DEPLOY_USER} --password-stdin ${CI_REGISTRY}
-  script:
-    - echo $CI_COMMIT_REF_SLUG
-    - tag=${CI_REGISTRY_IMAGE}/featurebase:linux-arm64-${CI_COMMIT_REF_SLUG}
-    - export SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct)
-    - docker build --build-arg GO_VERSION=$GOVERSION --build-arg ARCH=arm64 --build-arg SOURCE_DATE_EPOCH=$SOURCE_DATE_EPOCH -t $tag -f .gitlab/Dockerfile .
-    - docker push $tag
-    - echo Created docker featurebase image with tag "$tag"
-
-### start idk builds ###
-# building them all serially because otherwise you get container name conflicts.
-idk build_amd64:
-  stage: build
-  variables:
-    BUILD_NAME: build_${CI_COMMIT_SHA}_${CI_CONCURRENT_ID}
-  tags:
-    - shell
-  rules:
-    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
-  script:
-    - export SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct)
-    - cd ./idk/
-    - date
-    - make docker-build GOOS="linux" GOARCH="amd64" BUILD_CGO=1
-    - date
-    - make docker-build GOOS="darwin" GOARCH="amd64"
-    - date
-  artifacts:
-    paths:
-      - ./idk/build/*
-  needs:
-    # doesn't actually need this... just want it to start executing before *all* the tests finish
-    - job: run go tests
-
-idk build_arm64:
-  stage: build
-  tags:
-    - shell-arm64
-  rules:
-    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
-  script:
-    - export SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct)
-    - cd ./idk/
-    - date
-    - make docker-build GOOS="linux" GOARCH="arm64" BUILD_CGO=1 BUILD_NAME="linux-arm64"
-    - date
-    - make docker-build GOOS="darwin" GOARCH="arm64"
-    - date
-  artifacts:
-    paths:
-      - ./idk/build/*
-  needs:
-    # doesn't actually need this... just want it to start executing before *all* the tests finish
-    - job: run go tests
-
-# building them all serially because otherwise you get container name conflicts.
-# only do containers on default branch
-idk package_docker_all:
-  stage: build
-  tags:
-    - shell
-  rules:
-    - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web")'
-  script:
-    - export SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct)
-    - make docker-idk GOOS="linux" GOARCH="amd64"
-    - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
-    - make docker-idk-tag-push GOOS="linux" GOARCH="amd64"
-  needs:
-    - job: idk build_amd64
-    - job: idk build_arm64
-
-idk s3 dump:
-  stage: post build
-  allow_failure: false
-  variables:
-    PROFILE: "service-fb-ci"
-    AWS_SSH_PRIVATE_KEY: $AWS_FBCI_SSH_KEY
-    AWS_ACCESS_KEY_ID: $AWS_FBCI_ACCESS_KEY_ID
-    AWS_SECRET_ACCESS_KEY: $AWS_FBCI_SECRET_ACCESS_KEY
-  tags:
-    - shell
-  rules:
-    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
-  script:
-    - aws configure set aws_access_key_id $AWS_FBCI_ACCESS_KEY_ID
-    - aws configure set aws_secret_access_key $AWS_FBCI_SECRET_ACCESS_KEY
-    - aws configure set region "us-east-2"
-    - aws configure set aws_profile $PROFILE
-    - aws s3 cp ./idk/build/ s3://molecula-artifact-storage/idk/${CI_COMMIT_BRANCH}/${CI_COMMIT_SHORT_SHA}/ --recursive
-    - aws s3 cp ./idk/build/ s3://molecula-artifact-storage/idk/${CI_COMMIT_BRANCH}/_latest/ --recursive
-  needs:
-    - job: idk build_amd64
-    - job: idk build_arm64
-
-idk s3 dump tag:
-  stage: post build
-  variables:
-    PROFILE: "service-fb-ci"
-    AWS_SSH_PRIVATE_KEY: $AWS_FBCI_SSH_KEY
-    AWS_ACCESS_KEY_ID: $AWS_FBCI_ACCESS_KEY_ID
-    AWS_SECRET_ACCESS_KEY: $AWS_FBCI_SECRET_ACCESS_KEY
-    LOCATION: molecula-artifact-storage/idk/_tags
-  tags:
-    - shell
-  rules:
-    - if: '$CI_COMMIT_TAG != null && ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web")'
-  script:
-    - aws configure set aws_access_key_id $AWS_FBCI_ACCESS_KEY_ID
-    - aws configure set aws_secret_access_key $AWS_FBCI_SECRET_ACCESS_KEY
-    - aws configure set region "us-east-2"
-    - aws configure set aws_profile $PROFILE
-    - |
-      for goos in "darwin" "linux"; do
-        for goarch in "amd64" "arm64"; do
-          dir=idk-${CI_COMMIT_TAG}-${goos}-${goarch}
-          echo "Directory ${dir}"
-          mkdir ${dir}
-          mv ./idk/build/idk-${goos}-${goarch}/molecula-consumer-* ${dir}/
-          tar cvzf ${dir}.tar.gz ${dir}
-          aws s3 cp ${dir} s3://${LOCATION}/${CI_COMMIT_TAG}/${dir}/ --recursive
-          aws s3 cp ${dir}.tar.gz s3://${LOCATION}/${CI_COMMIT_TAG}/
-        done
-      done
-  needs:
-    - job: idk build_amd64
-    - job: idk build_arm64
-### end idk builds ###
-
-external lookup tests:
-  stage: integration
-  image: golang:$GOVERSION
-  extends: .go-cache
-  # TODO: no rules here, do we need to add the rules line?
-  variables:
-    POSTGRES_DB: $POSTGRES_DB
-    POSTGRES_USER: $POSTGRES_USER
-    POSTGRES_PASSWORD: $POSTGRES_PASSWORD
-    POSTGRES_HOST_AUTH_METHOD: trust
-  services:
-    - postgres:13.5
-  script:
-    - apt-get update --allow-releaseinfo-change -y
-    - apt-get install -y postgresql-client
-    - go test . -run "^TestExternalLookup" -externalLookupDSN postgresql://$POSTGRES_USER:$POSTGRES_PASSWORD@postgres/$POSTGRES_DB?sslmode=disable
-
-s3 dump:
-  stage: post build
-  variables:
-    PROFILE: "service-fb-ci"
-    AWS_SSH_PRIVATE_KEY: $AWS_FBCI_SSH_KEY
-    AWS_ACCESS_KEY_ID: $AWS_FBCI_ACCESS_KEY_ID
-    AWS_SECRET_ACCESS_KEY: $AWS_FBCI_SECRET_ACCESS_KEY
-  tags:
-    - shell
-  rules:
-    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "web"'
-  script:
-    - aws configure set aws_access_key_id $AWS_FBCI_ACCESS_KEY_ID
-    - aws configure set aws_secret_access_key $AWS_FBCI_SECRET_ACCESS_KEY
-    - aws configure set region "us-east-2"
-    - aws configure set aws_profile $PROFILE
-    - |
-      for goos in "darwin" "linux"; do
-        for goarch in "amd64" "arm64"; do
-          aws s3 cp featurebase_${goos}_${goarch} s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/${CI_COMMIT_SHORT_SHA}/featurebase_${goos}_${goarch}
-          aws s3 cp featurebase_${goos}_${goarch} s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/_latest/featurebase_${goos}_${goarch}
-          aws s3 cp ./build/fbsql_${goos}_${goarch} s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/${CI_COMMIT_SHORT_SHA}/fbsql_${goos}_${goarch}
-          aws s3 cp ./build/fbsql_${goos}_${goarch} s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/_latest/fbsql_${goos}_${goarch}
-        done
-      done
-  needs:
-    - job: build featurebase
-    - job: build fbsql amd64
-    - job: build fbsql arm64
-
-s3 dump tag:
-  stage: post build
-  variables:
-    PROFILE: "service-fb-ci"
-    AWS_SSH_PRIVATE_KEY: $AWS_FBCI_SSH_KEY
-    AWS_ACCESS_KEY_ID: $AWS_FBCI_ACCESS_KEY_ID
-    AWS_SECRET_ACCESS_KEY: $AWS_FBCI_SECRET_ACCESS_KEY
-    LOCATION: molecula-artifact-storage/featurebase/_tags
-  tags:
-    - shell
-  rules:
-    - if: '$CI_COMMIT_TAG != null && ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "web")'
-  script:
-    - aws configure set aws_access_key_id $AWS_FBCI_ACCESS_KEY_ID
-    - aws configure set aws_secret_access_key $AWS_FBCI_SECRET_ACCESS_KEY
-    - aws configure set region "us-east-2"
-    - aws configure set aws_profile $PROFILE
-    - |
-      for goos in "darwin" "linux"; do
-        for goarch in "amd64" "arm64"; do
-          dir=featurebase-${CI_COMMIT_TAG}-${goos}-${goarch}
-          echo "Directory ${dir}"
-          mkdir $dir
-          mv featurebase_${goos}_${goarch} ${dir}/featurebase
-          mv ./build/fbsql_${goos}_${goarch} ${dir}/fbsql
-          cp NOTICE install/featurebase.conf install/featurebase.*.service ${dir}/
-          tar cvzf ${dir}.tar.gz ${dir}
-          aws s3 cp ${dir} s3://${LOCATION}/${CI_COMMIT_TAG}/${dir}/ --recursive
-          aws s3 cp ${dir}.tar.gz s3://${LOCATION}/${CI_COMMIT_TAG}/
-        done
-      done
-
-  needs:
-    - job: build featurebase
-    - job: build fbsql amd64
-    - job: build fbsql arm64

.gitlab/Dockerfile

@@ -1,24 +0,0 @@
-FROM alpine:3.14.2
-
-LABEL maintainer "dev@molecula.com"
-LABEL org.opencontainers.image.authors="dev@molecula.com"
-
-ARG ARCH
-
-WORKDIR /
-
-RUN apk add --no-cache curl jq
-
-COPY NOTICE .
-COPY featurebase_linux_$ARCH featurebase
-
-RUN chmod ugo+x .
-EXPOSE 10101
-VOLUME /data
-
-ENV PILOSA_DATA_DIR /data
-ENV PILOSA_BIND 0.0.0.0:10101
-ENV PILOSA_BIND_GRPC 0.0.0.0:20101
-
-ENTRYPOINT ["/featurebase"]
-CMD ["server"]

.gitlab/batch-ci.yml

@@ -1,28 +0,0 @@
-run go tests batch:
-  variables:
-    PROJECT: batch_${CI_CONCURRENT_ID}
-  # this test relies on stuff that happens after build-lattice, which
-  # makes it pause the entire CI run waiting for this. we accept the
-  # small risk of wasting a build against the near certainty of spending
-  # five minutes running only one job.
-  stage: nonblocking
-  retry: 1
-  script:
-    - echo "Running test-all"
-    - cd ./batch/
-    - echo $PROJECT
-    - make build-featurebase
-    - make test-all
-  after_script:
-    - cd ./batch/
-    - make save-featurebase-logs
-    - make shutdown
-  artifacts:
-    paths:
-      - ./batch/testdata/*_coverage.out
-      - ./batch/testdata/*_logs.txt
-  tags:
-    - shell
-    - aws
-  needs:
-    - job: build amd container fb

Dockerfile-clustertests

@@ -20,12 +20,12 @@ RUN apt install -y docker.io
 ADD https://github.com/docker/compose/releases/latest/download/docker-compose-Linux-x86_64 /usr/local/bin/docker-compose
 RUN chmod +x /usr/local/bin/docker-compose
 
-WORKDIR /go/src/github.com/featurebasedb/featurebase/cmd/featurebase
+WORKDIR /go/src/github.com/molecula/featurebase/cmd/featurebase
 
 # generate an instrumented binary to allow for calculating code coverage for clustertests
 # the entrypoint for the binary is TestRunMain, which is wrapper for main
 RUN go test -covermode=atomic -coverpkg=../../... -c -tags testrunmain -o featurebase
-RUN cp /go/src/github.com/featurebasedb/featurebase/cmd/featurebase/featurebase /featurebase
+RUN cp /go/src/github.com/molecula/featurebase/cmd/featurebase/featurebase /featurebase
 
 COPY NOTICE /NOTICE
 

Dockerfile-clustertests-client

@@ -19,10 +19,10 @@ RUN apt install -y docker.io
 ADD https://github.com/docker/compose/releases/latest/download/docker-compose-Linux-x86_64 /usr/local/bin/docker-compose
 RUN chmod +x /usr/local/bin/docker-compose
 
-WORKDIR /go/src/github.com/featurebasedb/featurebase/cmd/featurebase
+WORKDIR /go/src/github.com/molecula/featurebase/cmd/featurebase
 
 RUN go test -covermode=atomic -coverpkg=../../... -c -tags testrunmain -o featurebase
-RUN cp /go/src/github.com/featurebasedb/featurebase/cmd/featurebase/featurebase /featurebase
+RUN cp /go/src/github.com/molecula/featurebase/cmd/featurebase/featurebase /featurebase
 
 
 COPY NOTICE /NOTICE
@@ -32,6 +32,6 @@ COPY ./internal/clustertests /go/src/github.com/featurebasedb/featurebase/intern
 EXPOSE 10101
 VOLUME /data
 
-WORKDIR /go/src/github.com/featurebasedb/featurebase
+WORKDIR /go/src/github.com/molecula/featurebase
 
 CMD ["/featurebase", "-test.run=TestRunMain", "-test.coverprofile=/results/coverage.out", "server", "--data-dir", "/data", "--bind", "http://0.0.0.0:10101"]

Dockerfile-dax

@@ -16,7 +16,7 @@ RUN make build FLAGS="-o build/featurebase" ${MAKE_FLAGS}
 ### FeatureBase runner ###
 ##########################
 
-FROM golang:alpine as runner
+FROM alpine:3.13.2 as runner
 
 LABEL maintainer "dev@featurebase.com"
 

Dockerfile-fbsql

@@ -1,48 +0,0 @@
-ARG GO_VERSION=1.19
-
-FROM golang:1.19-buster as builder
-
-WORKDIR /
-RUN apt-get update -y -qq && apt-get install -y -qq \
-  build-essential \
-  git \
-  musl-tools \
-  netcat \
-  unixodbc \
-  unixodbc-dev \
-  && rm -rf /var/lib/apt/lists/*
-RUN ["git", "clone", "https://github.com/edenhill/librdkafka.git"]
-WORKDIR /librdkafka
-RUN ./configure --prefix /usr && \
-  make && \
-  make install
-
-WORKDIR /featurebase
-
-COPY . .
-
-ARG MAKE_FLAGS
-ARG GO_BUILD_FLAGS
-ARG SOURCE_DATE_EPOCH
-
-WORKDIR /featurebase/
-
-ENV SOURCE_DATE_EPOCH=${SOURCE_DATE_EPOCH}
-RUN make build-fbsql GO_BUILD_FLAGS="-mod=vendor ${GO_BUILD_FLAGS}" ${MAKE_FLAGS}
-
-FROM ubuntu:20.04 as runner
-
-RUN apt-get update -y -qq && apt-get install -y -qq \
-  ca-certificates \
-  musl-tools \
-  netcat \
-  unixodbc-dev \
-  && rm -rf /var/lib/apt/lists/*
-
-COPY --from=builder /featurebase/fbsql /usr/local/bin/
-
-# Verify that the linker can find everything.
-FROM runner as linkcheck
-RUN if [ -e /usr/local/bin/fbsql ] ; then ldd /usr/local/bin/fbsql; fi
-
-FROM runner

LICENSE

@@ -1,201 +1,13 @@
-                                 Apache License
-                           Version 2.0, January 2004
-                        http://www.apache.org/licenses/
+Copyright 2023 Molecula Corp. All rights reserved.
 
-   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
 
-   1. Definitions.
+    http://www.apache.org/licenses/LICENSE-2.0
 
-      "License" shall mean the terms and conditions for use, reproduction,
-      and distribution as defined by Sections 1 through 9 of this document.
-
-      "Licensor" shall mean the copyright owner or entity authorized by
-      the copyright owner that is granting the License.
-
-      "Legal Entity" shall mean the union of the acting entity and all
-      other entities that control, are controlled by, or are under common
-      control with that entity. For the purposes of this definition,
-      "control" means (i) the power, direct or indirect, to cause the
-      direction or management of such entity, whether by contract or
-      otherwise, or (ii) ownership of fifty percent (50%) or more of the
-      outstanding shares, or (iii) beneficial ownership of such entity.
-
-      "You" (or "Your") shall mean an individual or Legal Entity
-      exercising permissions granted by this License.
-
-      "Source" form shall mean the preferred form for making modifications,
-      including but not limited to software source code, documentation
-      source, and configuration files.
-
-      "Object" form shall mean any form resulting from mechanical
-      transformation or translation of a Source form, including but
-      not limited to compiled object code, generated documentation,
-      and conversions to other media types.
-
-      "Work" shall mean the work of authorship, whether in Source or
-      Object form, made available under the License, as indicated by a
-      copyright notice that is included in or attached to the work
-      (an example is provided in the Appendix below).
-
-      "Derivative Works" shall mean any work, whether in Source or Object
-      form, that is based on (or derived from) the Work and for which the
-      editorial revisions, annotations, elaborations, or other modifications
-      represent, as a whole, an original work of authorship. For the purposes
-      of this License, Derivative Works shall not include works that remain
-      separable from, or merely link (or bind by name) to the interfaces of,
-      the Work and Derivative Works thereof.
-
-      "Contribution" shall mean any work of authorship, including
-      the original version of the Work and any modifications or additions
-      to that Work or Derivative Works thereof, that is intentionally
-      submitted to Licensor for inclusion in the Work by the copyright owner
-      or by an individual or Legal Entity authorized to submit on behalf of
-      the copyright owner. For the purposes of this definition, "submitted"
-      means any form of electronic, verbal, or written communication sent
-      to the Licensor or its representatives, including but not limited to
-      communication on electronic mailing lists, source code control systems,
-      and issue tracking systems that are managed by, or on behalf of, the
-      Licensor for the purpose of discussing and improving the Work, but
-      excluding communication that is conspicuously marked or otherwise
-      designated in writing by the copyright owner as "Not a Contribution."
-
-      "Contributor" shall mean Licensor and any individual or Legal Entity
-      on behalf of whom a Contribution has been received by Licensor and
-      subsequently incorporated within the Work.
-
-   2. Grant of Copyright License. Subject to the terms and conditions of
-      this License, each Contributor hereby grants to You a perpetual,
-      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-      copyright license to reproduce, prepare Derivative Works of,
-      publicly display, publicly perform, sublicense, and distribute the
-      Work and such Derivative Works in Source or Object form.
-
-   3. Grant of Patent License. Subject to the terms and conditions of
-      this License, each Contributor hereby grants to You a perpetual,
-      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-      (except as stated in this section) patent license to make, have made,
-      use, offer to sell, sell, import, and otherwise transfer the Work,
-      where such license applies only to those patent claims licensable
-      by such Contributor that are necessarily infringed by their
-      Contribution(s) alone or by combination of their Contribution(s)
-      with the Work to which such Contribution(s) was submitted. If You
-      institute patent litigation against any entity (including a
-      cross-claim or counterclaim in a lawsuit) alleging that the Work
-      or a Contribution incorporated within the Work constitutes direct
-      or contributory patent infringement, then any patent licenses
-      granted to You under this License for that Work shall terminate
-      as of the date such litigation is filed.
-
-   4. Redistribution. You may reproduce and distribute copies of the
-      Work or Derivative Works thereof in any medium, with or without
-      modifications, and in Source or Object form, provided that You
-      meet the following conditions:
-
-      (a) You must give any other recipients of the Work or
-          Derivative Works a copy of this License; and
-
-      (b) You must cause any modified files to carry prominent notices
-          stating that You changed the files; and
-
-      (c) You must retain, in the Source form of any Derivative Works
-          that You distribute, all copyright, patent, trademark, and
-          attribution notices from the Source form of the Work,
-          excluding those notices that do not pertain to any part of
-          the Derivative Works; and
-
-      (d) If the Work includes a "NOTICE" text file as part of its
-          distribution, then any Derivative Works that You distribute must
-          include a readable copy of the attribution notices contained
-          within such NOTICE file, excluding those notices that do not
-          pertain to any part of the Derivative Works, in at least one
-          of the following places: within a NOTICE text file distributed
-          as part of the Derivative Works; within the Source form or
-          documentation, if provided along with the Derivative Works; or,
-          within a display generated by the Derivative Works, if and
-          wherever such third-party notices normally appear. The contents
-          of the NOTICE file are for informational purposes only and
-          do not modify the License. You may add Your own attribution
-          notices within Derivative Works that You distribute, alongside
-          or as an addendum to the NOTICE text from the Work, provided
-          that such additional attribution notices cannot be construed
-          as modifying the License.
-
-      You may add Your own copyright statement to Your modifications and
-      may provide additional or different license terms and conditions
-      for use, reproduction, or distribution of Your modifications, or
-      for any such Derivative Works as a whole, provided Your use,
-      reproduction, and distribution of the Work otherwise complies with
-      the conditions stated in this License.
-
-   5. Submission of Contributions. Unless You explicitly state otherwise,
-      any Contribution intentionally submitted for inclusion in the Work
-      by You to the Licensor shall be under the terms and conditions of
-      this License, without any additional terms or conditions.
-      Notwithstanding the above, nothing herein shall supersede or modify
-      the terms of any separate license agreement you may have executed
-      with Licensor regarding such Contributions.
-
-   6. Trademarks. This License does not grant permission to use the trade
-      names, trademarks, service marks, or product names of the Licensor,
-      except as required for reasonable and customary use in describing the
-      origin of the Work and reproducing the content of the NOTICE file.
-
-   7. Disclaimer of Warranty. Unless required by applicable law or
-      agreed to in writing, Licensor provides the Work (and each
-      Contributor provides its Contributions) on an "AS IS" BASIS,
-      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-      implied, including, without limitation, any warranties or conditions
-      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
-      PARTICULAR PURPOSE. You are solely responsible for determining the
-      appropriateness of using or redistributing the Work and assume any
-      risks associated with Your exercise of permissions under this License.
-
-   8. Limitation of Liability. In no event and under no legal theory,
-      whether in tort (including negligence), contract, or otherwise,
-      unless required by applicable law (such as deliberate and grossly
-      negligent acts) or agreed to in writing, shall any Contributor be
-      liable to You for damages, including any direct, indirect, special,
-      incidental, or consequential damages of any character arising as a
-      result of this License or out of the use or inability to use the
-      Work (including but not limited to damages for loss of goodwill,
-      work stoppage, computer failure or malfunction, or any and all
-      other commercial damages or losses), even if such Contributor
-      has been advised of the possibility of such damages.
-
-   9. Accepting Warranty or Additional Liability. While redistributing
-      the Work or Derivative Works thereof, You may choose to offer,
-      and charge a fee for, acceptance of support, warranty, indemnity,
-      or other liability obligations and/or rights consistent with this
-      License. However, in accepting such obligations, You may act only
-      on Your own behalf and on Your sole responsibility, not on behalf
-      of any other Contributor, and only if You agree to indemnify,
-      defend, and hold each Contributor harmless for any liability
-      incurred by, or claims asserted against, such Contributor by reason
-      of your accepting any such warranty or additional liability.
-
-   END OF TERMS AND CONDITIONS
-
-   APPENDIX: How to apply the Apache License to your work.
-
-      To apply the Apache License to your work, attach the following
-      boilerplate notice, with the fields enclosed by brackets "[]"
-      replaced with your own identifying information. (Don't include
-      the brackets!)  The text should be enclosed in the appropriate
-      comment syntax for the file format. We also recommend that a
-      file or class name and description of purpose be included on the
-      same "printed page" as the copyright notice for easier
-      identification within third-party archives.
-
-   Copyright 2023 Molecula Corp. All rights reserved.
-
-   Licensed under the Apache License, Version 2.0 (the "License");
-   you may not use this file except in compliance with the License.
-   You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

Makefile

@@ -1,4 +1,4 @@
-.PHONY: build clean build-lattice cover cover-viz default docker docker-build docker-build-fbsql docker-tag-push generate generate-protoc generate-pql generate-statik generate-stringer install install-protoc-gen-gofast install-protoc install-statik install-peg test docker-login
+.PHONY: build clean build-lattice cover cover-viz default docker docker-build docker-tag-push generate generate-protoc generate-pql generate-statik generate-stringer install install-protoc-gen-gofast install-protoc install-statik install-peg test docker-login
 
 SHELL := /bin/bash
 VERSION := $(shell git describe --tags 2> /dev/null || echo unknown)
@@ -19,15 +19,11 @@ SHARD_WIDTH = 20
 COMMIT := $(shell git describe --exact-match >/dev/null 2>&1 || git rev-parse --short HEAD)
 LDFLAGS="-X github.com/featurebasedb/featurebase/v3.Version=$(VERSION) -X github.com/featurebasedb/featurebase/v3.BuildTime=$(BUILD_TIME) -X github.com/featurebasedb/featurebase/v3.Variant=$(VARIANT) -X github.com/featurebasedb/featurebase/v3.Commit=$(COMMIT) -X github.com/featurebasedb/featurebase/v3.TrialDeadline=$(TRIAL_DEADLINE)"
 GO_VERSION=1.19
-GO_BUILD_FLAGS=
 DOCKER_BUILD= # set to 1 to use `docker-build` instead of `build` when creating a release
 BUILD_TAGS += 
 TEST_TAGS = roaringparanoia
 TEST_TIMEOUT=10m
 RACE_TEST_TIMEOUT=10m
-# size in GB to use for ramdisk, ?= so you can override it with env
-# 4GB is not enough for `make test`, 8GB usually is.
-RAMDISK_SIZE ?= 8
 
 export GO111MODULE=on
 export GOPRIVATE=github.com/molecula
@@ -51,11 +47,11 @@ version:
 
 # We build a list of packages that omits the IDK and batch packages because
 # those packages require fancy environment setup.
-GOPACKAGES := $(shell $(GO) list ./... | grep -v "/v3/idk" | grep -v "/v3/batch")
+GOPACKAGES := $(shell $(GO) list ./... | grep -v "/idk" | grep -v "/batch")
 
 # Run test suite
 test:
-	$(GO) test $(GOPACKAGES) -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) -v -timeout $(TEST_TIMEOUT) -count=1
+	$(GO) test $(GOPACKAGES) -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) -v -timeout $(TEST_TIMEOUT)
 
 # Run test suite with race flag
 test-race:
@@ -81,18 +77,13 @@ testvsub:
 			echo; echo "999 done testing subpkg $$pkg"; \
 		done
 
-# make a $(RAMDISK_SIZE)GB RAMDisk. Speed up tests by running
-# them with TMPDIR=/mnt/ramdisk.
+# make a 2GB RAMDisk. Speed up tests by running them with RAMDISK=/mnt/ramdisk
 ramdisk-linux:
-	mount -o size=$(RAMDISK__SIZE)G -t tmpfs none /mnt/ramdisk
+	mount -o size=2G -t tmpfs none /mnt/ramdisk
 
-# make a $(RAMDISK_SIZE)GB RAMDisk. Speed up tests by running
-# them with TMPDIR=/Volumes/RAMDisk. This is more important on
-# OS X than it is on Linux, because there's performance issues
-# with fsync on OS X that can make the SSD slow down to moving-platters
-# drive speeds. Oops.
+# make a 2GB RAMDisk. Speed up tests by running them with RAMDISK=/Volumes/RAMDisk
 ramdisk-osx:
-	diskutil erasevolume HFS+ 'RAMDisk' $$(hdiutil attach -nobrowse -nomount ram://$$(expr 2097152 \* $(RAMDISK_SIZE)))
+	diskutil erasevolume HFS+ 'RAMDisk' `hdiutil attach -nobrowse -nomount ram://4194304`
 
 detach-ramdisk-osx:
 	hdiutil detach /Volumes/RAMDisk
@@ -117,13 +108,12 @@ cover:
 cover-viz: cover
 	$(GO) tool cover -html=build/coverage.out
 
-# Build featurebase
+# Compile Pilosa
 build:
 	$(GO) build -tags='$(BUILD_TAGS)' -ldflags $(LDFLAGS) $(FLAGS) ./cmd/featurebase
 
 package:
-	GOOS=$(GOOS) GOARCH=$(GOARCH) $(MAKE) build
-	GOOS=$(GOOS) GOARCH=$(GOARCH) $(MAKE) build-fbsql
+	GOOS=$(GOOS) GOARCH=$(GOARCH) FLAGS="-o featurebase" $(MAKE) build
 	GOARCH=$(GOARCH) VERSION=$(VERSION) nfpm package --packager deb --target featurebase.$(VERSION).$(GOARCH).deb
 	GOARCH=$(GOARCH) VERSION=$(VERSION) nfpm package --packager rpm --target featurebase.$(VERSION).$(GOARCH).rpm
 
@@ -154,7 +144,7 @@ authclustertests: vendor
 	CLUSTERTESTS_FB_ARGS=$(AUTH_ARGS) $(DOCKER_COMPOSE) -f internal/clustertests/docker-compose.yml down
 
 # Install FeatureBase and IDK
-install: install-featurebase install-idk install-fbsql
+install: install-featurebase install-idk
 
 install-featurebase:
 	$(GO) install -tags='$(BUILD_TAGS)' -ldflags $(LDFLAGS) $(FLAGS) ./cmd/featurebase
@@ -162,9 +152,6 @@ install-featurebase:
 install-idk:
 	$(MAKE) -C ./idk install
 
-install-fbsql:
-	CGO_ENABLED=1 $(GO) install ./cmd/fbsql
-
 # Build the lattice assets
 build-lattice:
 	docker build -t lattice:build ./lattice
@@ -233,13 +220,6 @@ docker-image-featurebase: vendor
 		--file Dockerfile-dax \
 		--tag dax/featurebase .
 
-docker-image-featurebase-linux-amd64: vendor
-	docker build \
-	    --build-arg GO_VERSION=$(GO_VERSION) \
-	    --platform linux/amd64 \
-	    --file Dockerfile-dax \
-	    --tag dax/featurebase .
-
 docker-image-featurebase-test: vendor
 	docker build \
 	    --build-arg GO_VERSION=$(GO_VERSION) \
@@ -266,12 +246,7 @@ docker-image-featurebase-quick: build-for-quick
 docker-image-datagen: vendor
 	docker build --tag dax/datagen --file Dockerfile-datagen .
 
-get-account-id:
-	$(eval AWS_ACCOUNTID := $(shell aws sts get-caller-identity --output=json | jq -r .Account))
-
-
 ecr-push-featurebase: docker-login
-	echo "Pushing to account $(AWS_ACCOUNTID), profile $(AWS_PROFILE)"
 	docker tag dax/featurebase:latest $(AWS_ACCOUNTID).dkr.ecr.us-east-2.amazonaws.com/dax/featurebase:latest
 	docker push $(AWS_ACCOUNTID).dkr.ecr.us-east-2.amazonaws.com/dax/featurebase:latest
 
@@ -279,7 +254,7 @@ ecr-push-datagen: docker-login
 	docker tag dax/datagen:latest $(AWS_ACCOUNTID).dkr.ecr.us-east-2.amazonaws.com/dax/datagen:latest
 	docker push $(AWS_ACCOUNTID).dkr.ecr.us-east-2.amazonaws.com/dax/datagen:latest
 
-docker-login: get-account-id
+docker-login:
 	aws ecr get-login-password --region us-east-2 | docker login --username AWS --password-stdin $(AWS_ACCOUNTID).dkr.ecr.us-east-2.amazonaws.com
 
 # Create docker image (alias)
@@ -353,53 +328,3 @@ test-external-lookup:
 
 bnf:
 	ebnf2railroad --no-overview-diagram  --no-optimizations ./sql3/sql3.ebnf
-
-#################################
-# fbsql builds in docker
-#################################
-
-# This allows multiple concurrent builds to happen in CI without
-# creating container name conflicts and such. (different BUILD_NAMEs
-# are passed in from gitlab-ci.yml)
-BUILD_NAME ?= fbsql-build
-
-LDFLAGS_STATIC="-linkmode external -extldflags \"-static\" -X 'github.com/featurebasedb/featurebase/v3/fbsql.Version=$(VERSION)' -X 'github.com/featurebasedb/featurebase/v3/fbsql.BuildTime=$(BUILD_TIME)' "
-
-UNAME_P := $(shell uname -p)
-BUILD_CGO ?= 0
-
-# Build fbsql
-build-fbsql:
-	@echo GOOS=$(GOOS) GOARCH=$(GOARCH) uname -p=$(UNAME_P) build_cgo=$(BUILD_CGO)
-ifeq ($(BUILD_CGO), 0)
-	make build-fbsql-non-cgo
-endif
-ifeq ($(BUILD_CGO), 1)
-	make build-fbsql-cgo
-endif
-
-build-fbsql-non-cgo:
-	CGO_ENABLED=0 $(GO) build -ldflags $(LDFLAGS) $(GO_BUILD_FLAGS) -o fbsql ./cmd/fbsql
-
-build-fbsql-cgo:
-ifeq ($(GOARCH), arm64)
-	CGO_ENABLED=1 $(GO) build -tags dynamic $(GO_BUILD_FLAGS) -o fbsql ./cmd/fbsql
-endif
-ifeq ($(GOARCH), amd64)
-	CC=/usr/bin/musl-gcc CGO_ENABLED=1 $(GO) build -tags "musl static" -ldflags $(LDFLAGS_STATIC) $(GO_BUILD_FLAGS) -o fbsql ./cmd/fbsql
-endif
-
-docker-build-fbsql: vendor
-	DOCKER_BUILDKIT=0 docker build \
-		--file Dockerfile-fbsql \
-	    --build-arg GO_VERSION=$(GO_VERSION) \
-	    --build-arg MAKE_FLAGS="GOOS=$(GOOS) GOARCH=$(GOARCH) BUILD_CGO=$(BUILD_CGO)" \
-	    --build-arg GO_BUILD_FLAGS=$(GO_BUILD_FLAGS) \
-		--build-arg SOURCE_DATE_EPOCH=$(SOURCE_DATE_EPOCH) \
-	    --target builder \
-	    --tag fbsql:$(BUILD_NAME) .
-	mkdir -p build
-	docker create --name $(BUILD_NAME) fbsql:$(BUILD_NAME)
-	docker cp $(BUILD_NAME):/featurebase/fbsql ./build/fbsql_$(GOOS)_$(GOARCH)
-	docker rm $(BUILD_NAME)
-

OPENSOURCE.md

@@ -1,45 +0,0 @@
-## User Contribution Guidelines for FeatureBase
-
-Thank you for your interest in contributing to FeatureBase! We appreciate your support in making this open-source project even better. Here are some guidelines to help you get started with contributing to FeatureBase:
-
-1. Familiarize Yourself with the Project:
-   - Visit the FeatureBase website at www.featurebase.com to understand the project's goals, capabilities, and features.
-   - Read the documentation available on the website, including the installation guide, configuration options, and data modeling concepts.
-   - Explore the codebase by cloning the repository and reviewing the source code.
-
-2. Join the Community:
-   - Visit the FeatureBase community page at https://www.featurebase.com/community to learn more about the project's community and how to get involved.
-   - Join the Discord server at https://discord.gg/FBn2vEp7Na to chat with other contributors and users, ask questions, and share your ideas.
-
-3. Set Up Your Development Environment:
-   - Ensure you have Go installed on your machine. Make sure your shell's search path includes the go/bin directory.
-   - Clone the FeatureBase repository or download it as a zip file from the repository's page.
-   - Follow the "Build FeatureBase Server from source" instructions in the README file to compile the server binary and the ingester binaries.
-
-4. Choose a Contribution Area:
-   - Identify the area you'd like to contribute to, such as bug fixes, new features, performance improvements, documentation updates, or community support.
-   - Check the issue tracker on the repository or the FeatureBase community for open issues or feature requests that align with your interests and skills. Alternatively, propose your own idea by creating a new issue.
-
-5. Create a New Branch:
-   - Before making any changes, create a new branch in the repository's Git repository. This branch will contain your contributions.
-   - Give your branch a descriptive name that reflects the nature of your contribution.
-
-6. Make Your Changes:
-   - Follow the coding style and conventions used in the existing codebase.
-   - Write clear and concise commit messages for each logical change.
-   - If you're introducing new features or modifying existing behavior, make sure to update the documentation to reflect the changes.
-
-7. Test Your Changes:
-   - Run the existing test suite to ensure that your modifications do not introduce any regressions.
-   - If applicable, write additional tests to cover the changes you made.
-   - Document any new testing procedures required for your contribution.
-
-8. Submitting Your Contribution:
-   - Push your branch to the main repository or create a fork and submit a pull request to the main repository.
-   - Provide a detailed description of your changes, including the problem you solved and the approach you took.
-   - Be responsive to any feedback or suggestions provided by the project maintainers or other contributors.
-   - Once your contribution is approved, it will be reviewed and merged into the main codebase.
-
-Please note that by contributing to FeatureBase, you agree that your contributions will be licensed under the Apache 2.0 license, which governs the project.
-
-Thank you for considering contributing to FeatureBase! Your contributions are valuable and help improve the project for everyone.

README.md

@@ -1,10 +1,4 @@
-# FeatureBase Community
-
-FeatureBase Community is now archived and no longer maintained.
-
-* [FeatureBase Community Help](https://github.com/FeatureBaseDB/FB-community-help)
-
-
+# FeatureBase
 
 ## Pilosa is now FeatureBase
 
@@ -16,8 +10,6 @@ For more information about FeatureBase, please visit [www.featurebase.com][HomeP
 
 ## Getting Started
 
-* [Learn how to install FeatureBase Community](https://github.com/FeatureBaseDB/FB-community-help/blob/main/docs/community/com-getstart/com-getstart-home.md)
-
 ### Build FeatureBase Server from source
 
 0. Install go. Ensure that your shell's search path includes the go/bin directory.
@@ -27,20 +19,42 @@ For more information about FeatureBase, please visit [www.featurebase.com][HomeP
 4. Run `featurebase server --handler.allowed-origins=http://localhost:3000` to run FeatureBase server with default settings (learn more about configuring FeatureBase at the link below). The `--handler.allowed-origins` parameter allows the standalone web UI to talk to the server; this can be omitted if the web UI is not needed.
 5. Run `curl localhost:10101/status` to verify the server is running and accessible.
 
+### Ingest Data and Query
+
+1. Run 
+```
+molecula-consumer-csv \
+    --index repository \
+    --header "language__ID_F,project_id__ID_F" \
+    --id-field project_id \
+    --batch-size 1000 \
+    --files example.csv
+```
+
+This will ingest the `example.csv` file into a FeatureBase table called `repository`. If the table does not exist, it will be automatically created. Learn more about ingesting into FeatureBase: [https://docs.featurebase.com/data-ingestion/enterprise/ingesters][Ingest]
+
+2. Query your data. 
+```
+curl localhost:10101/index/repository/query \
+     -X POST \
+     -d 'Row(example=5)'
+```
+Learn about supported [SQL][SQL], native [Pilosa Query Language (PQL)][PQL].
+
 ### Data Model
 
 Because FeatureBase is built on bitmaps, there is bit of a learning curve to grasp how your data is represented. 
+Data Model Guide: [https://docs.featurebase.com/data-modeling-guide/data-modeling][DataModel]
 
-* [Learn about Data Modeling](https://github.com/FeatureBaseDB/FB-community-help/blob/main/docs/concepts/concepts-home.md)
+### More Information
 
+Installation:[https://docs.featurebase.com/setting-up-featurebase/enterprise/installing-featurebase][Install]
 
-### Ingest Data and Query
-
-* [Learn how to ingest data from multiple data sources](https://github.com/FeatureBaseDB/FB-community-help/blob/main/docs/community/com-ingest/com-ingest-manage.md)
+Configuration: [https://docs.featurebase.com/setting-up-featurebase/enterprise/featurebase-configuration][Config]
 
 ## Community
 
-You can email us at community@featurebase.com and [learn more about contributing](https://github.com/FeatureBaseDB/featurebase/blob/master/OPENSOURCE.md).
+You can email us at community@featurebase.com or learn more about contributing at [https://www.featurebase.com/community][Community].
 
 Chat with us: [https://discord.gg/FBn2vEp7Na][Discord]
 
@@ -59,14 +73,13 @@ A lot has changed since the days of Pilosa. This list highlights some new capabi
 
 FeatureBase is licensed under the [Apache License, Version 2.0][License]
 
-[Community]: https://github.com/FeatureBaseDB/FB-community-help/tree/main
-[Install]:https://github.com/FeatureBaseDB/FB-community-help/blob/main/docs/community/com-getstart/com-getstart-home.md
-[Config]: https://github.com/FeatureBaseDB/FB-community-help/tree/main/docs/community/com-config
-[DataModel]: https://github.com/FeatureBaseDB/FB-community-help/blob/main/docs/concepts/concepts-home.md
+[Community]: http://www.featurebase.com/community?utm_campaign=Open%20Source&utm_source=GitHub
+[Config]: https://docs.featurebase.com/setting-up-featurebase/enterprise/featurebase-configuration?utm_campaign=Open%20Source&utm_source=GitHub
+[DataModel]: https://docs.featurebase.com/data-modeling-guide/data-modeling?utm_campaign=Open%20Source&utm_source=GitHub
 [Discord]: https://discord.gg/FBn2vEp7Na
 [HomePage]: http://featurebase.com?utm_campaign=Open%20Source&utm_source=GitHub
-[Ingest]: https://github.com/FeatureBaseDB/FB-community-help/blob/main/docs/community/com-ingest/com-ingest-manage.md
- 
+[Ingest]: https://docs.featurebase.com/data-ingestion/enterprise/ingesters?utm_campaign=Open%20Source&utm_source=GitHub
+[Install]: http://docs.featurebase.com/setting-up-featurebase/enterprise/installing-featurebase?utm_campaign=Open%20Source&utm_source=GitHub
 [License]: http://www.apache.org/licenses/LICENSE-2.0
-[PQL]: https://docs.featurebase.com/docs/pql-guide/pql-home/?utm_campaign=Open%20Source&utm_source=GitHub
-[SQL]: https://docs.featurebase.com/docs/sql-guide/sql-guide-home/?utm_campaign=Open%20Source&utm_source=GitHub
+[PQL]: http://docs.featurebase.com/data-querying/pql/introduction?utm_campaign=Open%20Source&utm_source=GitHub
+[SQL]: http://docs.featurebase.com/data-querying/sql?utm_campaign=Open%20Source&utm_source=GitHub

api.go

@@ -26,15 +26,15 @@ import (
 	"github.com/featurebasedb/featurebase/v3/dax"
 	"github.com/featurebasedb/featurebase/v3/dax/computer"
 	"github.com/featurebasedb/featurebase/v3/dax/storage"
-	"github.com/featurebasedb/featurebase/v3/disco"
 	"github.com/featurebasedb/featurebase/v3/logger"
+	"github.com/featurebasedb/featurebase/v3/disco"
 	"github.com/featurebasedb/featurebase/v3/rbf"
-	"github.com/prometheus/client_golang/prometheus"
 
 	//"github.com/featurebasedb/featurebase/v3/pg"
 	"github.com/featurebasedb/featurebase/v3/pql"
 	"github.com/featurebasedb/featurebase/v3/roaring"
 	planner_types "github.com/featurebasedb/featurebase/v3/sql3/planner/types"
+	"github.com/featurebasedb/featurebase/v3/stats"
 	"github.com/featurebasedb/featurebase/v3/tracing"
 	"github.com/pkg/errors"
 	"golang.org/x/sync/errgroup"
@@ -278,7 +278,7 @@ func (api *API) CreateIndex(ctx context.Context, indexName string, options Index
 		return nil, errors.Wrap(err, "creating index")
 	}
 
-	CounterCreateIndex.Inc()
+	api.holder.Stats.Count(MetricCreateIndex, 1, 1.0)
 	return index, nil
 }
 
@@ -319,7 +319,7 @@ func (api *API) DeleteDataframe(ctx context.Context, indexName string) error {
 		api.server.logger.Errorf("problem sending DeleteIndex message: %s", err)
 		return errors.Wrap(err, "sending DeleteIndex message")
 	}
-	CounterDeleteDataframe.Inc()
+	api.holder.Stats.Count(MetricDeleteDataframe, 1, 1.0)
 	return nil
 }
 
@@ -338,14 +338,6 @@ func (api *API) DeleteIndex(ctx context.Context, indexName string) error {
 	if err != nil {
 		return errors.Wrap(err, "deleting index")
 	}
-
-	// Remove from writelogger/snapshotter if serverless.
-	if api.isComputeNode {
-		if err := api.serverlessStorage.RemoveTable(dax.TableKey(indexName).QualifiedTableID()); err != nil {
-			return errors.Wrapf(err, "removing table from serverless storage: %s", indexName)
-		}
-	}
-
 	// Send the delete index message to all nodes.
 	err = api.server.SendSync(
 		&DeleteIndexMessage{
@@ -362,13 +354,13 @@ func (api *API) DeleteIndex(ctx context.Context, indexName string) error {
 			return errors.Wrap(err, "deleting id allocation for index")
 		}
 	}
-	CounterDeleteIndex.Inc()
+	api.holder.Stats.Count(MetricDeleteIndex, 1, 1.0)
 	return nil
 }
 
 // CreateField makes the named field in the named index with the given options.
-//
-// The resulting field will always have TrackExistence set.
+// This method currently only takes a single functional option, but that may be
+// changed in the future to support multiple options.
 func (api *API) CreateField(ctx context.Context, indexName string, fieldName string, opts ...FieldOption) (*Field, error) {
 	span, _ := tracing.StartSpanFromContext(ctx, "API.CreateField")
 	defer span.Finish()
@@ -381,11 +373,6 @@ func (api *API) CreateField(ctx context.Context, indexName string, fieldName str
 	// authN/Z info
 	requestUserID, _ := fbcontext.UserID(ctx) // requestUserID is "" if not in ctx
 
-	// newFieldOptions is also used in the path through the index creating
-	// a field from an update from DAX, so it can't assume it can always
-	// override this. But we're the call path for creating new fields, and
-	// new fields should always have TrackExistence on.
-	opts = append(opts, OptFieldTrackExistence())
 	// Apply and validate functional options.
 	fo, err := newFieldOptions(opts...)
 	if err != nil {
@@ -419,7 +406,7 @@ func (api *API) CreateField(ctx context.Context, indexName string, fieldName str
 		return nil, errors.Wrap(err, "sending CreateField message")
 	}
 
-	CounterCreateField.With(prometheus.Labels{"index": indexName})
+	api.holder.Stats.CountWithCustomTags(MetricCreateField, 1, 1.0, []string{fmt.Sprintf("index:%s", indexName)})
 	return field, nil
 }
 
@@ -499,9 +486,16 @@ func importWorker(importWork chan importJob) {
 	for j := range importWork {
 		err := func() (err0 error) {
 			for viewName, viewData := range j.req.Views {
-				viewName, err0 = j.field.cleanupViewName(viewName)
-				if err0 != nil {
-					return err0
+				// The logic here corresponds to the logic in fragment.cleanViewName().
+				// Unfortunately, the logic in that method is not completely exclusive
+				// (i.e. an "other" view named with format YYYYMMDD would be handled
+				// incorrectly). One way to address this would be to change the logic
+				// overall so there weren't conflicts. For now, we just
+				// rely on the field type to inform the intended view name.
+				if viewName == "" {
+					viewName = viewStandard
+				} else if j.field.Type() == FieldTypeTime {
+					viewName = fmt.Sprintf("%s_%s", viewStandard, viewName)
 				}
 				if len(viewData) == 0 {
 					return fmt.Errorf("no data to import for view: %s", viewName)
@@ -756,7 +750,7 @@ func (api *API) DeleteField(ctx context.Context, indexName string, fieldName str
 		api.server.logger.Errorf("problem sending DeleteField message: %s", err)
 		return errors.Wrap(err, "sending DeleteField message")
 	}
-	CounterDeleteField.With(prometheus.Labels{"index": indexName})
+	api.holder.Stats.CountWithCustomTags(MetricDeleteField, 1, 1.0, []string{fmt.Sprintf("index:%s", indexName)})
 	return nil
 }
 
@@ -788,7 +782,7 @@ func (api *API) DeleteAvailableShard(_ context.Context, indexName, fieldName str
 		api.server.logger.Errorf("problem sending DeleteAvailableShard message: %s", err)
 		return errors.Wrap(err, "sending DeleteAvailableShard message")
 	}
-	CounterDeleteAvailableShard.With(prometheus.Labels{"index": indexName}).Inc()
+	api.holder.Stats.CountWithCustomTags(MetricDeleteAvailableShard, 1, 1.0, []string{fmt.Sprintf("index:%s", indexName)})
 	return nil
 }
 
@@ -1314,6 +1308,7 @@ type ImportOptions struct {
 	Clear          bool
 	IgnoreKeyCheck bool
 	Presorted      bool
+	fullySorted    bool // format-aware sorting, internal use only please.
 	suppressLog    bool
 
 	// test Tx atomicity if > 0
@@ -1520,6 +1515,7 @@ func (api *API) ImportWithTx(ctx context.Context, qcx *Qcx, req *ImportRequest,
 		return errors.Wrap(err, "validating api method")
 	}
 
+	api.server.logger.Debugf("ImportWithTx: %v %v %v", req.Index, req.Field, req.Shard)
 	idx, field, err := api.indexField(req.Index, req.Field, req.Shard)
 	if err != nil {
 		return errors.Wrap(err, "getting index and field")
@@ -1638,12 +1634,6 @@ func (api *API) ImportWithTx(ctx context.Context, qcx *Qcx, req *ImportRequest,
 // across many fields in a single shard. It can both set and clear
 // bits and updates caches/bitDepth as appropriate, although only the
 // bitmap parts happen truly transactionally.
-//
-// This function does not attempt to do existence tracking, because
-// it can't; there's no way to distinguish empty sets from not setting
-// bits. As a result, users of this endpoint are responsible for
-// providing corrected existence views for fields with existence
-// tracking. Our batch API does that.
 func (api *API) ImportRoaringShard(ctx context.Context, indexName string, shard uint64, req *ImportRoaringShardRequest) error {
 	index, err := api.Index(ctx, indexName)
 	if err != nil {
@@ -1674,7 +1664,7 @@ func (api *API) ImportRoaringShard(ctx context.Context, indexName string, shard
 		}
 
 		fieldType := field.Options().Type
-		if viewUpdate.View, err1 = field.cleanupViewName(viewUpdate.View); err1 != nil {
+		if err1 = cleanupView(fieldType, &viewUpdate); err1 != nil {
 			return err1
 		}
 
@@ -1766,6 +1756,27 @@ func (api *API) ImportRoaringShard(ctx context.Context, indexName string, shard
 	return nil
 }
 
+func cleanupView(fieldType string, viewUpdate *RoaringUpdate) error {
+	// TODO wouldn't hurt to have consolidated logic somewhere for validating view names.
+	switch fieldType {
+	case FieldTypeSet, FieldTypeTime:
+		if viewUpdate.View == "" {
+			viewUpdate.View = "standard"
+		}
+		// add 'standard_' if we just have a time... this is how IDK works by default
+		if fieldType == FieldTypeTime && !strings.HasPrefix(viewUpdate.View, viewStandard) {
+			viewUpdate.View = fmt.Sprintf("%s_%s", viewStandard, viewUpdate.View)
+		}
+	case FieldTypeInt, FieldTypeDecimal, FieldTypeTimestamp:
+		if viewUpdate.View == "" {
+			viewUpdate.View = "bsig_" + viewUpdate.Field
+		} else if viewUpdate.View != "bsig_"+viewUpdate.Field {
+			return NewBadRequestError(errors.Errorf("invalid view name (%s) for field %s of type %s", viewUpdate.View, viewUpdate.Field, fieldType))
+		}
+	}
+	return nil
+}
+
 // ImportValue is a wrapper around the common code in ImportValueWithTx, which
 // currently just translates req.Clear into a clear ImportOption.
 func (api *API) ImportValue(ctx context.Context, qcx *Qcx, req *ImportValueRequest, opts ...ImportOption) error {
@@ -2019,20 +2030,21 @@ func (api *API) ImportValueWithTx(ctx context.Context, qcx *Qcx, req *ImportValu
 	return nil
 }
 
-func importExistenceColumns(qcx *Qcx, index *Index, columnIDs []uint64, shard uint64) (err0 error) {
+func importExistenceColumns(qcx *Qcx, index *Index, columnIDs []uint64, shard uint64) error {
 	ef := index.existenceField()
 	if ef == nil {
 		return nil
 	}
-	tx, finisher, err := qcx.GetTx(Txo{Write: true, Index: index, Shard: shard})
-	if err != nil {
-		return err
-	}
-	defer finisher(&err0)
-	// markExistingInView is simpler/faster than Import, but unusually, we use the
-	// standard view of the existence field, instead of the existence view of
-	// a specific field, when doing the index-wide update.
-	return ef.markExistingInView(tx, columnIDs, viewStandard, shard)
+
+	existenceRowIDs := make([]uint64, len(columnIDs))
+	// If we don't gratuitously hand-duplicate things in field.Import,
+	// the fact that fragment.bulkImport rewrites its row and column
+	// lists can burn us if we don't make a copy before doing the
+	// existence field write.
+	columnCopy := make([]uint64, len(columnIDs))
+	copy(columnCopy, columnIDs)
+	options := ImportOptions{}
+	return ef.Import(qcx, existenceRowIDs, columnCopy, nil, shard, &options)
 }
 
 // ShardDistribution returns an object representing the distribution of shards
@@ -2084,6 +2096,15 @@ func (api *API) AvailableShards(ctx context.Context, indexName string) (*roaring
 	return index.AvailableShards(false), nil
 }
 
+// StatsWithTags returns an instance of whatever implementation of StatsClient
+// pilosa is using with the given tags.
+func (api *API) StatsWithTags(tags []string) stats.StatsClient {
+	if api.holder == nil || api.cluster == nil {
+		return nil
+	}
+	return api.holder.Stats.WithTags(tags...)
+}
+
 // LongQueryTime returns the configured threshold for logging/statting
 // long running queries.
 func (api *API) LongQueryTime() time.Duration {
@@ -2370,19 +2391,19 @@ func (api *API) StartTransaction(ctx context.Context, id string, timeout time.Du
 	switch err {
 	case nil:
 		if exclusive {
-			CounterExclusiveTransactionRequest.Inc()
+			api.holder.Stats.Count(MetricExclusiveTransactionRequest, 1, 1.0)
 		} else {
-			CounterTransactionStart.Inc()
+			api.holder.Stats.Count(MetricTransactionStart, 1, 1.0)
 		}
 	case ErrTransactionExclusive:
 		if exclusive {
-			CounterExclusiveTransactionBlocked.Inc()
+			api.holder.Stats.Count(MetricExclusiveTransactionBlocked, 1, 1.0)
 		} else {
-			CounterTransactionBlocked.Inc()
+			api.holder.Stats.Count(MetricTransactionBlocked, 1, 1.0)
 		}
 	}
 	if exclusive && t != nil && t.Active {
-		CounterExclusiveTransactionActive.Inc()
+		api.holder.Stats.Count(MetricExclusiveTransactionActive, 1, 1.0)
 	}
 	return t, err
 }
@@ -2394,9 +2415,9 @@ func (api *API) FinishTransaction(ctx context.Context, id string, remote bool) (
 	t, err := api.server.FinishTransaction(ctx, id, remote)
 	if err == nil {
 		if t.Exclusive {
-			CounterExclusiveTransactionEnd.Inc()
+			api.holder.Stats.Count(MetricExclusiveTransactionEnd, 1, 1.0)
 		} else {
-			CounterTransactionEnd.Inc()
+			api.holder.Stats.Count(MetricTransactionEnd, 1, 1.0)
 		}
 	}
 	return t, err
@@ -2416,7 +2437,7 @@ func (api *API) GetTransaction(ctx context.Context, id string, remote bool) (*Tr
 	t, err := api.server.GetTransaction(ctx, id, remote)
 	if err == nil {
 		if t.Exclusive && t.Active {
-			CounterExclusiveTransactionActive.Inc()
+			api.holder.Stats.Count(MetricExclusiveTransactionActive, 1, 1.0)
 		}
 	}
 	return t, err
@@ -3025,10 +3046,6 @@ func (api *API) CompilePlan(ctx context.Context, q string) (planner_types.PlanOp
 	return api.server.CompileExecutionPlan(ctx, q)
 }
 
-func (api *API) RehydratePlanOperator(ctx context.Context, reader io.Reader) (planner_types.PlanOperator, error) {
-	return api.server.RehydratePlanOperator(ctx, reader)
-}
-
 func (api *API) RBFDebugInfo() map[string]*rbf.DebugInfo {
 	infos := make(map[string]*rbf.DebugInfo)
 
@@ -3050,9 +3067,9 @@ func (api *API) Directive(ctx context.Context, d *dax.Directive) error {
 }
 
 // DirectiveApplied returns true if the computer's current Directive has been
-// applied and is ready to be queried. This is temporary (primarily for tests)
-// and needs to be refactored as we improve the logic around
-// controller-to-computer communication.
+// applied and is ready to be queried. This it temporary (primarily for tests)
+// and needs to be refactored as we improve the logic around mds-to-computer
+// communication.
 func (api *API) DirectiveApplied(ctx context.Context) (bool, error) {
 	return api.holder.DirectiveApplied(), nil
 }
@@ -3065,7 +3082,7 @@ func (api *API) SnapshotShardData(ctx context.Context, req *dax.SnapshotShardDat
 	}
 	// TODO(jaffee) confirm this node is actually responsible for the given
 	// shard? Not sure we need to given that this request comes from
-	// the Controller, but might be a belt&suspenders situation.
+	// MDS, but might be a belt&suspenders situation.
 
 	qtid := req.TableKey.QualifiedTableID()
 
@@ -3296,14 +3313,6 @@ func shardInShards(i dax.ShardNum, s dax.ShardNums) bool {
 }
 
 type SchemaAPI interface {
-	CreateDatabase(context.Context, *dax.Database) error
-	DropDatabase(context.Context, dax.DatabaseID) error
-
-	DatabaseByName(ctx context.Context, dbname dax.DatabaseName) (*dax.Database, error)
-	DatabaseByID(ctx context.Context, dbid dax.DatabaseID) (*dax.Database, error)
-	SetDatabaseOption(ctx context.Context, dbid dax.DatabaseID, option string, value string) error
-	Databases(context.Context, ...dax.DatabaseID) ([]*dax.Database, error)
-
 	TableByName(ctx context.Context, tname dax.TableName) (*dax.Table, error)
 	TableByID(ctx context.Context, tid dax.TableID) (*dax.Table, error)
 	Tables(ctx context.Context) ([]*dax.Table, error)
@@ -3315,49 +3324,8 @@ type SchemaAPI interface {
 	DeleteField(ctx context.Context, tname dax.TableName, fname dax.FieldName) error
 }
 
-// Ensure type implements interface.
-var _ SchemaAPI = (*NopSchemaAPI)(nil)
-
-// NopSchemaAPI is a no-op implementation of the SchemaAPI.
-type NopSchemaAPI struct{}
-
-func (n *NopSchemaAPI) ClusterName() string {
-	return ""
-}
-func (n *NopSchemaAPI) CreateDatabase(context.Context, *dax.Database) error { return nil }
-func (n *NopSchemaAPI) DropDatabase(context.Context, dax.DatabaseID) error  { return nil }
-func (n *NopSchemaAPI) DatabaseByName(ctx context.Context, dbname dax.DatabaseName) (*dax.Database, error) {
-	return nil, nil
-}
-func (n *NopSchemaAPI) DatabaseByID(ctx context.Context, dbid dax.DatabaseID) (*dax.Database, error) {
-	return nil, nil
-}
-func (n *NopSchemaAPI) SetDatabaseOption(ctx context.Context, dbid dax.DatabaseID, option string, value string) error {
-	return nil
-}
-func (n *NopSchemaAPI) Databases(context.Context, ...dax.DatabaseID) ([]*dax.Database, error) {
-	return nil, nil
-}
-func (n *NopSchemaAPI) TableByName(ctx context.Context, tname dax.TableName) (*dax.Table, error) {
-	return nil, nil
-}
-func (n *NopSchemaAPI) TableByID(ctx context.Context, tid dax.TableID) (*dax.Table, error) {
-	return nil, nil
-}
-func (n *NopSchemaAPI) Tables(ctx context.Context) ([]*dax.Table, error) { return nil, nil }
-
-func (n *NopSchemaAPI) CreateTable(ctx context.Context, tbl *dax.Table) error { return nil }
-func (n *NopSchemaAPI) CreateField(ctx context.Context, tname dax.TableName, fld *dax.Field) error {
-	return nil
-}
-func (n *NopSchemaAPI) DeleteTable(ctx context.Context, tname dax.TableName) error { return nil }
-func (n *NopSchemaAPI) DeleteField(ctx context.Context, tname dax.TableName, fname dax.FieldName) error {
-	return nil
-}
-
 type ClusterNode struct {
 	ID        string
-	Type      string
 	State     string
 	URI       string
 	GRPCURI   string
@@ -3375,7 +3343,6 @@ type SystemAPI interface {
 	ClusterState() string
 	DataDir() string
 
-	NodeID() string
 	ClusterNodes() []ClusterNode
 }
 
@@ -3391,9 +3358,6 @@ type QueryAPI interface {
 	Query(ctx context.Context, req *QueryRequest) (QueryResponse, error)
 }
 
-// Ensure type implements interface.
-var _ SystemAPI = (*FeatureBaseSystemAPI)(nil)
-
 // FeatureBaseSystemAPI is a wrapper around pilosa.API. It implements the
 // SystemAPI interface
 type FeatureBaseSystemAPI struct {
@@ -3450,10 +3414,6 @@ func (fsapi *FeatureBaseSystemAPI) DataDir() string {
 	return fsapi.server.dataDir
 }
 
-func (fsapi *FeatureBaseSystemAPI) NodeID() string {
-	return fsapi.cluster.Node.ID
-}
-
 func (fsapi *FeatureBaseSystemAPI) ClusterNodes() []ClusterNode {
 	result := make([]ClusterNode, 0)
 
@@ -3472,54 +3432,3 @@ func (fsapi *FeatureBaseSystemAPI) ClusterNodes() []ClusterNode {
 
 	return result
 }
-
-// Ensure type implements interface.
-var _ SystemAPI = (*NopSystemAPI)(nil)
-
-// NopSystemAPI is a no-op implementation of the SystemAPI.
-type NopSystemAPI struct{}
-
-func (napi *NopSystemAPI) ClusterName() string {
-	return ""
-}
-
-func (napi *NopSystemAPI) Version() string {
-	return ""
-}
-
-func (napi *NopSystemAPI) PlatformDescription() string {
-	return ""
-}
-
-func (napi *NopSystemAPI) PlatformVersion() string {
-	return ""
-}
-
-func (napi *NopSystemAPI) ClusterNodeCount() int {
-	return 0
-}
-
-func (napi *NopSystemAPI) ClusterReplicaCount() int {
-	return 0
-}
-
-func (napi *NopSystemAPI) ShardWidth() int {
-	return 0
-}
-
-func (napi *NopSystemAPI) ClusterState() string {
-	return ""
-}
-
-func (napi *NopSystemAPI) DataDir() string {
-	return ""
-}
-
-func (napi *NopSystemAPI) NodeID() string {
-	return ""
-}
-
-func (napi *NopSystemAPI) ClusterNodes() []ClusterNode {
-	result := make([]ClusterNode, 0)
-	return result
-}

api_directive.go

@@ -35,22 +35,6 @@ func (api *API) ApplyDirective(ctx context.Context, d *dax.Directive) error {
 	// Handle the operations based on the directive method.
 	switch d.Method {
 	case dax.DirectiveMethodDiff:
-		// In order to prevent adding too much code specific to handling a diff
-		// directive (e.g. adding something like an `enactDirectiveDiff()`
-		// method), we are instead going to build a full Directive based on the
-		// diff, and then proceed normally as if we had received a full
-		// Directive. We do that by copying the previous Directive and then
-		// applying the diffs to the copy.
-		newD := previousDirective.Copy()
-
-		// Apply the diffs from the incoming Directive to the new, copied
-		// Directive.
-		newD.ApplyDiff(d)
-
-		// Now proceed with the new diff as if we had received it as a full diff.
-		d = newD
-
-	case dax.DirectiveMethodFull:
 		// pass: normal operation
 
 	case dax.DirectiveMethodReset:
@@ -58,6 +42,10 @@ func (api *API) ApplyDirective(ctx context.Context, d *dax.Directive) error {
 		if err := api.deleteAllIndexes(ctx); err != nil {
 			return errors.Wrap(err, "deleting all indexes")
 		}
+		if err := api.serverlessStorage.RemoveAll(); err != nil {
+			return errors.Wrap(err, "removing all managers")
+		}
+
 		// Set previousDirective to empty so the diff handles everything as new.
 		previousDirective = dax.Directive{}
 
@@ -76,7 +64,7 @@ func (api *API) ApplyDirective(ctx context.Context, d *dax.Directive) error {
 	// the "enactDirective" stage of ApplyDirective which validates against this
 	// cached Directive, so it's important that it be set before calling
 	// enactDirective(). An example: when loading partition data from the
-	// Writelogger, there are validations to ensure that the partition being
+	// WriteLogger, there are validations to ensure that the partition being
 	// loaded is meant to be handled by this node; that validation is done
 	// against the cached Directive.
 	// TODO(tlt): despite what this comment says, this logic is not sound; we
@@ -262,7 +250,7 @@ func (api *API) enactTables(ctx context.Context, fromD, toD *dax.Directive) erro
 	// Remove all indexes that are no longer part of the directive.
 	for _, tkey := range sc.removed() {
 		idx := string(tkey)
-		if err := api.DeleteIndex(ctx, idx); err != nil {
+		if err := api.holder.deleteIndex(idx); err != nil {
 			return errors.Wrapf(err, "deleting index: %s", tkey)
 		}
 	}
@@ -340,18 +328,7 @@ func (api *API) pushJobsTableKeys(ctx context.Context, jobs chan<- directiveJobT
 	// Get the diff between from/to directive.partitions.
 	partComp := newPartitionsComparer(fromD.TranslatePartitionsMap(), toPartitionsMap)
 
-	// Remove any partitions which are no longer assigned to this worker.
-	// TODO(tlt): currently, this is just removing the file lock on the
-	// resource; it's not actually removing the resource from the local
-	// computer. We should do that.
-	for tkey, partitions := range partComp.removed() {
-		qtid := tkey.QualifiedTableID()
-		for _, partition := range partitions {
-			api.serverlessStorage.RemoveTableKeyResource(qtid, partition)
-		}
-	}
-
-	// Loop over the partition map and load from Writelogger.
+	// Loop over the partition map and load from WriteLogger.
 	for tkey, partitions := range partComp.added() {
 		// Get index in order to find the translate stores (by partition) for
 		// the table.
@@ -438,18 +415,7 @@ func (api *API) pushJobsFieldKeys(ctx context.Context, jobs chan<- directiveJobT
 	// Get the diff between from/to directive.fields.
 	fieldComp := newFieldsComparer(fromD.TranslateFieldsMap(), toD.TranslateFieldsMap())
 
-	// Remove any field keys which are no longer assigned to this worker.
-	// TODO(tlt): currently, this is just removing the file lock on the
-	// resource; it's not actually removing the resource from the local
-	// computer. We should do that.
-	for tkey, fields := range fieldComp.removed() {
-		qtid := tkey.QualifiedTableID()
-		for _, field := range fields {
-			api.serverlessStorage.RemoveFieldKeyResource(qtid, field)
-		}
-	}
-
-	// Loop over the field map and load from Writelogger.
+	// Loop over the field map and load from WriteLogger.
 	for tkey, fields := range fieldComp.added() {
 		for _, field := range fields {
 			jobs <- directiveJobFieldKeys{
@@ -533,19 +499,7 @@ func (api *API) pushJobsShards(ctx context.Context, jobs chan<- directiveJobType
 	// Get the diff between from/to directive shards.
 	shardComp := newShardsComparer(fromD.ComputeShardsMap(), shardMap)
 
-	// Remove any shards which are no longer assigned to this worker.
-	// TODO(tlt): currently, this is just removing the file lock on the
-	// resource; it's not actually removing the resource from the local
-	// computer. We should do that.
-	for tkey, shards := range shardComp.removed() {
-		qtid := tkey.QualifiedTableID()
-		for _, shard := range shards {
-			partition := dax.PartitionNum(disco.ShardToShardPartition(string(tkey), uint64(shard), disco.DefaultPartitionN))
-			api.serverlessStorage.RemoveShardResource(qtid, partition, shard)
-		}
-	}
-
-	// Loop over the shard map and load from Writelogger.
+	// Loop over the shard map and load from WriteLogger.
 	for tkey, shards := range shardComp.added() {
 		for _, shard := range shards {
 			jobs <- directiveJobShards{
@@ -980,7 +934,7 @@ func createField(idx *Index, fld *dax.Field) error {
 		return errors.Wrapf(err, "creating field options from field: %s", fld.Name)
 	}
 
-	if _, err := idx.createNullableField(string(fld.Name), "", opts...); err != nil {
+	if _, err := idx.CreateField(string(fld.Name), "", opts...); err != nil {
 		return errors.Wrapf(err, "creating field on index: %s", fld.Name)
 	}
 	return nil

api_directive_test.go

@@ -20,17 +20,17 @@ func TestAPI_Directive(t *testing.T) {
 	api := c.GetPrimary().API
 	ctx := context.Background()
 
-	qdbid := dax.NewQualifiedDatabaseID("acme", "db1")
-	tbl1 := daxtest.TestQualifiedTableWithID(t, qdbid, "1", "tbl1", 12, false)
-	tbl2 := daxtest.TestQualifiedTableWithID(t, qdbid, "2", "tbl2", 12, false)
-	tbl3 := daxtest.TestQualifiedTableWithID(t, qdbid, "3", "tbl3", 12, false)
+	qual := dax.NewTableQualifier("acme", "db1")
+	tbl1 := daxtest.TestQualifiedTableWithID(t, qual, "1", "tbl1", 12, false)
+	tbl2 := daxtest.TestQualifiedTableWithID(t, qual, "2", "tbl2", 12, false)
+	tbl3 := daxtest.TestQualifiedTableWithID(t, qual, "3", "tbl3", 12, false)
 
 	t.Run("Schema", func(t *testing.T) {
 
 		// Empty directive (and empty holder).
 		{
 			d := &dax.Directive{
-				Method:  dax.DirectiveMethodFull,
+				Method:  dax.DirectiveMethodDiff,
 				Version: 1,
 			}
 			err := api.ApplyDirective(ctx, d)
@@ -41,7 +41,7 @@ func TestAPI_Directive(t *testing.T) {
 		// Add a new table.
 		{
 			d := &dax.Directive{
-				Method: dax.DirectiveMethodFull,
+				Method: dax.DirectiveMethodDiff,
 				Tables: []*dax.QualifiedTable{
 					tbl1,
 				},
@@ -55,7 +55,7 @@ func TestAPI_Directive(t *testing.T) {
 		// Add a new table, and keep the existing table.
 		{
 			d := &dax.Directive{
-				Method: dax.DirectiveMethodFull,
+				Method: dax.DirectiveMethodDiff,
 				Tables: []*dax.QualifiedTable{
 					tbl1,
 					tbl2,
@@ -70,7 +70,7 @@ func TestAPI_Directive(t *testing.T) {
 		// Add a new table and remove one of the existing tables.
 		{
 			d := &dax.Directive{
-				Method: dax.DirectiveMethodFull,
+				Method: dax.DirectiveMethodDiff,
 				Tables: []*dax.QualifiedTable{
 					tbl2,
 					tbl3,

api_test.go

@@ -837,7 +837,7 @@ func TestAPI_IDAlloc(t *testing.T) {
 			t.Fatalf("obtaining random bytes: %v", err)
 		}
 		ids3, err := primary.ReserveIDs(key, session, 0, 2)
-		var esync pilosa.IDOffsetDesyncError
+		var esync pilosa.ErrIDOffsetDesync
 		if errors.As(err, &esync) {
 			if esync.Requested != 0 {
 				t.Errorf("incorrect requested offset in error: provided %d but got %d", 0, esync.Requested)

apply.go

@@ -7,17 +7,16 @@ import (
 	"io"
 	"os"
 	"path/filepath"
-	"sort"
 	"strings"
 	"sync"
 
 	"github.com/apache/arrow/go/v10/arrow"
 	"github.com/apache/arrow/go/v10/arrow/array"
 	"github.com/apache/arrow/go/v10/arrow/memory"
+	"github.com/gomem/gomem/pkg/dataframe"
 	"github.com/featurebasedb/featurebase/v3/pql"
 	"github.com/featurebasedb/featurebase/v3/tracing"
 	"github.com/featurebasedb/featurebase/v3/vprint"
-	"github.com/gomem/gomem/pkg/dataframe"
 	"github.com/pkg/errors"
 
 	ivy "robpike.io/ivy/arrow"
@@ -95,7 +94,7 @@ func IvyReduce(reduceCode string, opCode string, opt *ExecOptions) (func(ctx con
 			col := value.ToArrowColumn(accumulator, pool)
 			return dataframe.NewDataFrameFromColumns(pool, []arrow.Column{*col})
 		}
-		// only actually reduce on the initiating node i hate the network
+		// only acutally reduce on the initiating node i hate the network
 		// over head but oh well
 		ctxIvy.AssignGlobal("_", accumulator)
 		ok, err := runIvyString(ctxIvy, reduceCode)
@@ -261,14 +260,14 @@ func (e *executor) executeApplyShard(ctx context.Context, qcx *Qcx, index string
 
 func NewShardFile(ctx context.Context, name string, mem memory.Allocator, e *executor) (*ShardFile, error) {
 	if !e.dataFrameExists(name) {
-		return &ShardFile{dest: name, executor: e, strings: make(map[key][]string)}, nil
+		return &ShardFile{dest: name, executor: e}, nil
 	}
 	// else read in existing
 	table, err := e.getDataTable(ctx, name, mem)
 	if err != nil {
 		return nil, err
 	}
-	return &ShardFile{table: table, schema: table.Schema(), dest: name, executor: e, strings: make(map[key][]string)}, nil
+	return &ShardFile{table: table, schema: table.Schema(), dest: name, executor: e}, nil
 }
 
 type NameType struct {
@@ -292,8 +291,6 @@ func cast(v interface{}) arrow.DataType {
 		return arrow.PrimitiveTypes.Float64
 	case float64:
 		return arrow.PrimitiveTypes.Float64
-	case *arrow.StringType:
-		return arrow.BinaryTypes.String
 	default:
 		vprint.VV("%T .... %v", v, v)
 	}
@@ -308,11 +305,6 @@ func (cr *ChangesetRequest) ArrowSchema() *arrow.Schema {
 	return arrow.NewSchema(fields, nil)
 }
 
-type key struct {
-	col   int
-	chunk int
-}
-
 type ShardFile struct {
 	table      arrow.Table
 	schema     *arrow.Schema
@@ -321,7 +313,6 @@ type ShardFile struct {
 	columns    []interface{}
 	dest       string
 	executor   *executor
-	strings    map[key][]string
 }
 
 func compareSchema(s1, s2 *arrow.Schema) bool {
@@ -374,8 +365,6 @@ func (sf *ShardFile) buildAppenders(maxid int64) {
 			sf.columns[i] = make([]int64, newSize)
 		case arrow.PrimitiveTypes.Float64:
 			sf.columns[i] = make([]float64, newSize)
-		case arrow.BinaryTypes.String:
-			sf.columns[i] = make([]string, newSize)
 		}
 	}
 	sf.added = newSize
@@ -392,11 +381,6 @@ func (sf *ShardFile) SetFloatValue(col int, row int64, val float64) {
 	v[row-sf.beforeRows] = val
 }
 
-func (sf *ShardFile) SetStringValue(col int, row int64, val string) {
-	v := sf.columns[col].([]string)
-	v[row-sf.beforeRows] = val
-}
-
 func (sf *ShardFile) Process(cs *ChangesetRequest) error {
 	err := sf.process(cs)
 	if err != nil {
@@ -410,34 +394,9 @@ func (sf *ShardFile) Process(cs *ChangesetRequest) error {
 	return os.Rename(rtemp+sf.executor.TableExtension(), sf.dest+sf.executor.TableExtension())
 }
 
-func (sf *ShardFile) LoadBlobs() error {
-	for col := 0; col < len(sf.schema.Fields()); col++ {
-		column := sf.table.Column(col)
-		switch column.DataType() {
-		case arrow.BinaryTypes.String:
-			for i, chunk := range column.Data().Chunks() {
-				stringData := chunk.(*array.String)
-				k := key{col: col, chunk: i}
-				for j := 0; j < stringData.Len(); j++ {
-					v := stringData.Value(j)
-					sf.strings[k] = append(sf.strings[k], v)
-
-				}
-			}
-		}
-	}
-	return nil
-}
-
-func (sf *ShardFile) ReplaceString(col, chunk, l int, s string) {
-	sf.strings[key{col: col, chunk: chunk}][l] = s
-}
-
 func (sf *ShardFile) process(cs *ChangesetRequest) error {
 	offset := 0
 	if sf.table != nil {
-		// need to load blobs prior
-		sf.LoadBlobs()
 		column := sf.table.Column(0)
 		resolver := dataframe.NewChunkResolver(column)
 		for i, rowid := range cs.ShardIds {
@@ -455,10 +414,6 @@ func (sf *ShardFile) process(cs *ChangesetRequest) error {
 				case arrow.PrimitiveTypes.Float64:
 					v := column.Data().Chunk(chunk).(*array.Float64).Float64Values()
 					v[l] = cs.Columns[col].([]float64)[i]
-				case arrow.BinaryTypes.String:
-					// TODO(twg) 2023/01/09 How to update existing?
-					new := cs.Columns[col].([]string)[i]
-					sf.ReplaceString(col, chunk, l, new)
 				default:
 					panic(fmt.Sprintf("Unknown Type %v", column.DataType()))
 				}
@@ -478,8 +433,6 @@ func (sf *ShardFile) process(cs *ChangesetRequest) error {
 					sf.SetIntValue(col, rowid, cs.Columns[col].([]int64)[i])
 				case arrow.PrimitiveTypes.Float64:
 					sf.SetFloatValue(col, rowid, cs.Columns[col].([]float64)[i])
-				case arrow.BinaryTypes.String:
-					sf.SetStringValue(col, rowid, cs.Columns[col].([]string)[i])
 				default:
 					panic(fmt.Sprintf("2 Unknown Type %v", sf.schema.Field(col).Type))
 				}
@@ -490,65 +443,15 @@ func (sf *ShardFile) process(cs *ChangesetRequest) error {
 	return nil
 }
 
-type twoSlices struct {
-	id_slice    []int
-	lists_slice [][]string
-}
-
-type SortByOther twoSlices
-
-func (sbo SortByOther) Len() int {
-	return len(sbo.id_slice)
-}
-
-func (sbo SortByOther) Swap(i, j int) {
-	sbo.id_slice[i], sbo.id_slice[j] = sbo.id_slice[j], sbo.id_slice[i]
-	sbo.lists_slice[i], sbo.lists_slice[j] = sbo.lists_slice[j], sbo.lists_slice[i]
-}
-
-func (sbo SortByOther) Less(i, j int) bool {
-	return sbo.id_slice[i] < sbo.id_slice[j]
-}
-
-func (sf *ShardFile) buildFromStrings(idx int, mem memory.Allocator) []arrow.Array {
-	ids := make([]int, 0)
-	lists := make([][]string, 0)
-	for k, v := range sf.strings {
-		if k.col == idx { // ugh not ordered :(
-			ids = append(ids, k.chunk)
-			lists = append(lists, v)
-		}
-	}
-	// sort ids/lists
-	parts := twoSlices{id_slice: ids, lists_slice: lists}
-	sort.Sort(SortByOther(parts))
-
-	builder := array.NewStringBuilder(mem)
-	chunks := make([]arrow.Array, 0)
-	for _, v := range parts.lists_slice {
-		builder.AppendValues(v, nil)
-		newChunk := builder.NewArray()
-		chunks = append(chunks, newChunk)
-	}
-	return chunks
-}
-
 func (sf *ShardFile) Save(name string) error {
 	parts := make([]arrow.Array, 0)
 	mem := memory.NewGoAllocator()
 	for col := 0; col < len(sf.schema.Fields()); col++ {
 		chunks := make([]arrow.Array, 0)
 		if sf.table != nil {
-			// we append if there was existing file
+			// we append if there was existing parquet file
 			column := sf.table.Column(col)
-			// if primitive type
-			switch column.DataType() {
-			case arrow.BinaryTypes.String:
-				chunks = sf.buildFromStrings(col, mem)
-			default:
-				chunks = append(chunks, column.Data().Chunks()...)
-			}
-			// else binary type
+			chunks = append(chunks, column.Data().Chunks()...)
 		}
 		switch sf.schema.Field(col).Type {
 		case arrow.PrimitiveTypes.Int64:
@@ -577,18 +480,6 @@ func (sf *ShardFile) Save(name string) error {
 				return err
 			}
 			parts = append(parts, record)
-		case arrow.BinaryTypes.String:
-			if sf.added > 0 {
-				fbuild := array.NewStringBuilder(mem)
-				fbuild.AppendValues(sf.columns[col].([]string), nil) // TODO(twg) 2022/09/28 need to handle null
-				newChunk := fbuild.NewArray()
-				chunks = append(chunks, newChunk)
-			}
-			record, err := array.Concatenate(chunks, mem)
-			if err != nil {
-				return err
-			}
-			parts = append(parts, record)
 		default:
 			vprint.VV("UNKNOWN %T", sf.schema.Field(col).Type)
 		}

arrow.go

@@ -17,9 +17,9 @@ import (
 	"github.com/apache/arrow/go/v10/parquet"
 	"github.com/apache/arrow/go/v10/parquet/file"
 	"github.com/apache/arrow/go/v10/parquet/pqarrow"
+	"github.com/gomem/gomem/pkg/dataframe"
 	"github.com/featurebasedb/featurebase/v3/pql"
 	"github.com/featurebasedb/featurebase/v3/tracing"
-	"github.com/gomem/gomem/pkg/dataframe"
 	"github.com/pkg/errors"
 )
 
@@ -60,7 +60,7 @@ func (e *executor) executeArrow(ctx context.Context, qcx *Qcx, index string, c *
 		mu.Unlock()
 		return e.executeArrowShard(ctx, qcx, index, c, shard, pool, columnFilter)
 	}
-	tables := make([]*BasicTable, 0)
+	tables := make([]*basicTable, 0)
 
 	reduceFn := func(ctx context.Context, prev, v interface{}) interface{} {
 		mu.Lock()
@@ -70,7 +70,7 @@ func (e *executor) executeArrow(ctx context.Context, qcx *Qcx, index string, c *
 			return prev
 		}
 		switch t := v.(type) {
-		case *BasicTable:
+		case *basicTable:
 
 			if t.resolver != nil {
 				mu.Lock()
@@ -93,113 +93,107 @@ func (e *executor) executeArrow(ctx context.Context, qcx *Qcx, index string, c *
 		return nil, err
 	}
 	if len(tables) == 0 {
-		return &BasicTable{name: "empty"}, nil
+		return &basicTable{name: "empty"}, nil
 	}
 	tbl := Concat(tables[0].Schema(), tables, pool)
 	r := dataframe.NewChunkResolver(tbl.Column(0))
-	return &BasicTable{resolver: &r, table: tbl}, nil
+	return &basicTable{resolver: &r, table: tbl}, nil
 }
 
-type BasicTable struct {
+type basicTable struct {
 	resolver dataframe.Resolver
 	table    arrow.Table
 	filtered bool
 	name     string
 }
 
-func (st *BasicTable) Name() string {
+func (st *basicTable) Name() string {
 	return st.name
 }
 
-func (st *BasicTable) Schema() *arrow.Schema {
+func (st *basicTable) Schema() *arrow.Schema {
 	if st.table != nil {
 		return st.table.Schema()
 	}
 	return &arrow.Schema{}
 }
 
-func (st *BasicTable) IsFiltered() bool {
+func (st *basicTable) IsFiltered() bool {
 	return st.filtered
 }
 
-func (st *BasicTable) NumRows() int64 {
+func (st *basicTable) NumRows() int64 {
 	if st.resolver == nil {
 		return 0
 	}
 	return int64(st.resolver.NumRows())
 }
 
-func (st *BasicTable) NumCols() int64 {
+func (st *basicTable) NumCols() int64 {
 	if st.table != nil {
 		return st.table.NumCols()
 	}
 	return 0
 }
 
-func (st *BasicTable) Column(i int) *arrow.Column {
+func (st *basicTable) Column(i int) *arrow.Column {
 	if st.table != nil {
 		return st.table.Column(i)
 	}
 	return nil
 }
 
-func (st *BasicTable) Retain() {
+func (st *basicTable) Retain() {
 	if st.table != nil {
 		st.table.Retain()
 	}
 }
 
-func (st *BasicTable) Release() {
+func (st *basicTable) Release() {
 	if st.table != nil {
 		st.table.Retain()
 	}
 }
 
-func (st *BasicTable) Get(column, row int) interface{} {
+func (st *basicTable) Get(column, row int) interface{} {
 	field := st.Schema().Field(column)
 	c, i := st.resolver.Resolve(row)
-	nullable := field.Nullable
 
 	chunk := st.Column(column).Data().Chunk(c)
-	// TODO(twg) 2023/01/26 potential NULL support?
-	if nullable && chunk.IsNull(i) {
-		return nil
-	}
 	switch field.Type.(type) {
-	case *arrow.BooleanType:
-		return chunk.(*array.Boolean).Value(i)
+	// case *arrow.BooleanType:
+	//	v := chunk.(*array.Boolean).BooleanValues()
+	//	return v[i]
 	case *arrow.Int8Type:
 		v := chunk.(*array.Int8).Int8Values()
-		return int64(v[i])
+		return v[i]
 	case *arrow.Int16Type:
 		v := chunk.(*array.Int16).Int16Values()
-		return int64(v[i])
+		return v[i]
 	case *arrow.Int32Type:
 		v := chunk.(*array.Int32).Int32Values()
-		return int64(v[i])
+		return v[i]
 	case *arrow.Int64Type:
 		v := chunk.(*array.Int64).Int64Values()
-		return int64(v[i])
+		return v[i]
 	case *arrow.Uint8Type:
 		v := chunk.(*array.Uint8).Uint8Values()
-		return uint64(v[i])
+		return v[i]
 	case *arrow.Uint16Type:
 		v := chunk.(*array.Uint16).Uint16Values()
-		return uint64(v[i])
+		return v[i]
 	case *arrow.Uint32Type:
 		v := chunk.(*array.Uint32).Uint32Values()
-		return uint64(v[i])
+		return v[i]
 	case *arrow.Uint64Type:
 		v := chunk.(*array.Uint64).Uint64Values()
 		return v[i]
 	case *arrow.Float32Type:
 		v := chunk.(*array.Float32).Float32Values()
-		return float64(v[i])
+		return v[i]
 	case *arrow.Float64Type:
 		v := chunk.(*array.Float64).Float64Values()
 		return v[i]
-	case *arrow.StringType:
-		return chunk.(*array.String).Value(i)
 	}
 	return 0
 }
@@ -229,10 +223,8 @@ func builderFrom(mem memory.Allocator, dt arrow.DataType, size int64) array.Buil
 		bldr = array.NewFloat32Builder(mem)
 	case *arrow.Float64Type:
 		bldr = array.NewFloat64Builder(mem)
-	case *arrow.StringType:
-		bldr = array.NewStringBuilder(mem)
 	default:
-		panic(fmt.Errorf("builderFrom: invalid Arrow type %v", dt))
+		panic(fmt.Errorf("npy2root: invalid Arrow type %v", dt))
 	}
 	bldr.Reserve(int(size))
 	return bldr
@@ -262,14 +254,12 @@ func appendData(bldr array.Builder, v interface{}) {
 		bldr.Append(v.(float32))
 	case *array.Float64Builder:
 		bldr.Append(v.(float64))
-	case *array.StringBuilder:
-		bldr.Append(v.(string))
 	default:
-		panic(fmt.Errorf("appendData: invalid Arrow builder type %T", bldr))
+		panic(fmt.Errorf("npy2root: invalid Arrow builder type %T", bldr))
 	}
 }
 
-func Concat(schema *arrow.Schema, tables []*BasicTable, mem memory.Allocator) arrow.Table {
+func Concat(schema *arrow.Schema, tables []*basicTable, mem memory.Allocator) arrow.Table {
 	if len(tables) == 1 {
 		if !tables[0].IsFiltered() {
 			return tables[0]
@@ -311,7 +301,7 @@ func Concat(schema *arrow.Schema, tables []*BasicTable, mem memory.Allocator) ar
 	return array.NewTable(schema, cols, -1)
 }
 
-func (st *BasicTable) MarshalJSON() ([]byte, error) {
+func (st *basicTable) MarshalJSON() ([]byte, error) {
 	results := make(map[string]interface{})
 	n := 0
 	if st.table != nil {
@@ -330,10 +320,10 @@ func (st *BasicTable) MarshalJSON() ([]byte, error) {
 	return json.Marshal(results)
 }
 
-func BasicTableFromArrow(table arrow.Table, mem memory.Allocator) *BasicTable {
+func BasicTableFromArrow(table arrow.Table, mem memory.Allocator) *basicTable {
 	col := table.Column(0)
 	r := dataframe.NewChunkResolver(col)
-	return &BasicTable{resolver: &r, table: table}
+	return &basicTable{resolver: &r, table: table}
 }
 
 func filterColumns(filters []string, table arrow.Table) arrow.Table {
@@ -363,7 +353,7 @@ func filterColumns(filters []string, table arrow.Table) arrow.Table {
 	return array.NewTable(filterdSchema, cols, table.NumRows())
 }
 
-func (e *executor) executeArrowShard(ctx context.Context, qcx *Qcx, index string, c *pql.Call, shard uint64, pool memory.Allocator, columnFilter []string) (*BasicTable, error) {
+func (e *executor) executeArrowShard(ctx context.Context, qcx *Qcx, index string, c *pql.Call, shard uint64, pool memory.Allocator, columnFilter []string) (*basicTable, error) {
 	name := fmt.Sprintf("a. %v", shard)
 	span, _ := tracing.StartSpanFromContext(ctx, "Executor.executeArrowShard")
 	defer span.Finish()
@@ -377,7 +367,7 @@ func (e *executor) executeArrowShard(ctx context.Context, qcx *Qcx, index string
 		filter = row
 		if !filter.Any() {
 			// no need to actuall run the query for its not operating against any values
-			return &BasicTable{name: name}, nil
+			return &basicTable{name: name}, nil
 		}
 	}
 	//
@@ -391,7 +381,7 @@ func (e *executor) executeArrowShard(ctx context.Context, qcx *Qcx, index string
 	fname := idx.GetDataFramePath(shard)
 
 	if !e.dataFrameExists(fname) {
-		return &BasicTable{name: name}, nil
+		return &basicTable{name: name}, nil
 	}
 
 	table, err := e.getDataTable(ctx, fname, pool)
@@ -411,7 +401,7 @@ func (e *executor) executeArrowShard(ctx context.Context, qcx *Qcx, index string
 	resolver = &p
 	if filter != nil {
 		if len(ids) == 0 {
-			return &BasicTable{name: name}, nil
+			return &basicTable{name: name}, nil
 		}
 		resolver, err = filterDataframe(resolver, pool, ids)
 		if err != nil {
@@ -419,7 +409,7 @@ func (e *executor) executeArrowShard(ctx context.Context, qcx *Qcx, index string
 		}
 	}
 	table.Retain()
-	return &BasicTable{resolver: resolver, table: table, filtered: filter != nil, name: name}, nil
+	return &basicTable{resolver: resolver, table: table, filtered: filter != nil, name: name}, nil
 }
 
 func (e *executor) dataFrameExists(fname string) bool {
@@ -478,7 +468,7 @@ func readTableArrow(filename string, mem memory.Allocator) (arrow.Table, error)
 		return nil, err
 	}
 	defer rr.Close()
-	records := make([]arrow.Record, rr.NumRecords())
+	records := make([]arrow.Record, rr.NumRecords(), rr.NumRecords())
 	i := 0
 	for {
 		rec, err := rr.Read()

batch/Dockerfile-test

@@ -2,10 +2,10 @@ ARG GO_VERSION=1.19
 
 FROM golang:${GO_VERSION}
 
-WORKDIR /go/src/github.com/featurebasedb/featurebase/
+WORKDIR /go/src/github.com/molecula/featurebase/
 
 COPY . .
 
-WORKDIR /go/src/github.com/featurebasedb/featurebase/batch/
+WORKDIR /go/src/github.com/molecula/featurebase/batch/
 
 CMD ["go","test","-v","-mod=vendor","-tags=odbc,dynamic","./..."]

batch/batch.go

@@ -23,7 +23,6 @@ import (
 const (
 	DefaultKeyTranslateBatchSize = 100000
 	existenceFieldName           = "_exists"
-	existenceViewName            = "existence" // this should match top level featurebase viewExistence
 )
 
 // TODO if using column translation, column ids might get way out of
@@ -574,11 +573,7 @@ func (b *Batch) Add(rec Row) error {
 		case int64:
 			b.values[field.Name] = append(b.values[field.Name], val)
 		case []string:
-			// note that a length of 0 can be valid, and represents an
-			// empty set. an empty set counts as a non-NULL value for
-			// SQL purposes -- it means the existence view bit should
-			// get set.
-			if val == nil {
+			if len(val) == 0 {
 				continue
 			}
 			rowIDSets, ok := b.rowIDSets[field.Name]
@@ -613,11 +608,7 @@ func (b *Batch) Add(rec Row) error {
 			}
 			b.rowIDSets[field.Name] = append(rowIDSets, rowIDs)
 		case []uint64:
-			// note that a length of 0 can be valid, and represents an
-			// empty set. an empty set counts as a non-NULL value for
-			// SQL purposes -- it means the existence view bit should
-			// get set.
-			if val == nil {
+			if len(val) == 0 {
 				continue
 			}
 			rowIDSets, ok := b.rowIDSets[field.Name]
@@ -672,9 +663,6 @@ func (b *Batch) Add(rec Row) error {
 
 	for i, uval := range rec.Clears {
 		field := b.header[i]
-		if field.Options.Type == featurebase.FieldTypeMutex && uval != nil {
-			return errors.Errorf("individual-bit clears not allowed on mutex fields; use nil to clear a mutex")
-		}
 		if _, ok := b.clearRowIDs[i]; !ok {
 			b.clearRowIDs[i] = make(map[int]uint64)
 		}
@@ -767,7 +755,7 @@ func (b *Batch) Import() error {
 		}()
 	}
 	defer func() {
-		featurebase.SummaryBatchImportDurationSeconds.Observe(time.Since(start).Seconds())
+		b.importer.StatsTiming(MetricBatchImportDurationSeconds, time.Since(start), 1.0)
 	}()
 
 	size := len(b.ids)
@@ -840,7 +828,7 @@ func (b *Batch) Flush() error {
 		if err != nil {
 			b.log.Errorf("error finishing transaction: %v. trns: %+v", err, trnsl)
 		}
-		featurebase.SummaryBatchFlushDurationSeconds.Observe(time.Since(start).Seconds())
+		b.importer.StatsTiming(MetricBatchFlushDurationSeconds, time.Since(start), 1.0)
 	}()
 
 	importStart := time.Now()
@@ -1200,7 +1188,7 @@ func (b *Batch) doImportShardTransactional(frags, clearFrags fragments) error {
 		}
 	}
 
-	featurebase.SummaryBatchShardImportBuildRequestsSeconds.Observe(time.Since(start).Seconds())
+	b.importer.StatsTiming(MetricBatchShardImportBuildRequestsSeconds, time.Since(start), 1.0)
 	start = time.Now()
 	eg := egpool.Group{PoolSize: 20}
 	for shard, request := range requests {
@@ -1212,7 +1200,7 @@ func (b *Batch) doImportShardTransactional(frags, clearFrags fragments) error {
 	}
 	err := eg.Wait()
 	dur := time.Since(start)
-	featurebase.SummaryBatchImportDurationSeconds.Observe(dur.Seconds())
+	b.importer.StatsTiming(MetricBatchShardImportDurationSeconds, dur, 1.0)
 	b.log.Printf("import shard took: %v\n", dur)
 	return errors.Wrap(err, "doing shard-transactional imports")
 }
@@ -1257,7 +1245,7 @@ func (b *Batch) doImport(frags, clearFrags fragments) error {
 			}
 
 			ferr := b.importer.ImportRoaringBitmap(ctx, b.tbl.ID, fld, shard, viewMap, false)
-			b.log.Debugf("imp-roar    field: %s, shard:%d, views:%d %v", field, shard, len(viewMap), time.Since(starty))
+			b.log.Debugf("imp-roar     %s,shard:%d,views:%d %v", field, shard, len(clearViewMap), time.Since(starty))
 			return errors.Wrapf(ferr, "importing data for %s", field)
 		})
 	}
@@ -1355,7 +1343,6 @@ func (b *Batch) makeFragments(frags, clearFrags fragments) (fragments, fragments
 		curShard := ^uint64(0) // impossible sentinel value for shard.
 		var curBM *roaring.Bitmap
 		var clearBM *roaring.Bitmap
-		var existCurBM *roaring.Bitmap
 		for j := range b.ids {
 			col := b.ids[j]
 			row := nilSentinel
@@ -1368,12 +1355,8 @@ func (b *Batch) makeFragments(frags, clearFrags fragments) (fragments, fragments
 
 			if col/shardWidth != curShard {
 				curShard = col / shardWidth
-				// the API treats "" as standard
 				curBM = frags.GetOrCreate(curShard, field.Name, "")
 				clearBM = clearFrags.GetOrCreate(curShard, field.Name, "")
-				if opts.ActuallyTrackingExistence() {
-					existCurBM = frags.GetOrCreate(curShard, field.Name, existenceViewName)
-				}
 			}
 			if row != nilSentinel {
 				// TODO this is super ugly, but we want to avoid setting
@@ -1383,9 +1366,6 @@ func (b *Batch) makeFragments(frags, clearFrags fragments) (fragments, fragments
 				// the NoStandardView case would be great.
 				if !(opts.Type == featurebase.FieldTypeTime && opts.NoStandardView) {
 					curBM.DirectAdd(row*shardWidth + (col % shardWidth))
-					if opts.ActuallyTrackingExistence() {
-						existCurBM.DirectAdd(col % shardWidth)
-					}
 				}
 				if opts.Type == featurebase.FieldTypeTime {
 					views, err := b.times[j].views(opts.TimeQuantum)
@@ -1406,16 +1386,6 @@ func (b *Batch) makeFragments(frags, clearFrags fragments) (fragments, fragments
 				// we want to make sure that at this point, the "set"
 				// fragments don't contain the bit that we're clearing
 				curBM.DirectRemoveN(clearRow*shardWidth + (col % shardWidth))
-				// Because this is RowIDs, not RowIDSets, there's only one
-				// bit. We should not be setting the existence bit based on
-				// this value, if we're actually clearing it. This doesn't
-				// mean we will clear an existing existence bit, though.
-				// The case where we would clear an existence bit is the
-				// case where someone specified row[mutexField].Clears = nil,
-				// which is far from here.
-				if opts.ActuallyTrackingExistence() {
-					existCurBM.DirectRemoveN(col % shardWidth)
-				}
 			}
 		}
 	}
@@ -1434,23 +1404,14 @@ func (b *Batch) makeFragments(frags, clearFrags fragments) (fragments, fragments
 		opts := field.Options
 		curShard := ^uint64(0) // impossible sentinel value for shard.
 		var curBM *roaring.Bitmap
-		var existCurBM *roaring.Bitmap
 		for j := range b.ids {
 			col, rowIDs := b.ids[j], rowIDSets[j]
+			if len(rowIDs) == 0 {
+				continue
+			}
 			if col/shardWidth != curShard {
 				curShard = col / shardWidth
 				curBM = frags.GetOrCreate(curShard, fname, "")
-				if opts.ActuallyTrackingExistence() {
-					existCurBM = frags.GetOrCreate(curShard, fname, existenceViewName)
-				}
-			}
-			if len(rowIDs) == 0 {
-				// you can validly specify an empty set, which is not the same as a null,
-				// but which still ought to set the existence bit if we're tracking that.
-				if opts.ActuallyTrackingExistence() && rowIDs != nil {
-					existCurBM.DirectAdd(col % shardWidth)
-				}
-				continue
 			}
 			// TODO this is super ugly, but we want to avoid setting
 			// bits on the standard view in the specific case when
@@ -1461,9 +1422,6 @@ func (b *Batch) makeFragments(frags, clearFrags fragments) (fragments, fragments
 				for _, row := range rowIDs {
 					curBM.DirectAdd(row*shardWidth + (col % shardWidth))
 				}
-				if opts.ActuallyTrackingExistence() {
-					existCurBM.DirectAdd(col % shardWidth)
-				}
 			}
 			if opts.Type == featurebase.FieldTypeTime {
 				views, err := b.times[j].views(opts.TimeQuantum)
@@ -1591,11 +1549,6 @@ func (b *Batch) makeSingleValFragments(frags, clearFrags fragments) (fragments,
 		shard := ids[0] / shardWidth
 		bitmap := frags.GetOrCreate(shard, field.Name, "standard")
 		clearBM := clearFrags.GetOrCreate(shard, field.Name, "standard")
-		var existBM, existClearBM *roaring.Bitmap
-		if field.Options.ActuallyTrackingExistence() {
-			existBM = frags.GetOrCreate(shard, field.Name, existenceViewName)
-			existClearBM = clearFrags.GetOrCreate(shard, field.Name, existenceViewName)
-		}
 		for i, id := range ids {
 			if i+1 < len(ids) {
 				// we only want the last value set for each id
@@ -1608,10 +1561,6 @@ func (b *Batch) makeSingleValFragments(frags, clearFrags fragments) (fragments,
 				shard = id / shardWidth
 				bitmap = frags.GetOrCreate(shard, field.Name, "standard")
 				clearBM = clearFrags.GetOrCreate(shard, field.Name, "standard")
-				if field.Options.ActuallyTrackingExistence() {
-					existBM = frags.GetOrCreate(shard, field.Name, existenceViewName)
-					existClearBM = clearFrags.GetOrCreate(shard, field.Name, existenceViewName)
-				}
 			}
 			fragmentColumn := id % shardWidth
 			clearBM.Add(fragmentColumn) // Will use this to clear columns.
@@ -1619,11 +1568,6 @@ func (b *Batch) makeSingleValFragments(frags, clearFrags fragments) (fragments,
 				// clearSentinel is used for deletion
 				// so this value should only be added if its not clearSentinel
 				bitmap.Add(row*shardWidth + fragmentColumn)
-				if field.Options.ActuallyTrackingExistence() {
-					existBM.Add(fragmentColumn)
-				}
-			} else if field.Options.ActuallyTrackingExistence() {
-				existClearBM.Add(fragmentColumn)
 			}
 		}
 	}
@@ -1652,11 +1596,6 @@ func (b *Batch) makeSingleValFragments(frags, clearFrags fragments) (fragments,
 			fragmentColumn := recID % shardWidth
 
 			clearBM.Add(fragmentColumn)
-			if field.Options.ActuallyTrackingExistence() {
-				existClearBM := clearFrags.GetOrCreate(shard, field.Name, existenceViewName)
-
-				existClearBM.Add(fragmentColumn)
-			}
 		}
 	}
 
@@ -1679,10 +1618,6 @@ func (b *Batch) makeSingleValFragments(frags, clearFrags fragments) (fragments,
 
 			fragmentColumn := recID % shardWidth
 			clearBM.Add(fragmentColumn)
-			if field.Options.ActuallyTrackingExistence() {
-				exist := frags.GetOrCreate(shard, field.Name, existenceViewName)
-				exist.Add(fragmentColumn)
-			}
 
 			if boolVal {
 				bitmap.Add(trueRowOffset + fragmentColumn)
@@ -1768,7 +1703,7 @@ func (b *Batch) importValueData() error {
 					start := time.Now()
 					fld := featurebase.FieldInfoToField(field)
 					err := b.importer.DoImport(ctx, b.tbl.ID, fld, shard, path, data)
-					b.log.Debugf("imp-vals    field: %s, shard: %d, data: %d %v", field.Name, shard, len(data), time.Since(start))
+					b.log.Debugf("imp-vals %s,shard:%d,data:%d %v", field, shard, len(data), time.Since(start))
 					return errors.Wrapf(err, "importing values for field = %s", field.Name)
 				})
 				startIdx = i

batch/batch_test.go

@@ -103,12 +103,6 @@ func testStringSliceCombos(t *testing.T, importer featurebase.Importer, sapi fea
 		Index: idx.Name,
 		Query: "TopN(a1, n=10)",
 	})
-	if resp.Err != nil {
-		t.Fatalf("unexpected error from TopN query: %v", resp.Err)
-	}
-	if len(resp.Results) < 1 {
-		t.Fatalf("expected non-empty result set, got empty results")
-	}
 	pairsField, ok := resp.Results[0].(*featurebase.PairsField)
 	assert.True(t, ok, "wrong return type: %T", resp.Results[0])
 
@@ -514,11 +508,10 @@ func testStringSliceEmptyAndNil(t *testing.T, importer featurebase.Importer, sap
 			{
 				Name: "strslice",
 				Options: featurebase.FieldOptions{
-					Type:           featurebase.FieldTypeSet,
-					Keys:           true,
-					CacheType:      featurebase.CacheTypeRanked,
-					CacheSize:      100,
-					TrackExistence: true,
+					Type:      featurebase.FieldTypeSet,
+					Keys:      true,
+					CacheType: featurebase.CacheTypeRanked,
+					CacheSize: 100,
 				},
 			},
 		},
@@ -618,14 +611,6 @@ func testStringSliceEmptyAndNil(t *testing.T, importer featurebase.Importer, sap
 			pql: "Row(strslice='z')",
 			exp: []uint64{2},
 		},
-		{
-			pql: "Row(strslice==null)",
-			exp: []uint64{1},
-		},
-		{
-			pql: "Row(strslice!=null)",
-			exp: []uint64{0, 2, 3, 4},
-		},
 	}
 	for i, test := range tests {
 		t.Run(fmt.Sprintf("test-%d", i), func(t *testing.T) {
@@ -2060,7 +2045,7 @@ func mutexClearRegression(t *testing.T, importer featurebase.Importer, sapi feat
 	}
 
 	col := uint64(0)
-	row := uint64(0)
+	row := uint64(1)
 	for i := uint64(0); i <= 21; i++ {
 		col = (i%2+1)*featurebase.ShardWidth + i%5
 		row = i % 3
@@ -2141,7 +2126,7 @@ func mutexNilClearID(t *testing.T, importer featurebase.Importer, sapi featureba
 	}
 
 	col := uint64(0)
-	row := uint64(0)
+	row := uint64(1)
 	// populate mutex with some data
 	for i := uint64(0); i < 11; i++ {
 		col = (i%2+1)*featurebase.ShardWidth + i%5
@@ -2356,58 +2341,3 @@ func testImportBatchBools(t *testing.T, importer featurebase.Importer, sapi feat
 	assert.True(t, ok, "wrong return type: %T", resp.Results[0])
 	assert.Equal(t, uint64(2), count)
 }
-
-func TestConvert(t *testing.T) {
-	t.Run("timestampToInt", func(t *testing.T) {
-		tests := []struct {
-			unit TimeUnit
-			ts   string
-			exp  int64
-		}{
-			{unit: "s", ts: "2022-01-01T00:00:00Z", exp: 1640995200},
-			{unit: "ms", ts: "2022-01-01T00:00:00Z", exp: 1640995200000},
-			{unit: "us", ts: "2022-01-01T00:00:00Z", exp: 1640995200000000},
-			{unit: "ns", ts: "2022-01-01T00:00:00Z", exp: 1640995200000000000},
-			{unit: "x", ts: "2022-01-01T00:00:00Z", exp: 0},
-		}
-		for i, test := range tests {
-			t.Run(fmt.Sprintf("test-%d", i), func(t *testing.T) {
-				ts, err := time.Parse(time.RFC3339, test.ts)
-				assert.NoError(t, err)
-				v := timestampToInt(test.unit, ts)
-				assert.Equal(t, test.exp, v)
-			})
-		}
-	})
-
-	t.Run("Int64ToTimestamp", func(t *testing.T) {
-		tests := []struct {
-			unit  TimeUnit
-			epoch string
-			val   int64
-			exp   time.Time
-		}{
-			{
-				unit:  "ms",
-				epoch: "2022-01-01T00:00:00Z",
-				val:   0,
-				exp:   time.Date(2022, 1, 1, 0, 0, 0, 0, time.UTC),
-			},
-			{
-				unit:  "s",
-				epoch: "2022-01-01T00:00:00Z",
-				val:   86400,
-				exp:   time.Date(2022, 1, 2, 0, 0, 0, 0, time.UTC),
-			},
-		}
-		for i, test := range tests {
-			t.Run(fmt.Sprintf("test-%d", i), func(t *testing.T) {
-				epoch, err := time.Parse(time.RFC3339, test.epoch)
-				assert.NoError(t, err)
-				ts, err := Int64ToTimestamp(test.unit, epoch, test.val)
-				assert.NoError(t, err)
-				assert.Equal(t, test.exp, ts)
-			})
-		}
-	})
-}

batch/batcher.go

@@ -1,20 +0,0 @@
-package batch
-
-import (
-	"time"
-
-	"github.com/featurebasedb/featurebase/v3/dax"
-)
-
-// Batcher is an interface implemented by anything which can allocate new
-// batches.
-type Batcher interface {
-	NewBatch(cfg Config, tbl *dax.Table, fields []*dax.Field) (RecordBatch, error)
-}
-
-// Config is the configuration options passed to NewBatch for any implementation
-// of the Batcher interface.
-type Config struct {
-	Size         int
-	MaxStaleness time.Duration
-}

batch/egpool/egpool.go

@@ -58,11 +58,11 @@ func (eg *Group) err(err error) {
 	eg.errs = append(eg.errs, err)
 }
 
-type PanicError struct {
+type ErrPanic struct {
 	Value interface{}
 }
 
-func (p PanicError) Error() string {
+func (p ErrPanic) Error() string {
 	return fmt.Sprintf("panic: %v", p.Value)
 }
 
@@ -77,7 +77,7 @@ func (eg *Group) processJobs() {
 	defer func() {
 		if !finished {
 			if p := recover(); p != nil {
-				eg.err(PanicError{p})
+				eg.err(ErrPanic{p})
 			} else {
 				eg.err(ErrGoexit)
 			}

batch/metrics.go

@@ -1,3 +1,28 @@
 // Copyright 2022 Molecula Corp. (DBA FeatureBase).
 // SPDX-License-Identifier: Apache-2.0
 package batch
+
+const (
+	// MetricBatchImportDurationSeconds records the full time of the
+	// RecordBatch.Import call. This includes starting and finishing a
+	// transaction, doing key translation, building fragments locally,
+	// importing all data, and resetting internal structures.
+	MetricBatchImportDurationSeconds = "batch_import_duration_seconds"
+
+	// MetricBatchFlushDurationSeconds records the full time for
+	// RecordBatch.Flush (if splitBatchMode is in use). This includes
+	// starting and finishing a transaction, importing all data, and
+	// resetting internal structures.
+	MetricBatchFlushDurationSeconds = "batch_flush_duration_seconds"
+
+	// MetricBatchShardImportBuildRequestsSeconds is the time it takes
+	// after making fragments to build the shard-transactional request
+	// objects (but not actually import them or do any network activity).
+	MetricBatchShardImportBuildRequestsSeconds = "batch_shard_import_build_requests_seconds"
+
+	// MetricBatchShardImportDurationSeconds is the time it takes to
+	// import all data for all shards in the batch using the
+	// shard-transactional endpoint. This does not include the time it
+	// takes to build the requests locally.
+	MetricBatchShardImportDurationSeconds = "batch_shard_import_duration_seconds"
+)

buffer/filebuffer.go

@@ -1,110 +0,0 @@
-package buffer
-
-import (
-	"bytes"
-	"io"
-	"io/ioutil"
-	"os"
-	"sync"
-)
-
-// NewFileBuffer returns a file buffer which will use an in-memory buffer, until `max` bytes have been written, at which point it will write the contents of memory to a file, and continue writing future data to the file.
-// The file will be written to `temp` directory. The buffer fulfills the io.Reader and io.Writer interface
-func NewFileBuffer(max int, temp string) *FileBuffer {
-	return &FileBuffer{max: max, tempDir: temp}
-}
-
-type FileBuffer struct {
-	max     int
-	buf     bytes.Buffer
-	file    *os.File
-	tempDir string
-	reading bool
-	files   []*os.File
-	mu      sync.Mutex
-}
-
-func (fb *FileBuffer) Write(p []byte) (n int, err error) {
-	if fb.reading {
-		panic("cannot write after read")
-	}
-	if fb.file != nil {
-		return fb.file.Write(p)
-	}
-	n, err = fb.buf.Write(p)
-	if err != nil {
-		return
-	}
-	if fb.buf.Len() > fb.max {
-		fb.file, err = ioutil.TempFile(fb.tempDir, "filebuffer-")
-		if err != nil {
-			return
-		}
-		_, err = io.Copy(fb.file, &fb.buf)
-		fb.buf.Reset()
-	}
-	return
-}
-
-func (fb *FileBuffer) Len() (int64, error) {
-	if fb.file == nil {
-		return int64(fb.buf.Len()), nil
-	}
-	fi, err := fb.file.Stat()
-	if err != nil {
-		return 0, err
-	}
-
-	return fi.Size(), nil
-}
-
-func (fb *FileBuffer) Read(p []byte) (n int, err error) {
-	if fb.file != nil {
-		if !fb.reading {
-			fb.reading = true
-			_, err = fb.file.Seek(0, 0)
-			if err != nil {
-				return
-			}
-		}
-		return fb.file.Read(p)
-	}
-	fb.reading = true
-	return fb.buf.Read(p)
-}
-
-func (fb *FileBuffer) Close() error {
-	if fb.file != nil {
-		name := fb.file.Name()
-		if err := fb.file.Close(); err != nil {
-			return err
-		}
-		for _, f := range fb.files {
-			f.Close()
-		}
-		fb.files = fb.files[:0]
-		fb.file = nil
-		return os.Remove(name)
-	}
-	return nil
-}
-
-func (fb *FileBuffer) Reset() error {
-	fb.mu.Lock()
-	defer fb.mu.Unlock()
-	fb.reading = false
-	fb.buf.Reset()
-	return fb.Close()
-}
-
-func (fb *FileBuffer) NewReader() (io.Reader, error) {
-	fb.mu.Lock()
-	defer fb.mu.Unlock()
-	fb.reading = true
-	if fb.file == nil {
-		return bytes.NewReader(fb.buf.Bytes()), nil
-	}
-	f, err := os.OpenFile(fb.file.Name(), os.O_RDONLY, 0)
-	fb.files = append(fb.files, f)
-	return f, err
-}

bufferpool/page.go

@@ -53,10 +53,10 @@ const PAGE_PREV_POINTER_OFFSET = 12 // offset 12, length 4, end 16
 const PAGE_NEXT_POINTER_OFFSET = 16 // offset 16, length 4, end 20
 const PAGE_SLOTS_START_OFFSET = 20  // offset 20
 
-// PAGE_SLOT_LENGTH is the size of the page slot key/value.
+// page slots
 //
-// key offset int16    	//offset 0, length 2, end 2
-// value offset int16	    //offset 2, length 2, end 4
+//	key offset int16    	//offset 0, length 2, end 2
+//	value offset int16	    //offset 2, length 2, end 4
 const PAGE_SLOT_LENGTH = 4
 
 // Page represents a page on disk

cache.go

@@ -13,6 +13,7 @@ import (
 
 	"github.com/featurebasedb/featurebase/v3/lru"
 	pb "github.com/featurebasedb/featurebase/v3/proto"
+	"github.com/featurebasedb/featurebase/v3/stats"
 	"github.com/pkg/errors"
 )
 
@@ -40,6 +41,9 @@ type cache interface {
 	// Returns an ordered list of the top ranked bitmaps.
 	Top() []bitmapPair
 
+	// SetStats defines the stats client used in the cache.
+	SetStats(s stats.StatsClient)
+
 	// Clear removes everything from the cache. If possible it should leave allocated structures in place to be reused.
 	Clear()
 }
@@ -48,6 +52,7 @@ type cache interface {
 type lruCache struct {
 	cache  *lru.Cache
 	counts map[uint64]uint64
+	stats  stats.StatsClient
 	// maxEntries is saved to support Clear which recreates the cache.
 	maxEntries uint32
 }
@@ -57,6 +62,7 @@ func newLRUCache(maxEntries uint32) *lruCache {
 	c := &lruCache{
 		cache:      lru.New(int(maxEntries)),
 		counts:     make(map[uint64]uint64),
+		stats:      stats.NopStatsClient,
 		maxEntries: maxEntries,
 	}
 	c.cache.OnEvicted = c.onEvicted
@@ -114,6 +120,11 @@ func (c *lruCache) Top() []bitmapPair {
 	return a
 }
 
+// SetStats defines the stats client used in the cache.
+func (c *lruCache) SetStats(s stats.StatsClient) {
+	c.stats = s
+}
+
 func (c *lruCache) Clear() {
 	for k := range c.counts {
 		delete(c.counts, k)
@@ -147,6 +158,8 @@ type rankCache struct {
 
 	// thresholdValue is the value of the last item in the cache
 	thresholdValue uint64
+
+	stats stats.StatsClient
 }
 
 // NewRankCache returns a new instance of RankCache.
@@ -155,6 +168,7 @@ func NewRankCache(maxEntries uint32) *rankCache {
 		maxEntries:      maxEntries,
 		thresholdBuffer: int(thresholdFactor * float64(maxEntries)),
 		entries:         make(map[uint64]uint64),
+		stats:           stats.NopStatsClient,
 	}
 }
 
@@ -215,7 +229,7 @@ func (c *rankCache) BulkAdd(id uint64, n uint64) {
 	// as this can take up an upbounded amount of memory. This is especially
 	// true when restoring shards as all rows will be added.
 	if len(c.entries) > int(2*c.maxEntries) {
-		CounterRecalculateCache.Inc()
+		c.stats.Count(MetricRecalculateCache, 1, 1.0)
 		c.recalculate()
 	}
 }
@@ -260,7 +274,7 @@ func (c *rankCache) Invalidate() {
 func (c *rankCache) Recalculate() {
 	c.mu.Lock()
 	defer c.mu.Unlock()
-	CounterRecalculateCache.Inc()
+	c.stats.Count(MetricRecalculateCache, 1, 1.0)
 	c.recalculate()
 }
 
@@ -272,12 +286,12 @@ func (c *rankCache) invalidate() {
 		// This is somewhat necessary for now since recalculation is not cheap.
 		// The cache will remain flagged as dirty and will be recalculated if Top is called.
 		// This may cause unexpected memory growth, so record it in metrics for debugging purposes.
-		CounterInvalidateCacheSkipped.Inc()
+		c.stats.Count(MetricInvalidateCacheSkipped, 1, 1.0)
 		// Ensure that we're marked as dirty even if we weren't otherwise.
 		c.dirty = true
 		return
 	}
-	CounterInvalidateCache.Inc()
+	c.stats.Count(MetricInvalidateCache, 1, 1.0)
 	c.recalculate()
 }
 
@@ -303,7 +317,7 @@ func (c *rankCache) recalculate() {
 
 	// Store the count of the item at the threshold index.
 	length := len(c.rankings)
-	GaugeRankCacheLength.Set(float64(length))
+	c.stats.Gauge(MetricRankCacheLength, float64(length), 1.0)
 
 	var removeItems []bitmapPair // cached, ordered list
 	if length > int(c.maxEntries) {
@@ -319,7 +333,7 @@ func (c *rankCache) recalculate() {
 
 	// If size is larger than the threshold then trim it.
 	if len(c.entries) > c.thresholdBuffer {
-		CounterCacheThresholdReached.Inc()
+		c.stats.Count(MetricCacheThresholdReached, 1, 1.0)
 		for _, pair := range removeItems {
 			delete(c.entries, pair.ID)
 		}
@@ -329,6 +343,11 @@ func (c *rankCache) recalculate() {
 	c.dirty = false
 }
 
+// SetStats defines the stats client used in the cache.
+func (c *rankCache) SetStats(s stats.StatsClient) {
+	c.stats = s
+}
+
 // Top returns an ordered list of pairs.
 func (c *rankCache) Top() []bitmapPair {
 	c.mu.Lock()
@@ -336,7 +355,7 @@ func (c *rankCache) Top() []bitmapPair {
 
 	if c.dirty {
 		// The cache is dirty, so we need to recalculate it to get a consistent view.
-		CounterReadDirtyCache.Inc()
+		c.stats.Count(MetricReadDirtyCache, 1, 1.0)
 		c.recalculate()
 	}
 
@@ -587,21 +606,25 @@ func (p uint64Slice) Len() int           { return len(p) }
 func (p uint64Slice) Less(i, j int) bool { return p[i] < p[j] }
 
 // nopCache represents a no-op Cache implementation.
-type nopCache struct{}
+type nopCache struct {
+	stats stats.StatsClient
+}
 
 // Ensure NopCache implements Cache.
-var globalNopCache cache = nopCache{}
+var globalNopCache cache = nopCache{
+	stats: stats.NopStatsClient,
+}
 
 func (c nopCache) Add(uint64, uint64)     {}
 func (c nopCache) BulkAdd(uint64, uint64) {}
 func (c nopCache) Get(uint64) uint64      { return 0 }
 func (c nopCache) IDs() []uint64          { return []uint64{} }
 
-func (c nopCache) Invalidate()  {}
-func (c nopCache) Len() int     { return 0 }
-func (c nopCache) Recalculate() {}
-
-func (c nopCache) Clear() {}
+func (c nopCache) Invalidate()                {}
+func (c nopCache) Len() int                   { return 0 }
+func (c nopCache) Recalculate()               {}
+func (c nopCache) SetStats(stats.StatsClient) {}
+func (c nopCache) Clear()                     {}
 
 func (c nopCache) Top() []bitmapPair {
 	return []bitmapPair{}

cache_test.go

@@ -6,7 +6,7 @@ import (
 	"reflect"
 	"testing"
 
-	pilosa "github.com/featurebasedb/featurebase/v3"
+	"github.com/featurebasedb/featurebase/v3"
 )
 
 // Ensure cache stays constrained to its configured size.
@@ -62,7 +62,7 @@ func TestCache_Rank_Dirty(t *testing.T) {
 		cache.Add(v.ID, v.Count)
 	}
 
-	var got []pair //nolint:prealloc
+	var got []pair
 	for _, p := range cache.Top() {
 		got = append(got, pair(p))
 	}

catcher.go

@@ -124,17 +124,6 @@ func (c *catcherTx) Remove(index, field, view string, shard uint64, a ...uint64)
 	return c.b.Remove(index, field, view, shard, a...)
 }
 
-func (c *catcherTx) Removed(index, field, view string, shard uint64, a ...uint64) (changed []uint64, err error) {
-
-	defer func() {
-		if r := recover(); r != nil {
-			vprint.AlwaysPrintf("see Removed() PanicOn '%v' at '%v'", r, vprint.Stack())
-			vprint.PanicOn(r)
-		}
-	}()
-	return c.b.Removed(index, field, view, shard, a...)
-}
-
 func (c *catcherTx) Contains(index, field, view string, shard uint64, key uint64) (exists bool, err error) {
 
 	defer func() {

cli/Makefile

@@ -1,15 +0,0 @@
-.PHONY: test testv test-integration testv-integration
-
-GO=go
-
-test:
-	$(GO) test ./... -short
-
-testv:
-	$(GO) test -v ./... -short
-
-test-integration:
-	$(GO) test . -count 1 -timeout 20m -run TestCLIIntegration/$(RUN)
-
-testv-integration:
-	$(GO) test -v . -count 1 -timeout 20m -run TestCLIIntegration/$(RUN)

cli/batch/inserter.go

@@ -1,8 +0,0 @@
-package batch
-
-// Inserter can be implemented by anything which can handle a SQL statement
-// representing a write operation. An example is `BULK INSERT`. The Insert()
-// method on this interface does not return any results other than an error.
-type Inserter interface {
-	Insert(sql string) error
-}

cli/batch/sql.go

@@ -1,215 +0,0 @@
-package batch
-
-import (
-	"encoding/json"
-	"fmt"
-	"strings"
-	"time"
-
-	fbbatch "github.com/featurebasedb/featurebase/v3/batch"
-	"github.com/featurebasedb/featurebase/v3/dax"
-	"github.com/featurebasedb/featurebase/v3/errors"
-	"github.com/featurebasedb/featurebase/v3/pql"
-)
-
-// Ensure type implements interface.
-var _ fbbatch.Batcher = (*sqlBatcher)(nil)
-
-type sqlBatcher struct {
-	inserter Inserter
-	fields   []*dax.Field
-}
-
-func NewSQLBatcher(i Inserter, flds []*dax.Field) *sqlBatcher {
-	return &sqlBatcher{
-		inserter: i,
-		fields:   flds,
-	}
-}
-
-func (b *sqlBatcher) NewBatch(cfg fbbatch.Config, tbl *dax.Table, flds []*dax.Field) (fbbatch.RecordBatch, error) {
-	fields := flds
-	if b.fields != nil {
-		fields = b.fields
-	}
-	return &sqlBatch{
-		table:        tbl,
-		fields:       fields,
-		size:         cfg.Size,
-		maxStaleness: cfg.MaxStaleness,
-		ids:          make([]interface{}, 0, cfg.Size),
-		rows:         make([][]interface{}, 0, cfg.Size),
-		inserter:     b.inserter,
-	}, nil
-}
-
-// Ensure type implements interface.
-var _ fbbatch.RecordBatch = (*sqlBatch)(nil)
-
-type sqlBatch struct {
-	table  *dax.Table
-	fields []*dax.Field
-	size   int
-
-	ids  []interface{}
-	rows [][]interface{}
-
-	// staleTime tracks the time the first record of the batch was inserted
-	// plus the maxStaleness, in order to raise ErrBatchNowStale if the
-	// maxStaleness has elapsed
-	staleTime    time.Time
-	maxStaleness time.Duration
-
-	// inserter handles SQL INSERT statements generated for each batch.
-	inserter Inserter
-}
-
-func (b *sqlBatch) Add(rec fbbatch.Row) error {
-	// Clear rec.Values and rec.Clears upon return.
-	defer func() {
-		for i := range rec.Values {
-			rec.Values[i] = nil
-		}
-		for k := range rec.Clears {
-			delete(rec.Clears, k)
-		}
-	}()
-
-	if len(b.ids) == cap(b.ids) {
-		return fbbatch.ErrBatchAlreadyFull
-	}
-	if len(rec.Values) != len(b.fields) {
-		return errors.Errorf("record needs to match up with batch fields, got %d fields and %d record", len(b.fields), len(rec.Values))
-	}
-
-	// Append the ID to b.ids.
-	b.ids = append(b.ids, rec.ID)
-
-	// Convert decimal fields (which come in as int64, along with the scale in
-	// field) to pql.Decimal.
-	for i, fld := range b.fields {
-		switch b.fields[i].Type {
-		case dax.BaseTypeDecimal:
-			if val, ok := rec.Values[i].(int64); ok {
-				rec.Values[i] = pql.NewDecimal(val, fld.Options.Scale)
-			}
-		case dax.BaseTypeTimestamp:
-			if val, ok := rec.Values[i].(int64); ok {
-				ts := time.Unix(val, 0)
-				rec.Values[i] = ts.Format(time.RFC3339)
-			}
-		}
-	}
-
-	// Append the record to b.rows.
-	vals := make([]interface{}, 0, len(rec.Values))
-	vals = append(vals, rec.Values...)
-	b.rows = append(b.rows, vals)
-
-	// Check for batch full or stale.
-	if len(b.ids) == cap(b.ids) {
-		return fbbatch.ErrBatchNowFull
-	}
-	if b.maxStaleness != time.Duration(0) { // set maxStaleness to 0 to disable staleness checking
-		if len(b.ids) == 1 {
-			b.staleTime = time.Now().Add(b.maxStaleness)
-		} else if time.Now().After(b.staleTime) {
-			return fbbatch.ErrBatchNowStale
-		}
-	}
-	return nil
-}
-
-func (b *sqlBatch) Import() error {
-	if len(b.rows) == 0 {
-		return nil
-	}
-
-	// Construct the BULK INSERT statement based on the table and fields.
-	sql, err := buildBulkInsert(b.table, b.fields, b.ids, b.rows)
-	if err != nil {
-		return errors.Wrap(err, "building bulk insert statement")
-	}
-
-	// Reset batch data.
-	b.reset()
-
-	// Submit the SQL statement.
-	return b.inserter.Insert(sql)
-}
-
-func (b *sqlBatch) reset() {
-	b.ids = b.ids[:0]
-	b.rows = b.rows[:0]
-}
-
-func (b *sqlBatch) Len() int {
-	return len(b.rows)
-}
-
-func (b *sqlBatch) Flush() error {
-	return nil
-}
-
-func buildBulkInsert(tbl *dax.Table, fields []*dax.Field, ids []interface{}, rows [][]interface{}) (string, error) {
-	// Validation.
-	if tbl.Name == "" {
-		return "", errors.New(errors.ErrUncoded, "table name is required")
-	} else if len(fields) == 0 {
-		return "", errors.New(errors.ErrUncoded, "at least one field is required")
-	}
-
-	var sb strings.Builder
-
-	sb.WriteString(`BULK INSERT INTO `)
-	sb.WriteString(string(tbl.Name))
-	sb.WriteString(` (_id,`)
-
-	flds := make([]string, 0, len(fields))
-	maps := make([]string, 0, len(fields))
-	for i := range fields {
-		flds = append(flds, string(fields[i].Name))
-		maps = append(maps, fmt.Sprintf("'$.col_%d' %s", i, fields[i].FullType()))
-	}
-	// Fields
-	sb.WriteString(strings.Join(flds, ","))
-
-	// MAP
-	keyType := dax.BaseTypeID
-	if tbl.StringKeys() {
-		keyType = dax.BaseTypeString
-	}
-	sb.WriteString(`) MAP ('$._id' `)
-	sb.WriteString(keyType)
-	sb.WriteString(`,`)
-	sb.WriteString(strings.Join(maps, ","))
-	sb.WriteString(`) FROM x'`)
-
-	// Row values.
-
-	// m is a map representing a single row to be marshalled and added to the
-	// bulk insert as one line in the NDJSON payload. We re-use the map for each
-	// row.
-	m := make(map[string]interface{})
-	for i := range rows {
-		// Write the ID value.
-		m[string(dax.PrimaryKeyFieldName)] = ids[i]
-		// Write the rest of the data values.
-		for col := range rows[i] {
-			m[fmt.Sprintf("col_%d", col)] = rows[i][col]
-		}
-
-		// Marshal the map to json and add to the sql statement.
-		if j, err := json.Marshal(m); err != nil {
-			return "", errors.Wrap(err, "marshalling row to json")
-		} else {
-			sb.Write(j)
-			sb.WriteString("\n")
-		}
-	}
-
-	// WITH
-	sb.WriteString(fmt.Sprintf(`' WITH BATCHSIZE %d FORMAT 'NDJSON' INPUT 'STREAM'`, len(rows)))
-
-	return sb.String(), nil
-}

cli/batch/sql_test.go

@@ -1,47 +0,0 @@
-package batch
-
-import (
-	"testing"
-
-	"github.com/featurebasedb/featurebase/v3/dax"
-	"github.com/stretchr/testify/assert"
-)
-
-func TestBatchSQL(t *testing.T) {
-	tbl := &dax.Table{
-		Name: "foo",
-	}
-	fields := []*dax.Field{
-		{
-			Name: "name",
-			Type: dax.BaseTypeString,
-		},
-		{
-			Name: "age",
-			Type: dax.BaseTypeInt,
-		},
-	}
-	ids := []interface{}{
-		0, 1, 2,
-	}
-	rows := [][]interface{}{
-		{
-			[]interface{}{"Alice", int64(11)},
-		},
-		{
-			[]interface{}{"Bob", int64(22)},
-		},
-		{
-			[]interface{}{"Carl,Comma", int64(33)},
-		},
-	}
-
-	s, err := buildBulkInsert(tbl, fields, ids, rows)
-	assert.NoError(t, err)
-
-	exp := `BULK INSERT INTO foo (_id,name,age) MAP ('$._id' id,'$.col_0' string,'$.col_1' int) FROM x'{"_id":0,"col_0":["Alice",11]}
-{"_id":1,"col_0":["Bob",22]}
-{"_id":2,"col_0":["Carl,Comma",33]}
-' WITH BATCHSIZE 3 FORMAT 'NDJSON' INPUT 'STREAM'`
-	assert.Equal(t, exp, s)
-}

cli/buffer.go

@@ -1,91 +0,0 @@
-package cli
-
-import (
-	"io"
-	"strings"
-
-	"github.com/featurebasedb/featurebase/v3/errors"
-)
-
-// buffer is a query buffer for SQL statements. Note that this is not a query
-// buffer as you would find on a database server (buffering query results).
-// Rather, this buffers the working SQL statement. The buffer has two
-// components: the buffer of query parts making up the working, incomplete SQL
-// statement, and the last completed SQL statement submitted to the Queryer.
-type buffer struct {
-	parts     []queryPart
-	lastQuery query
-
-	hasBatchFile bool
-}
-
-func newBuffer() *buffer {
-	return &buffer{}
-}
-
-// addPart adds the given queryPart to the buffer. If the part is of type
-// `partTerminator` (which is generally singified in the CLI by a ";"), the
-// buffer will finalize the query and return it. In all other cases, the
-// returned query is nil.
-func (b *buffer) addPart(part queryPart) (query, error) {
-	// Check for part type compatibility. For example, multiple batchFile parts
-	// are not allowed in the same query.
-	switch part.(type) {
-	case *partBatchFile:
-		if b.hasBatchFile {
-			return nil, errors.Errorf("multiple batch files in one query is not supported")
-		}
-		b.hasBatchFile = true
-	case *partTerminator:
-		return b.finalize(), nil
-	}
-
-	b.parts = append(b.parts, part)
-	return nil, nil
-}
-
-// finalize copies the contents (queryParts) of buffer to lastQuery and then
-// resets the buffer. It returns the query that was finalized.
-func (b *buffer) finalize() query {
-	q := make(query, len(b.parts))
-	copy(q, b.parts)
-	b.lastQuery = q
-	b.reset()
-	return q
-
-}
-
-// print returns the contents of the buffer as a string. This is generally used
-// to visually inspect the state of the buffer (for example, when a user issues
-// a `\p` meta-command in the CLI).
-func (b *buffer) print() string {
-	if len(b.parts) > 0 {
-		return query(b.parts).String()
-	} else if b.lastQuery != nil {
-		return b.lastQuery.String() + ";"
-	}
-	return "Query buffer is empty."
-}
-
-// reset clears the buffer. It returns a message which may optionally be used to
-// display to a user.
-func (b *buffer) reset() string {
-	b.parts = b.parts[:0]
-	b.hasBatchFile = false
-	return "Query buffer reset (cleared)."
-}
-
-func (b *buffer) Reader() io.Reader {
-	if len(b.parts) > 0 {
-		return query(b.parts).Reader()
-	} else if b.lastQuery != nil {
-		r := b.lastQuery.Reader()
-		// TODO(tlt): terminating the query here results in a line feed just
-		// before the semi-colon (for example, when you print out the query
-		// buffer using `\w [FILE]`). The removal and re-introduction of line
-		// feeds is kind of a mess.
-		term := strings.NewReader(";")
-		return io.MultiReader(r, term)
-	}
-	return strings.NewReader("")
-}

cli/cli.go

@@ -1,816 +0,0 @@
-// Package cli contains a FeatureBase command line interface.
-package cli
-
-import (
-	"context"
-	"fmt"
-	"io"
-	"net/http"
-	"os"
-	"path/filepath"
-	"strings"
-	"time"
-
-	"github.com/chzyer/readline"
-	featurebase "github.com/featurebasedb/featurebase/v3"
-	"github.com/featurebasedb/featurebase/v3/cli/batch"
-	"github.com/featurebasedb/featurebase/v3/cli/fbcloud"
-	"github.com/featurebasedb/featurebase/v3/errors"
-	"github.com/featurebasedb/featurebase/v3/logger"
-)
-
-const (
-	defaultHost     string = "localhost"
-	defaultClientID string = "6i2gs7mu215ab23cnvmshdoq6t" // production Cognito client ID
-	defaultRegion   string = "us-east-2"
-	terminationChar string = ";"
-	nullValue       string = "NULL"
-)
-
-var (
-	Stdin  io.ReadCloser = os.Stdin
-	Stdout io.Writer     = os.Stdout
-	Stderr io.Writer     = os.Stderr
-)
-
-var splash string = fmt.Sprintf(`FeatureBase CLI (%s)
-Type "\q" to quit.
-`, featurebase.Version)
-
-// Ensure type implments interfaces.
-var _ printer = (*Command)(nil)
-var _ batch.Inserter = (*Command)(nil)
-
-type Command struct {
-	host string
-	port string
-
-	splitter   *splitter
-	buffer     *buffer
-	workingDir *workingDir
-
-	organizationID string
-	database       string
-	databaseID     string
-	databaseName   string
-
-	Queryer Queryer `json:"-"`
-
-	stdin  io.ReadCloser `json:"-"`
-	stdout io.Writer     `json:"-"`
-	stderr io.Writer     `json:"-"`
-
-	// output is where actual results are written. This might point to stdout,
-	// or to a file, based on the current configuration.
-	output       io.Writer `json:"-"`
-	writeOptions *writeOptions
-
-	Config *Config `json:"config"`
-
-	historyPath string
-
-	// Commands contains optional commands provided via one or more `-c` (or
-	// `--command`) flags. If this is non-empty, the cli will run in
-	// non-interactive mode; i.e. it will quit after the command is complete.
-	Commands []string `json:"commands"`
-
-	// Files contains optional files provided via one or more `-f` (or `--file`)
-	// flags. If this is non-empty, the cli will run in non-interactive mode;
-	// i.e. it will quit after the command is complete.
-	Files []string `json:"files"`
-
-	// variables holds the variables created with the \set meta-command.
-	variables map[string]string
-
-	// nonInteractiveMode is set to true when fbsql is running in
-	// non-ineracative mode. And example of this is when the user has provided a
-	// `-c` flag in the command line.
-	nonInteractiveMode bool
-
-	// quit gets closed when Run should stop listening for input.
-	quit chan struct{}
-}
-
-func NewCommand(logdest logger.Logger) *Command {
-	variables := make(map[string]string)
-
-	return &Command{
-		Config: &Config{
-			Host: defaultHost,
-			Port: "",
-
-			OrganizationID: "",
-			Database:       "",
-
-			CloudAuth: CloudAuthConfig{
-				ClientID: defaultClientID,
-				Region:   defaultRegion,
-				Email:    "",
-				Password: "",
-			},
-
-			HistoryPath: "",
-
-			CSV: false,
-		},
-
-		buffer:     newBuffer(),
-		splitter:   newSplitter(newReplacer(variables)),
-		workingDir: newWorkingDir(),
-
-		stdin:  Stdin,
-		stdout: Stdout,
-		stderr: Stderr,
-
-		output:       Stdout,
-		writeOptions: defaultWriteOptions(),
-
-		variables: variables,
-
-		quit: make(chan struct{}),
-	}
-}
-
-// SetStdin sets stdin. This is useful for initial configuration in tests.
-func (cmd *Command) SetStdin(rc io.ReadCloser) {
-	cmd.stdin = rc
-}
-
-// SetStdout sets both stdout and output to the value provided. This is useful
-// for initial configuration in tests.
-func (cmd *Command) SetStdout(w io.Writer) {
-	cmd.stdout = w
-	cmd.output = w
-}
-
-// SetStderr sets stderr. This is useful for initial configuration in tests.
-func (cmd *Command) SetStderr(w io.Writer) {
-	cmd.stderr = w
-}
-
-// Run is the main entry-point to the CLI.
-func (cmd *Command) Run(ctx context.Context) error {
-	if err := cmd.run(ctx); err != nil {
-		cmd.Errorf(err.Error() + "\n")
-		return err
-	}
-	return nil
-}
-
-// run is effectively wrapped by the Run() method, but it's split out this way
-// so that run() can simply return errors, rather than worrying about how errors
-// should be printed; printing errors returned by run() is left up to the Run()
-// method.
-func (cmd *Command) run(ctx context.Context) error {
-	if err := cmd.setupConfig(); err != nil {
-		return errors.Wrap(err, "setting up config")
-	}
-
-	// Check to see if Command needs to run in non-interactive mode.
-	if len(cmd.Commands) > 0 ||
-		len(cmd.Files) > 0 ||
-		cmd.Config.KafkaConfig != "" ||
-		cmd.Config.CSV {
-		cmd.nonInteractiveMode = true
-	}
-
-	// Print the splash message.
-	if !cmd.nonInteractiveMode {
-		cmd.Printf(splash)
-	}
-
-	if err := cmd.setupClient(); err != nil {
-		return errors.Wrap(err, "setting up client")
-	}
-
-	// Print the connection info.
-	if !cmd.nonInteractiveMode {
-		cmd.printConnInfo()
-	}
-
-	if err := cmd.connectToDatabase(cmd.database); err != nil {
-		cmd.Errorf(errors.Wrap(err, "connecting to database").Error() + "\n")
-		// We intentionally do not return err here.
-	}
-
-	// Run in non-interactive mode based on flags and configuration.
-	// This includes either handling `-c` and/or `-f` flags, or handling a
-	// `--kafka-config` flag.
-	if len(cmd.Commands) > 0 || len(cmd.Files) > 0 {
-		// Run Commands.
-		for _, line := range cmd.Commands {
-			if err := cmd.handleLine(line); err != nil {
-				return errors.Wrapf(err, "handling line: %s", line)
-			}
-		}
-
-		// Run Files.
-		for _, fname := range cmd.Files {
-			if _, err := executeFile(cmd, fname); err != nil {
-				return errors.Wrapf(err, "executing file: %s", fname)
-			}
-		}
-
-		return nil
-	} else if cmd.Config.KafkaConfig != "" {
-		runner, err := cmd.newKafkaRunner(cmd.Config.KafkaConfig)
-		if err != nil {
-			return errors.Wrap(err, "getting new kafka runner")
-		}
-		if err := runner.Main.Run(); err != nil {
-			return errors.Wrap(err, "running kafka")
-		}
-		return nil
-	}
-
-	// From this point on, we should be in interactive mode.
-
-	// Set up history for saving user input.
-	cmd.setupHistory()
-
-	rl, err := readline.NewEx(&readline.Config{
-		Prompt:                 cmd.prompt(false),
-		HistoryFile:            cmd.historyPath,
-		HistoryLimit:           100000,
-		DisableAutoSaveHistory: true,
-
-		Stdin:  cmd.stdin,
-		Stdout: cmd.stdout,
-		Stderr: cmd.stderr,
-	})
-	if err != nil {
-		return errors.Wrap(err, "getting readline")
-	}
-	defer rl.Close()
-
-	// inMidCommand indicates whether a partial command has been received and
-	// we're still waiting for a termination character.
-	var inMidCommand bool
-
-	for {
-		rl.SetPrompt(cmd.prompt(inMidCommand))
-
-		// Read user provided input.
-		line, err := rl.Readline()
-		if err == readline.ErrInterrupt {
-			inMidCommand = false
-			cmd.buffer.reset()
-			continue
-		} else if err != nil {
-			return errors.Wrap(err, "reading line")
-		}
-
-		// We append a line feed at the end of each line because at this point
-		// we have effectively stripped any intentional line feeds (since we are
-		// reading a line at a time), and we don't want to do that. An example
-		// of an intentional line feed is in a BULK INSERT CSV STREAM like this
-		// example:
-		//
-		// bulk replace
-		// 	 into foo (_id, age)
-		// 	 map (0 id, 1 int)
-		// from
-		// 	 x'3,33
-		// 	 4,44
-		// 	 5,55'
-		// with
-		// 	 format 'CSV'
-		// 	 input 'STREAM';
-		//
-		// We want to preserve the line feeds that are contained in the x''
-		// block; those are intentional as they demarc records within the csv.
-		qps, mcs, err := cmd.splitter.split(line + "\n")
-		if err != nil {
-			cmd.Errorf("error splitting line: %s\n", err)
-			continue
-		}
-
-		// Save line in the history.
-		if err := rl.SaveHistory(line); err != nil {
-			cmd.Errorf("Couldn't save history: %v\n", err)
-		}
-
-		// This is wrapped in an anonymous function so we can capture any
-		// errors, ignore the rest of the line, and return back to a prompt.
-		if err := func() error {
-			for i := range qps {
-				if qry, err := cmd.buffer.addPart(qps[i]); err != nil {
-					return errors.Wrap(err, "adding part to buffer")
-				} else if qry != nil {
-					if err := cmd.executeAndWriteQuery(qry); err != nil {
-						return errors.Wrap(err, "executing query")
-					}
-					// In addition to saving each line in the history, we also
-					// save each successful query.
-					if err := rl.SaveHistory(qry.String() + ";"); err != nil {
-						cmd.Errorf("Couldn't save query in history: %v\n", err)
-					}
-					inMidCommand = false
-				} else {
-					inMidCommand = true
-				}
-			}
-			return nil
-		}(); err != nil {
-			cmd.Errorf(err.Error() + "\n")
-			inMidCommand = false
-			continue
-		}
-
-		// This is wrapped in an anonymous function so we can capture any
-		// errors, ignore the rest of the line, and return back to a prompt.
-		if err := func() error {
-			for i := range mcs {
-				action, err := mcs[i].execute(cmd)
-				if err != nil {
-					return errors.Wrap(err, "executing meta command")
-				}
-				switch action {
-				case actionQuit:
-					close(cmd.quit)
-					return nil
-				case actionReset:
-					inMidCommand = false
-				}
-			}
-			return nil
-		}(); err != nil {
-			cmd.Errorf(err.Error() + "\n")
-			inMidCommand = false
-			continue
-		}
-
-		select {
-		case <-cmd.quit:
-			if err := cmd.close(); err != nil {
-				cmd.Errorf("closing: %s\n", err)
-			}
-			return nil
-		default:
-			// pass
-		}
-	}
-}
-
-// prompt constructs the prompt that the user sees based on the currently
-// connected database and whether the user is in the middle of a sql statement.
-func (cmd *Command) prompt(mid bool) string {
-	db := "fbsql" // default prompt when a database is not set.
-	if cmd.databaseName != "" {
-		db = cmd.databaseName
-	}
-
-	if mid {
-		return strings.Repeat(" ", len(db)) + "-# "
-	}
-	return db + "=# "
-}
-
-// close is called upon quitting. It should close any remaining open file
-// handles used by the CLICommand.
-func (cmd *Command) close() error {
-	return cmd.closeOutput()
-}
-
-// setupConfig sets up private struct members based on values provided via the
-// configuration flags.
-func (cmd *Command) setupConfig() error {
-	if cmd.Config == nil {
-		return nil
-	}
-
-	cmd.host = cmd.Config.Host
-	cmd.port = cmd.Config.Port
-
-	cmd.organizationID = cmd.Config.OrganizationID
-	cmd.database = cmd.Config.Database
-
-	cmd.historyPath = cmd.Config.HistoryPath
-
-	// Apply any pset flag arguments.
-	for _, pset := range cmd.Config.PSets {
-		if err := cmd.applyPSet(pset); err != nil {
-			return errors.Wrapf(err, "applying pset: %s", pset)
-		}
-	}
-
-	// If running with the `--csv` flag, configure things to ensure the output
-	// is correct (i.e. that it's just the csv).
-	if cmd.Config.CSV {
-		cmd.writeOptions.format = formatCSV
-	}
-
-	return nil
-}
-
-// applyPSet takes a pset string of the form `arg` or `arg=val` and applies it
-// as if the user had run `\pset arg val`. The only difference is that applying
-// pset here suppresses any output to stdout.
-func (cmd *Command) applyPSet(pset string) error {
-	// We expect arg to be one of the folowing formats:
-	// arg
-	// arg=val
-	args := strings.SplitN(pset, "=", 2)
-
-	// This is kind of hacky, but until we re-think the metaCommand interface to
-	// take a printer interface somewhere (so we can pass in the nopPrinter
-	// here), we're just going to discard stdout for the duration of this apply,
-	// and then set stdout back to its previous writer after the apply.
-	hold := cmd.stdout
-	cmd.stdout = io.Discard
-	defer func() {
-		cmd.stdout = hold
-	}()
-
-	_, err := newMetaPSet(args).execute(cmd)
-	return err
-}
-
-func (cmd *Command) executeAndWriteQuery(qry query) error {
-	queryResponse, err := cmd.executeQuery(qry)
-	if err != nil {
-		if errors.Is(err, ErrOrganizationRequired) {
-			// Print an error message and return nil, effectively aborting any
-			// further writes for this query.
-			cmd.Errorf("Organization required. Use \\org to set an organization.\n")
-			return nil
-		}
-		return errors.Wrap(err, "making query")
-	}
-	if err := writeOutput(queryResponse, cmd.writeOptions, cmd.output, cmd.stdout, cmd.stderr); err != nil {
-		return errors.Wrap(err, "writing out response")
-	}
-
-	return nil
-}
-
-func (cmd *Command) executeQuery(qry query) (*featurebase.WireQueryResponse, error) {
-	wqr, err := cmd.Queryer.Query(cmd.organizationID, cmd.databaseID, qry.Reader())
-	if err != nil {
-		return nil, errors.Wrap(err, "executing query")
-	}
-
-	// If we're running in non-interactive mode, we need to check the error that
-	// comes back in the WireQueryResponse. If there's an error, we want to
-	// return it now (rather than just printing it later) so that we immediately
-	// stop any further execution of commands.
-	if cmd.nonInteractiveMode && wqr.Error != "" {
-		return nil, errors.Errorf(wqr.Error)
-	}
-
-	return wqr, nil
-}
-
-// printer is an interface which encapsulates the methods used to print output
-// to the various io.Writers.
-type printer interface {
-	Printf(format string, a ...any)
-	Outputf(format string, a ...any)
-	Errorf(format string, a ...any)
-}
-
-type nopPrinter struct{}
-
-func newNopPrinter() *nopPrinter {
-	return &nopPrinter{}
-}
-
-func (n *nopPrinter) Printf(format string, a ...any)  {}
-func (n *nopPrinter) Outputf(format string, a ...any) {}
-func (n *nopPrinter) Errorf(format string, a ...any)  {}
-
-// Printf is a helper method which sends the given payload to stdout.
-func (cmd *Command) Printf(format string, a ...any) {
-	out := fmt.Sprintf(format, a...)
-	cmd.stdout.Write([]byte(out))
-}
-
-// Outputf is a helper method which sends the given payload to output.
-func (cmd *Command) Outputf(format string, a ...any) {
-	out := fmt.Sprintf(format, a...)
-	cmd.output.Write([]byte(out))
-}
-
-// Errorf is a helper method which sends the given payload to stderr.
-func (cmd *Command) Errorf(format string, a ...any) {
-	out := fmt.Sprintf(format, a...)
-	cmd.stderr.Write([]byte(out))
-}
-
-func (cmd *Command) setupHistory() {
-	// If HistoryPath has already been configured (i.e. with a command flag),
-	// don't bother setting up the default in the home directory.
-	if cmd.historyPath != "" {
-		return
-	}
-
-	historyPath := ""
-	if home, err := os.UserHomeDir(); err != nil {
-		cmd.Errorf("Error getting home directory, command history persistence will be disabled: %v\n", err)
-	} else {
-		historyDir := filepath.Join(home, ".featurebase")
-		err := os.MkdirAll(historyDir, 0o750)
-		if err != nil {
-			cmd.Errorf("Creating directory for history: %v\n", err)
-		} else {
-			historyPath = filepath.Join(historyDir, "fbsql_history")
-		}
-	}
-	cmd.historyPath = historyPath
-}
-
-// printConnInfo displays the currently set host.
-// TODO(tlt): extend this to be the output of the /conninfo meta-command.
-func (cmd *Command) printConnInfo() {
-	cmd.Printf("Host: %s\n", hostPort(cmd.host, cmd.port))
-}
-
-func (cmd *Command) connectToDatabase(dbName string) error {
-	var p printer = cmd
-	if cmd.nonInteractiveMode {
-		p = newNopPrinter()
-	}
-
-	// Providing a blank ("") or hyphen ("-") dbName is the equivalent of
-	// disconnecting from the current database. We support the hyphen option
-	// because calling the `\c` meta-command without an argument is how you
-	// print the current connection.
-	switch dbName {
-	case "-", "":
-		cmd.databaseID = ""
-		cmd.databaseName = ""
-		p.Printf(cmd.connectionMessage())
-		return nil
-	}
-
-	// Look up dbID based on dbName.
-	wqr, err := cmd.executeQuery(newRawQuery("SHOW DATABASES"))
-	if err != nil {
-		return errors.Wrap(err, "executing query")
-	}
-
-	for _, db := range wqr.Data {
-		// 0: _id
-		// 1: name
-		if db[1] == dbName {
-			cmd.databaseName = dbName
-			cmd.databaseID = db[0].(string)
-			p.Printf(cmd.connectionMessage())
-			return nil
-		}
-	}
-	return errors.Errorf("invalid database: %s", dbName)
-}
-
-func (cmd *Command) orgMessage() string {
-	if cmd.organizationID == "" {
-		return "You have not set an organization.\n"
-	}
-	return fmt.Sprintf("You have set organization \"%s\".\n", cmd.organizationID)
-}
-
-func (cmd *Command) connectionMessage() string {
-	if cmd.databaseName == "" {
-		return "You are not connected to a database.\n"
-	}
-	return fmt.Sprintf("You are now connected to database \"%s\" (%s).\n", cmd.databaseName, cmd.databaseID)
-}
-
-func (cmd *Command) setupClient() error {
-	// If the Queryer has already been set (in tests for example), don't bother
-	// trying to detect it.
-	if cmd.Queryer != nil {
-		return nil
-	}
-
-	var p printer = cmd
-	if cmd.nonInteractiveMode {
-		p = newNopPrinter()
-	}
-
-	if strings.TrimSpace(cmd.host) == "" {
-		return errors.Errorf("no host provided\n")
-	}
-
-	if !strings.HasPrefix(cmd.host, "http") {
-		cmd.host = "http://" + cmd.host
-	}
-
-	typ, err := cmd.detectFBType()
-	if err != nil {
-		return errors.Wrap(err, "detecting FeatureBase deployment type")
-	}
-
-	switch typ {
-	case featurebaseTypeOnPremClassic:
-		p.Printf("Detected on-prem, classic deployment.\n")
-		cmd.Queryer = &standardQueryer{
-			Host: cmd.host,
-			Port: cmd.port,
-		}
-	case featurebaseTypeOnPremServerless:
-		p.Printf("Detected on-prem, serverless deployment.\n")
-		cmd.Queryer = &serverlessQueryer{
-			Host: cmd.host,
-			Port: cmd.port,
-		}
-	case featurebaseTypeCloud:
-		p.Printf("Detected cloud deployment.\n")
-
-		cmd.Queryer = &fbcloud.Queryer{
-			Host: hostPort(cmd.host, cmd.port),
-
-			ClientID: cmd.Config.CloudAuth.ClientID,
-			Region:   cmd.Config.CloudAuth.Region,
-			Email:    cmd.Config.CloudAuth.Email,
-			Password: cmd.Config.CloudAuth.Password,
-		}
-	case featurebaseTypeUnknown:
-		p.Printf("Could not detect deployment\n")
-		// cmd.Queryer = &nopQueryer{}
-		// Instead of using a no-op queryer when the type can't be detected, we
-		// default to using a cloud queryer.
-		cmd.Queryer = &fbcloud.Queryer{
-			Host: hostPort(cmd.host, cmd.port),
-
-			ClientID: cmd.Config.CloudAuth.ClientID,
-			Region:   cmd.Config.CloudAuth.Region,
-			Email:    cmd.Config.CloudAuth.Email,
-			Password: cmd.Config.CloudAuth.Password,
-		}
-	default:
-		return errors.Errorf("unknown type: %s", typ)
-	}
-	return nil
-}
-
-type featurebaseType string
-
-const (
-	featurebaseTypeUnknown          featurebaseType = "unknown"            // unknown
-	featurebaseTypeOnPremClassic    featurebaseType = "on-prem-standard"   // on-prem, classic
-	featurebaseTypeOnPremServerless featurebaseType = "on-prem-serverless" // on-prem, serverless
-	featurebaseTypeCloud            featurebaseType = "cloud"              // cloud, (both classic and serverless)?
-)
-
-func hostPort(host, port string) string {
-	if port == "" {
-		return host
-	}
-	return host + ":" + port
-}
-
-// detectFBType determines if we're talking to standalone FeatureBase
-// or FeatureBase Cloud
-func (cmd *Command) detectFBType() (featurebaseType, error) {
-	type trial struct {
-		port   string
-		health string
-		typ    featurebaseType
-	}
-
-	// trials is populated with the url/endpoints to try in order to detect if a
-	// process is running there which can support the cli requests.
-	trials := []trial{}
-
-	var clientTimeout time.Duration
-	if cmd.port != "" {
-		clientTimeout = 100 * time.Millisecond
-		trials = append(trials,
-			// on-prem, serverless
-			trial{
-				port:   cmd.port,
-				health: "/queryer/health",
-				typ:    featurebaseTypeOnPremServerless,
-			},
-			// on-prem, classic
-			trial{
-				port:   cmd.port,
-				health: "/status",
-				typ:    featurebaseTypeOnPremClassic,
-			},
-		)
-	} else if strings.HasPrefix(cmd.host, "https") {
-		// https suggesting we might be connecting to a cloud host
-		clientTimeout = 1 * time.Second
-		trials = append(trials,
-			// cloud
-			trial{
-				port:   "",
-				health: "/health",
-				typ:    featurebaseTypeCloud,
-			},
-		)
-	} else {
-		// Try default ports just in case.
-		clientTimeout = 100 * time.Millisecond
-		trials = append(trials,
-			// on-prem, serverless
-			trial{
-				port:   "8080",
-				health: "/queryer/health",
-				typ:    featurebaseTypeOnPremServerless,
-			},
-			// on-prem, classic
-			trial{
-				port:   "10101",
-				health: "/status",
-				typ:    featurebaseTypeOnPremClassic,
-			},
-		)
-	}
-
-	client := http.Client{
-		Timeout: clientTimeout,
-	}
-	for _, trial := range trials {
-		url := hostPort(cmd.host, trial.port) + trial.health
-		if resp, err := client.Get(url); err != nil {
-			continue
-		} else if resp.StatusCode/100 == 2 {
-			cmd.port = trial.port
-			return trial.typ, nil
-		}
-	}
-
-	return featurebaseTypeUnknown, nil
-}
-
-func (cmd *Command) closeOutput() error {
-	if cmd.output == nil {
-		return nil
-	}
-
-	if closer, ok := cmd.output.(io.Closer); ok {
-		return closer.Close()
-	}
-
-	return nil
-}
-
-func (cmd *Command) handleLine(line string) error {
-	// For single-line command handling, we handle either a meta-command, or
-	// query parts, but not both. The logic is that any line which begins with
-	// "\" will be handled as a meta-command, otherwise it will be handled as a
-	// query.
-	if len(line) == 0 {
-		return nil
-	} else if line[0] == byte('\\') {
-		return cmd.handleLineAsMetaCommand(line)
-	} else {
-		return cmd.handleLineAsQueryParts(line)
-	}
-}
-
-func (cmd *Command) handleLineAsMetaCommand(line string) error {
-	_, mcs, err := cmd.splitter.split(line)
-	if err != nil {
-		return errors.Wrapf(err, "splitting line")
-	}
-
-	for i := range mcs {
-		_, err := mcs[i].execute(cmd)
-		if err != nil {
-			return errors.Wrap(err, "executing meta command")
-		}
-	}
-
-	return nil
-}
-
-func (cmd *Command) handleLineAsQueryParts(line string) error {
-	qps, mcs, err := cmd.splitter.split(line)
-	if err != nil {
-		return errors.Wrapf(err, "splitting line")
-	} else if len(mcs) > 0 {
-		return errors.Errorf("--command does not support meta-commands")
-	}
-
-	// Add a termintor part to the end of []queryPart. We do this because the
-	// command is coming in from the --command flag, it may not end with a
-	// semi-colon, but we still want to execute it.
-	if len(qps) > 0 {
-		if _, ok := qps[len(qps)-1].(*partTerminator); !ok {
-			qps = append(qps, newPartTerminator())
-		}
-	}
-
-	for i := range qps {
-		if qry, err := cmd.buffer.addPart(qps[i]); err != nil {
-			return errors.Wrap(err, "adding part to buffer")
-		} else if qry != nil {
-			if err := cmd.executeAndWriteQuery(qry); err != nil {
-				return errors.Wrap(err, "executing query")
-			}
-		}
-	}
-	return nil
-}
-
-func (cmd *Command) Insert(sql string) error {
-	wqr, err := cmd.executeQuery(newRawQuery(sql))
-	if wqr.Error != "" {
-		return errors.Errorf(wqr.Error)
-	}
-	return err
-}

cli/cli_integration_test.go

@@ -1,257 +0,0 @@
-package cli_test
-
-import (
-	"bufio"
-	"context"
-	"fmt"
-	"os"
-	"strings"
-	"testing"
-	"time"
-
-	"github.com/featurebasedb/featurebase/v3/cli"
-	"github.com/featurebasedb/featurebase/v3/dax/server/test"
-	"github.com/featurebasedb/featurebase/v3/logger"
-	"github.com/stretchr/testify/require"
-)
-
-func TestCLIIntegration(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skipping integration test")
-	}
-
-	ctx := context.Background()
-
-	t.Run("Stubbed Framework", func(t *testing.T) {
-		mc := test.MustRunManagedCommand(t)
-		defer mc.Close()
-
-		addr := mc.Address()
-
-		capture := newCapture(t)
-
-		comparer := newComparer(t)
-		comparer.run()
-
-		fbsql := cli.NewCommand(logger.StderrLogger)
-		fbsql.SetStdin(capture)
-		fbsql.SetStdout(comparer)
-		fbsql.SetStderr(comparer)
-
-		fbsql.Config = &cli.Config{
-			Host: addr.Host(),
-			Port: fmt.Sprintf("%d", addr.Port()),
-		}
-
-		// Run fbsql in a goroutine so we can continue to send it commands
-		// below.
-		didQuit := make(chan struct{})
-		go func() {
-			require.NoError(t, fbsql.Run(ctx))
-			close(didQuit)
-		}()
-
-		// testFiles reference files located in the cli/testdata directory. All
-		// tests should be placed there; other than adding another test file to
-		// this list, you probably shouldn't be editing this file unless you are
-		// trying to modify the way the test framework itself works.
-		testFiles := []string{
-			"setup",
-			"database",
-			"table",
-			// the tests below may be dependent on the previous tests, which do
-			// setup and some shared database and table creation.
-			"query_buffer",
-			// meta commands
-			"meta_bang",
-			"meta_cd",
-			"meta_echo",
-			"meta_describe",
-			"meta_file",
-			"meta_pset_border",
-			"meta_pset_expanded",
-			"meta_pset_format_csv",
-			"meta_pset_tuples_only",
-			"meta_include",
-			"meta_output",
-			"meta_set",
-			"meta_timing",
-			"meta_write",
-		}
-
-		for _, testFile := range testFiles {
-			t.Run(testFile, func(t *testing.T) {
-				f, err := os.Open("testdata/" + testFile)
-				require.NoError(t, err)
-
-				scanner := bufio.NewScanner(f)
-				var lineNo int
-				for scanner.Scan() {
-					line := scanner.Text()
-					lineNo++
-
-					// Empty lines and comments (//) are ignored.
-					if line == "" {
-						continue
-					} else if strings.HasPrefix(line, "//") {
-						continue
-					}
-
-					parts := strings.SplitN(line, ":", 2)
-
-					switch parts[0] {
-					case "SEND":
-						v := ""
-						if len(parts) == 2 {
-							v = parts[1]
-						}
-						capture.sendLine(v)
-					case "EXPECT":
-						v := ""
-						if len(parts) == 2 {
-							v = parts[1]
-						}
-						comparer.expectLine(v, testFile, lineNo)
-					case "EXPECTCOMP":
-						if len(parts) == 2 {
-							comps := strings.SplitN(parts[1], ":", 2)
-							v := ""
-							if len(comps) == 2 {
-								v = comps[1]
-							}
-							comparer.expectLineComp(comparator(comps[0]), v, testFile, lineNo)
-						} else {
-							t.Errorf("unexpected line: %s[%d]:%s", testFile, lineNo, line)
-						}
-					default:
-						t.Errorf("unexpected line: %s[%d]:%s", testFile, lineNo, line)
-					}
-				}
-				require.NoError(t, scanner.Err())
-			})
-		}
-
-		// End with quit to ensure that fbsql closes without error.
-		capture.sendLine(`\q`)
-
-		// Ensure fbsql quits cleanly.
-		select {
-		case <-didQuit:
-		case <-time.After(time.Second):
-			t.Fatalf("expected fbsql to quit")
-		}
-	})
-}
-
-// compare is used to compare fbsql output written to its Stdout with expected
-// lines.
-type comparer struct {
-	t       *testing.T
-	out     chan byte
-	outline chan []byte
-	exp     chan []byte
-}
-
-func newComparer(t *testing.T) *comparer {
-	return &comparer{
-		t:       t,
-		out:     make(chan byte, 1024),
-		outline: make(chan []byte, 128),
-		exp:     make(chan []byte, 1024),
-	}
-}
-
-func (c *comparer) run() {
-	// Read bytes off output, and for every line (designated by a line feed "\n"),
-	// push the line onto the outline channel.
-	go func() {
-		var line []byte
-		for {
-			b := <-c.out
-			if b == byte('\n') {
-				c.outline <- line
-				line = []byte{}
-				continue
-			}
-			line = append(line, b)
-		}
-	}()
-}
-
-type comparator string
-
-const (
-	compEquals     = "Equals"
-	compHasPrefix  = "HasPrefix"
-	compWithFormat = "WithFormat"
-)
-
-// expectLine is a convenience method which calls expectLineComp with the compEq
-// comparator and the given line.
-func (c *comparer) expectLine(line string, fileName string, lineNo int) {
-	c.expectLineComp(compEquals, line, fileName, lineNo)
-}
-
-// expectLineComp reads the next line from the outline channel and compares it
-// with the given `line`. A comparator can be provided to inform how the lines
-// should be compared (for example, the compHasPrefix comparator will just
-// compare the beginning part of the outline).
-func (c *comparer) expectLineComp(comp comparator, line string, fileName string, lineNo int) {
-	var outline []byte
-	select {
-	case outline = <-c.outline:
-	case <-time.After(10 * time.Second):
-		// TODO(tlt): this is 10 seconds to account for the fb_views creation on
-		// a local mac. This should really be something like 2 seconds. Put this
-		// back to 2 once fb_views issue is addressed.
-		c.t.Fatalf("expected output line %s[%d]: >%s<", fileName, lineNo, line)
-	}
-
-	// msg is included in any require which fails.
-	msg := []interface{}{"exp: %s[%d], got: >%s<", fileName, lineNo, outline}
-
-	switch comp {
-	case compEquals:
-		require.Equal(c.t, []byte(line), outline, msg...)
-	case compHasPrefix:
-		require.True(c.t, strings.HasPrefix(string(outline), line), msg...)
-	case compWithFormat:
-		require.True(c.t, compareByteSlices(outline, []byte(line)), msg...)
-	default:
-		c.t.Fatalf("invalid comparator: %s", comp)
-	}
-}
-
-func (c *comparer) Write(b []byte) (n int, err error) {
-	for i := range b {
-		c.out <- b[i]
-	}
-	return len(b), err
-}
-
-// compareByteSlices compares a byte slice s with another byte slice format and
-// returns true if they are the same. It will accept underscore as a
-// single-character wildcard anywhere in slice format.
-func compareByteSlices(s, format []byte) bool {
-	// Replace some helpers in format before comparing.
-	f := string(format)
-	f = strings.ReplaceAll(f, `{uuid}`, `________-____-____-____-____________`)
-	f = strings.ReplaceAll(f, `{timestamp}`, `____-__-__T__:__:__Z`)
-	format = []byte(f)
-
-	if len(s) != len(format) {
-		return false
-	}
-
-	for i := range s {
-		if format[i] == '_' {
-			continue
-		}
-		if s[i] != format[i] {
-			// log.Printf("DEBUG: characters differ: (%d): '%v' != '%v'", i, s[i], format[i])
-			return false
-		}
-	}
-
-	return true
-}

cli/cli_test.go

@@ -1,199 +0,0 @@
-package cli_test
-
-import (
-	"context"
-	"io"
-	"strings"
-	"sync"
-	"testing"
-	"time"
-
-	featurebase "github.com/featurebasedb/featurebase/v3"
-	"github.com/featurebasedb/featurebase/v3/cli"
-	"github.com/featurebasedb/featurebase/v3/logger"
-	"github.com/pkg/errors"
-	"github.com/stretchr/testify/assert"
-)
-
-func TestCLI(t *testing.T) {
-	t.Run("Input", func(t *testing.T) {
-		ctx := context.Background()
-
-		capture := newCapture(t)
-
-		cli := cli.NewCommand(logger.StderrLogger)
-		cli.SetStdin(capture)
-		cli.SetStdout(capture)
-		cli.Queryer = capture
-
-		go func() {
-			assert.NoError(t, cli.Run(ctx))
-		}()
-
-		none := []string{}
-
-		// One statement, one line.
-		capture.Assert("one;", []string{"one\n"})
-
-		// One statement, multiple lines.
-		capture.Assert("one", none)
-		capture.Assert(" two ", none)
-		capture.Assert("three;", []string{"one\ntwo\nthree\n"})
-
-		// Multiple statements, one line.
-		capture.Assert("foo; bar;", []string{"foo\n", "bar\n"})
-
-		// Multiple statements, multiple lines.
-		capture.Assert("a1", none)
-		capture.Assert("a2; b1", []string{"a1\na2\n"})
-		capture.Assert("b2;", []string{"b1\nb2\n"})
-
-		// Blank lines.
-		capture.Assert("one", none)
-		capture.Assert("", none)
-		capture.Assert("three;", []string{"one\nthree\n"})
-
-		// Just a semi-colon.
-		capture.Assert(";", []string{""})
-
-		// Multi-line with just a semi-colon.
-		capture.Assert("one", none)
-		capture.Assert(";", []string{"one\n"})
-
-		// Ensure a clean exit with no errors.
-		assert.NoError(t, capture.Exit())
-	})
-}
-
-////////////////////////////////////////////////////////
-
-// Ensure type implementes interface.
-var _ io.ReadCloser = (*capture)(nil)
-var _ io.Writer = (*capture)(nil)
-var _ cli.Queryer = (*capture)(nil)
-
-// capture implements the various CLI interfaces in order to capture test input
-// and submit it as though that input were being read from the command line. It
-// also captures calls made to the Queryer.Query method and ensures the sql they
-// contain is expected.
-type capture struct {
-	t *testing.T
-
-	// ch is a channel of strings (one line at a time) of CLI input.
-	ch chan string
-
-	mu   sync.RWMutex
-	sqls []string
-
-	// queryDone will receive an event any time the Query method is called and
-	// has completed. This is to tell the Assert method that it's safe to
-	// compare the sqls slice.
-	queryDone chan struct{}
-
-	asserting chan struct{}
-
-	err error
-}
-
-func newCapture(t *testing.T) *capture {
-	return &capture{
-		t:         t,
-		ch:        make(chan string),
-		sqls:      make([]string, 0),
-		queryDone: make(chan struct{}),
-	}
-}
-
-func (c *capture) Exit() error {
-	c.sendLine(`\q`)
-	c.mu.RLock()
-	defer c.mu.RUnlock()
-	return c.err
-}
-
-func (c *capture) Assert(in string, out []string) {
-	c.asserting = make(chan struct{})
-
-	c.sendLine(in)
-
-	// Wait for the CLI command to complete processing the input and send the
-	// sql to Query() by blocking on the queryDone channel. Because Query gets
-	// called for every sql statement in the input, an input resulting in
-	// multiple sql statements needs to wait for all expected queries to
-	// complete. A timeout is included to this so it doesn't deadlock in the
-	// case where Query is expected to be called, but isn't; after the timeout,
-	// the test should fail completely. In summary: we wait on queryDone the
-	// number of sql statements we expect. If we receive fewer than expected,
-	// the timeout will occur. If we receive more than expected, the Query()
-	// method will effectively deadlock, reach its own timout, then write to
-	// capture.err, which will be reported upon Exit().
-	for range out {
-		select {
-		case <-c.queryDone:
-		case <-time.After(2 * time.Second):
-			c.t.Fatalf("expected Query() to be called")
-		}
-	}
-
-	close(c.asserting)
-
-	c.mu.Lock()
-	defer c.mu.Unlock()
-
-	assert.Equal(c.t, out, c.sqls)
-
-	// Reset the slice.
-	c.sqls = c.sqls[:0]
-
-}
-
-// sendLine sends the given string as a line input to the CLI command. It
-// appends a line feed to the end of string in order to mimic the user hitting
-// the return key.
-func (c *capture) sendLine(s string) {
-	// Add a line feed before putting s on the channel in order to mimic the
-	// user hitting the return key.
-	c.ch <- s + "\n"
-}
-
-// Read is read by the CLI in place of user input. It effectively sends lines of
-// input to the CLI, getting each line to be sent off the channel.
-func (c *capture) Read(b []byte) (n int, err error) {
-	s := <-c.ch
-	return strings.NewReader(s).Read(b)
-}
-
-func (c *capture) Close() error {
-	close(c.ch)
-	return nil
-}
-
-// Write is called with anything written to output. This would included results
-// from calling Query() under normal, non-testing conditions, as well as other
-// informational text sent to output, such as the splash message.
-func (c *capture) Write(b []byte) (n int, err error) {
-	return 0, nil
-}
-
-// Query is called by the CLI command once a full SQL statement is received
-// (signified by the terminator: `;`).
-func (c *capture) Query(org string, db string, sql io.Reader) (*featurebase.WireQueryResponse, error) {
-	tmpBuf := new(strings.Builder)
-	_, err := io.Copy(tmpBuf, sql)
-	if err != nil {
-		return nil, err
-	}
-
-	c.mu.Lock()
-	c.sqls = append(c.sqls, tmpBuf.String())
-	c.mu.Unlock()
-
-	select {
-	case c.queryDone <- struct{}{}:
-	case <-c.asserting:
-		c.mu.Lock()
-		c.err = errors.Errorf("unexpected query: %s", sql)
-		c.mu.Unlock()
-	}
-	return &featurebase.WireQueryResponse{}, nil
-}

cli/config.go

@@ -1,31 +0,0 @@
-package cli
-
-// Config represents the configuration for the command.
-type Config struct {
-	Host string `json:"host"`
-	Port string `json:"port"`
-
-	OrganizationID string `json:"org-id"`
-	Database       string `json:"db"`
-
-	// CloudAuth
-	CloudAuth CloudAuthConfig `json:"cloud-auth"`
-
-	// Kafka
-	KafkaConfig string `json:"kafka-config"`
-
-	HistoryPath string `json:"history-path"`
-
-	// CSV (Comma-Separated Values) table output mode.
-	CSV bool `json:"csv"`
-
-	// PSet takes one or more pset arguments of the form: `--pset=VAR[=ARG]`.
-	PSets []string `json:"pset"`
-}
-
-type CloudAuthConfig struct {
-	ClientID string `json:"client-id"`
-	Region   string `json:"region"`
-	Email    string `json:"email"`
-	Password string `json:"password"`
-}

cli/errors.go

@@ -1,16 +0,0 @@
-package cli
-
-import (
-	"github.com/featurebasedb/featurebase/v3/errors"
-)
-
-const (
-	ErrOrganizationRequired errors.Code = "OrganizationRequired"
-)
-
-func NewErrOrganizationRequired() error {
-	return errors.New(
-		ErrOrganizationRequired,
-		"organization required",
-	)
-}

cli/kafka.go

@@ -1,70 +0,0 @@
-package cli
-
-import (
-	"fmt"
-
-	"github.com/featurebasedb/featurebase/v3/cli/batch"
-	"github.com/featurebasedb/featurebase/v3/cli/kafka"
-	"github.com/featurebasedb/featurebase/v3/errors"
-	"github.com/spf13/viper"
-)
-
-func (cmd *Command) newKafkaRunner(cfgFile string) (*kafka.Runner, error) {
-	// Read the kafka config file.
-	v := viper.New()
-	v.SetConfigFile(cfgFile)
-	v.SetConfigType("toml")
-	err := v.ReadInConfig()
-	if err != nil {
-		return nil, fmt.Errorf("error reading configuration file '%s': %v", cfgFile, err)
-	}
-
-	cfg := kafka.Config{}
-	if err := v.Unmarshal(&cfg); err != nil {
-		return nil, errors.Wrap(err, "unmarshalling config")
-	}
-
-	if err := kafka.ValidateConfig(cfg); err != nil {
-		return nil, errors.Wrap(err, "validating config")
-	}
-
-	// Create a new config with defaults.
-
-	// Look up fields based on table provided in the config.
-	wqr, err := cmd.executeQuery(newRawQuery("SHOW COLUMNS FROM " + cfg.Table))
-	if err != nil {
-		return nil, errors.Wrap(err, "executing query")
-	}
-
-	scr, err := wqr.ShowColumnsResponse()
-	if err != nil {
-		return nil, errors.Wrap(err, "getting show columns from wire query response")
-	}
-
-	// If no fields were provided in the config, use the fields defined on the
-	// table and assume a 1-to-1 mapping of source to destination.
-	if len(cfg.Fields) == 0 {
-		cfg.Fields = kafka.FieldsToConfig(scr.Fields)
-	} else {
-		cfg.Fields, err = kafka.CheckFieldCompatibility(cfg.Fields, scr)
-		if err != nil {
-			return nil, errors.Wrap(err, "validating config fields")
-		}
-	}
-
-	idkCfg, err := kafka.ConvertConfig(cfg)
-	if err != nil {
-		return nil, errors.Wrap(err, "cleaning config")
-	}
-
-	flds, err := kafka.ConfigToFields(cfg)
-	if err != nil {
-		return nil, errors.Wrap(err, "getting fields from config")
-	}
-
-	return kafka.NewRunner(
-		idkCfg,
-		batch.NewSQLBatcher(cmd, flds),
-		cmd.stderr,
-	), nil
-}

cli/kafka/config.go

@@ -1,251 +0,0 @@
-package kafka
-
-import (
-	"fmt"
-	"time"
-
-	featurebase "github.com/featurebasedb/featurebase/v3"
-	"github.com/featurebasedb/featurebase/v3/dax"
-	"github.com/featurebasedb/featurebase/v3/idk"
-	"github.com/pkg/errors"
-)
-
-// Config is the user-facing configuration for kafka support in the CLI. This is
-// unmarshalled from the the toml config file supplied by the user.
-type Config struct {
-	Hosts  []string `mapstructure:"hosts" help:"Kafka hosts."`
-	Group  string   `mapstructure:"group" help:"Kafka group."`
-	Topics []string `mapstructure:"topics" help:"Kafka topics to read from."`
-
-	BatchSize         int           `mapstructure:"batch-size" help:"Batch size."`
-	BatchMaxStaleness time.Duration `mapstructure:"batch-max-staleness" help:"Maximum length of time that the oldest record in a batch can exist before flushing the batch. Note that this can potentially stack with timeouts waiting for the source."`
-	Timeout           time.Duration `mapstructure:"timeout" help:"Time to wait for more records from Kafka before flushing a batch. 0 to disable."`
-
-	Table  string  `mapstructure:"table" help:"Destination table name."`
-	Fields []Field `mapstructure:"fields"`
-}
-
-// Field is a user-facing configuration field.
-type Field struct {
-	Name       string   `mapstructure:"name"`
-	SourceType string   `mapstructure:"source-type"`
-	SourcePath []string `mapstructure:"source-path"`
-	PrimaryKey bool     `mapstructure:"primary-key"`
-}
-
-// ConfigForIDK represents Config converted to values suitable for IDK. In
-// particular, the idk.RawField is used in parsing the schema in IDK.
-type ConfigForIDK struct {
-	Hosts  []string
-	Group  string
-	Topics []string
-
-	BatchSize         int
-	BatchMaxStaleness time.Duration
-	Timeout           time.Duration
-
-	Table   string
-	IDField string
-	Fields  []idk.RawField
-}
-
-// ValidateConfig validates the config is usable.
-func ValidateConfig(c Config) error {
-	if c.Table == "" {
-		return errors.Errorf("table is required")
-	} else if len(c.Topics) == 0 {
-		return errors.Errorf("at least one topic is required")
-	} else if len(c.Fields) > 0 {
-		// We only need to do these checks if any fields are specified at all.
-		// If no fields are specified, that's ok because then we default to
-		// using fields based off the existing table.
-		if len(c.Fields) < 2 {
-			return errors.Errorf("at least two fields are required (one should be a primary key)")
-		} else {
-			var found int
-			for i := range c.Fields {
-				if c.Fields[i].PrimaryKey {
-					found++
-				}
-				if c.Fields[i].Name == "" {
-					return errors.Errorf("a name attribute (which isn't equal to \"\") should exist for all fields")
-				}
-			}
-			if found != 1 {
-				return errors.Errorf("exactly one primary key field is required")
-			}
-		}
-	}
-	return nil
-}
-
-// ConvertConfig converts a Config to one that suitable for IDK.
-func ConvertConfig(c Config) (ConfigForIDK, error) {
-	// Set a default kafka host in case one isn't provided.
-	hosts := []string{"localhost:9092"}
-	if len(c.Hosts) > 0 {
-		hosts = c.Hosts
-	}
-
-	// Copy all the shared members from Config to ConfigForIDK.
-	out := ConfigForIDK{
-		Hosts:             hosts,
-		Group:             c.Group,
-		Topics:            c.Topics,
-		BatchSize:         c.BatchSize,
-		BatchMaxStaleness: c.BatchMaxStaleness,
-		Timeout:           c.Timeout,
-		Table:             c.Table,
-	}
-
-	if len(c.Fields) == 0 {
-		return out, errors.New("fields cannot be empty")
-	}
-
-	// rawFields wil be the same as c.Fields, but possibly enhanced.
-	rawFields := make([]idk.RawField, 0, len(c.Fields))
-
-	var foundPK bool
-	for _, fld := range c.Fields {
-		if fld.PrimaryKey {
-			out.IDField = fld.Name
-			foundPK = true
-		}
-
-		typ, quals, err := dax.SplitFieldType(fld.SourceType)
-		if err != nil {
-			return out, errors.Wrap(err, "getting base type")
-		}
-
-		rawFld := idk.RawField{
-			Name: fld.Name,
-			Type: string(typ),
-			Path: fld.SourcePath,
-		}
-		// If a SourcePath wasn't provided, default to using the field name.
-		if len(rawFld.Path) == 0 {
-			rawFld.Path = []string{fld.Name}
-		}
-
-		switch typ {
-		case dax.BaseTypeInt:
-			// We don't have to handle min/max because we don't create the table.
-		case dax.BaseTypeDecimal:
-			if len(quals) != 1 {
-				return out, errors.Errorf("expected decimal scale")
-			}
-			rawFld.Config = []byte(fmt.Sprintf(`{"scale":%d}`, quals[0]))
-		case dax.BaseTypeID:
-			rawFld.Config = []byte("{\"mutex\":true}")
-		case dax.BaseTypeIDSet:
-			rawFld.Type = "ids"
-		case dax.BaseTypeString:
-			rawFld.Config = []byte("{\"mutex\":true}")
-		case dax.BaseTypeStringSet:
-			rawFld.Type = "strings"
-		case dax.BaseTypeTimestamp:
-			// No timestamp options are handled for now.
-		}
-
-		rawFields = append(rawFields, rawFld)
-	}
-	if !foundPK {
-		return out, errors.New("primary-key not found in fields")
-	}
-
-	out.Fields = rawFields
-
-	return out, nil
-}
-
-// ConfigToFields returns a list of *dax.Field based on the IDField and Fields
-// in the Config.
-func ConfigToFields(c Config) ([]*dax.Field, error) {
-	// We don't know if a primary key will be found, so we can't set the
-	// capacity to `len(c.Fields)-1`.
-	out := make([]*dax.Field, 0, len(c.Fields))
-
-	for _, fld := range c.Fields {
-		if fld.PrimaryKey {
-			continue
-		}
-		typ, quals, err := dax.SplitFieldType(fld.SourceType)
-		if err != nil {
-			return nil, errors.Wrap(err, "splitting field type")
-		}
-		dfld := &dax.Field{
-			Name: dax.FieldName(fld.Name),
-			Type: typ,
-		}
-		switch typ {
-		case dax.BaseTypeDecimal:
-			if len(quals) != 1 {
-				return nil, errors.Errorf("expected decimal scale")
-			}
-			scale, ok := quals[0].(int64)
-			if !ok {
-				return nil, errors.Errorf("invalid decimal scale: %v", quals[0])
-			}
-			dfld.Options.Scale = scale
-		}
-		out = append(out, dfld)
-	}
-
-	return out, nil
-}
-
-// FieldsToConfig returns a Config.Fields based on a list of *dax.Field.
-func FieldsToConfig(flds []*dax.Field) []Field {
-	out := make([]Field, 0, len(flds))
-	for _, fld := range flds {
-		out = append(out, Field{
-			Name:       string(fld.Name),
-			SourceType: fld.FullType(),
-			PrimaryKey: fld.IsPrimaryKey(),
-		})
-	}
-	return out
-}
-
-// CheckFieldCompatibility ensures that the fields provided in the kafka config
-// are compatible with the fields in the existing table. It returns a copy of
-// the kafka config fields with empty values defaulted to the table field
-// configuration.
-func CheckFieldCompatibility(cflds []Field, scr *featurebase.ShowColumnsResponse) ([]Field, error) {
-	out := make([]Field, len(cflds))
-	for i, cfld := range cflds {
-		out[i] = cfld
-		cfldName := dax.FieldName(cfld.Name)
-
-		// Primary key field.
-		if cfld.PrimaryKey {
-			f := scr.Field(dax.PrimaryKeyFieldName)
-			if f == nil {
-				return nil, dax.NewErrFieldDoesNotExist(dax.PrimaryKeyFieldName) // It should be impossible to hit this.
-			}
-			if out[i].SourceType == "" {
-				if f.StringKeys() {
-					out[i].SourceType = dax.BaseTypeString
-				} else {
-					out[i].SourceType = dax.BaseTypeID
-				}
-			}
-			continue
-		}
-
-		// Non primary key fields.
-		if cfldName == dax.PrimaryKeyFieldName {
-			return nil, errors.Errorf("field named '%s' must be a primary key", dax.PrimaryKeyFieldName)
-		}
-
-		f := scr.Field(cfldName)
-		if f == nil {
-			return nil, dax.NewErrFieldDoesNotExist(cfldName)
-		}
-
-		if out[i].SourceType == "" {
-			out[i].SourceType = f.FullType()
-		}
-	}
-	return out, nil
-}

cli/kafka/runner.go

@@ -1,67 +0,0 @@
-package kafka
-
-import (
-	"io"
-	"time"
-
-	fbbatch "github.com/featurebasedb/featurebase/v3/batch"
-	"github.com/featurebasedb/featurebase/v3/errors"
-	"github.com/featurebasedb/featurebase/v3/idk"
-	"github.com/featurebasedb/featurebase/v3/idk/kafka_static"
-	"github.com/featurebasedb/featurebase/v3/logger"
-)
-
-// Runner is a CLI-specific kafka consumer. It's similar to
-// idk.kafka_static.Main in that it embeds idk.Main and contains additional
-// functionality specific to its use case.
-type Runner struct {
-	idk.Main   `flag:"!embed"`
-	KafkaHosts []string       `help:"Comma separated list of host:port pairs for Kafka."`
-	Group      string         `help:"Kafka group."`
-	Topics     []string       `help:"Kafka topics to read from."`
-	Timeout    time.Duration  `help:"Time to wait for more records from Kafka before flushing a batch. 0 to disable."`
-	Header     []idk.RawField `help:"Header configuration."`
-}
-
-func NewRunner(cfg ConfigForIDK, batcher fbbatch.Batcher, logWriter io.Writer) *Runner {
-	idkMain := idk.NewMain()
-	idkMain.IDField = cfg.IDField
-	idkMain.Index = cfg.Table
-	idkMain.Batcher = batcher
-	idkMain.BatchSize = cfg.BatchSize
-	idkMain.BatchMaxStaleness = cfg.BatchMaxStaleness
-	idkMain.SetBasic()
-	idkMain.SetLog(logger.NewStandardLogger(logWriter))
-
-	kr := &Runner{
-		Main:       *idkMain,
-		KafkaHosts: cfg.Hosts,
-		Group:      cfg.Group,
-		Topics:     cfg.Topics,
-		Header:     cfg.Fields,
-		Timeout:    cfg.Timeout,
-	}
-	kr.OffsetMode = true
-	kr.Main.Namespace = "cli_kafka_runner"
-	kr.Main.Pprof = "" // don't initialize pprof until we actually use it in tests
-	kr.NewSource = func() (idk.Source, error) {
-		source := kafka_static.NewSource()
-		source.Hosts = kr.KafkaHosts
-		source.Group = kr.Group
-		source.Topics = kr.Topics
-		source.Log = kr.Main.Log()
-		// source.TLS = m.KafkaTLS
-		source.Timeout = kr.Timeout
-		// source.SkipOld = m.SkipOld
-		source.HeaderFields = kr.Header
-		// source.S3Region = m.S3Region
-		// source.AllowMissingFields = m.AllowMissingFields
-
-		err := source.Open()
-		if err != nil {
-			return nil, errors.Wrap(err, "opening source")
-		}
-		return source, nil
-	}
-	return kr
-}

cli/parts.go

@@ -1,136 +0,0 @@
-package cli
-
-import (
-	"fmt"
-	"io"
-	"os"
-	"strings"
-)
-
-// query is a collection of queryParts which, when applied together, make up an
-// executable SQL query.
-type query []queryPart
-
-func (q query) String() string {
-	var sb strings.Builder
-	for i := range q {
-		sb.WriteString(q[i].String())
-		if i < len(q)-1 {
-			sb.WriteRune('\n')
-		}
-	}
-	return sb.String()
-}
-
-// Reader returns the query as an io.Reader so that it can be passed to, for
-// example, http.Post().
-func (q query) Reader() io.Reader {
-	readers := make([]io.Reader, 0, len(q))
-	for i := range q {
-		readers = append(readers, q[i].Reader())
-	}
-	return io.MultiReader(readers...)
-}
-
-// queryPart is an interface representing anything which can use to build up a
-// query.
-type queryPart interface {
-	fmt.Stringer
-	Reader() io.Reader
-}
-
-func newRawQuery(s string) query {
-	return []queryPart{
-		newPartRaw(s),
-	}
-}
-
-// ////////////////////////////////////////////////////////////////////////////
-// raw
-// ////////////////////////////////////////////////////////////////////////////
-
-// Ensure type implements interface.
-var _ queryPart = (*partRaw)(nil)
-
-type partRaw struct {
-	raw string
-}
-
-func newPartRaw(s string) *partRaw {
-	return &partRaw{
-		raw: s,
-	}
-}
-
-func (p *partRaw) Reader() io.Reader {
-	return strings.NewReader(p.raw + "\n")
-}
-
-func (p *partRaw) String() string {
-	return p.raw
-}
-
-// ////////////////////////////////////////////////////////////////////////////
-// file
-// ////////////////////////////////////////////////////////////////////////////
-
-// Ensure type implements interface.
-var _ queryPart = (*partFile)(nil)
-
-type partFile struct {
-	file *os.File
-}
-
-func newPartFile(f *os.File) *partFile {
-	return &partFile{
-		file: f,
-	}
-}
-
-func (p *partFile) Reader() io.Reader {
-	return p.file
-}
-
-func (p *partFile) String() string {
-	return fmt.Sprintf("[file: %s]", p.file.Name())
-}
-
-// ////////////////////////////////////////////////////////////////////////////
-// batch file
-// ////////////////////////////////////////////////////////////////////////////
-
-// Ensure type implements interface.
-var _ queryPart = (*partBatchFile)(nil)
-
-type partBatchFile struct {
-	file *os.File
-}
-
-func (p *partBatchFile) Reader() io.Reader {
-	return p.file
-}
-
-func (p *partBatchFile) String() string {
-	return p.file.Name()
-}
-
-// ////////////////////////////////////////////////////////////////////////////
-// terminator (i.e. ";")
-// ////////////////////////////////////////////////////////////////////////////
-
-// Ensure type implements interface.
-var _ queryPart = (*partTerminator)(nil)
-
-type partTerminator struct{}
-
-func newPartTerminator() *partTerminator {
-	return &partTerminator{}
-}
-
-func (p *partTerminator) Reader() io.Reader {
-	return nil
-}
-
-func (p *partTerminator) String() string {
-	return terminationChar
-}

cli/queryer.go

@@ -1,108 +0,0 @@
-package cli
-
-import (
-	"fmt"
-	"io"
-	"net/http"
-	"time"
-
-	featurebase "github.com/featurebasedb/featurebase/v3"
-	"github.com/pkg/errors"
-)
-
-type Queryer interface {
-	Query(org string, db string, sql io.Reader) (*featurebase.WireQueryResponse, error)
-}
-
-// Ensure type implements interface.
-var _ Queryer = (*nopQueryer)(nil)
-
-type nopQueryer struct{}
-
-func (qryr *nopQueryer) Query(org string, db string, sql io.Reader) (*featurebase.WireQueryResponse, error) {
-	return nil, errors.Errorf("no-op queryer")
-}
-
-// Ensure type implements interface.
-var _ Queryer = (*standardQueryer)(nil)
-
-// standardQueryer supports a standard featurebase deployment hitting the /sql
-// endpoint with a payload containing only the sql statement.
-type standardQueryer struct {
-	Host string
-	Port string
-}
-
-func (qryr *standardQueryer) Query(org string, db string, sql io.Reader) (*featurebase.WireQueryResponse, error) {
-	url := fmt.Sprintf("%s/sql", hostPort(qryr.Host, qryr.Port))
-
-	resp, err := http.Post(url, "application/json", sql)
-	if err != nil {
-		return nil, errors.Wrapf(err, "posting query")
-	}
-
-	fullbod, err := io.ReadAll(resp.Body)
-	if err != nil {
-		return nil, errors.Wrap(err, "reading response")
-	}
-
-	sqlResponse := &featurebase.WireQueryResponse{}
-	// TODO(tlt): switch this back once all responses are typed
-	// TODO(twg) 2023/03/01 using json.Number to decode large ints so care must be made
-	// if err := json.Unmarshal(fullbod, sqlResponse); err != nil {
-	if err := sqlResponse.UnmarshalJSONTyped(fullbod, true); err != nil {
-		return nil, errors.Wrapf(err, "unmarshaling query response, body:\n'%s'\n", fullbod)
-	}
-
-	return sqlResponse, nil
-}
-
-// Ensure type implements interface.
-var _ Queryer = (*serverlessQueryer)(nil)
-
-// serverlessQueryer is similar to the standardQueryer except that it hits a
-// different endpoint, and its payload is database-aware.
-type serverlessQueryer struct {
-	Host string
-	Port string
-}
-
-func (qryr *serverlessQueryer) Query(org string, db string, sql io.Reader) (*featurebase.WireQueryResponse, error) {
-	if org == "" {
-		return nil, NewErrOrganizationRequired()
-	}
-
-	url := fmt.Sprintf("%s/queryer/databases/%s/sql", hostPort(qryr.Host, qryr.Port), db)
-	if db == "" {
-		url = fmt.Sprintf("%s/queryer/sql", hostPort(qryr.Host, qryr.Port))
-	}
-
-	client := &http.Client{
-		Timeout: time.Second * 30,
-	}
-
-	req, err := http.NewRequest(http.MethodPost, url, sql)
-	if err != nil {
-		return nil, errors.Wrap(err, "creating new post request")
-	}
-	req.Header.Add("Content-Type", "text/plain")
-	req.Header.Add("OrganizationID", org)
-
-	var resp *http.Response
-	if resp, err = client.Do(req); err != nil {
-		return nil, errors.Wrap(err, "executing post request")
-	}
-
-	fullbod, err := io.ReadAll(resp.Body)
-	if err != nil {
-		return nil, errors.Wrap(err, "reading response")
-	}
-	sqlResponse := &featurebase.WireQueryResponse{}
-	// TODO(tlt): switch this back once all responses are typed
-	// if err := json.Unmarshal(fullbod, sqlResponse); err != nil {
-	if err := sqlResponse.UnmarshalJSONTyped(fullbod, true); err != nil {
-		return nil, errors.Wrapf(err, "unmarshaling query response, body:\n'%s'\n", fullbod)
-	}
-
-	return sqlResponse, nil
-}

cli/replacer.go

@@ -1,109 +0,0 @@
-package cli
-
-import (
-	"strings"
-
-	"github.com/benhoyt/goawk/lexer"
-)
-
-// replacer can replace parts of a string based on some rules and the provided
-// map[string]string. For example, the Command can replace strings with values
-// in its `variables` map.
-type replacer struct {
-	m map[string]string
-}
-
-func newReplacer(m map[string]string) *replacer {
-	return &replacer{
-		m: m,
-	}
-}
-
-// replace replaces all instances of the string pattern `:key` with the value at
-// m[key]. For example we want something like this:
-//
-// GIVEN: `start :one,:'two', :"three" ::four ::`
-//
-// with map
-//
-//	map[string]string{
-//		"one":   "repl1",
-//		"three": "repl3",
-//	}
-//
-// WANT: `start repl1,:'two', "repl3" ::four ::`
-func (r *replacer) replace(s string) string {
-	// If no variables have been added to the map, there's no need to parse the
-	// string for variable replacement.
-	if len(r.m) == 0 {
-		return s
-	}
-
-	line := []byte(s)
-	lex := lexer.NewLexer(line)
-
-	// finger contains the index into line at the start of non-variable text
-	// that we want to include, as-is in the output.
-	var finger int
-
-	// sb builds the string which will be the final output.
-	var sb strings.Builder
-	for {
-		pos, tok, _ := lex.Scan()
-
-		switch tok {
-		case lexer.COLON:
-			// last is the last normal character position before the colon.
-			last := pos.Column - 1
-
-			// Get the next byte to see if the colon value is quoted, and if so,
-			// whether its has single or double quotes.
-			b := lex.PeekByte()
-
-			// padding is the amount of padding we have to consider around the
-			// variable name. If the variable is not quoted, it doesn't require
-			// any padding. But if it has quotes, it needs 2 characters of
-			// paddings to accomodate the quotes.
-			padding := 0
-
-			// quote holds the character to use to quote the final, replaced
-			// output value. Because the lexer doesn't tell us how a certain
-			// `string` token was quoted, we need to keep track of that here so
-			// we can put them back.
-			quote := ""
-			switch b {
-			case byte('\''): // single quote
-				quote = `'`
-				padding = 2
-			case byte('"'): // double quote
-				quote = `"`
-				padding = 2
-			}
-
-			pos, tok, key := lex.Scan()
-			switch tok {
-			case lexer.NAME, lexer.STRING:
-				// Write the normal text up to the variable replacement
-				// position.
-				sb.Write(line[finger:last])
-
-				if v, ok := r.m[key]; ok {
-					// Write replaced variable with the quotes it had.
-					sb.WriteString(quote + v + quote)
-				} else {
-					// Since the variable was not found in the map, just write
-					// back what was already there.
-					sb.WriteString(":" + quote + key + quote)
-				}
-
-				// Reset finger to point to the next position after the
-				// variable.
-				finger = pos.Column + len(key) + padding - 1
-			}
-		case lexer.EOF:
-			// Write the remainder of the string and return.
-			sb.Write(line[finger:])
-			return sb.String()
-		}
-	}
-}

cli/replacer_test.go

@@ -1,109 +0,0 @@
-package cli
-
-import (
-	"fmt"
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-)
-
-func TestReplacer(t *testing.T) {
-	t.Run("general replace function", func(t *testing.T) {
-
-		m := map[string]string{
-			"v1": "newVone",
-			"v2": "newVtwo",
-		}
-
-		tests := []struct {
-			s   string
-			m   map[string]string
-			exp string
-		}{
-			{
-				// no variables present
-				s:   "foo",
-				m:   m,
-				exp: "foo",
-			},
-			{
-				// variable prefix, but not in map
-				s:   ":foo",
-				m:   m,
-				exp: ":foo",
-			},
-			{
-				// variable name match, but missing prefix
-				s:   "v1",
-				m:   m,
-				exp: "v1",
-			},
-			{
-				// variable name match
-				s:   ":v1",
-				m:   m,
-				exp: "newVone",
-			},
-			{
-				// two variables, the same, no space
-				s:   ":v1:v1",
-				m:   m,
-				exp: "newVonenewVone",
-			},
-			{
-				// two variables, different, no space
-				s:   ":v1:v2",
-				m:   m,
-				exp: "newVonenewVtwo",
-			},
-			{
-				// two variables, different, spaces
-				s:   ":v1  :v2",
-				m:   m,
-				exp: "newVone  newVtwo",
-			},
-			{
-				// one variable, one non-variable, no space
-				s:   ":v1:foo",
-				m:   m,
-				exp: "newVone:foo",
-			},
-			{
-				// one non-variable, one variable, no space
-				s:   "foo:v1",
-				m:   m,
-				exp: "foonewVone",
-			},
-			{
-				// two variables, different, comma
-				s:   ":v1, :v2",
-				m:   m,
-				exp: "newVone, newVtwo",
-			},
-			{
-				// single quotes
-				s:   ":'v1'",
-				m:   m,
-				exp: "'newVone'",
-			},
-			{
-				// double quotes
-				s:   `:"v2"`,
-				m:   m,
-				exp: `"newVtwo"`,
-			},
-			{
-				// more quotes
-				s:   `start :v1,:'two', :"v2" ::four :: `,
-				m:   m,
-				exp: `start newVone,:'two', "newVtwo" ::four :: `,
-			},
-		}
-		for i, test := range tests {
-			t.Run(fmt.Sprintf("test-%d", i), func(t *testing.T) {
-				replacer := newReplacer(test.m)
-				assert.Equal(t, test.exp, replacer.replace(test.s))
-			})
-		}
-	})
-}

cli/splitter.go

@@ -1,133 +0,0 @@
-package cli
-
-import (
-	"strings"
-
-	"github.com/pkg/errors"
-)
-
-// splitter is a line splitter which splits a line into queryParts and
-// metaCommands. It may not be necessary to have this be a separate struct since
-// it contains no members and just has the one `split()` method, but here we
-// are.
-type splitter struct {
-	replacer *replacer
-}
-
-func newSplitter(r *replacer) *splitter {
-	return &splitter{
-		replacer: r,
-	}
-}
-
-// split splits the given line into queryParts and metaCommands.
-// If a metaCommand is found, everything after that is considered either arguments to that
-// metaCommand, or additional metaCommands. In other words, queryParts can not follow
-// metaCommands in the same line.
-//
-// A line can contain any of the following patterns:
-// 1- [queryParts...]: "select * from tbl; select"
-// 2- [metaCommands...]: "\! pwd \q"
-// 3- [queryParts...][metaCommands...]: "select * from \i file.sql"
-func (s *splitter) split(line string) ([]queryPart, []metaCommand, error) {
-	// Look for a comment line.
-	if strings.HasPrefix(line, "--") {
-		return nil, nil, nil
-	}
-
-	// Look for a meta command.
-	parts := strings.SplitN(line, `\`, 2)
-
-	switch len(parts) {
-	case 1:
-		// slice of queryParts (pattern 1)
-		if qps, err := s.splitQueryParts(strings.TrimSpace(parts[0])); err != nil {
-			return nil, nil, errors.Wrap(err, "splitting query parts")
-		} else {
-			return qps, nil, nil
-		}
-	case 2:
-		// slice of parts + slice of meta commands (pattern 3)
-		// or
-		// slice of meta commands (pattern 2)
-		qps, err := s.splitQueryParts(strings.TrimSpace(parts[0]))
-		if err != nil {
-			return nil, nil, errors.Wrap(err, "splitting query parts")
-		}
-
-		mcs, err := s.splitMetaCommands(strings.TrimSpace(parts[1]))
-		if err != nil {
-			return nil, nil, errors.Wrap(err, "splitting meta commands")
-		}
-
-		return qps, mcs, nil
-	}
-
-	return nil, nil, nil
-}
-
-func (s *splitter) splitQueryParts(line string) ([]queryPart, error) {
-	if line == "" {
-		return nil, nil
-	}
-
-	// Look for a termination character;
-	parts := strings.Split(line, terminationChar)
-
-	// Do variable replacement.
-	for i := range parts {
-		parts[i] = s.replacer.replace(parts[i])
-	}
-
-	if len(parts) == 1 {
-		part0 := strings.TrimSpace(parts[0])
-		return []queryPart{
-			newPartRaw(part0),
-		}, nil
-	}
-
-	qps := make([]queryPart, 0)
-	for i := range parts {
-		part := strings.TrimSpace(parts[i])
-		if part == "" {
-			// If the line starts with a ";", treat it as a terminator for a
-			// previous line.
-			if i == 0 {
-				qps = append(qps, &partTerminator{})
-			}
-			continue
-		}
-		qps = append(qps, newPartRaw(part))
-		if i < len(parts)-1 {
-			qps = append(qps, &partTerminator{})
-		}
-	}
-
-	return qps, nil
-}
-
-func (s *splitter) splitMetaCommands(in string) ([]metaCommand, error) {
-	parts := strings.Split(in, `\`)
-	if len(parts) == 1 {
-		mc, err := splitMetaCommand(parts[0], s.replacer)
-		if err != nil {
-			return nil, errors.Wrapf(err, "splitting meta command: %s", parts[0])
-		}
-		return []metaCommand{mc}, nil
-	}
-
-	mcs := make([]metaCommand, 0)
-	for i := range parts {
-		part := strings.TrimSpace(parts[i])
-		if part == "" {
-			continue
-		}
-		mc, err := splitMetaCommand(part, s.replacer)
-		if err != nil {
-			return nil, errors.Wrapf(err, "splitting meta command: %s", part)
-		}
-		mcs = append(mcs, mc)
-	}
-
-	return mcs, nil
-}

cli/splitter_test.go

@@ -1,146 +0,0 @@
-package cli
-
-import (
-	"fmt"
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-)
-
-func TestSplitter(t *testing.T) {
-	s := newSplitter(newReplacer(nil))
-	t.Run("Split", func(t *testing.T) {
-		tests := []struct {
-			line            string
-			expQueryParts   []queryPart
-			expMetaCommands []metaCommand
-			expError        string
-		}{
-			{
-				line: `foo`,
-				expQueryParts: []queryPart{
-					newPartRaw("foo"),
-				},
-			},
-			{
-				line: `foo;`,
-				expQueryParts: []queryPart{
-					newPartRaw("foo"),
-					newPartTerminator(),
-				},
-			},
-			{
-				line: `foo; `,
-				expQueryParts: []queryPart{
-					newPartRaw("foo"),
-					newPartTerminator(),
-				},
-			},
-			{
-				line: `foo; ; `,
-				expQueryParts: []queryPart{
-					newPartRaw("foo"),
-					newPartTerminator(),
-				},
-			},
-			{
-				line: `foo; bar`,
-				expQueryParts: []queryPart{
-					newPartRaw("foo"),
-					newPartTerminator(),
-					newPartRaw("bar"),
-				},
-			},
-			{
-				line: `foo; bar;`,
-				expQueryParts: []queryPart{
-					newPartRaw("foo"),
-					newPartTerminator(),
-					newPartRaw("bar"),
-					newPartTerminator(),
-				},
-			},
-			{
-				line: `\q`,
-				expMetaCommands: []metaCommand{
-					&metaQuit{},
-				},
-			},
-			{
-				line: ` \p`,
-				expMetaCommands: []metaCommand{
-					&metaPrint{},
-				},
-			},
-			{
-				line: `\q \p`,
-				expMetaCommands: []metaCommand{
-					&metaQuit{},
-					&metaPrint{},
-				},
-			},
-			{
-				line: `\q \p arg1 arg2`,
-				expMetaCommands: []metaCommand{
-					&metaQuit{},
-					&metaPrint{},
-				},
-			},
-			{
-				line: `\set`,
-				expMetaCommands: []metaCommand{
-					&metaSet{
-						args: []string{},
-					},
-				},
-			},
-			{
-				line: `\set arg1 arg2`,
-				expMetaCommands: []metaCommand{
-					&metaSet{
-						args: []string{"arg1", "arg2"},
-					},
-				},
-			},
-			{
-				line: `\set 'arg1' 'arg2'`,
-				expMetaCommands: []metaCommand{
-					&metaSet{
-						args: []string{"arg1", "arg2"},
-					},
-				},
-			},
-			{
-				line: `\set 'arg1' '"arg2"'`,
-				expMetaCommands: []metaCommand{
-					&metaSet{
-						args: []string{"arg1", "\"arg2\""},
-					},
-				},
-			},
-			{
-				line:     `\`,
-				expError: "unsupported meta-command:",
-			},
-			{
-				line:     `\xyzxyz`,
-				expError: "unsupported meta-command:",
-			},
-		}
-		for i, tt := range tests {
-			t.Run(fmt.Sprintf("test-%d-%s", i, tt.line), func(t *testing.T) {
-				qps, mcs, err := s.split(tt.line)
-				if tt.expError != "" {
-					if assert.Error(t, err) {
-						assert.Contains(t, err.Error(), tt.expError)
-					}
-					return
-				}
-
-				assert.NoError(t, err)
-				assert.ElementsMatch(t, tt.expQueryParts, qps)
-				assert.ElementsMatch(t, tt.expMetaCommands, mcs)
-			})
-		}
-	})
-}

cli/testdata/database

@@ -1,53 +0,0 @@
-// Show databases now that we have set org.
-SEND:SHOW DATABASES;
-EXPECT:+-----+------+-------+------------+------------+------------+-------+-------------+
-EXPECT:| _id | name | owner | updated_by | created_at | updated_at | units | description |
-EXPECT:+-----+------+-------+------------+------------+------------+-------+-------------+
-EXPECT:+-----+------+-------+------------+------------+------------+-------+-------------+
-EXPECT:
-
-// Create db1.
-SEND:CREATE DATABASE db1 WITH UNITS 1;
-EXPECT:
-
-// List databases via SHOW DATABASES.
-SEND:SHOW DATABASES;
-EXPECT:+--------------------------------------+------+-------+------------+----------------------+----------------------+-------+-------------+
-EXPECT:| _id                                  | name | owner | updated_by | created_at           | updated_at           | units | description |
-EXPECT:+--------------------------------------+------+-------+------------+----------------------+----------------------+-------+-------------+
-EXPECTCOMP:WithFormat:| {uuid} | db1  |       |            | {timestamp} | {timestamp} |     1 |             |
-EXPECT:+--------------------------------------+------+-------+------------+----------------------+----------------------+-------+-------------+
-EXPECT:
-
-// List databases via SHOW DATABASES.
-SEND:\l
-EXPECT:+--------------------------------------+------+-------+------------+----------------------+----------------------+-------+-------------+
-EXPECT:| _id                                  | name | owner | updated_by | created_at           | updated_at           | units | description |
-EXPECT:+--------------------------------------+------+-------+------------+----------------------+----------------------+-------+-------------+
-EXPECTCOMP:WithFormat:| {uuid} | db1  |       |            | {timestamp} | {timestamp} |     1 |             |
-EXPECT:+--------------------------------------+------+-------+------------+----------------------+----------------------+-------+-------------+
-EXPECT:
-
-// Check database connection.
-SEND:\c
-EXPECT:You are not connected to a database.
-
-// Try connecting to an invalid database.
-SEND:\c invalid
-EXPECT:executing meta command: invalid database: invalid
-
-// Try connecting with too many arguments.
-SEND:\c db1 extra
-EXPECT:executing meta command: meta command 'connect' takes zero or one argument
-
-// Connect to a database.
-SEND:\c db1
-EXPECTCOMP:WithFormat:You are now connected to database "db1" ({uuid}).
-
-// Disconnect from the current database.
-SEND:\c -
-EXPECT:You are not connected to a database.
-
-// Connect to a database again.
-SEND:\c db1
-EXPECTCOMP:WithFormat:You are now connected to database "db1" ({uuid}).

cli/testdata/famous.csv

@@ -1,10 +0,0 @@
-
-"Id", "Name", "Short description", "Gender", "Country", "Occupation", "Birth year", "Death year", "Manner of death", "Age of death"
-1, "George Washington", "1st president of the United States (1732–1799)", "Male", "United States of America; Kingdom of Great Britain", "Politician", "1732", "1799", "natural causes", "67"
-2, "Douglas Adams", "English writer and humorist", "Male", "United Kingdom", "Artist", "1952", "2001", "natural causes", "49"
-3, "Abraham Lincoln", "16th president of the United States (1809-1865)", "Male", "United States of America", "Politician", "1809", "1865", "homicide", "56"
-4, "Wolfgang Amadeus Mozart", "Austrian composer of the Classical period", "Male", "Archduchy of Austria; Archbishopric of Salzburg", "Artist", "1756", "1791", "0", "35"
-5, "Ludwig van Beethoven", "German classical and romantic composer", "Male", "Holy Roman Empire; Austrian Empire", "Artist", "1770", "1827", "0", "57"
-6, "Jean-François Champollion", "French classical scholar", "Male", "Kingdom of France; First French Empire", "Egyptologist", "1790", "1832", "natural causes", "42"
-7, "Paul Morand", "French writer", "Male", "France", "Artist", "1888", "1976", "0", "88"
-8, "Claude Monet", "French impressionist painter (1840-1926)", "Male", "France", "Artist", "1840", "1926", "natural causes", "86"

cli/testdata/meta_bang

@@ -1,8 +0,0 @@
-SEND:\! echo 'foo'
-EXPECT:foo
-
-SEND:\! echo "foo"
-EXPECT:"foo"
-
-SEND:\!
-EXPECT:executing meta command: meta command '!' requires at least one argument

cli/testdata/meta_cd

@@ -1,17 +0,0 @@
-// Make a directory so we can test \cd'ing into it.
-SEND:\! mkdir cli-test-dir
-SEND:\cd cli-test-dir
-SEND:\cd ..
-SEND:\! rmdir cli-test-dir
-
-// TODO(tlt): before we do this, we should implement the ability to execute
-// commands in a \set like:
-// \set homedir `pwd`
-// then we can store what directory we're in so we can move back to it 
-// at the end of the test
-// Switch to home directory.
-// SEND:\cd
-
-// Expect error on extra argument to \cd.
-SEND:\cd dir extra
-EXPECT:executing meta command: meta command 'cd' takes zero or one argument

cli/testdata/meta_describe

@@ -1,33 +0,0 @@
-// TODO(tlt): we can't run this test until we get the system tables under control (i.e. sorted). Currently, fb_views is in a map with users, so the following can fail 50% of the time.
-// Show tables for database by calling describe with no args.
-// SEND:\d
-// EXPECT:+-------------------------+-------------------------+-------+------------+----------------------+----------------------+-------+------------+------------------------+
-// EXPECT:| _id                     | name                    | owner | updated_by | created_at           | updated_at           | keys  | space_used | description            |
-// EXPECT:+-------------------------+-------------------------+-------+------------+----------------------+----------------------+-------+------------+------------------------+
-// EXPECTCOMP:WithFormat:| fb_veiws                | fb_views                |       |            | {timestamp} | {timestamp} | true  |          0 | system table for views |
-// EXPECTCOMP:WithFormat:| users                   | users                   |       |            | {timestamp} | {timestamp} | false |          0 |                        |
-// EXPECTCOMP:WithFormat:| fb_____________________ | fb_____________________ |       |            | {timestamp} | {timestamp} | false |          0 |                        |
-// EXPECTCOMP:WithFormat:| fb_____________________ | fb_____________________ |       |            | {timestamp} | {timestamp} | false |          0 |                        |
-// EXPECTCOMP:WithFormat:| fb_____________________ | fb_____________________ |       |            | {timestamp} | {timestamp} | false |          0 |                        |
-// EXPECTCOMP:WithFormat:| fb_____________________ | fb_____________________ |       |            | {timestamp} | {timestamp} | false |          0 |                        |
-// EXPECTCOMP:WithFormat:| fb_____________________ | fb_____________________ |       |            | {timestamp} | {timestamp} | false |          0 |                        |
-// EXPECT:+-------------------------+-------------------------+-------+------------+----------------------+----------------------+-------+------------+------------------------+
-// EXPECT:
-
-// Show columns for table.
-SEND:\d users
-EXPECT:+------+------+--------+----------------------+-------+------------+------------+-------+----------------------+---------------------+----------+-------+-------------+-----+
-EXPECT:| _id  | name | type   | created_at           | keys  | cache_type | cache_size | scale |                  min |                 max | timeunit | epoch | timequantum | ttl |
-EXPECT:+------+------+--------+----------------------+-------+------------+------------+-------+----------------------+---------------------+----------+-------+-------------+-----+
-EXPECTCOMP:WithFormat:| _id  | _id  | id     | {timestamp} | false |            |          0 |     0 |                    0 |                   0 |          |     0 |             | 0s  |
-EXPECTCOMP:WithFormat:| name | name | string | {timestamp} | true  | ranked     |      50000 |     0 |                    0 |                   0 |          |     0 |             | 0s  |
-EXPECTCOMP:WithFormat:| age  | age  | int    | {timestamp} | false |            |          0 |     0 | -9223372036854775808 | 9223372036854775807 |          |     0 |             | 0s  |
-EXPECT:+------+------+--------+----------------------+-------+------------+------------+-------+----------------------+---------------------+----------+-------+-------------+-----+
-EXPECT:
-
-// Show columns for an invalid table.
-SEND:\d invalid
-EXPECT:Error: compiling plan: [1:19] table 'invalid' not found
-
-SEND:\d users extra
-EXPECT:executing meta command: meta command 'describe' takes zero or one argument

cli/testdata/meta_echo

@@ -1,6 +0,0 @@
-SEND:\echo
-EXPECT:
-
-// Simple \echo.
-SEND:\echo foo bar
-EXPECT:foo bar

cli/testdata/meta_file

@@ -1,70 +0,0 @@
-// Create a table.
-SEND:CREATE TABLE famous (
-SEND:    _id ID,
-SEND:    name STRING,
-SEND:    description STRING,
-SEND:    gender STRING,
-SEND:    country STRING,
-SEND:    occupation STRING,
-SEND:    birth_year INT min -32767 max 32767,
-SEND:    death_year INT min -32767 max 32767,
-SEND:    death_manner STRING,
-SEND:    birth_age INT min -32767 max 32767
-SEND:);
-EXPECT:
-
-// Open bulk insert.
-SEND:BULK INSERT
-SEND:INTO famous (_id, name, description, gender, country, occupation,
-SEND:    birth_year, death_year, death_manner, birth_age )
-SEND:MAP(0 INT,
-SEND:1 STRING,
-SEND:2 STRING,
-SEND:3 STRING,
-SEND:4 STRING,
-SEND:5 STRING,
-SEND:6 INT,
-SEND:7 INT,
-SEND:8 STRING,
-SEND:9 INT )
-SEND:FROM
-SEND:    x'
-
-// Call \file
-SEND:\file testdata/famous.csv
-
-// Close bulk insert.
-SEND:'
-SEND:WITH
-SEND:    BATCHSIZE 100000
-SEND:    FORMAT 'CSV'
-SEND:    INPUT 'STREAM'
-SEND:    HEADER_ROW;
-EXPECT:
-
-// Query table to ensure we have data.
-SEND:SELECT * FROM famous;
-EXPECT:+-----+---------------------------+-------------------------------------------------+--------+----------------------------------------------------+--------------+------------+------------+----------------+-----------+
-EXPECT:| _id | name                      | description                                     | gender | country                                            | occupation   | birth_year | death_year | death_manner   | birth_age |
-EXPECT:+-----+---------------------------+-------------------------------------------------+--------+----------------------------------------------------+--------------+------------+------------+----------------+-----------+
-EXPECT:|   1 | George Washington         | 1st president of the United States (1732–1799)  | Male   | United States of America; Kingdom of Great Britain | Politician   |       1732 |       1799 | natural causes |        67 |
-EXPECT:|   2 | Douglas Adams             | English writer and humorist                     | Male   | United Kingdom                                     | Artist       |       1952 |       2001 | natural causes |        49 |
-EXPECT:|   3 | Abraham Lincoln           | 16th president of the United States (1809-1865) | Male   | United States of America                           | Politician   |       1809 |       1865 | homicide       |        56 |
-EXPECT:|   4 | Wolfgang Amadeus Mozart   | Austrian composer of the Classical period       | Male   | Archduchy of Austria; Archbishopric of Salzburg    | Artist       |       1756 |       1791 | 0              |        35 |
-EXPECT:|   5 | Ludwig van Beethoven      | German classical and romantic composer          | Male   | Holy Roman Empire; Austrian Empire                 | Artist       |       1770 |       1827 | 0              |        57 |
-EXPECT:|   6 | Jean-François Champollion | French classical scholar                        | Male   | Kingdom of France; First French Empire             | Egyptologist |       1790 |       1832 | natural causes |        42 |
-EXPECT:|   7 | Paul Morand               | French writer                                   | Male   | France                                             | Artist       |       1888 |       1976 | 0              |        88 |
-EXPECT:|   8 | Claude Monet              | French impressionist painter (1840-1926)        | Male   | France                                             | Artist       |       1840 |       1926 | natural causes |        86 |
-EXPECT:+-----+---------------------------+-------------------------------------------------+--------+----------------------------------------------------+--------------+------------+------------+----------------+-----------+
-EXPECT:
-
-// TODO(tlt): dropping the table seems to cause problems.
-// Drop the table.
-//SEND:DROP TABLE famous;
-
-// Ensure that invalid aruments (none or too many) return an error.
-SEND:\file
-EXPECT:executing meta command: meta command 'file' requires exactly one argument
-SEND:\file filename extra
-EXPECT:executing meta command: meta command 'file' requires exactly one argument
-

cli/testdata/meta_include

@@ -1,24 +0,0 @@
-// Include with no argument should error.
-SEND:\i
-EXPECT:executing meta command: meta command 'include' requires exactly one argument
-
-// Include with too many arguments should error.
-SEND:\include testdata/people.sql extra
-EXPECT:executing meta command: meta command 'include' requires exactly one argument
-
-// Invalid file should error.
-SEND:\include invalid.file
-EXPECT:executing meta command: opening file: invalid.file: open invalid.file: no such file or directory
-
-SEND:\include testdata/people.sql
-EXPECT:
-EXPECT:
-EXPECT:+-----+------+-----+
-EXPECT:| _id | name | age |
-EXPECT:+-----+------+-----+
-EXPECT:|   1 | Amy  |  42 |
-EXPECT:|   2 | Bob  |  27 |
-EXPECT:|   3 | Carl |  33 |
-EXPECT:+-----+------+-----+
-EXPECT:
-EXPECT:mix in a meta command

cli/testdata/meta_output

@@ -1,67 +0,0 @@
-SEND:SELECT * FROM users;
-EXPECT:+-----+-------+-----+
-EXPECT:| _id | name  | age |
-EXPECT:+-----+-------+-----+
-EXPECT:|   1 | Anne  |  38 |
-EXPECT:|   2 | Bill  |  23 |
-EXPECT:|   3 | Cindy |  64 |
-EXPECT:+-----+-------+-----+
-EXPECT:
-
-// Redirect output to a file.
-SEND:\o test-output-file
-SEND:SELECT * FROM users;
-
-// Ensure the output went to the file.
-SEND:\! cat test-output-file
-EXPECT:+-----+-------+-----+
-EXPECT:| _id | name  | age |
-EXPECT:+-----+-------+-----+
-EXPECT:|   1 | Anne  |  38 |
-EXPECT:|   2 | Bill  |  23 |
-EXPECT:|   3 | Cindy |  64 |
-EXPECT:+-----+-------+-----+
-EXPECT:
-
-// Let's test some qecho stuff here while we're at it.
-SEND:\qecho string with "double quotes"
-SEND:\qecho -n one
-SEND:\qecho -n two
-SEND:\qecho three
-SEND:\qecho four
-
-SEND:\! cat test-output-file
-EXPECT:+-----+-------+-----+
-EXPECT:| _id | name  | age |
-EXPECT:+-----+-------+-----+
-EXPECT:|   1 | Anne  |  38 |
-EXPECT:|   2 | Bill  |  23 |
-EXPECT:|   3 | Cindy |  64 |
-EXPECT:+-----+-------+-----+
-EXPECT:
-EXPECT:string with "double quotes"
-EXPECT:onetwothree
-EXPECT:four
-
-// And \warn messages should still go to stderr, not the file.
-SEND:\warn a warning string
-EXPECT:a warning string
-
-// Remove the file.
-SEND:\! rm test-output-file
-
-// Set the output back to stdout.
-SEND:\o
-SEND:SELECT * FROM users;
-EXPECT:+-----+-------+-----+
-EXPECT:| _id | name  | age |
-EXPECT:+-----+-------+-----+
-EXPECT:|   1 | Anne  |  38 |
-EXPECT:|   2 | Bill  |  23 |
-EXPECT:|   3 | Cindy |  64 |
-EXPECT:+-----+-------+-----+
-EXPECT:
-
-// Ensure extra arguments to \output causes an error.
-SEND:\o filename extra
-EXPECT:executing meta command: meta command 'output' takes zero or one argument

cli/testdata/meta_pset_border

@@ -1,52 +0,0 @@
-SEND:\pset border 2
-EXPECT:Border style is 2.
-
-SEND:SELECT 1 as foo, 'baz' as bar;
-EXPECT:+-----+-----+
-EXPECT:| foo | bar |
-EXPECT:+-----+-----+
-EXPECT:|   1 | baz |
-EXPECT:+-----+-----+
-EXPECT:
-
-SEND:\pset border
-EXPECT:Border style is 2.
-
-SEND:\pset border 999
-EXPECT:Border style is 0.
-
-SEND:\pset border 1
-EXPECT:Border style is 1.
-
-SEND:SELECT 1 as foo, 'baz' as bar;
-EXPECT: foo | bar 
-EXPECT:-----+-----
-EXPECT:   1 | baz 
-EXPECT:
-
-SEND:\pset border 2
-EXPECT:Border style is 2.
-
-SEND:SELECT 1 as foo, 'baz' as bar;
-EXPECT:+-----+-----+
-EXPECT:| foo | bar |
-EXPECT:+-----+-----+
-EXPECT:|   1 | baz |
-EXPECT:+-----+-----+
-EXPECT:
-
-SEND:\pset border 0
-EXPECT:Border style is 0.
-
-SEND:SELECT 1 as foo, 'baz' as bar;
-EXPECT:foo bar
-EXPECT:--- ---
-EXPECT:  1 baz
-EXPECT:
-
-SEND:\pset border 1 extra
-EXPECT:executing meta command: meta command 'pset' takes zero, one, or two arguments
-
-// Set border back to the testing default.
-SEND:\pset border 2
-EXPECT:Border style is 2.

cli/testdata/meta_pset_expanded

@@ -1,35 +0,0 @@
-// Set to off.
-SEND:\pset expanded off
-EXPECT:Expanded display is off.
-
-// Set to on.
-SEND:\pset expanded on
-EXPECT:Expanded display is on.
-
-// Toggle to off.
-SEND:\pset expanded
-EXPECT:Expanded display is off.
-
-// Toggle to on.
-SEND:\pset expanded
-EXPECT:Expanded display is on.
-
-// Set to something invalid.
-SEND:\pset expanded invalid
-EXPECT:executing meta command: unrecognized value "invalid" for "expanded": Boolean expected
-
-// Ensure expanded shows results vertically.
-SEND:SELECT 1 as foo, 'baz' as bar;
-EXPECT:+-----+-----+
-EXPECT:| foo | 1   |
-EXPECT:| bar | baz |
-EXPECT:+-----+-----+
-EXPECT:
-
-// Set back to off as we started.
-SEND:\pset expanded off
-EXPECT:Expanded display is off.
-
-// make sure the \x meta-command returns expected errors
-SEND:\x on extra
-EXPECT:executing meta command: meta command 'expanded' takes zero or one argument

cli/testdata/meta_pset_format_csv

@@ -1,47 +0,0 @@
-SEND:\pset format csv
-EXPECT:Output format is csv.
-
-SEND:SELECT * FROM users;
-EXPECT:_id,name,age
-EXPECT:1,Anne,38
-EXPECT:2,Bill,23
-EXPECT:3,Cindy,64
-
-// Exclude headers.
-SEND:\t on
-EXPECT:Tuples only is on.
-
-SEND:SELECT * FROM users;
-EXPECT:1,Anne,38
-EXPECT:2,Bill,23
-EXPECT:3,Cindy,64
-
-// Reset headers.
-SEND:\t off
-EXPECT:Tuples only is off.
-
-// Set expanded to on.
-SEND:\x on
-EXPECT:Expanded display is on.
-
-SEND:SELECT * FROM users;
-EXPECT:_id,1
-EXPECT:name,Anne
-EXPECT:age,38
-EXPECT:_id,2
-EXPECT:name,Bill
-EXPECT:age,23
-EXPECT:_id,3
-EXPECT:name,Cindy
-EXPECT:age,64
-
-// Set expanded back to off.
-SEND:\x off
-EXPECT:Expanded display is off.
-
-// Set format back to aligned as we started.
-SEND:\pset format aligned
-EXPECT:Output format is aligned.
-
-SEND:\pset format invalid
-EXPECT:executing meta command: \pset: allowed formats are aligned, csv

cli/testdata/meta_pset_tuples_only

@@ -1,34 +0,0 @@
-// Set to off.
-SEND:\pset tuples_only off
-EXPECT:Tuples only is off.
-
-// Set to on.
-SEND:\pset tuples_only on
-EXPECT:Tuples only is on.
-
-// Toggle to off.
-SEND:\pset tuples_only
-EXPECT:Tuples only is off.
-
-// Toggle to on.
-SEND:\pset tuples_only
-EXPECT:Tuples only is on.
-
-// Set to something invalid.
-SEND:\pset tuples_only invalid
-EXPECT:executing meta command: unrecognized value "invalid" for "tuples_only": Boolean expected
-
-// Ensure tuples_only shows only tuples.
-SEND:SELECT 1 as foo, 'baz' as bar;
-EXPECT:+---+-----+
-EXPECT:| 1 | baz |
-EXPECT:+---+-----+
-EXPECT:
-
-// Set back to off as we started.
-SEND:\pset tuples_only off
-EXPECT:Tuples only is off.
-
-// make sure the \t meta-command returns expected errors
-SEND:\t off extra
-EXPECT:executing meta command: meta command 'tuples_only' takes zero or one argument

cli/testdata/meta_set

@@ -1,31 +0,0 @@
-SEND:\set
-
-SEND:\set var1 foo
-SEND:\set
-EXPECT:var1 = 'foo'
-
-SEND:\set var2 bar
-SEND:\set
-EXPECT:var1 = 'foo'
-EXPECT:var2 = 'bar'
-
-SEND:\set var3 zoo
-SEND:\set
-EXPECT:var1 = 'foo'
-EXPECT:var2 = 'bar'
-EXPECT:var3 = 'zoo'
-
-SEND:\unset
-EXPECT:\unset: missing required argument
-
-SEND:\unset non-existent-key
-
-SEND:\unset var1
-SEND:\set
-EXPECT:var2 = 'bar'
-EXPECT:var3 = 'zoo'
-
-SEND:\unset var2 extra
-EXPECT:\unset: extra argument "extra" ignored
-SEND:\set
-EXPECT:var3 = 'zoo'

cli/testdata/meta_timing

@@ -1,50 +0,0 @@
-// Start by ensuring timing is off.
-SEND:\timing off
-EXPECT:Timing is off.
-
-// Set timing on.
-SEND:\timing on
-EXPECT:Timing is on.
-
-// Toggle timing.
-SEND:\timing
-EXPECT:Timing is off.
-
-// Toggle timing again.
-SEND:\timing
-EXPECT:Timing is on.
-
-// Send extra argument to \timing.
-SEND:\timing on extra
-EXPECT:executing meta command: meta command 'timing' takes zero or one argument
-
-
-SEND:SELECT * FROM users;
-EXPECT:+-----+-------+-----+
-EXPECT:| _id | name  | age |
-EXPECT:+-----+-------+-----+
-EXPECT:|   1 | Anne  |  38 |
-EXPECT:|   2 | Bill  |  23 |
-EXPECT:|   3 | Cindy |  64 |
-EXPECT:+-----+-------+-----+
-EXPECT:
-EXPECTCOMP:HasPrefix:Execution time:
-
-// Turn timing back off.
-SEND:\timing off
-EXPECT:Timing is off.
-
-// Ensure we don't get timing.
-SEND:SELECT * FROM users;
-EXPECT:+-----+-------+-----+
-EXPECT:| _id | name  | age |
-EXPECT:+-----+-------+-----+
-EXPECT:|   1 | Anne  |  38 |
-EXPECT:|   2 | Bill  |  23 |
-EXPECT:|   3 | Cindy |  64 |
-EXPECT:+-----+-------+-----+
-EXPECT:
-
-// Ensure an invalid timing value returns an error.
-SEND:\timing invalid
-EXPECT:executing meta command: unrecognized value "invalid" for "\timing": Boolean expected

cli/testdata/meta_write

@@ -1,24 +0,0 @@
-// Make sure there's something in the query buffer.
-// This is left unterminated because we don't need to execute the query;
-// we just need there to be something in the buffer.
-SEND:SELECT * FROM invalid-table
-SEND:\write query-buffer-contents
-
-// Reset the buffer.
-SEND:\r
-EXPECT:Query buffer reset (cleared).
-
-// Read from the file.
-SEND:\! cat query-buffer-contents
-EXPECT:SELECT * FROM invalid-table
-
-// Remove the file.
-SEND:\! rm query-buffer-contents
-
-// Send \write with no arguments.
-SEND:\write
-EXPECT:\w: missing required argument
-
-// Send \write with extra arguments.
-SEND:\write filename extra
-EXPECT:executing meta command: meta command 'w' exactly one argument

cli/testdata/people.sql

@@ -1,12 +0,0 @@
--- Create a table.
-create table people (_id id, name string, age int);
-
--- Insert some values.
-insert into people values (1, 'Amy', 42), (2, 'Bob', 27), (3, 'Carl', 33);
-
--- Get all rows from the table.
-select * from people;
-
--- Mix in a meta-command to show that both are supported
--- in the include file.
-\echo mix in a meta command

cli/testdata/query_buffer

@@ -1,40 +0,0 @@
-SEND:select 1 as foo;
-EXPECT:+-----+
-EXPECT:| foo |
-EXPECT:+-----+
-EXPECT:|   1 |
-EXPECT:+-----+
-EXPECT:
-
-SEND:\p
-EXPECT:select 1 as foo;
-
-SEND:select 2
-SEND:\p
-EXPECT:select 2
-SEND:\r
-EXPECT:Query buffer reset (cleared).
-
-SEND:\p
-EXPECT:select 1 as foo;
-
-SEND:select 3
-SEND:\p
-EXPECT:select 3
-
-SEND:as foo
-SEND:\p
-EXPECT:select 3
-EXPECT:as foo
-
-SEND:;
-EXPECT:+-----+
-EXPECT:| foo |
-EXPECT:+-----+
-EXPECT:|   3 |
-EXPECT:+-----+
-EXPECT:
-
-SEND:\p
-EXPECT:select 3
-EXPECT:as foo;

cli/testdata/setup

@@ -1,51 +0,0 @@
-// Startup splash.
-EXPECT:FeatureBase CLI ()
-EXPECT:Type "\q" to quit.
-EXPECT:Detected on-prem, serverless deployment.
-EXPECTCOMP:HasPrefix:Host: http://localhost:
-EXPECT:You are not connected to a database.
-
-// Show databases.
-SEND:SHOW DATABASES;
-EXPECT:Organization required. Use \org to set an organization.
-
-// Get current org.
-SEND:\org
-EXPECT:You have not set an organization.
-
-// Set org.
-SEND:\org acme
-EXPECT:You have set organization "acme".
-
-// Try to set org with too many arguments.
-SEND:\org acme extra
-EXPECT:executing meta command: meta command 'org' takes zero or one argument
-
-// Set location to UTC so that expected timestamp size is consistent.
-// Without this, a test running locally in may have a timestamp that
-// ends in a timezone offset such as `-06:00`, while one running as UTC
-// will have `Z`. Since these string lengths differ, our generic
-// {timestamp} comparison will fail.
-SEND:\pset location UTC
-EXPECT:Location is UTC.
-
-// Set an invalid location.
-SEND:\pset location invalid
-EXPECT:executing meta command: loading location: invalid: unknown time zone invalid
-
-// Try to set location with too many arguments.
-SEND:\pset location UTC extra
-EXPECT:executing meta command: meta command 'pset' takes zero, one, or two arguments
-
-// Set border to 2 for testing because it makes it easier to visually see
-// what the tests are expecting (because lines don't end in spaces).
-SEND:\pset border 2
-EXPECT:Border style is 2.
-
-// Check the state of pset.
-SEND:\pset
-EXPECT:border      2
-EXPECT:expanded    off
-EXPECT:format      aligned
-EXPECT:location    UTC
-EXPECT:tuples_only off

cli/testdata/table

@@ -1,72 +0,0 @@
-// Show tables for database using SHOW TABLES WITH SYSTEM.
-SEND:SHOW TABLES WITH SYSTEM;
-EXPECT:+-------------------------+-------------------------+-------+------------+----------------------+----------------------+-------+------------+-------------+
-EXPECT:| _id                     | name                    | owner | updated_by | created_at           | updated_at           | keys  | space_used | description |
-EXPECT:+-------------------------+-------------------------+-------+------------+----------------------+----------------------+-------+------------+-------------+
-EXPECTCOMP:WithFormat:| fb_____________________ | fb_____________________ |       |            | {timestamp} | {timestamp} | false |          0 |             |
-EXPECTCOMP:WithFormat:| fb_____________________ | fb_____________________ |       |            | {timestamp} | {timestamp} | false |          0 |             |
-EXPECTCOMP:WithFormat:| fb_____________________ | fb_____________________ |       |            | {timestamp} | {timestamp} | false |          0 |             |
-EXPECTCOMP:WithFormat:| fb_____________________ | fb_____________________ |       |            | {timestamp} | {timestamp} | false |          0 |             |
-EXPECTCOMP:WithFormat:| fb_____________________ | fb_____________________ |       |            | {timestamp} | {timestamp} | false |          0 |             |
-EXPECT:+-------------------------+-------------------------+-------+------------+----------------------+----------------------+-------+------------+-------------+
-EXPECT:
-
-// Show tables for database using \d.
-SEND:\d
-EXPECT:+-------------------------+-------------------------+-------+------------+----------------------+----------------------+-------+------------+-------------+
-EXPECT:| _id                     | name                    | owner | updated_by | created_at           | updated_at           | keys  | space_used | description |
-EXPECT:+-------------------------+-------------------------+-------+------------+----------------------+----------------------+-------+------------+-------------+
-EXPECTCOMP:WithFormat:| fb_____________________ | fb_____________________ |       |            | {timestamp} | {timestamp} | false |          0 |             |
-EXPECTCOMP:WithFormat:| fb_____________________ | fb_____________________ |       |            | {timestamp} | {timestamp} | false |          0 |             |
-EXPECTCOMP:WithFormat:| fb_____________________ | fb_____________________ |       |            | {timestamp} | {timestamp} | false |          0 |             |
-EXPECTCOMP:WithFormat:| fb_____________________ | fb_____________________ |       |            | {timestamp} | {timestamp} | false |          0 |             |
-EXPECTCOMP:WithFormat:| fb_____________________ | fb_____________________ |       |            | {timestamp} | {timestamp} | false |          0 |             |
-EXPECT:+-------------------------+-------------------------+-------+------------+----------------------+----------------------+-------+------------+-------------+
-EXPECT:
-
-// Show tables for database using SHOW TABLES.
-SEND:SHOW TABLES;
-EXPECT:+-----+------+-------+------------+------------+------------+------+------------+-------------+
-EXPECT:| _id | name | owner | updated_by | created_at | updated_at | keys | space_used | description |
-EXPECT:+-----+------+-------+------------+------------+------------+------+------------+-------------+
-EXPECT:+-----+------+-------+------------+------------+------------+------+------------+-------------+
-EXPECT:
-
-
-// Show tables for database using \dt.
-SEND:\dt
-EXPECT:+-----+------+-------+------------+------------+------------+------+------------+-------------+
-EXPECT:| _id | name | owner | updated_by | created_at | updated_at | keys | space_used | description |
-EXPECT:+-----+------+-------+------------+------------+------------+------+------------+-------------+
-EXPECT:+-----+------+-------+------------+------------+------------+------+------------+-------------+
-EXPECT:
-
-// Create a table. That can be used for general testing.
-SEND:CREATE TABLE users (_id id, name string, age int);
-EXPECT:
-SEND:INSERT INTO users VALUES (1, 'Anne', 38), (2, 'Bill', 23), (3, 'Cindy', 64);
-EXPECT:
-
-
-// Show tables for database to get the newly created table.
-SEND:\dt
-EXPECT:+-------+-------+-------+------------+----------------------+----------------------+-------+------------+-------------+
-EXPECT:| _id   | name  | owner | updated_by | created_at           | updated_at           | keys  | space_used | description |
-EXPECT:+-------+-------+-------+------------+----------------------+----------------------+-------+------------+-------------+
-EXPECTCOMP:WithFormat:| users | users |       |            | {timestamp} | {timestamp} | false |          0 |             |
-EXPECT:+-------+-------+-------+------------+----------------------+----------------------+-------+------------+-------------+
-EXPECT:
-
-// We don't select from users until AFTER we check SHOW TABLES above because
-// running this creates the fb_views sytem table which has a description.
-// And it's annoying to mask out all of the description fields because we
-// don't know which row fb_views will fall into.
-SEND:SELECT * FROM users;
-EXPECT:+-----+-------+-----+
-EXPECT:| _id | name  | age |
-EXPECT:+-----+-------+-----+
-EXPECT:|   1 | Anne  |  38 |
-EXPECT:|   2 | Bill  |  23 |
-EXPECT:|   3 | Cindy |  64 |
-EXPECT:+-----+-------+-----+
-EXPECT:

cli/workingdir.go

@@ -1,21 +0,0 @@
-package cli
-
-import (
-	"os"
-)
-
-// workingDir was originally set up with the intention of using it to maintain a
-// reference to the current working directory. But it turns out we haven't
-// really needed that so far. The `cd()` method is unsed in one of the meta
-// commands, but we could probably just call `os.Chdir()` directly there. With
-// that said, I'm leaving it here for now until we're abosolutely sure we don't
-// need to use this for other directory/file handling functionality.
-type workingDir struct{}
-
-func newWorkingDir() *workingDir {
-	return &workingDir{}
-}
-
-func (wd *workingDir) cd(dir string) error {
-	return os.Chdir(dir)
-}

cli/writer.go

@@ -1,294 +0,0 @@
-package cli
-
-import (
-	"encoding/csv"
-	"fmt"
-	"io"
-	"log"
-	"time"
-
-	featurebase "github.com/featurebasedb/featurebase/v3"
-	"github.com/jedib0t/go-pretty/table"
-	"github.com/jedib0t/go-pretty/text"
-	"github.com/pkg/errors"
-)
-
-// writeOptions contains user configuration options which describe how to write
-// the query output.
-type writeOptions struct {
-	border     int
-	expanded   bool
-	format     string
-	location   *time.Location
-	timing     bool
-	tuplesOnly bool
-}
-
-const (
-	formatAligned = "aligned"
-	formatCSV     = "csv"
-)
-
-func defaultWriteOptions() *writeOptions {
-	return &writeOptions{
-		border:     1,
-		expanded:   false,
-		format:     formatAligned,
-		location:   time.Local,
-		timing:     false,
-		tuplesOnly: false,
-	}
-}
-
-// writeOutput writes the query response, taking the format into consideration.
-// It sends query output to qOut, non-error informational output (such as query
-// timing) to wOut, and errors to wErr.
-func writeOutput(r *featurebase.WireQueryResponse, opts *writeOptions, qOut io.Writer, wOut io.Writer, wErr io.Writer) error {
-	if r == nil {
-		return errors.New("attempt to write out nil response")
-	}
-	if r.Error != "" {
-		if _, err := wErr.Write([]byte("Error: " + r.Error + "\n")); err != nil {
-			return errors.Wrapf(err, "writing error: %s", r.Error)
-		}
-		return writeWarnings(r, wErr)
-	}
-
-	switch opts.format {
-	case formatAligned:
-		if err := writeTable(r, opts, qOut); err != nil {
-			return errors.Wrap(err, "writing table")
-		}
-
-		// Add some white space after query results.
-		qOut.Write([]byte("\n"))
-
-	case formatCSV:
-		if err := writeCSV(r, opts, qOut); err != nil {
-			return errors.Wrap(err, "writing csv")
-		}
-
-	default:
-		return errors.Errorf("invalid format: %s", opts.format)
-	}
-
-	if err := writeWarnings(r, wErr); err != nil {
-		return err
-	}
-
-	// Timing.
-	if opts.timing {
-		if _, err := wOut.Write([]byte(fmt.Sprintf("Execution time: %dμs\n", r.ExecutionTime))); err != nil {
-			return errors.Wrapf(err, "writing execution time: %s", r.Error)
-		}
-	}
-
-	return nil
-}
-
-// writeCSV writes the WireQueryResponse to qOut as csv.
-func writeCSV(r *featurebase.WireQueryResponse, opts *writeOptions, qOut io.Writer) error {
-	w := csv.NewWriter(qOut)
-
-	if opts.expanded {
-		// Expanded csv
-
-		// rec is used to write the row as a slice of strings. It is reused to
-		// avoid unnecessary memory allocation.
-		rec := make([]string, 2)
-
-		for _, row := range r.Data {
-			cleanRow(row, opts)
-			for i, col := range r.Schema.Fields {
-				rec[0] = string(col.Name)
-				rec[1] = fmt.Sprintf("%v", row[i])
-
-				// Write the record.
-				if err := w.Write(rec); err != nil {
-					log.Fatalln("error writing expanded record to csv:", err)
-				}
-			}
-		}
-
-	} else {
-		// Normal csv (i.e. NOT expanded)
-
-		// Write the schema.
-		if !opts.tuplesOnly {
-			header := make([]string, 0, len(r.Schema.Fields))
-			for i := range r.Schema.Fields {
-				header = append(header, string(r.Schema.Fields[i].Name))
-			}
-			if err := w.Write(header); err != nil {
-				return errors.Wrapf(err, "error writing header to csv")
-			}
-		}
-
-		// Write the records.
-
-		// rec is used to write the row as a slice of strings. It is reused to
-		// avoid unnecessary memory allocation.
-		rec := make([]string, len(r.Schema.Fields))
-
-		for _, row := range r.Data {
-			cleanRow(row, opts)
-			for i := range row {
-				rec[i] = fmt.Sprintf("%v", row[i])
-			}
-			if err := w.Write(rec); err != nil {
-				log.Fatalln("error writing record to csv:", err)
-			}
-		}
-	}
-
-	// Write any buffered data to the underlying writer (standard output).
-	w.Flush()
-
-	return w.Error()
-}
-
-// writeTable writes the WireQueryResponse to qOut in a tabular format.
-func writeTable(r *featurebase.WireQueryResponse, opts *writeOptions, qOut io.Writer) error {
-	t := table.NewWriter()
-	t.SetOutputMirror(qOut)
-	switch opts.border {
-	case 0:
-		t.SetStyle(styleBorder0)
-	case 1:
-		t.SetStyle(styleBorder1)
-	default:
-		t.SetStyle(styleBorder2)
-		// In expanded mode with a border, we need borders between each record.
-		if opts.expanded {
-			t.Style().Options.SeparateRows = true
-		}
-	}
-
-	// Don't uppercase the header values.
-	t.Style().Format.Header = text.FormatDefault
-
-	if opts.expanded {
-		// Expanded table
-		for _, row := range r.Data {
-			cleanRow(row, opts)
-			colRow := make([]interface{}, 2)
-			scolRow := make([]string, 2)
-			div := "\n"
-			for i, col := range r.Schema.Fields {
-				if i == len(r.Schema.Fields)-1 {
-					div = ""
-				}
-				scolRow[0] += fmt.Sprintf("%s%s", col.Name, div)
-				scolRow[1] += fmt.Sprintf("%v%s", row[i], div)
-			}
-			colRow[0] = scolRow[0]
-			colRow[1] = scolRow[1]
-			t.AppendRow(table.Row(colRow[:]))
-		}
-	} else {
-		// Normal table (i.e. NOT expanded)
-		if !opts.tuplesOnly {
-			t.AppendHeader(schemaToRow(r.Schema))
-		}
-		for _, row := range r.Data {
-			cleanRow(row, opts)
-			t.AppendRow(table.Row(row))
-		}
-	}
-	t.Render()
-
-	return nil
-}
-
-// cleanRow loops through all the columns of row and modifies its value based on
-// type.
-//
-// If the value is nil, replace it with a null string; go-pretty doesn't expect
-// nil pointers in the data values.
-//
-// If the value is a time.Time, we want to print it using RFC3339Nano to be
-// consistent with everything else.
-func cleanRow(row []interface{}, opts *writeOptions) {
-	for i := range row {
-		switch v := row[i].(type) {
-		case nil:
-			row[i] = nullValue
-		case time.Time:
-			row[i] = v.In(opts.location).Format(time.RFC3339Nano)
-		}
-	}
-}
-
-func schemaToRow(schema featurebase.WireQuerySchema) []interface{} {
-	ret := make([]interface{}, len(schema.Fields))
-	for i, field := range schema.Fields {
-		ret[i] = field.Name
-	}
-	return ret
-}
-
-func writeWarnings(r *featurebase.WireQueryResponse, w io.Writer) error {
-	if len(r.Warnings) == 0 {
-		return nil
-	}
-
-	if _, err := w.Write([]byte("\n")); err != nil {
-		return errors.Wrapf(err, "writing line feed")
-	}
-	for _, warning := range r.Warnings {
-		if _, err := w.Write([]byte("Warning: " + warning + "\n")); err != nil {
-			return errors.Wrapf(err, "writing warning: %s", warning)
-		}
-	}
-	return nil
-}
-
-var styleBorder2 table.Style = table.StyleDefault
-
-var styleBorder1 table.Style = table.Style{
-	Name:   "StyleBorder1",
-	Box:    table.StyleBoxDefault,
-	Color:  table.ColorOptionsDefault,
-	Format: table.FormatOptionsDefault,
-	Options: table.Options{
-		DrawBorder:      false,
-		SeparateColumns: true,
-		SeparateFooter:  true,
-		SeparateHeader:  true,
-		SeparateRows:    false,
-	},
-	Title: table.TitleOptionsDefault,
-}
-
-var styleBorder0 table.Style = table.Style{
-	Name: "StyleBorder0",
-	Box: table.BoxStyle{
-		BottomLeft:       "+",
-		BottomRight:      "+",
-		BottomSeparator:  "+",
-		Left:             "|",
-		LeftSeparator:    "+",
-		MiddleHorizontal: "-",
-		MiddleSeparator:  " ",
-		MiddleVertical:   " ",
-		PaddingLeft:      "",
-		PaddingRight:     "",
-		PageSeparator:    "\n",
-		Right:            "|",
-		RightSeparator:   "+",
-		TopLeft:          "+",
-		TopRight:         "+",
-		TopSeparator:     "+",
-		UnfinishedRow:    " ~",
-	},
-	Color:  table.ColorOptionsDefault,
-	Format: table.FormatOptionsDefault,
-	Options: table.Options{
-		DrawBorder:      false,
-		SeparateColumns: true,
-		SeparateFooter:  true,
-		SeparateHeader:  true,
-		SeparateRows:    false,
-	},
-	Title: table.TitleOptionsDefault,
-}

cli/writer_test.go

@@ -1,188 +0,0 @@
-package cli
-
-import (
-	"bytes"
-	"fmt"
-	"strings"
-	"testing"
-
-	featurebase "github.com/featurebasedb/featurebase/v3"
-	dax "github.com/featurebasedb/featurebase/v3/dax"
-	"github.com/stretchr/testify/assert"
-)
-
-func TestWriter(t *testing.T) {
-	t.Run("writeTable", func(t *testing.T) {
-		wqr := &featurebase.WireQueryResponse{
-			Schema: featurebase.WireQuerySchema{
-				Fields: []*featurebase.WireQueryField{
-					{Name: "_id", Type: dax.BaseTypeID},
-					{Name: "name", Type: dax.BaseTypeString},
-					{Name: "age", Type: dax.BaseTypeInt},
-				},
-			},
-			Data: [][]interface{}{
-				{1, "Amy", 44},
-				{2, "Bob", 32},
-				{3, "Cindy", 28},
-			},
-		}
-
-		// TODO(tlt): used for debugging
-		// format := defaultWriteOptions()
-		// assert.NoError(t, writeTable(wqr, format, os.Stdout, os.Stdout, os.Stdout))
-		// return
-
-		tests := []struct {
-			format  *writeOptions
-			expQOut string
-			expOut  string
-			expErr  string
-		}{
-			{
-				// default format
-				format: defaultWriteOptions(),
-				expQOut: stringOfLines(
-					" _id | name  | age ",
-					"-----+-------+-----",
-					"   1 | Amy   |  44 ",
-					"   2 | Bob   |  32 ",
-					"   3 | Cindy |  28 ",
-					"",
-				),
-				expOut: "",
-				expErr: "",
-			},
-			{
-				// timing on
-				format: &writeOptions{
-					border:     1,
-					expanded:   false,
-					format:     formatAligned,
-					timing:     true,
-					tuplesOnly: false,
-				},
-				expQOut: stringOfLines(
-					" _id | name  | age ",
-					"-----+-------+-----",
-					"   1 | Amy   |  44 ",
-					"   2 | Bob   |  32 ",
-					"   3 | Cindy |  28 ",
-					"",
-				),
-				expOut: "Execution time: 0μs\n",
-				expErr: "",
-			},
-			{
-				// format.border = 2 (or higher)
-				format: &writeOptions{
-					border:     2,
-					expanded:   false,
-					format:     formatAligned,
-					timing:     false,
-					tuplesOnly: false,
-				},
-				expQOut: stringOfLines(
-					"+-----+-------+-----+",
-					"| _id | name  | age |",
-					"+-----+-------+-----+",
-					"|   1 | Amy   |  44 |",
-					"|   2 | Bob   |  32 |",
-					"|   3 | Cindy |  28 |",
-					"+-----+-------+-----+",
-					"",
-				),
-				expOut: "",
-				expErr: "",
-			},
-			{
-				// format.border = 0
-				format: &writeOptions{
-					border:     0,
-					expanded:   false,
-					format:     formatAligned,
-					timing:     false,
-					tuplesOnly: false,
-				},
-				expQOut: stringOfLines(
-					"_id name  age",
-					"--- ----- ---",
-					"  1 Amy    44",
-					"  2 Bob    32",
-					"  3 Cindy  28",
-					"",
-				),
-				expOut: "",
-				expErr: "",
-			},
-			{
-				// format.tuplesOnly = true
-				format: &writeOptions{
-					border:     1,
-					expanded:   false,
-					format:     formatAligned,
-					timing:     false,
-					tuplesOnly: true,
-				},
-				expQOut: stringOfLines(
-					" 1 | Amy   | 44 ",
-					" 2 | Bob   | 32 ",
-					" 3 | Cindy | 28 ",
-					"",
-				),
-				expOut: "",
-				expErr: "",
-			},
-			{
-				// format.border = 2, expanded
-				format: &writeOptions{
-					border:     2,
-					expanded:   true,
-					format:     formatAligned,
-					timing:     false,
-					tuplesOnly: false,
-				},
-				expQOut: stringOfLines(
-					"+------+-------+",
-					"| _id  | 1     |",
-					"| name | Amy   |",
-					"| age  | 44    |",
-					"+------+-------+",
-					"| _id  | 2     |",
-					"| name | Bob   |",
-					"| age  | 32    |",
-					"+------+-------+",
-					"| _id  | 3     |",
-					"| name | Cindy |",
-					"| age  | 28    |",
-					"+------+-------+",
-					"",
-				),
-				expOut: "",
-				expErr: "",
-			},
-		}
-		for i, test := range tests {
-			t.Run(fmt.Sprintf("test-%d", i), func(t *testing.T) {
-				// Set up buffers to capture the output.
-				qOut := bytes.NewBuffer(make([]byte, 0, 100000))
-				wOut := bytes.NewBuffer(make([]byte, 0, 100000))
-				wErr := bytes.NewBuffer(make([]byte, 0, 100000))
-
-				assert.NoError(t, writeOutput(wqr, test.format, qOut, wOut, wErr))
-
-				assert.Equal(t, test.expQOut, qOut.String())
-				assert.Equal(t, test.expOut, wOut.String())
-				assert.Equal(t, test.expErr, wErr.String())
-			})
-		}
-	})
-}
-
-func stringOfLines(lines ...string) string {
-	var sb strings.Builder
-	for _, line := range lines {
-		sb.WriteString(line + "\n")
-	}
-	return sb.String()
-}

client/api.go

@@ -26,26 +26,6 @@ func NewSchemaAPI(c *Client) *schemaAPI {
 	}
 }
 
-func (s *schemaAPI) CreateDatabase(context.Context, *dax.Database) error {
-	return errors.Errorf("unimplemented: schemaAPI.CreateDatabase()")
-}
-func (s *schemaAPI) DropDatabase(context.Context, dax.DatabaseID) error {
-	return errors.Errorf("unimplemented: schemaAPI.DropDatabase()")
-}
-
-func (s *schemaAPI) DatabaseByName(ctx context.Context, dbname dax.DatabaseName) (*dax.Database, error) {
-	return nil, errors.Errorf("unimplemented: schemaAPI.DatabaseByName()")
-}
-func (s *schemaAPI) DatabaseByID(ctx context.Context, dbid dax.DatabaseID) (*dax.Database, error) {
-	return nil, errors.Errorf("unimplemented: schemaAPI.DatabaseByID()")
-}
-func (s *schemaAPI) SetDatabaseOption(ctx context.Context, dbid dax.DatabaseID, option string, value string) error {
-	return nil
-}
-func (s *schemaAPI) Databases(context.Context, ...dax.DatabaseID) ([]*dax.Database, error) {
-	return nil, errors.Errorf("unimplemented: schemaAPI.Databases()")
-}
-
 func (s *schemaAPI) TableByName(ctx context.Context, tname dax.TableName) (*dax.Table, error) {
 	return nil, errors.New(errors.ErrUncoded, "schemaAPI.TableByName not implemented")
 }

client/client.go

@@ -20,6 +20,7 @@ import (
 	"sync"
 	"time"
 
+	"github.com/golang/protobuf/proto" //nolint:staticcheck
 	pilosa "github.com/featurebasedb/featurebase/v3"
 	"github.com/featurebasedb/featurebase/v3/client/types"
 	fbproto "github.com/featurebasedb/featurebase/v3/encoding/proto" // TODO use this everywhere and get rid of proto import
@@ -28,8 +29,8 @@ import (
 	"github.com/featurebasedb/featurebase/v3/pb"
 	"github.com/featurebasedb/featurebase/v3/pql"
 	"github.com/featurebasedb/featurebase/v3/roaring"
+	"github.com/featurebasedb/featurebase/v3/stats"
 	"github.com/featurebasedb/featurebase/v3/vprint"
-	"github.com/golang/protobuf/proto" //nolint:staticcheck
 	"github.com/opentracing/opentracing-go"
 	"github.com/pkg/errors"
 	"golang.org/x/sync/errgroup"
@@ -51,6 +52,7 @@ type Client struct {
 	manualFragmentNode *fragmentNode
 	manualServerURI    *pnet.URI
 	tracer             opentracing.Tracer
+	Stats              stats.StatsClient
 	// An exponential backoff algorithm retries requests exponentially (if an HTTP request fails),
 	// increasing the waiting time between retries up to a maximum backoff time.
 	maxBackoff time.Duration
@@ -209,6 +211,11 @@ func newClientWithOptions(options *ClientOptions) *Client {
 	} else {
 		c.tracer = options.tracer
 	}
+	if options.stats == nil {
+		c.Stats = stats.NopStatsClient
+	} else {
+		c.Stats = options.stats
+	}
 
 	c.maxRetries = *options.retries
 	c.maxBackoff = 2 * time.Minute
@@ -1352,6 +1359,7 @@ type ClientOptions struct {
 	manualServerAddress bool
 	tracer              opentracing.Tracer
 	retries             *int
+	stats               stats.StatsClient
 	nat                 map[pnet.URI]pnet.URI
 	pathPrefix          string
 }
@@ -1437,6 +1445,14 @@ func OptClientRetries(retries int) ClientOption {
 	}
 }
 
+// OptClientStatsClient sets a stats client, such as Prometheus
+func OptClientStatsClient(stats stats.StatsClient) ClientOption {
+	return func(options *ClientOptions) error {
+		options.stats = stats
+		return nil
+	}
+}
+
 // OptClientNAT sets a NAT map used to translate the advertised URI to something
 // else (for example, when accessing pilosa running in docker).
 func OptClientNAT(nat map[string]string) ClientOption {

client/importer.go

@@ -328,3 +328,7 @@ func (i *importer) EncodeImport(ctx context.Context, tid dax.TableID, fld *dax.F
 func (i *importer) DoImport(ctx context.Context, tid dax.TableID, fld *dax.Field, shard uint64, path string, data []byte) error {
 	return i.client.DoImport(string(tid), shard, path, data)
 }
+
+func (i *importer) StatsTiming(name string, value time.Duration, rate float64) {
+	i.client.Stats.Timing(name, value, rate)
+}

cluster.go

@@ -26,7 +26,7 @@ const (
 )
 
 // cluster represents a collection of nodes.
-type cluster struct { //nolint: maligned
+type cluster struct { // nolint: maligned
 	noder disco.Noder
 
 	id   string

cmd/auth_token.go

@@ -16,7 +16,7 @@ func newAuthTokenCommand(logdest logger.Logger) *cobra.Command {
 		Long: `
 Retrieves an auth-token for use in authenticating with FeatureBase from the configured identity provider.
 `,
-		RunE: UsageErrorWrapper(cmd),
+		RunE: usageErrorWrapper(cmd),
 	}
 
 	flags := ccmd.Flags()

cmd/backup.go

@@ -16,7 +16,7 @@ func newBackupCommand(logdest logger.Logger) *cobra.Command {
 		Long: `
 Backs up a FeatureBase server to a local, tar-formatted snapshot file.
 `,
-		RunE: UsageErrorWrapper(cmd),
+		RunE: usageErrorWrapper(cmd),
 	}
 
 	flags := ccmd.Flags()

cmd/backup_tar.go

@@ -16,7 +16,7 @@ func newBackupTarCommand(logdest io.Writer) *cobra.Command {
 		Long: `
 Backs up a FeatureBase server to a local, tar-formatted snapshot file.
 `,
-		RunE: UsageErrorWrapper(cmd),
+		RunE: usageErrorWrapper(cmd),
 	}
 
 	flags := ccmd.Flags()
@@ -28,7 +28,6 @@ Backs up a FeatureBase server to a local, tar-formatted snapshot file.
 	ctl.SetTLSConfig(flags, "", &cmd.TLS.CertificatePath, &cmd.TLS.CertificateKeyPath, &cmd.TLS.CACertPath, &cmd.TLS.SkipVerify, &cmd.TLS.EnableClientVerification)
 	flags.StringVar(&cmd.AuthToken, "auth-token", "", "Authentication token")
 	flags.StringVar(&cmd.HeaderTimeoutStr, "header-timeout", cmd.HeaderTimeoutStr, "Length of time to wait for initial HTTP response before giving up.")
-	flags.StringVar(&cmd.TempDir, "temp-dir", cmd.TempDir, "Location of temporary spillover files. The default is the system's default (usually /tmp)")
 
 	return ccmd
 }

cmd/chksum.go

@@ -3,15 +3,13 @@
 package cmd
 
 import (
-	"os"
-
 	"github.com/featurebasedb/featurebase/v3/ctl"
 	"github.com/featurebasedb/featurebase/v3/logger"
 	"github.com/spf13/cobra"
 )
 
 func newChkSumCommand(logdest logger.Logger) *cobra.Command {
-	cmd := ctl.NewChkSumCommand(logdest, os.Stdout)
+	cmd := ctl.NewChkSumCommand(logdest)
 	ccmd := &cobra.Command{
 		Use:   "chksum",
 		Short: "Digital signature of FeatureBase data",
@@ -19,7 +17,7 @@ func newChkSumCommand(logdest logger.Logger) *cobra.Command {
 			Generates a digital signature of all the data associated with a provided FeatureBase server
 			WARNING: could be slow if high cardinality fields exist
 `,
-		RunE: UsageErrorWrapper(cmd),
+		RunE: usageErrorWrapper(cmd),
 	}
 
 	flags := ccmd.Flags()

cmd/cli.go

@@ -0,0 +1,35 @@
+// Copyright 2021 Molecula Corp. All rights reserved.
+package cmd
+
+import (
+	"github.com/featurebasedb/featurebase/v3/ctl"
+	"github.com/featurebasedb/featurebase/v3/logger"
+	"github.com/spf13/cobra"
+)
+
+var cli *ctl.CLICommand
+
+// newCLICommand runs the FeatureBase CLI subcommand for ingesting bulk data.
+func newCLICommand(logdest logger.Logger) *cobra.Command {
+	cli = ctl.NewCLICommand(logdest)
+	cliCmd := &cobra.Command{
+		Use:   "cli",
+		Short: "Query FB with SQL3 from the command line",
+		Long:  ``,
+		RunE:  usageErrorWrapper(cli),
+	}
+
+	flags := cliCmd.Flags()
+	flags.StringVarP(&cli.Host, "host", "", cli.Host, "hostname of FeatureBase.")
+	flags.StringVarP(&cli.Port, "port", "", cli.Port, "port of FeatureBase.")
+	flags.StringVar(&cli.HistoryPath, "history-path", cli.HistoryPath, "path for history files.")
+	flags.StringVar(&cli.OrganizationID, "org-id", cli.OrganizationID, "OrganizationID.")
+	flags.StringVar(&cli.DatabaseID, "db-id", cli.DatabaseID, "DatabaseID.")
+
+	flags.StringVar(&cli.ClientID, "client-id", cli.ClientID, "Cognito Client ID for FeatureBase Cloud access.")
+	flags.StringVar(&cli.Region, "region", cli.Region, "Cloud region for FeatureBase Cloud access (e.g. us-east-2).")
+	flags.StringVar(&cli.Email, "email", cli.Email, "Email address for FeatureBase Cloud access.")
+	flags.StringVar(&cli.Password, "password", cli.Password, "Password for FeatureBase Cloud access.")
+
+	return cliCmd
+}

cmd/dataframe-csv-loader.go

@@ -15,7 +15,7 @@ func newDataframeCsvLoaderCommand(logdest logger.Logger) *cobra.Command {
 		Short: "load dataframe integer and floating point values into featurebase",
 		Long: `
 `,
-		RunE: UsageErrorWrapper(cmd),
+		RunE: usageErrorWrapper(cmd),
 	}
 	flags := loaderCmd.Flags()
 	flags.StringVar(&cmd.Path, "csv", "", "path to csv input file")
@@ -24,7 +24,6 @@ func newDataframeCsvLoaderCommand(logdest logger.Logger) *cobra.Command {
 	flags.StringVar(&cmd.AuthToken, "auth-token", "", "Authentication token")
 	flags.StringVar(&cmd.Index, "index", "", "Destination Index. ")
 	flags.IntVar(&cmd.MaxCapacity, "buffer", 0, "Maximum size of of the line buffer defaults to go bufio default ")
-	flags.IntVar(&cmd.BatchSize, "batch-size", 1048576, "Maximum number of records to send in a single batch ")
 	ctl.SetTLSConfig(
 		flags, "",
 		&cmd.TLS.CertificatePath,

cmd/export.go

@@ -26,7 +26,7 @@ The format of the CSV file is:
 
 The file does not contain any headers.
 `,
-		RunE: UsageErrorWrapper(Exporter),
+		RunE: usageErrorWrapper(Exporter),
 	}
 	flags := exportCmd.Flags()
 

cmd/fbsql/main.go

@@ -1,66 +0,0 @@
-// Copyright 2021 Molecula Corp. All rights reserved.
-package main
-
-import (
-	"io"
-	"os"
-
-	"github.com/featurebasedb/featurebase/v3/cli"
-	"github.com/featurebasedb/featurebase/v3/cmd"
-	"github.com/featurebasedb/featurebase/v3/logger"
-	"github.com/spf13/cobra"
-	"github.com/spf13/viper"
-)
-
-func main() {
-	command := newCLICommand(os.Stderr)
-	command.Execute()
-}
-
-// newCLICommand runs the FeatureBase CLI subcommand.
-func newCLICommand(stderr io.Writer) *cobra.Command {
-	logdest := logger.NewStandardLogger(stderr)
-	cliCmd := cli.NewCommand(logdest)
-	cobraCmd := &cobra.Command{
-		Use:   "fbsql",
-		Short: "Query FeatureBase with SQL from the command line",
-		Long:  ``,
-		RunE:  cmd.UsageErrorWrapper(cliCmd),
-		PersistentPreRunE: func(cobraCmd *cobra.Command, args []string) error {
-			v := viper.New()
-			return cmd.SetAllConfig(v, cobraCmd.Flags(), "FBSQL")
-		},
-		SilenceErrors: true,
-	}
-
-	// Attach flags to the command.
-	buildFlags(cobraCmd, cliCmd)
-	return cobraCmd
-}
-
-// buildFlags attaches a set of flags to the command for a cli instance.
-func buildFlags(cmd *cobra.Command, cliCmd *cli.Command) {
-	flags := cmd.Flags()
-
-	// Base struct flags.
-	flags.StringSliceVarP(&cliCmd.Commands, "command", "c", cliCmd.Commands, "Command to run in non-interactive mode. Provide multiple flags to execute more than one command. All `--command` flags run before all `--file` flags.")
-	flags.StringSliceVarP(&cliCmd.Files, "file", "f", cliCmd.Files, "File to run in non-interactive mode. Provide multiple flags to execute more than one file. All `--command` flags run before all `--file` flags.")
-
-	// Config flags.
-	flags.StringVarP(&cliCmd.Config.Host, "host", "", cliCmd.Config.Host, "hostname of FeatureBase.")
-	flags.StringVarP(&cliCmd.Config.Port, "port", "p", cliCmd.Config.Port, "port of FeatureBase.")
-	flags.StringVar(&cliCmd.Config.HistoryPath, "history-path", cliCmd.Config.HistoryPath, "path for history files.")
-	flags.StringVar(&cliCmd.Config.OrganizationID, "org-id", cliCmd.Config.OrganizationID, "OrganizationID.")
-	flags.StringVarP(&cliCmd.Config.Database, "dbname", "d", cliCmd.Config.Database, "Name of the database to connect to.")
-
-	flags.StringVar(&cliCmd.Config.CloudAuth.ClientID, "client-id", cliCmd.Config.CloudAuth.ClientID, "Cognito Client ID for FeatureBase Cloud access.")
-	flags.StringVar(&cliCmd.Config.CloudAuth.Region, "region", cliCmd.Config.CloudAuth.Region, "Cloud region for FeatureBase Cloud access (e.g. us-east-2).")
-	flags.StringVar(&cliCmd.Config.CloudAuth.Email, "email", cliCmd.Config.CloudAuth.Email, "Email address for FeatureBase Cloud access.")
-	flags.StringVar(&cliCmd.Config.CloudAuth.Password, "password", cliCmd.Config.CloudAuth.Password, "Password for FeatureBase Cloud access.")
-
-	flags.StringVar(&cliCmd.Config.KafkaConfig, "kafka-config", cliCmd.Config.KafkaConfig, "Kafka configuration file to read from.")
-	flags.BoolVar(&cliCmd.Config.CSV, "csv", cliCmd.Config.CSV, "CSV (Comma-Separated Values) table output mode.")
-	flags.StringSliceVar(&cliCmd.Config.PSets, "pset", cliCmd.Config.PSets, "Set printing option VAR to ARG (see \\pset command). Use form: --pset=VAR[=ARG]")
-
-	flags.String("config", "", "Configuration file to read from.")
-}

cmd/generate_config.go

@@ -18,7 +18,7 @@ func newGenerateConfigCommand(logdest logger.Logger) *cobra.Command {
 		Short: "Print the default configuration.",
 		Long: `generate-config prints the default configuration to stdout
 `,
-		RunE: UsageErrorWrapper(generateConf),
+		RunE: usageErrorWrapper(generateConf),
 	}
 
 	return confCmd

cmd/keygen.go

@@ -16,7 +16,7 @@ func newKeygenCommand(logdest logger.Logger) *cobra.Command {
 		Long: `
 Generate secret key to configure FeatureBase for Authentication.
 `,
-		RunE: UsageErrorWrapper(cmd),
+		RunE: usageErrorWrapper(cmd),
 	}
 
 	flags := ccmd.Flags()

cmd/parquet-info.go

@@ -1,33 +0,0 @@
-// Copyright 2022 Molecula Corp. (DBA FeatureBase).
-// SPDX-License-Identifier: Apache-2.0
-package cmd
-
-import (
-	"fmt"
-
-	"github.com/featurebasedb/featurebase/v3/ctl"
-	"github.com/featurebasedb/featurebase/v3/logger"
-	"github.com/spf13/cobra"
-)
-
-func newParquetInfoCommand(logdest logger.Logger) *cobra.Command {
-	c := ctl.NewParquetInfoCommand(logdest)
-	cmd := &cobra.Command{
-		Use:   "parquet-info PATH|URL",
-		Short: "Inspect Parquet Files.",
-		Long: `
-Displays schema and sample data from the specified file
-`,
-		Args: func(cmd *cobra.Command, args []string) error {
-			if len(args) == 0 {
-				return fmt.Errorf("data directory path required")
-			} else if len(args) > 1 {
-				return fmt.Errorf("too many command line arguments")
-			}
-			c.Path = args[0]
-			return nil
-		},
-		RunE: UsageErrorWrapper(c),
-	}
-	return cmd
-}

cmd/presort.go

@@ -1,32 +0,0 @@
-// Copyright 2021 Molecula Corp. All rights reserved.
-package cmd
-
-import (
-	"github.com/featurebasedb/featurebase/v3/ctl"
-	"github.com/featurebasedb/featurebase/v3/logger"
-	"github.com/spf13/cobra"
-)
-
-func newPreSortCommand(logdest logger.Logger) *cobra.Command {
-	cmd := ctl.NewPreSortCommand(logdest)
-	ccmd := &cobra.Command{
-		Use:   "presort",
-		Short: "Sort records within files into files by FB partition for more efficient ingest",
-		Long: `
-Takes all input files and writes PartitionN numbered files to a directory, where each file contains only records that will go into the partition it is named for.
-`,
-		RunE: UsageErrorWrapper(cmd),
-	}
-
-	flags := ccmd.Flags()
-	flags.StringVarP(&cmd.File, "file", "", "", "Input file or directory.")
-	flags.StringVarP(&cmd.Table, "table", "", "", "Name of table (used to hash keys to determine partition).")
-	flags.StringVarP(&cmd.Type, "type", "", cmd.Type, "Input file type (csv or ndjson).")
-	flags.StringSliceVar(&cmd.PrimaryKeyFields, "primary-key-fields", []string{}, "Names of primary key fields. For CSV there must be a header row and these are pulled from there.")
-	flags.IntVar(&cmd.PartitionN, "partition-n", cmd.PartitionN, "Number of partitions.")
-	flags.StringVarP(&cmd.OutputDir, "output-dir", "", cmd.OutputDir, "Directory name to write output to.")
-	flags.StringVarP(&cmd.PrimaryKeySeparator, "primary-key-separator", "", cmd.PrimaryKeySeparator, "Separator to write in between primary key fields, can be empty.")
-	flags.IntVar(&cmd.JobSize, "job-size", cmd.JobSize, "Number of lines to put into each job (purely a performance tuning parameter, only supported by ndjson mode).")
-	flags.IntVar(&cmd.NumWorkers, "num-workers", cmd.NumWorkers, "Number of parallel worker routines doing decode->hash->encode. Only supported by ndjson mode.")
-	return ccmd
-}

cmd/rbf.go

@@ -44,7 +44,7 @@ Executes a consistency check on an RBF data directory.
 			c.Path = args[0]
 			return nil
 		},
-		RunE: UsageErrorWrapper(c),
+		RunE: usageErrorWrapper(c),
 	}
 	return cmd
 }
@@ -76,7 +76,7 @@ Dumps the raw hex data for one or more RBF pages.
 
 			return nil
 		},
-		RunE: UsageErrorWrapper(c),
+		RunE: usageErrorWrapper(c),
 	}
 	return cmd
 }
@@ -98,7 +98,7 @@ Prints a line for every page in the database with its type/status.
 			c.Path = args[0]
 			return nil
 		},
-		RunE: UsageErrorWrapper(c),
+		RunE: usageErrorWrapper(c),
 	}
 
 	flags := cmd.Flags()
@@ -133,7 +133,7 @@ Prints the header & cell data for one or more pages.
 
 			return nil
 		},
-		RunE: UsageErrorWrapper(c),
+		RunE: usageErrorWrapper(c),
 	}
 	return cmd
 }

cmd/restore.go

@@ -16,7 +16,7 @@ func newRestoreCommand(logdest logger.Logger) *cobra.Command {
 		Long: `
 The Restore command will take a backup archive and restore it to a new, clean cluster.
 `,
-		RunE: UsageErrorWrapper(cmd),
+		RunE: usageErrorWrapper(cmd),
 	}
 	flags := restoreCmd.Flags()
 	flags.StringVarP(&cmd.Path, "source", "s", "", "backup file; specify '-' to restore from stdin tar stream")

cmd/restore_tar.go

@@ -15,7 +15,7 @@ func newRestoreTarCommand(logdest logger.Logger) *cobra.Command {
 		Long: `
 The Restore command will take a tar-formatted backup archive and restore it to a new, clean cluster.
 `,
-		RunE: UsageErrorWrapper(cmd),
+		RunE: usageErrorWrapper(cmd),
 	}
 	flags := restoreCmd.Flags()
 	flags.StringVarP(&cmd.Path, "source", "s", "", "backup file; specify '-' to restore from stdin tar stream")
@@ -23,8 +23,6 @@ The Restore command will take a tar-formatted backup archive and restore it to a
 	flags.DurationVar(&cmd.RetryPeriod, "retry-period", cmd.RetryPeriod, "Length of time after HTTP request failure to continue retrying request.")
 	flags.StringVar(&cmd.Pprof, "pprof", cmd.Pprof, "host:port to listen for profiling requests at /debug/pprof and /debug/fgprof.")
 	flags.StringVar(&cmd.AuthToken, "auth-token", "", "Authentication token")
-	flags.StringVar(&cmd.TempDir, "temp-dir", cmd.TempDir, "Location of temporary spillover files. The default is the system's default (usually /tmp)")
-
 	ctl.SetTLSConfig(
 		flags, "",
 		&cmd.TLS.CertificatePath,

cmd/root.go

@@ -25,11 +25,11 @@ type runner interface {
 	Run(context.Context) error
 }
 
-// UsageErrorWrapper takes a thing with a Run(context) error, and produces
+// usageErrorWrapper takes a thing with a Run(context) error, and produces
 // a func(*cobra.Command, []string) error from it which will run that
 // command, and then set Cobra's SilenceUsage flag unless the returned
 // error errors.Is() a ctl.UsageError.
-func UsageErrorWrapper(inner runner) func(*cobra.Command, []string) error {
+func usageErrorWrapper(inner runner) func(*cobra.Command, []string) error {
 	return func(c *cobra.Command, args []string) error {
 		return considerUsageError(c, inner.Run(context.Background()))
 	}
@@ -41,7 +41,7 @@ func UsageErrorWrapper(inner runner) func(*cobra.Command, []string) error {
 // wrappers.
 func considerUsageError(cmd *cobra.Command, err error) error {
 	cmd.SilenceErrors = true
-	if !errors.Is(err, ctl.ErrUsage) {
+	if !errors.Is(err, ctl.UsageError) {
 		cmd.SilenceUsage = true
 	}
 	return err
@@ -63,12 +63,11 @@ at https://docs.featurebase.com/.
 ` + pilosa.VersionInfo(true) + "\n",
 		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
 			v := viper.New()
-
-			switch cmd.Use {
-			case "dax":
+			if cmd.Use == "dax" {
 				v.Set("future.rename", true) // always use FEATUREBASE env for dax
 			}
-			if err := SetAllConfig(v, cmd.Flags(), ""); err != nil {
+			err := setAllConfig(v, cmd.Flags())
+			if err != nil {
 				return err
 			}
 
@@ -105,37 +104,35 @@ at https://docs.featurebase.com/.
 	rc.AddCommand(newServeCmd(stderr))
 	rc.AddCommand(newHolderCmd(stderr))
 	rc.AddCommand(newKeygenCommand(logdest))
+	rc.AddCommand(newCLICommand(logdest))
 	rc.AddCommand(newDAXCommand(stderr))
 	rc.AddCommand(newDataframeCsvLoaderCommand(logdest))
-	rc.AddCommand(newPreSortCommand(logdest))
-	rc.AddCommand(newParquetInfoCommand(logdest))
 
 	rc.SetOutput(stderr)
 	return rc
 }
 
-// SetAllConfig takes a FlagSet to be the definition of all configuration
+// setAllConfig takes a FlagSet to be the definition of all configuration
 // options, as well as their defaults. It then reads from the command line, the
 // environment, and a config file (if specified), and applies the configuration
 // in that priority order. Since each flag in the set contains a pointer to
-// where its value should be stored, SetAllConfig can directly modify the value
+// where its value should be stored, setAllConfig can directly modify the value
 // of each config variable.
 //
-// SetAllConfig looks for environment variables which are capitalized versions
+// setAllConfig looks for environment variables which are capitalized versions
 // of the flag names with dashes replaced by underscores, and prefixed with
 // envPrefix plus an underscore.
-func SetAllConfig(v *viper.Viper, flags *pflag.FlagSet, envPrefix string) error { // nolint: unparam
+func setAllConfig(v *viper.Viper, flags *pflag.FlagSet) error { // nolint: unparam
 	// add cmd line flag def to viper
 	err := v.BindPFlags(flags)
 	if err != nil {
 		return err
 	}
 
-	if envPrefix == "" {
-		envPrefix = "PILOSA"
-		if v.GetBool("future.rename") {
-			envPrefix = "FEATUREBASE"
-		}
+	envPrefix := "PILOSA"
+	rename := v.GetBool("future.rename")
+	if rename {
+		envPrefix = "FEATUREBASE"
 	}
 
 	// add env to viper

cmd/server.go

@@ -5,10 +5,19 @@ package cmd
 import (
 	"io"
 
+	"gopkg.in/DataDog/dd-trace-go.v1/ddtrace/opentracer"
+	"gopkg.in/DataDog/dd-trace-go.v1/ddtrace/tracer"
+
+
 	"github.com/featurebasedb/featurebase/v3/ctl"
 	"github.com/featurebasedb/featurebase/v3/server"
+	"github.com/featurebasedb/featurebase/v3/tracing"
+	"github.com/featurebasedb/featurebase/v3/tracing/opentracing"
 	"github.com/pkg/errors"
 	"github.com/spf13/cobra"
+	jaegercfg "github.com/uber/jaeger-client-go/config"
+
+	"gopkg.in/DataDog/dd-trace-go.v1/profiler"
 )
 
 // Server is global so that tests can control and verify it.
@@ -48,6 +57,7 @@ func newServeCmd(stderr io.Writer) *cobra.Command {
 		Use:   "server",
 		Short: "Run FeatureBase.",
 		Long: `featurebase server runs FeatureBase.
+
 It will load existing data from the configured
 directory and start listening for client connections
 on the configured port.`,
@@ -56,7 +66,64 @@ on the configured port.`,
 			if err := Server.Start(); err != nil {
 				return considerUsageError(cmd, errors.Wrap(err, "running server"))
 			}
+			// anything past here is definitely not a usage error
+			cmd.SilenceErrors = true
+			cmd.SilenceUsage = true
+			if Server.Config.DataDog.Enable {
+				opts := make([]profiler.ProfileType, 0)
+				if Server.Config.DataDog.CPUProfile {
+					opts = append(opts, profiler.CPUProfile)
+				}
+				if Server.Config.DataDog.HeapProfile {
+					opts = append(opts, profiler.HeapProfile)
+				}
+				if Server.Config.DataDog.BlockProfile {
+					opts = append(opts, profiler.BlockProfile)
+				}
+				if Server.Config.DataDog.GoroutineProfile {
+					opts = append(opts, profiler.GoroutineProfile)
+				}
+				if Server.Config.DataDog.MutexProfile {
+					opts = append(opts, profiler.MutexProfile)
+				}
+				err := profiler.Start(
+					profiler.WithService(Server.Config.DataDog.Service),
+					profiler.WithEnv(Server.Config.DataDog.Env),
+					profiler.WithVersion(Server.Config.DataDog.Version),
+					profiler.WithTags(Server.Config.DataDog.Tags),
+					profiler.WithProfileTypes(
+						opts...,
+					),
+				)
+				if err != nil {
+					return errors.Wrap(err, "starting datadog")
+				}
+				defer profiler.Stop()
+			}
 
+			if Server.Config.Tracing.SamplerType != "off" {
+				// Initialize tracing in the command since it is global.
+				var cfg jaegercfg.Configuration
+				cfg.ServiceName = "pilosa"
+				cfg.Sampler = &jaegercfg.SamplerConfig{
+					Type:  Server.Config.Tracing.SamplerType,
+					Param: Server.Config.Tracing.SamplerParam,
+				}
+				cfg.Reporter = &jaegercfg.ReporterConfig{
+					LocalAgentHostPort: Server.Config.Tracing.AgentHostPort,
+				}
+				tracer, closer, err := cfg.NewTracer()
+				if err != nil {
+					return errors.Wrap(err, "initializing jaeger tracer")
+				}
+				defer closer.Close()
+				tracing.GlobalTracer = opentracing.NewTracer(tracer, Server.Logger())
+
+			} else if Server.Config.DataDog.EnableTracing { // Give preference to legacy support of jaeger
+				t := opentracer.New(tracer.WithServiceName(Server.Config.DataDog.Service))
+				defer tracer.Stop()
+				tracing.GlobalTracer = opentracing.NewTracer(t, Server.Logger())
+			}
 			return errors.Wrap(Server.Wait(), "waiting on Server")
 		},
 	}

cmd/server_test.go

@@ -9,11 +9,11 @@ import (
 	"testing"
 	"time"
 
+	"github.com/felixge/fgprof"
 	"github.com/featurebasedb/featurebase/v3/cmd"
 	_ "github.com/featurebasedb/featurebase/v3/test"
 	"github.com/featurebasedb/featurebase/v3/testhook"
 	"github.com/featurebasedb/featurebase/v3/toml"
-	"github.com/felixge/fgprof"
 	"github.com/pkg/errors"
 )
 

ctl/backup.go

@@ -10,6 +10,7 @@ import (
 	"fmt"
 	"io"
 	"io/fs"
+	"log"
 	"os"
 	"path/filepath"
 	"time"
@@ -94,13 +95,13 @@ func (cmd *BackupCommand) Run(ctx context.Context) (err error) {
 
 	// Validate arguments.
 	if cmd.OutputDir == "" {
-		return fmt.Errorf("%w: -o flag required", ErrUsage)
+		return fmt.Errorf("%w: -o flag required", UsageError)
 	} else if cmd.Concurrency <= 0 {
-		return fmt.Errorf("%w: concurrency must be at least one", ErrUsage)
+		return fmt.Errorf("%w: concurrency must be at least one", UsageError)
 	}
 	if cmd.HeaderTimeoutStr != "" {
 		if dur, err := time.ParseDuration(cmd.HeaderTimeoutStr); err != nil {
-			return fmt.Errorf("%w: could not parse '%s' as a duration: %v", ErrUsage, cmd.HeaderTimeoutStr, err)
+			return fmt.Errorf("%w: could not parse '%s' as a duration: %v", UsageError, cmd.HeaderTimeoutStr, err)
 		} else {
 			cmd.HeaderTimeout = dur
 		}
@@ -548,7 +549,7 @@ func (cmd *BackupCommand) backupShardDataframe(ctx context.Context, indexName st
 	resp, err := client.GetDataframeShard(ctx, indexName, shard)
 	// no error if doesn't exist
 	if err != nil {
-		return fmt.Errorf("getting dataframe: %w", err)
+		log.Fatal(err)
 	}
 	defer resp.Body.Close()
 	if resp.StatusCode == 404 {

ctl/backup_tar.go

@@ -3,11 +3,13 @@ package ctl
 
 import (
 	"archive/tar"
+	"bytes"
 	"context"
 	"crypto/tls"
 	"encoding/json"
 	"fmt"
 	"io"
+	"log"
 	"os"
 	"path"
 	"path/filepath"
@@ -15,7 +17,6 @@ import (
 
 	pilosa "github.com/featurebasedb/featurebase/v3"
 	"github.com/featurebasedb/featurebase/v3/authn"
-	"github.com/featurebasedb/featurebase/v3/buffer"
 	"github.com/featurebasedb/featurebase/v3/disco"
 	"github.com/featurebasedb/featurebase/v3/encoding/proto"
 	"github.com/featurebasedb/featurebase/v3/logger"
@@ -36,9 +37,6 @@ type BackupTarCommand struct { // nolint: maligned
 	// Path to write the backup to.
 	OutputPath string
 
-	// TempDir location of scratch files
-	TempDir string
-
 	// Amount of time after first failed request to continue retrying.
 	RetryPeriod time.Duration `json:"retry-period"`
 
@@ -82,7 +80,7 @@ func (cmd *BackupTarCommand) Run(ctx context.Context) (err error) {
 	logdest := cmd.Logger()
 	// Validate arguments.
 	if cmd.OutputPath == "" {
-		return fmt.Errorf("%w: -o flag required", ErrUsage)
+		return fmt.Errorf("%w: -o flag required", UsageError)
 	}
 	useStdout := cmd.OutputPath == "-"
 	if useStdout && cmd.logwriter == os.Stdout {
@@ -100,7 +98,7 @@ func (cmd *BackupTarCommand) Run(ctx context.Context) (err error) {
 
 	if cmd.HeaderTimeoutStr != "" {
 		if dur, err := time.ParseDuration(cmd.HeaderTimeoutStr); err != nil {
-			return fmt.Errorf("%w: could not parse '%s' as a duration: %v", ErrUsage, cmd.HeaderTimeoutStr, err)
+			return fmt.Errorf("%w: could not parse '%s' as a duration: %v", UsageError, cmd.HeaderTimeoutStr, err)
 		} else {
 			cmd.HeaderTimeout = dur
 		}
@@ -231,7 +229,25 @@ func (cmd *BackupTarCommand) backupTarIDAllocData(ctx context.Context, tw *tar.W
 	}
 	defer rc.Close()
 
-	return writeToTar(tw, "idalloc", rc, cmd.TempDir)
+	// Read to buffer to determine size.
+	var buf bytes.Buffer
+	if _, err := buf.ReadFrom(rc); err != nil {
+		return fmt.Errorf("copying id alloc data to memory: %w", err)
+	}
+
+	// Build header & copy data to archive.
+	if err = tw.WriteHeader(&tar.Header{
+		Name:    "idalloc",
+		Mode:    0o666,
+		Size:    int64(buf.Len()),
+		ModTime: time.Now(),
+	}); err != nil {
+		return err
+	} else if _, err := io.Copy(tw, &buf); err != nil {
+		return fmt.Errorf("copying id alloc data to archive: %w", err)
+	}
+
+	return nil
 }
 
 // backupTarIndex backs up all shards for a given index.
@@ -252,7 +268,7 @@ func (cmd *BackupTarCommand) backupTarIndex(ctx context.Context, tw *tar.Writer,
 	}
 
 	if ii.Options.Keys {
-		// Back up translation data after bitmap data so we ensurean translate all data.
+		// Back up translation data after bitmap data so we ensure we can translate all data.
 		if err := cmd.backupTarIndexTranslateData(ctx, tw, ii.Name); err != nil {
 			return err
 		}
@@ -314,7 +330,27 @@ func (cmd *BackupTarCommand) backupTarShardNode(ctx context.Context, tw *tar.Wri
 		return fmt.Errorf("fetching shard reader: %w", err)
 	}
 	defer rc.Close()
-	return writeToTar(tw, filename, rc, cmd.TempDir)
+
+	// Read to buffer to determine size.
+	// TODO: Provide size via the reader itself.
+	var buf bytes.Buffer
+	if _, err := buf.ReadFrom(rc); err != nil {
+		return fmt.Errorf("copying shard data to memory: %w", err)
+	}
+
+	// Build header & copy data to archive.
+	if err = tw.WriteHeader(&tar.Header{
+		Name:    filename,
+		Mode:    0o666,
+		Size:    int64(buf.Len()),
+		ModTime: time.Now(),
+	}); err != nil {
+		return err
+	} else if _, err := io.Copy(tw, &buf); err != nil {
+		return fmt.Errorf("copying shard data to archive: %w", err)
+	}
+
+	return nil
 }
 
 func (cmd *BackupTarCommand) backupTarShardDataframe(ctx context.Context, tw *tar.Writer, indexName string, shard uint64, node *disco.Node) error {
@@ -328,7 +364,7 @@ func (cmd *BackupTarCommand) backupTarShardDataframe(ctx context.Context, tw *ta
 	resp, err := client.GetDataframeShard(ctx, indexName, shard)
 	// no error if doesn't exist
 	if err != nil {
-		return fmt.Errorf("getting dataframe: %w", err)
+		log.Fatal(err)
 	}
 	defer resp.Body.Close()
 	if resp.StatusCode == 404 {
@@ -337,7 +373,24 @@ func (cmd *BackupTarCommand) backupTarShardDataframe(ctx context.Context, tw *ta
 	}
 
 	filename := filepath.Join("indexes", indexName, "dataframe", fmt.Sprintf("%04d", shard))
-	return writeToTar(tw, filename, resp.Body, cmd.TempDir)
+	logger.Printf("writing %v", filename)
+	var buf bytes.Buffer
+	if _, err := buf.ReadFrom(resp.Body); err != nil {
+		return fmt.Errorf("copying shard data to memory: %w", err)
+	}
+
+	// Build header & copy data to archive.
+	if err = tw.WriteHeader(&tar.Header{
+		Name:    filename,
+		Mode:    0o666,
+		Size:    int64(buf.Len()),
+		ModTime: time.Now(),
+	}); err != nil {
+		return err
+	} else if _, err := io.Copy(tw, &buf); err != nil {
+		return fmt.Errorf("copying shard data to archive: %w", err)
+	}
+	return nil
 }
 
 func (cmd *BackupTarCommand) backupTarIndexTranslateData(ctx context.Context, tw *tar.Writer, name string) error {
@@ -363,7 +416,25 @@ func (cmd *BackupTarCommand) backupTarIndexPartitionTranslateData(ctx context.Co
 	}
 	defer rc.Close()
 
-	return writeToTar(tw, path.Join("indexes", name, "translate", fmt.Sprintf("%04d", partitionID)), rc, cmd.TempDir)
+	// Read to buffer to determine size.
+	var buf bytes.Buffer
+	if _, err := buf.ReadFrom(rc); err != nil {
+		return fmt.Errorf("copying translate data to memory: %w", err)
+	}
+
+	// Build header & copy data to archive.
+	if err = tw.WriteHeader(&tar.Header{
+		Name:    path.Join("indexes", name, "translate", fmt.Sprintf("%04d", partitionID)),
+		Mode:    0o666,
+		Size:    int64(buf.Len()),
+		ModTime: time.Now(),
+	}); err != nil {
+		return err
+	} else if _, err := io.Copy(tw, &buf); err != nil {
+		return fmt.Errorf("copying translate data to archive: %w", err)
+	}
+
+	return nil
 }
 
 func (cmd *BackupTarCommand) backupTarFieldTranslateData(ctx context.Context, tw *tar.Writer, indexName, fieldName string) error {
@@ -377,42 +448,27 @@ func (cmd *BackupTarCommand) backupTarFieldTranslateData(ctx context.Context, tw
 		return fmt.Errorf("fetching translate data reader: %w", err)
 	}
 	defer rc.Close()
-	return writeToTar(tw, path.Join("indexes", indexName, "fields", fieldName, "translate"), rc, cmd.TempDir)
-}
 
-func (cmd *BackupTarCommand) TLSHost() string { return cmd.Host }
-
-func (cmd *BackupTarCommand) TLSConfiguration() server.TLSConfig { return cmd.TLS }
-
-func writeToTar(tw *tar.Writer, entryName string, rc io.Reader, tmpDir string) error {
-	spillFile, err := os.CreateTemp("", "spill")
-	if err != nil {
-		return fmt.Errorf("creating temp file : %w", err)
-	}
-	defer func() {
-		spillFile.Close()
-		os.Remove(spillFile.Name())
-	}()
-	mb512 := 2 << 29
-	buf := buffer.NewFileBuffer(mb512, tmpDir)
-	defer buf.Close()
-
-	n, err := io.Copy(buf, rc)
 	// Read to buffer to determine size.
-	if err != nil {
+	var buf bytes.Buffer
+	if _, err := buf.ReadFrom(rc); err != nil {
 		return fmt.Errorf("copying translate data to memory: %w", err)
 	}
 
 	// Build header & copy data to archive.
 	if err = tw.WriteHeader(&tar.Header{
-		Name:    entryName,
+		Name:    path.Join("indexes", indexName, "fields", fieldName, "translate"),
 		Mode:    0o666,
-		Size:    n,
+		Size:    int64(buf.Len()),
 		ModTime: time.Now(),
 	}); err != nil {
 		return err
-	} else if _, err := io.Copy(tw, buf); err != nil {
+	} else if _, err := io.Copy(tw, &buf); err != nil {
 		return fmt.Errorf("copying translate data to archive: %w", err)
 	}
 	return nil
 }
+
+func (cmd *BackupTarCommand) TLSHost() string { return cmd.Host }
+
+func (cmd *BackupTarCommand) TLSConfiguration() server.TLSConfig { return cmd.TLS }

ctl/backup_test.go

@@ -15,19 +15,19 @@ func TestBackupCommand_Run(t *testing.T) {
 	cm := NewBackupCommand(cmLog)
 	cm.OutputDir = ""
 	err := cm.Run(context.Background())
-	if !errors.Is(err, ErrUsage) {
+	if !errors.Is(err, UsageError) {
 		t.Fatalf("expected usage error, got %v", err)
 	}
 	cm.OutputDir = "foo"
 	cm.Concurrency = 0
 	err = cm.Run(context.Background())
-	if !errors.Is(err, ErrUsage) {
+	if !errors.Is(err, UsageError) {
 		t.Fatalf("expected usage error, got %v", err)
 	}
 	cm.Concurrency = 1
 	cm.HeaderTimeoutStr = "until the cat wakes up"
 	err = cm.Run(context.Background())
-	if !errors.Is(err, ErrUsage) {
+	if !errors.Is(err, UsageError) {
 		t.Fatalf("expected usage error, got %v", err)
 	}
 }

ctl/chksum.go

@@ -7,6 +7,7 @@ import (
 	"crypto/tls"
 	"fmt"
 	"io"
+	"os"
 
 	"github.com/cespare/xxhash"
 	pilosa "github.com/featurebasedb/featurebase/v3"
@@ -37,9 +38,9 @@ func (cmd *ChkSumCommand) Logger() logger.Logger {
 }
 
 // NewChkSumCommand returns a new instance of BackupCommand.
-func NewChkSumCommand(logdest logger.Logger, stdout io.Writer) *ChkSumCommand {
+func NewChkSumCommand(logdest logger.Logger) *ChkSumCommand {
 	return &ChkSumCommand{
-		stdout:  stdout,
+		stdout:  os.Stdout,
 		logDest: logdest,
 	}
 }