.github/workflows/cd.yml

@@ -1,77 +0,0 @@
-name: CD
-
-on:
-  push:
-    branches: ["master"]
-
-  workflow_dispatch:
-
-jobs:
-  validate:
-    runs-on: ubuntu-latest
-
-    steps:
-      # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
-      - uses: actions/checkout@v3
-
-      - uses: actions/setup-go@v3
-        with:
-          go-version: "^1.19.1"
-
-      - run: go version
-
-      - name: golangci-lint
-        uses: golangci/golangci-lint-action@v3
-        with:
-          args: --timeout=5m
-
-      - name: go vet
-        run: go vet ./...
-
-      - name: test
-        run: go test ./...
-
-  release:
-    needs: validate
-    runs-on: ubuntu-latest
-    outputs:
-      version: ${{ steps.semrel.outputs.version }}
-
-    steps:
-      - name: go-semantic-release
-        id: semrel
-        uses: go-semantic-release/action@v1.17.0
-        with:
-          github-token: ${{ secrets.GITHUB_TOKEN }}
-
-  build:
-    needs: release
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        goos:
-          - "darwin"
-          - "linux"
-          - "windows"
-        goarch:
-          - "amd64"
-          - "arm64"
-
-    steps:
-      - uses: actions/checkout@v3
-
-      - uses: actions/setup-go@v3
-        with:
-          go-version: "^1.19.1"
-
-      - name: build
-        run: GOOS=${{ matrix.goos }} GOARCH=${{ matrix.goarch }} go build -o ./build/${{ matrix.goos }}-${{ matrix.goarch }}
-
-      - name: Upload binaries to release
-        uses: svenstaro/upload-release-action@v2
-        with:
-          repo_token: ${{ secrets.GITHUB_TOKEN }}
-          file: ./build/${{ matrix.goos }}-${{ matrix.goarch }}
-          asset_name: ${{ matrix.goos }}-${{ matrix.goarch }}
-          tag: ${{ github.ref }}
-

.github/workflows/ci.yml

@@ -1,57 +0,0 @@
-# This is a basic workflow to help you get started with Actions
-
-name: CI
-
-# Controls when the workflow will run
-on:
-  pull_request:
-    branches: ["master"]
-
-  # Allows you to run this workflow manually from the Actions tab
-  workflow_dispatch:
-
-# A workflow run is made up of one or more jobs that can run sequentially or in parallel
-jobs:
-  validate-title:
-    name: Validate PR title
-    runs-on: ubuntu-latest
-    steps:
-      - uses: go-semantic-release/action@v1
-        id: semrel
-        with:
-          github-token: ${{ secrets.GITHUB_TOKEN }}
-          dry: true
-
-  # golangci-lint must be run separately from "validate" as there are go mod issues if you run it after the vet
-  golangci:
-    name: lint
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/setup-go@v3
-        with:
-          go-version: 1.19
-      - uses: actions/checkout@v3
-      - name: golangci-lint
-        uses: golangci/golangci-lint-action@v3
-        with:
-          # Optional: version of golangci-lint to use in form of v1.2 or v1.2.3 or `latest` to use the latest version
-          # version: v1.29
-          args: --timeout=8m
-
-  validate:
-    name: Code Checks
-    runs-on: ubuntu-latest
-
-    # Steps represent a sequence of tasks that will be executed as part of the job
-    steps:
-      # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
-      - uses: actions/checkout@v3
-
-      - uses: actions/setup-go@v3
-        with:
-          go-version: "^1.19.1"
-
-      - run: go version
-
-      - name: go vet
-        run: go vet ./...

.github/workflows/release.yml

@@ -1,25 +0,0 @@
-name: Release
-
-on: workflow_dispatch
-
-jobs:
-  build:
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        goos:
-          - "darwin"
-          - "linux"
-        goarch:
-          - "amd64"
-          - "arm64"
-
-    steps:
-      - uses: actions/checkout@v3
-
-      - uses: actions/setup-go@v3
-        with:
-          go-version: "^1.19.1"
-
-      - name: build
-        run: GOOS=${{ matrix.goos }} GOARCH=${{ matrix.goarch }} go build -o ./build/${{ matrix.goos }}-${{ matrix.goarch }}

.gitignore

@@ -1,87 +1,2 @@
 default.etcd/
 *.test
-vendor
-.protoc-gen-gofast
-.DS_Store
-build
-*~
-release-pilosa-fsck.*.*.tar.gz
-/log.*
-/tourna.log.*
-pilosa
-/featurebase
-*.dot
-.idea/
-.*.swp
-.terraform/
-*.tfstate
-launch.json
-.terraform.lock.hcl
-__pycache__/
-report.xml
-outputs.json
-builds/
-*.tfstate.backup
-.vscode
-
-batch/testdata/batch*.out
-
-idk/testdata/idk*.out
-idk/testenv/certs/*
-
-# copy of .gitignore from archived idk repo
-# Compiled Object files, Static and Dynamic libs (Shared Objects)
-*.o
-*.a
-*.so
-
-# Folders
-_obj
-_test
-
-# Architecture specific extensions/prefixes
-*.[568vq]
-[568vq].out
-
-*.cgo1.go
-*.cgo2.c
-_cgo_defun.c
-_cgo_gotypes.go
-_cgo_export.*
-
-_testmain.go
-
-*.exe
-*.test
-*.prof
-
-vendor
-
-.terraform
-terraform.tfstate*
-
-bin
-build
-testenv
-.pulled
-
-pilosa-sec-data-idk
-
-.idea/
-tags.dot
-*.log
-*.swp
-*__debug_bin
-
-# SQL3
-/sql3/sql3.html
-
-staticcheck.conf
-
-
-.quick
-dax/dax-data
-
-coverage-from-docker
-*.client_id.txt
-

.gitlab/.gitlab-ci.yml

@@ -1,747 +0,0 @@
-# You will see a couple of instances of:
-# PKG_LIST=$(go list ./... | grep -Ev 'internal/clustertests|simulacraData|batch|idk|v3/dax/test/dax' | paste -s -d, -)
-# This gets us a package list, comma-separated, which excludes the batch
-# and IDK tests, and a couple of subdirs with specialized stuff. We can
-# then use this with -coverpkg, or we can use ${PKG_LIST//,/ } to get a
-# space-separated list for use with `go test` to run the tests for those
-# directories only.
-
-include:
-  - local: /.gitlab/batch-ci.yml
-  - template: Security/SAST.gitlab-ci.yml
-  - template: Security/License-Scanning.gitlab-ci.yml
-  - template: Security/Dependency-Scanning.gitlab-ci.yml
-
-default:
-  retry:
-    max: 2 # This is confusing but this means "3 runs at max".
-    when:
-      - unknown_failure
-      - api_failure
-      - runner_system_failure
-      - job_execution_timeout
-      - stuck_or_timeout_failure
-
-variables:
-  GOVERSION: "1.19.3"
-  GOFUTURE: "latest"
-  CI_IMAGE: "${CI_REGISTRY_IMAGE}/ci-builder:0.0.1"
-  CI_PRE_CLONE_SCRIPT: |
-    set -x
-
-stages:
-  - ci_image_build
-  - lint
-  - test
-  - build
-  - post build
-  - integration
-  - gauntlet
-  - performance
-  - nonblocking
-  - cleanup_build
-
-gosec-sast:
-  allow_failure: false
-  before_script:
-    - export GOPRIVATE=github.com/molecula/*
-    - apk add openssh-client
-    - eval $(ssh-agent -s)
-    - echo "$FB_SSH_PRIVATE_KEY" | tr -d '\r' | ssh-add -
-    - mkdir -p ~/.ssh
-    - chmod 700 ~/.ssh
-    - git config --global --add url."ssh://git@github.com/".insteadOf "https://github.com/"
-    - ssh-keygen -F github.com || echo "$SSH_KNOWN_HOSTS_HASHED" >> ~/.ssh/known_hosts
-    - chmod 644 ~/.ssh/known_hosts
-  script:
-    ## securego/gosec works for scanning, but not converting to the gitlab report format.
-    - go install github.com/securego/gosec/v2/cmd/gosec@v2.12.0
-    - gosec -fmt=json -out=gosec.json -tests ./... || true
-    ## gitlab's wrapper for gosec works for converting, but not for scanning.
-    - go install 'gitlab.com/gitlab-org/security-products/analyzers/gosec@v1.4.0'
-    - gosec convert gosec.json > gl-sast-report.json
-
-.go-cache:
-  variables:
-    GOPATH: $CI_PROJECT_DIR/.go
-  before_script:
-    - mkdir -p .go
-  cache:
-    # this caching strategy makes it so each branch uses the same cache
-    key: "$CI_COMMIT_REF_SLUG"
-    paths:
-      - .go/pkg/mod/
-
-smoke build:
-  image: golang:$GOVERSION
-  extends: .go-cache
-  stage: lint
-  allow_failure: false
-  rules:
-    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
-  script:
-    - echo "Let's just see if it compiles... (sometimes the linter gives unclear errors if it doesn't)"
-    - go build ./...
-
-golangci-lint:
-  image: golangci/golangci-lint:v1.46.2
-  extends: .go-cache
-  stage: lint
-  allow_failure: true
-  rules:
-    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
-  script:
-    - echo "Checking for issues in new code"
-    - golangci-lint run -v --timeout=8m
-
-go mod tidy:
-  stage: lint
-  image: golang:$GOVERSION
-  extends: .go-cache
-  rules:
-    - if: '$CI_COMMIT_TAG == null && ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web")'
-  script:
-    - go mod tidy
-    - git diff --exit-code -- go.mod go.sum
-
-build lattice:
-  stage: test
-  image: node:14
-  variables:
-    AWS_PROFILE: "service-fb-ci"
-    AWS_ACCESS_KEY_ID: $AWS_FBCI_ACCESS_KEY_ID
-    AWS_SECRET_ACCESS_KEY: $AWS_FBCI_SECRET_ACCESS_KEY
-    CI: "false"
-  rules:
-    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
-  before_script:
-    - curl -sS "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
-    - unzip -qq awscliv2.zip
-    - ./aws/install
-    - aws --version
-    - aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID
-    - aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY
-    - aws configure set region "us-east-2"
-    - aws configure set aws_profile $AWS_PROFILE
-    - aws sts get-caller-identity # ensure we have a valid AWS login
-  script:
-    - cd lattice
-    - cache=$(find . -type f -print0 | sort -z | xargs -0 sha1sum | sha1sum | cut -d ' ' -f 1)
-    - echo "'$cache'"
-    - echo "looking for s3://molecula-artifact-storage/lattice/$cache/build.tar.gz"
-    # if is for if we had a cache object in S3
-    # else is for if we didn't have a cache object (and have to build).
-    - |
-      if aws s3api head-object --bucket molecula-artifact-storage --key "lattice/$cache/build.tar.gz"; then
-        # download object, extract, name the folder `build`
-        aws s3 cp "s3://molecula-artifact-storage/lattice/$cache/build.tar.gz" build.tar.gz
-        tar -xf build.tar.gz
-      else # cache file not found
-        yarn install --frozen-lockfile # CI needs to enforce that the lockfile doesn't need to be updated
-        yarn build
-        tar -czvf "$cache.tar.gz" build/
-        aws s3 mv "$cache.tar.gz" "s3://molecula-artifact-storage/lattice/$cache/build.tar.gz"
-        touch "$CI_COMMIT_SHA"
-        aws s3 mv "$CI_COMMIT_SHA" "s3://molecula-artifact-storage/lattice/$cache/$CI_COMMIT_SHA"
-      fi
-    - | # Ensure that we have build directory after the caching step
-      if [ ! -d build ]; then
-        echo "no build directory, erroring out" || exit 1
-      fi
-    - mv build ../
-    - cd ../
-    - rm -r lattice
-    - mv build lattice
-    - tar -czvf lattice.tar.gz lattice
-  artifacts:
-    paths:
-      - lattice.tar.gz
-
-build featurebase:
-  stage: test
-  image: golang:$GOVERSION
-  extends: .go-cache
-  rules:
-    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
-  script:
-    - rm -r lattice
-    - tar -xvf lattice.tar.gz
-    - go install github.com/rakyll/statik@v0.1.7
-    - $GOPATH/bin/statik -src=lattice
-    - export SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct)
-    - |
-      for goos in "darwin" "linux"; do
-        for goarch in "amd64" "arm64"; do
-          GOOS="${goos}" GOARCH="${goarch}" make build FLAGS="-o featurebase_${goos}_${goarch}"
-        done
-      done
-  artifacts:
-    paths:
-      - featurebase_*
-  needs:
-    - job: build lattice
-
-build fbsql amd64:
-  stage: test
-  variables:
-    BUILD_NAME: build_${CI_COMMIT_SHA}_${CI_CONCURRENT_ID}
-  tags:
-    - shell
-  rules:
-    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
-  script:
-    - export SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct)
-    - date
-    - GOOS="linux" GOARCH="amd64" make docker-build-fbsql BUILD_CGO=1
-    - GOOS="darwin" GOARCH="amd64" make docker-build-fbsql
-  artifacts:
-    paths:
-      - ./build/fbsql_*
-
-build fbsql arm64:
-  stage: test
-  variables:
-    BUILD_NAME: build_${CI_COMMIT_SHA}_${CI_CONCURRENT_ID}
-  tags:
-    - shell-arm64
-  rules:
-    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
-  script:
-    - export SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct)
-    - date
-    - GOOS="linux" GOARCH="arm64" make docker-build-fbsql BUILD_CGO=1
-    - GOOS="darwin" GOARCH="arm64" make docker-build-fbsql
-  artifacts:
-    paths:
-      - ./build/fbsql_*
-
-build amd container fb:
-  stage: test
-  tags:
-    - shell
-  rules:
-    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
-  before_script:
-    - echo "${DOCKER_DEPLOY_TOKEN}" | docker login -u ${DOCKER_DEPLOY_USER} --password-stdin ${CI_REGISTRY}
-  script:
-    - tag=${CI_REGISTRY_IMAGE}/featurebase:linux-amd64-${CI_COMMIT_REF_SLUG}
-    - export SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct)
-    - docker build --build-arg GO_VERSION=$GOVERSION --build-arg ARCH=amd64 --build-arg SOURCE_DATE_EPOCH=$SOURCE_DATE_EPOCH -t $tag -f .gitlab/Dockerfile .
-    - docker push $tag
-    - echo Created docker featurebase image with tag "$tag"
-  needs:
-    - job: build featurebase
-
-run jest tests:
-  stage: test
-  image: node:14
-  variables:
-    CI: "true"
-  rules:
-    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
-  script:
-    - echo "Testing lattice..."
-    - cd lattice
-    - npm install --force
-    - npm test -- --coverage --testResultsProcessor=jest-sonar-reporter
-  artifacts:
-    paths:
-      - lattice/coverage/lcov.info
-
-# We run go test -race on all the standard packages, skipping the ones that have their
-# own separate tests. we spin this off as nonblocking because it used to take a really
-# long time and even now it's pretty slow.
-run go tests race:
-  stage: nonblocking # don't let this job block any other jobs because it takes much longer than the other tests.
-  image: golang:$GOVERSION
-  extends: .go-cache
-  variables:
-    FEATUREBASE_CONTROLLER_CONFIG_SQLDB_DATABASE: run_go_tests_race
-    POSTGRES_DB: run_go_tests_race
-    FEATUREBASE_CONTROLLER_CONFIG_SQLDB_USER: postgres
-    POSTGRES_USER: postgres
-    FEATUREBASE_CONTROLLER_CONFIG_SQLDB_PASSWORD: $POSTGRES_PASSWORD
-    POSTGRES_PASSWORD: $POSTGRES_PASSWORD
-    FEATUREBASE_CONTROLLER_CONFIG_SQLDB_HOST: postgres
-  services:
-    - postgres:14.7
-  rules:
-    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
-  needs: ["smoke build"] # we do block on smoke build though bc it's pretty dumb to test stuff if it doesn't build
-  script:
-    - echo "Running featurebase race tests..."
-    - PKG_LIST=$(go list ./... | grep -Ev 'internal/clustertests|simulacraData|batch|idk|v3/dax/test/dax' | paste -s -d, -)
-    - export TMPDIR=/mnt/ramdisk/test-$CI_JOB_ID
-    - mkdir -p $TMPDIR
-    - go test -race -v -timeout=10m ${PKG_LIST//,/ }
-  after_script:
-    - rm -rf /mnt/ramdisk/test-$CI_JOB_ID
-  tags:
-    - docker
-
-# We run our base tests against $GOVERSION (a reasonably current version that we trust)
-# and use shardwidth22 for them. This gives us a canary for things breaking for
-# unusual shard widths.
-run go tests:
-  stage: test
-  image: golang:$GOVERSION
-  extends: .go-cache
-  variables:
-    FEATUREBASE_CONTROLLER_CONFIG_SQLDB_DATABASE: run_go_tests
-    POSTGRES_DB: run_go_tests
-    FEATUREBASE_CONTROLLER_CONFIG_SQLDB_USER: postgres
-    POSTGRES_USER: postgres
-    FEATUREBASE_CONTROLLER_CONFIG_SQLDB_PASSWORD: $POSTGRES_PASSWORD
-    POSTGRES_PASSWORD: $POSTGRES_PASSWORD
-    FEATUREBASE_CONTROLLER_CONFIG_SQLDB_HOST: postgres
-  services:
-    - postgres:14.7
-  rules:
-    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
-  script:
-    - echo "Running featurebase unit tests..."
-    - PKG_LIST=$(go list ./... | grep -Ev 'internal/clustertests|simulacraData|batch|idk|v3/dax/test/dax' | paste -s -d, -)
-    - export TMPDIR=/mnt/ramdisk/test-$CI_JOB_ID
-    - mkdir -p $TMPDIR
-    - go test -tags=shardwidth22 -timeout=10m -coverprofile=coverage.out -covermode=atomic -coverpkg=${PKG_LIST} ${PKG_LIST//,/ }
-  after_script:
-    - rm -rf /mnt/ramdisk/test-$CI_JOB_ID
-  artifacts:
-    paths:
-      - coverage.out
-  tags:
-    - docker
-
-run go tests dax/test/dax:
-  stage: test
-  image: golang:$GOVERSION
-  extends: .go-cache
-  tags:
-    - docker
-  variables:
-    FEATUREBASE_CONTROLLER_CONFIG_SQLDB_DATABASE: run_go_tests_dax
-    POSTGRES_DB: run_go_tests_dax
-    FEATUREBASE_CONTROLLER_CONFIG_SQLDB_USER: postgres
-    POSTGRES_USER: postgres
-    FEATUREBASE_CONTROLLER_CONFIG_SQLDB_PASSWORD: $POSTGRES_PASSWORD
-    POSTGRES_PASSWORD: $POSTGRES_PASSWORD
-    FEATUREBASE_CONTROLLER_CONFIG_SQLDB_HOST: postgres
-  services:
-    - postgres:14.7
-  rules:
-    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
-  script:
-    - echo "Building FB and Datagen docker images for DAX tests"
-    - PKG_LIST=$(go list ./... | grep -Ev 'internal/clustertests|simulacraData' | paste -s -d, -)
-    - export TMPDIR=/mnt/ramdisk/test-$CI_JOB_ID
-    - mkdir -p $TMPDIR
-    - go test -coverprofile=coverage-dax-integration.out -covermode=atomic -coverpkg=${PKG_LIST} -timeout=20m ./dax/test/dax
-  after_script:
-    - rm -rf /mnt/ramdisk/test-$CI_JOB_ID
-  artifacts:
-    paths:
-      - coverage-dax-integration.out
-
-# idk tests
-run go tests idk race:
-  variables:
-    PROJECT: race_${CI_CONCURRENT_ID}
-  stage: nonblocking
-  retry: 1
-  script:
-    - echo "Running test-all-race"
-    - cd ./idk/
-    - echo $PROJECT
-    - echo $CI_COMMIT_REF_SLUG
-    - BRANCH_NAME=${CI_COMMIT_REF_SLUG} IDK_FEATUREBASE_TAG=${CI_COMMIT_TAG} IDK_FEATUREBASE_HASH=${CI_COMMIT_SHA} make test-all-race
-  after_script:
-    - cd ./idk/
-    - make save-pilosa-logs
-    - make shutdown
-  artifacts:
-    paths:
-      - ./idk/testdata/*_coverage.out
-  tags:
-    - shell
-    - aws
-  needs:
-    - job: build amd container fb
-
-run go tests idk shard transactional:
-  variables:
-    IDK_DEFAULT_SHARD_TRANSACTIONAL: 1
-    PROJECT: shardttrans_${CI_CONCURRENT_ID}
-  stage: nonblocking
-  retry: 1
-  script:
-    - echo "Running shard transactional tests"
-    - cd ./idk/
-    - echo $PROJECT
-    - echo $CI_COMMIT_REF_SLUG
-    - BRANCH_NAME=${CI_COMMIT_REF_SLUG} IDK_FEATUREBASE_TAG=${CI_COMMIT_TAG} IDK_FEATUREBASE_HASH=${CI_COMMIT_SHA} make test-all
-  after_script:
-    - cd ./idk/
-    - make save-pilosa-logs
-    - make shutdown
-  artifacts:
-    paths:
-      - ./idk/testdata/*_coverage.out
-      - ./idk/testdata/*_logs.txt
-  tags:
-    - shell
-    - aws
-  needs:
-    - job: build amd container fb
-
-run go tests idk 533:
-  variables:
-    USERNAME: fb-idk-access
-    PROJECT: test533_${CI_CONCURRENT_ID}
-  stage: nonblocking
-  retry: 1
-  script:
-    - echo "Running confluent 5.3.3 test-all"
-    - cd ./idk/
-    - echo $PROJECT
-    - echo $CI_COMMIT_REF_SLUG
-    - CONFLUENT_VERSION=5.3.3 BRANCH_NAME=${CI_COMMIT_REF_SLUG} IDK_FEATUREBASE_TAG=${CI_COMMIT_TAG} IDK_FEATUREBASE_HASH=${CI_COMMIT_SHA} make test-all
-  after_script:
-    - cd ./idk/
-    - make save-pilosa-logs
-    - make shutdown
-  tags:
-    - shell
-    - aws
-  artifacts:
-    paths:
-      - ./idk/testdata/*_coverage.out
-  needs:
-    - job: build amd container fb
-
-run go tests idk sasl:
-  variables:
-    PROJECT: sasl_${CI_CONCURRENT_ID}
-  stage: nonblocking
-  retry: 1
-  script:
-    - echo "Running test-all-kafka-sasl"
-    - cd ./idk/
-    - echo $PROJECT
-    - echo $CI_COMMIT_REF_SLUG
-    - BRANCH_NAME=${CI_COMMIT_REF_SLUG} IDK_FEATUREBASE_TAG=${CI_COMMIT_TAG} IDK_FEATUREBASE_HASH=${CI_COMMIT_SHA} make test-all-kafka-sasl
-  after_script:
-    - cd ./idk/
-    - make save-pilosa-logs
-    - make shutdown
-  tags:
-    - shell
-    - aws
-  artifacts:
-    paths:
-      - ./idk/testdata/*_coverage.out
-  needs:
-    - job: build amd container fb
-
-upload to sonarcloud:
-  stage: nonblocking
-  image: sonarsource/sonar-scanner-cli:4.7
-  variables:
-    SONAR_TOKEN: $SONAR_TOKEN
-  rules:
-    - if: '$CI_COMMIT_TAG == null && ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web")'
-  script:
-    - sonar-scanner -Dsonar.projectKey=molecula_featurebase -Dsonar.organization=molecula -Dsonar.sources=. -Dsonar.host.url=https://sonarcloud.io -Dsonar.go.coverage.reportPaths=coverage*.out,results/coverage*out,idk/testdata/*coverage.out,batch/testdata/*coverage.out,coverage-from-docker/*.out -Dsonar.javascript.lcov.reportPaths=lattice/coverage/lcov.info
-  needs:
-    - job: run go tests
-    - job: run jest tests
-    - job: external lookup tests
-    - job: run go tests idk race
-      optional: true
-    - job: run go tests idk shard transactional
-      optional: true
-    - job: run go tests idk sasl
-      optional: true
-    - job: run go tests idk 533
-      optional: true
-    - job: run go tests batch
-      optional: true
-    - job: run go tests dax/test/dax
-      optional: true
-
-package for linux amd64:
-  stage: build
-  image: golang:$GOVERSION
-  extends: .go-cache
-  rules:
-    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
-  variables:
-    GOOS: "linux"
-    GOARCH: "amd64"
-  script:
-    - echo 'deb [trusted=yes] https://repo.goreleaser.com/apt/ /' | tee /etc/apt/sources.list.d/goreleaser.list
-    - apt update && apt install nfpm=2.11.3
-    - make package
-  artifacts:
-    paths:
-      - "*.deb"
-      - "*.rpm"
-
-upload_artifacts_to_nexus:
-  stage: post build
-  image: golang:$GOVERSION
-  rules:
-    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
-  script:
-    - curl -v --user "$NEXUS_YUM_CREDS" --upload-file *.arm64.rpm https://nexus.molecula.com/repository/molecula-yum/release/
-    - curl -v --user "$NEXUS_YUM_CREDS" --upload-file *.amd64.rpm https://nexus.molecula.com/repository/molecula-yum/release/
-  dependencies:
-    - package for linux amd64
-    - package for linux arm64
-
-trigger_m-cloud-images:
-  stage: post build
-  rules:
-    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
-  needs:
-    - upload_artifacts_to_nexus
-  trigger: molecula/m-cloud-images
-
-package for linux arm64:
-  stage: build
-  image: golang:$GOVERSION
-  extends: .go-cache
-  rules:
-    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
-  variables:
-    GOOS: "linux"
-    GOARCH: "arm64"
-  script:
-    - echo 'deb [trusted=yes] https://repo.goreleaser.com/apt/ /' | tee /etc/apt/sources.list.d/goreleaser.list
-    - apt update && apt install nfpm=2.11.3
-    - make package
-  artifacts:
-    paths:
-      - "*.deb"
-      - "*.rpm"
-
-build arm container fb:
-  stage: build
-  needs:
-    - "build featurebase"
-  tags:
-    - shell
-  rules:
-    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
-  before_script:
-    - echo "${DOCKER_DEPLOY_TOKEN}" | docker login -u ${DOCKER_DEPLOY_USER} --password-stdin ${CI_REGISTRY}
-  script:
-    - echo $CI_COMMIT_REF_SLUG
-    - tag=${CI_REGISTRY_IMAGE}/featurebase:linux-arm64-${CI_COMMIT_REF_SLUG}
-    - export SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct)
-    - docker build --build-arg GO_VERSION=$GOVERSION --build-arg ARCH=arm64 --build-arg SOURCE_DATE_EPOCH=$SOURCE_DATE_EPOCH -t $tag -f .gitlab/Dockerfile .
-    - docker push $tag
-    - echo Created docker featurebase image with tag "$tag"
-
-### start idk builds ###
-# building them all serially because otherwise you get container name conflicts.
-idk build_amd64:
-  stage: build
-  variables:
-    BUILD_NAME: build_${CI_COMMIT_SHA}_${CI_CONCURRENT_ID}
-  tags:
-    - shell
-  rules:
-    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
-  script:
-    - export SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct)
-    - cd ./idk/
-    - date
-    - make docker-build GOOS="linux" GOARCH="amd64" BUILD_CGO=1
-    - date
-    - make docker-build GOOS="darwin" GOARCH="amd64"
-    - date
-  artifacts:
-    paths:
-      - ./idk/build/*
-  needs:
-    # doesn't actually need this... just want it to start executing before *all* the tests finish
-    - job: run go tests
-
-idk build_arm64:
-  stage: build
-  tags:
-    - shell-arm64
-  rules:
-    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
-  script:
-    - export SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct)
-    - cd ./idk/
-    - date
-    - make docker-build GOOS="linux" GOARCH="arm64" BUILD_CGO=1 BUILD_NAME="linux-arm64"
-    - date
-    - make docker-build GOOS="darwin" GOARCH="arm64"
-    - date
-  artifacts:
-    paths:
-      - ./idk/build/*
-  needs:
-    # doesn't actually need this... just want it to start executing before *all* the tests finish
-    - job: run go tests
-
-# building them all serially because otherwise you get container name conflicts.
-# only do containers on default branch
-idk package_docker_all:
-  stage: build
-  tags:
-    - shell
-  rules:
-    - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web")'
-  script:
-    - export SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct)
-    - make docker-idk GOOS="linux" GOARCH="amd64"
-    - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
-    - make docker-idk-tag-push GOOS="linux" GOARCH="amd64"
-  needs:
-    - job: idk build_amd64
-    - job: idk build_arm64
-
-idk s3 dump:
-  stage: post build
-  allow_failure: false
-  variables:
-    PROFILE: "service-fb-ci"
-    AWS_SSH_PRIVATE_KEY: $AWS_FBCI_SSH_KEY
-    AWS_ACCESS_KEY_ID: $AWS_FBCI_ACCESS_KEY_ID
-    AWS_SECRET_ACCESS_KEY: $AWS_FBCI_SECRET_ACCESS_KEY
-  tags:
-    - shell
-  rules:
-    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
-  script:
-    - aws configure set aws_access_key_id $AWS_FBCI_ACCESS_KEY_ID
-    - aws configure set aws_secret_access_key $AWS_FBCI_SECRET_ACCESS_KEY
-    - aws configure set region "us-east-2"
-    - aws configure set aws_profile $PROFILE
-    - aws s3 cp ./idk/build/ s3://molecula-artifact-storage/idk/${CI_COMMIT_BRANCH}/${CI_COMMIT_SHORT_SHA}/ --recursive
-    - aws s3 cp ./idk/build/ s3://molecula-artifact-storage/idk/${CI_COMMIT_BRANCH}/_latest/ --recursive
-  needs:
-    - job: idk build_amd64
-    - job: idk build_arm64
-
-idk s3 dump tag:
-  stage: post build
-  variables:
-    PROFILE: "service-fb-ci"
-    AWS_SSH_PRIVATE_KEY: $AWS_FBCI_SSH_KEY
-    AWS_ACCESS_KEY_ID: $AWS_FBCI_ACCESS_KEY_ID
-    AWS_SECRET_ACCESS_KEY: $AWS_FBCI_SECRET_ACCESS_KEY
-    LOCATION: molecula-artifact-storage/idk/_tags
-  tags:
-    - shell
-  rules:
-    - if: '$CI_COMMIT_TAG != null && ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web")'
-  script:
-    - aws configure set aws_access_key_id $AWS_FBCI_ACCESS_KEY_ID
-    - aws configure set aws_secret_access_key $AWS_FBCI_SECRET_ACCESS_KEY
-    - aws configure set region "us-east-2"
-    - aws configure set aws_profile $PROFILE
-    - |
-      for goos in "darwin" "linux"; do
-        for goarch in "amd64" "arm64"; do
-          dir=idk-${CI_COMMIT_TAG}-${goos}-${goarch}
-          echo "Directory ${dir}"
-          mkdir ${dir}
-          mv ./idk/build/idk-${goos}-${goarch}/molecula-consumer-* ${dir}/
-          tar cvzf ${dir}.tar.gz ${dir}
-          aws s3 cp ${dir} s3://${LOCATION}/${CI_COMMIT_TAG}/${dir}/ --recursive
-          aws s3 cp ${dir}.tar.gz s3://${LOCATION}/${CI_COMMIT_TAG}/
-        done
-      done
-  needs:
-    - job: idk build_amd64
-    - job: idk build_arm64
-### end idk builds ###
-
-external lookup tests:
-  stage: integration
-  image: golang:$GOVERSION
-  extends: .go-cache
-  # TODO: no rules here, do we need to add the rules line?
-  variables:
-    POSTGRES_DB: $POSTGRES_DB
-    POSTGRES_USER: $POSTGRES_USER
-    POSTGRES_PASSWORD: $POSTGRES_PASSWORD
-    POSTGRES_HOST_AUTH_METHOD: trust
-  services:
-    - postgres:13.5
-  script:
-    - apt-get update --allow-releaseinfo-change -y
-    - apt-get install -y postgresql-client
-    - go test . -run "^TestExternalLookup" -externalLookupDSN postgresql://$POSTGRES_USER:$POSTGRES_PASSWORD@postgres/$POSTGRES_DB?sslmode=disable
-
-s3 dump:
-  stage: post build
-  variables:
-    PROFILE: "service-fb-ci"
-    AWS_SSH_PRIVATE_KEY: $AWS_FBCI_SSH_KEY
-    AWS_ACCESS_KEY_ID: $AWS_FBCI_ACCESS_KEY_ID
-    AWS_SECRET_ACCESS_KEY: $AWS_FBCI_SECRET_ACCESS_KEY
-  tags:
-    - shell
-  rules:
-    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "web"'
-  script:
-    - aws configure set aws_access_key_id $AWS_FBCI_ACCESS_KEY_ID
-    - aws configure set aws_secret_access_key $AWS_FBCI_SECRET_ACCESS_KEY
-    - aws configure set region "us-east-2"
-    - aws configure set aws_profile $PROFILE
-    - |
-      for goos in "darwin" "linux"; do
-        for goarch in "amd64" "arm64"; do
-          aws s3 cp featurebase_${goos}_${goarch} s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/${CI_COMMIT_SHORT_SHA}/featurebase_${goos}_${goarch}
-          aws s3 cp featurebase_${goos}_${goarch} s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/_latest/featurebase_${goos}_${goarch}
-          aws s3 cp ./build/fbsql_${goos}_${goarch} s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/${CI_COMMIT_SHORT_SHA}/fbsql_${goos}_${goarch}
-          aws s3 cp ./build/fbsql_${goos}_${goarch} s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/_latest/fbsql_${goos}_${goarch}
-        done
-      done
-  needs:
-    - job: build featurebase
-    - job: build fbsql amd64
-    - job: build fbsql arm64
-
-s3 dump tag:
-  stage: post build
-  variables:
-    PROFILE: "service-fb-ci"
-    AWS_SSH_PRIVATE_KEY: $AWS_FBCI_SSH_KEY
-    AWS_ACCESS_KEY_ID: $AWS_FBCI_ACCESS_KEY_ID
-    AWS_SECRET_ACCESS_KEY: $AWS_FBCI_SECRET_ACCESS_KEY
-    LOCATION: molecula-artifact-storage/featurebase/_tags
-  tags:
-    - shell
-  rules:
-    - if: '$CI_COMMIT_TAG != null && ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "web")'
-  script:
-    - aws configure set aws_access_key_id $AWS_FBCI_ACCESS_KEY_ID
-    - aws configure set aws_secret_access_key $AWS_FBCI_SECRET_ACCESS_KEY
-    - aws configure set region "us-east-2"
-    - aws configure set aws_profile $PROFILE
-    - |
-      for goos in "darwin" "linux"; do
-        for goarch in "amd64" "arm64"; do
-          dir=featurebase-${CI_COMMIT_TAG}-${goos}-${goarch}
-          echo "Directory ${dir}"
-          mkdir $dir
-          mv featurebase_${goos}_${goarch} ${dir}/featurebase
-          mv ./build/fbsql_${goos}_${goarch} ${dir}/fbsql
-          cp NOTICE install/featurebase.conf install/featurebase.*.service ${dir}/
-          tar cvzf ${dir}.tar.gz ${dir}
-          aws s3 cp ${dir} s3://${LOCATION}/${CI_COMMIT_TAG}/${dir}/ --recursive
-          aws s3 cp ${dir}.tar.gz s3://${LOCATION}/${CI_COMMIT_TAG}/
-        done
-      done
-
-  needs:
-    - job: build featurebase
-    - job: build fbsql amd64
-    - job: build fbsql arm64

.gitlab/Dockerfile

@@ -1,24 +0,0 @@
-FROM alpine:3.14.2
-
-LABEL maintainer "dev@molecula.com"
-LABEL org.opencontainers.image.authors="dev@molecula.com"
-
-ARG ARCH
-
-WORKDIR /
-
-RUN apk add --no-cache curl jq
-
-COPY NOTICE .
-COPY featurebase_linux_$ARCH featurebase
-
-RUN chmod ugo+x .
-EXPOSE 10101
-VOLUME /data
-
-ENV PILOSA_DATA_DIR /data
-ENV PILOSA_BIND 0.0.0.0:10101
-ENV PILOSA_BIND_GRPC 0.0.0.0:20101
-
-ENTRYPOINT ["/featurebase"]
-CMD ["server"]

.gitlab/batch-ci.yml

@@ -1,28 +0,0 @@
-run go tests batch:
-  variables:
-    PROJECT: batch_${CI_CONCURRENT_ID}
-  # this test relies on stuff that happens after build-lattice, which
-  # makes it pause the entire CI run waiting for this. we accept the
-  # small risk of wasting a build against the near certainty of spending
-  # five minutes running only one job.
-  stage: nonblocking
-  retry: 1
-  script:
-    - echo "Running test-all"
-    - cd ./batch/
-    - echo $PROJECT
-    - make build-featurebase
-    - make test-all
-  after_script:
-    - cd ./batch/
-    - make save-featurebase-logs
-    - make shutdown
-  artifacts:
-    paths:
-      - ./batch/testdata/*_coverage.out
-      - ./batch/testdata/*_logs.txt
-  tags:
-    - shell
-    - aws
-  needs:
-    - job: build amd container fb

.golangci.yml

@@ -1,83 +0,0 @@
-run:
-  deadline: 5m
-  timeout: 5m
-  skip-dirs-use-default: true
-  #skip the protobuf generated files
-  skip-dirs:
-    - pb
-    - proto
-  skip-files:
-    - pql/pql.peg.go
-linters:
-  enable:
-    # Recommended to be enabled by default (https://golangci-lint.run).
-    # - errcheck (lots to fix)
-    - gosimple
-    - govet
-    - ineffassign
-    - staticcheck
-    - typecheck
-    # - unused (about 20 to fix)
-
-    # Additional linters we choose to enable.
-    # - bodyclose (lots to fix, but we should)
-    - errchkjson
-    - errname
-    - gofmt
-    # - misspell (lots to fix, but we should)
-    - prealloc
-    # - predeclared (20 to fix)
-    # - stylecheck (quite a lot to fix, but we should definitely work on this)
-    - stylecheck
-    # - unconvert (not at all critical, but makes for cleaner code)
-  enable-all: false
-  disable-all: true
-
-output:
-  # colored-line-number|line-number|json|tab|checkstyle|code-climate, default is "colored-line-number"
-  format: tab
-  # print lines of code with issue, default is true
-  print-issued-lines: true
-  # print linter name in the end of issue text, default is true
-  print-linter-name: true
-
-linters-settings:
-  gofmt:
-    simplify: true
-  govet:
-    # report about shadowed variables
-    check-shadowing: true
-
-
-    # settings per analyzer
-    settings:
-      printf: # analyzer name, run `go tool vet help` to see all analyzers
-        funcs: # run `go tool vet help printf` to see available settings for `printf` analyzer
-          - (github.com/golangci/golangci-lint/pkg/logutils.Log).Infof
-          - (github.com/golangci/golangci-lint/pkg/logutils.Log).Warnf
-          - (github.com/golangci/golangci-lint/pkg/logutils.Log).Errorf
-          - (github.com/golangci/golangci-lint/pkg/logutils.Log).Fatalf
-
-    # enable or disable analyzers by name
-    # run `go tool vet help` to see all analyzers
-    enable:
-      - atomicalign
-    enable-all: false
-    disable:
-      - shadow
-    disable-all: false
-
-  stylecheck:
-    # ST1000: at least one file in a package should have a package comment
-    # ST1003: golang naming standards
-    # ST1016: methods on the same type should have the same receiver name
-    # ST1020: comment on exported function
-    checks: ["all", "-ST1000", "-ST1003", "-ST1016", "-ST1020"]
-
-issues:
-  exclude-use-default: false
-  max-issues-per-linter: 0
-  max-same-issues: 0
-  exclude:
-    - 'declaration of "(err|ctx)" shadows declaration at'
-    - 'Error return value of .(.*\.Help|.*\.MarkFlagRequired|(os\.)?std(out|err)\..*|.*Close|.*Flush|os\.Remove(All)?|.*printf?|os\.(Un)?Setenv). is not checked'

CODE_OF_CONDUCT.md

@@ -1,133 +0,0 @@
-
-# Contributor Covenant Code of Conduct
-
-## Our Pledge
-
-We as members, contributors, and leaders pledge to make participation in our
-community a harassment-free experience for everyone, regardless of age, body
-size, visible or invisible disability, ethnicity, sex characteristics, gender
-identity and expression, level of experience, education, socio-economic status,
-nationality, personal appearance, race, caste, color, religion, or sexual
-identity and orientation.
-
-We pledge to act and interact in ways that contribute to an open, welcoming,
-diverse, inclusive, and healthy community.
-
-## Our Standards
-
-Examples of behavior that contributes to a positive environment for our
-community include:
-
-* Demonstrating empathy and kindness toward other people
-* Being respectful of differing opinions, viewpoints, and experiences
-* Giving and gracefully accepting constructive feedback
-* Accepting responsibility and apologizing to those affected by our mistakes,
-  and learning from the experience
-* Focusing on what is best not just for us as individuals, but for the overall
-  community
-
-Examples of unacceptable behavior include:
-
-* The use of sexualized language or imagery, and sexual attention or advances of
-  any kind
-* Trolling, insulting or derogatory comments, and personal or political attacks
-* Public or private harassment
-* Publishing others' private information, such as a physical or email address,
-  without their explicit permission
-* Other conduct which could reasonably be considered inappropriate in a
-  professional setting
-
-## Enforcement Responsibilities
-
-Community leaders are responsible for clarifying and enforcing our standards of
-acceptable behavior and will take appropriate and fair corrective action in
-response to any behavior that they deem inappropriate, threatening, offensive,
-or harmful.
-
-Community leaders have the right and responsibility to remove, edit, or reject
-comments, commits, code, wiki edits, issues, and other contributions that are
-not aligned to this Code of Conduct, and will communicate reasons for moderation
-decisions when appropriate.
-
-## Scope
-
-This Code of Conduct applies within all community spaces, and also applies when
-an individual is officially representing the community in public spaces.
-Examples of representing our community include using an official e-mail address,
-posting via an official social media account, or acting as an appointed
-representative at an online or offline event.
-
-## Enforcement
-
-Instances of abusive, harassing, or otherwise unacceptable behavior may be
-reported to the community leaders responsible for enforcement at
-community@featurebase.com.
-All complaints will be reviewed and investigated promptly and fairly.
-
-All community leaders are obligated to respect the privacy and security of the
-reporter of any incident.
-
-## Enforcement Guidelines
-
-Community leaders will follow these Community Impact Guidelines in determining
-the consequences for any action they deem in violation of this Code of Conduct:
-
-### 1. Correction
-
-**Community Impact**: Use of inappropriate language or other behavior deemed
-unprofessional or unwelcome in the community.
-
-**Consequence**: A private, written warning from community leaders, providing
-clarity around the nature of the violation and an explanation of why the
-behavior was inappropriate. A public apology may be requested.
-
-### 2. Warning
-
-**Community Impact**: A violation through a single incident or series of
-actions.
-
-**Consequence**: A warning with consequences for continued behavior. No
-interaction with the people involved, including unsolicited interaction with
-those enforcing the Code of Conduct, for a specified period of time. This
-includes avoiding interactions in community spaces as well as external channels
-like social media. Violating these terms may lead to a temporary or permanent
-ban.
-
-### 3. Temporary Ban
-
-**Community Impact**: A serious violation of community standards, including
-sustained inappropriate behavior.
-
-**Consequence**: A temporary ban from any sort of interaction or public
-communication with the community for a specified period of time. No public or
-private interaction with the people involved, including unsolicited interaction
-with those enforcing the Code of Conduct, is allowed during this period.
-Violating these terms may lead to a permanent ban.
-
-### 4. Permanent Ban
-
-**Community Impact**: Demonstrating a pattern of violation of community
-standards, including sustained inappropriate behavior, harassment of an
-individual, or aggression toward or disparagement of classes of individuals.
-
-**Consequence**: A permanent ban from any sort of public interaction within the
-community.
-
-## Attribution
-
-This Code of Conduct is adapted from the [Contributor Covenant][homepage],
-version 2.1, available at
-[https://www.contributor-covenant.org/version/2/1/code_of_conduct.html][v2.1].
-
-Community Impact Guidelines were inspired by
-[Mozilla's code of conduct enforcement ladder][Mozilla CoC].
-
-For answers to common questions about this code of conduct, see the FAQ at
-[https://www.contributor-covenant.org/faq][FAQ]. Translations are available at
-[https://www.contributor-covenant.org/translations][translations].
-
-[homepage]: https://www.contributor-covenant.org
-[v2.1]: https://www.contributor-covenant.org/version/2/1/code_of_conduct.html
-[Mozilla CoC]: https://github.com/mozilla/diversity
-[FAQ]: https://www.contributor-covenant.org/faq
-[translations]: https://www.contributor-covenant.org/translations

Dockerfile

@@ -1,58 +0,0 @@
-ARG GO_VERSION=latest
-
-#######################
-### Lattice builder ###
-#######################
-
-FROM ghcr.io/featurebasedb/nodejs:0.0.1 as lattice-builder
-WORKDIR /lattice
-
-COPY lattice/package.json ./
-COPY lattice/yarn.lock ./
-RUN yarn install
-
-COPY lattice ./
-RUN yarn build
-
-######################
-### Pilosa builder ###
-######################
-
-FROM golang:${GO_VERSION} as pilosa-builder
-ARG MAKE_FLAGS
-ARG SOURCE_DATE_EPOCH
-
-WORKDIR /pilosa
-
-RUN go install github.com/rakyll/statik@v0.1.7
-
-COPY . ./
-COPY --from=lattice-builder /lattice/build /lattice
-RUN /go/bin/statik -src=/lattice -dest=/pilosa
-
-ENV SOURCE_DATE_EPOCH=${SOURCE_DATE_EPOCH}
-RUN make build FLAGS="-o build/featurebase" ${MAKE_FLAGS}
-
-#####################
-### Pilosa runner ###
-#####################
-
-FROM alpine:3.13.2 as runner
-
-LABEL maintainer "dev@molecula.com"
-
-RUN apk add --no-cache curl jq tree
-
-COPY --from=pilosa-builder /pilosa/build/featurebase /
-
-COPY NOTICE /NOTICE
-
-EXPOSE 10101
-VOLUME /data
-
-ENV PILOSA_DATA_DIR /data
-ENV PILOSA_BIND 0.0.0.0:10101
-ENV PILOSA_BIND_GRPC 0.0.0.0:20101
-
-ENTRYPOINT ["/featurebase"]
-CMD ["server"]

Dockerfile-clustertests

@@ -1,38 +0,0 @@
-# This Dockerfile is used for cluster testing - it produces a much larger image
-# and includes all of Go as well as some utilities.
-
-FROM golang:1.19
-
-LABEL maintainer "dev@pilosa.com"
-
-COPY . /go/src/github.com/featurebasedb/featurebase/
-
-
-# download pumba for fault injection
-ADD https://github.com/alexei-led/pumba/releases/download/0.6.0/pumba_linux_amd64 /pumba
-RUN chmod +x /pumba
-
-# add docker client to pause/unpause nodes
-RUN apt update
-RUN apt install -y docker.io
-
-# add docker-compose so tests can use it for stuff
-ADD https://github.com/docker/compose/releases/latest/download/docker-compose-Linux-x86_64 /usr/local/bin/docker-compose
-RUN chmod +x /usr/local/bin/docker-compose
-
-WORKDIR /go/src/github.com/featurebasedb/featurebase/cmd/featurebase
-
-# generate an instrumented binary to allow for calculating code coverage for clustertests
-# the entrypoint for the binary is TestRunMain, which is wrapper for main
-RUN go test -covermode=atomic -coverpkg=../../... -c -tags testrunmain -o featurebase
-RUN cp /go/src/github.com/featurebasedb/featurebase/cmd/featurebase/featurebase /featurebase
-
-COPY NOTICE /NOTICE
-
-EXPOSE 10101
-VOLUME /data
-
-
-# use e.g. "-test.coverprofile=/results/coverage.out"
-CMD ["/featurebase", "-test.run=TestRunMain", "server"]
-

Dockerfile-clustertests-client

@@ -1,37 +0,0 @@
-# This Dockerfile is used for cluster testing - it produces a much larger image
-# and includes all of Go as well as some utilities.
-
-FROM golang:1.19
-
-LABEL maintainer "dev@pilosa.com"
-
-COPY . /go/src/github.com/featurebasedb/featurebase/
-
-# download pumba for fault injection
-ADD https://github.com/alexei-led/pumba/releases/download/0.6.0/pumba_linux_amd64 /pumba
-RUN chmod +x /pumba
-
-# add docker client to pause/unpause nodes
-RUN apt update
-RUN apt install -y docker.io
-
-# add docker-compose so tests can use it for stuff
-ADD https://github.com/docker/compose/releases/latest/download/docker-compose-Linux-x86_64 /usr/local/bin/docker-compose
-RUN chmod +x /usr/local/bin/docker-compose
-
-WORKDIR /go/src/github.com/featurebasedb/featurebase/cmd/featurebase
-
-RUN go test -covermode=atomic -coverpkg=../../... -c -tags testrunmain -o featurebase
-RUN cp /go/src/github.com/featurebasedb/featurebase/cmd/featurebase/featurebase /featurebase
-
-
-COPY NOTICE /NOTICE
-
-COPY ./internal/clustertests /go/src/github.com/featurebasedb/featurebase/internal/clustertests
-
-EXPOSE 10101
-VOLUME /data
-
-WORKDIR /go/src/github.com/featurebasedb/featurebase
-
-CMD ["/featurebase", "-test.run=TestRunMain", "-test.coverprofile=/results/coverage.out", "server", "--data-dir", "/data", "--bind", "http://0.0.0.0:10101"]

Dockerfile-datagen

@@ -1,47 +0,0 @@
-# syntax=docker/dockerfile:1
-
-##########################
-### datagen builder    ###
-##########################
-
-FROM golang:alpine as builder
-
-WORKDIR /featurebase
-
-COPY . ./
-
-RUN apk add --no-cache build-base bash git make librdkafka pkgconfig
-
-# install librdkafka
-RUN git clone https://github.com/edenhill/librdkafka.git
-RUN cd librdkafka && ./configure --prefix /usr && make && make install
-
-ENV PKG_CONFIG_PATH=/usr/lib/pkgconfig/
-
-RUN cd idk && make build-datagen
-
-# ENTRYPOINT ["tail", "-f", "/dev/null"]
-
-#########################
-### datagen runner    ###
-#########################
-
-FROM alpine:3.15.3 as runner
-
-WORKDIR /
-
-LABEL maintainer "dev@molecula.com"
-
-RUN apk add --no-cache curl jq
-
-COPY --from=builder /featurebase/idk/build/datagen /bin/
-COPY --from=builder /usr/lib/librdkafka* /usr/lib/
-COPY idk/datagen/testdata/* /testdata/
-
-EXPOSE 8080
-
-# VOLUME /data
-# ENV ADDR 0.0.0.0:8080
-
-#ENTRYPOINT ["sleep", "infinity"]
-ENTRYPOINT ["datagen"]

Dockerfile-dax

@@ -1,32 +0,0 @@
-ARG GO_VERSION=latest
-
-
-###########################
-### FeatureBase Builder ###
-###########################
-
-FROM golang:${GO_VERSION} as featurebase-builder
-ARG MAKE_FLAGS
-WORKDIR /fb
-
-COPY . ./
-RUN make build FLAGS="-o build/featurebase" ${MAKE_FLAGS}
-
-##########################
-### FeatureBase runner ###
-##########################
-
-FROM golang:alpine as runner
-
-LABEL maintainer "dev@featurebase.com"
-
-RUN apk add --no-cache curl jq tree
-
-COPY --from=featurebase-builder /fb/build/featurebase /
-
-COPY NOTICE /NOTICE
-
-EXPOSE 8080
-
-ENTRYPOINT ["/featurebase"]
-CMD ["dax"]

Dockerfile-dax-quick

@@ -1,18 +0,0 @@
-ARG GO_VERSION=latest
-
-##########################
-### FeatureBase runner ###
-##########################
-
-FROM alpine:3.13.2 as runner
-
-LABEL maintainer "dev@featurebase.com"
-
-RUN apk add --no-cache curl jq tree
-
-COPY ./fb_linux /featurebase
-
-EXPOSE 8080
-
-ENTRYPOINT ["/featurebase"]
-CMD ["dax"]

Dockerfile-fbsql

@@ -1,48 +0,0 @@
-ARG GO_VERSION=1.19
-
-FROM golang:1.19-buster as builder
-
-WORKDIR /
-RUN apt-get update -y -qq && apt-get install -y -qq \
-  build-essential \
-  git \
-  musl-tools \
-  netcat \
-  unixodbc \
-  unixodbc-dev \
-  && rm -rf /var/lib/apt/lists/*
-RUN ["git", "clone", "https://github.com/edenhill/librdkafka.git"]
-WORKDIR /librdkafka
-RUN ./configure --prefix /usr && \
-  make && \
-  make install
-
-WORKDIR /featurebase
-
-COPY . .
-
-ARG MAKE_FLAGS
-ARG GO_BUILD_FLAGS
-ARG SOURCE_DATE_EPOCH
-
-WORKDIR /featurebase/
-
-ENV SOURCE_DATE_EPOCH=${SOURCE_DATE_EPOCH}
-RUN make build-fbsql GO_BUILD_FLAGS="-mod=vendor ${GO_BUILD_FLAGS}" ${MAKE_FLAGS}
-
-FROM ubuntu:20.04 as runner
-
-RUN apt-get update -y -qq && apt-get install -y -qq \
-  ca-certificates \
-  musl-tools \
-  netcat \
-  unixodbc-dev \
-  && rm -rf /var/lib/apt/lists/*
-
-COPY --from=builder /featurebase/fbsql /usr/local/bin/
-
-# Verify that the linker can find everything.
-FROM runner as linkcheck
-RUN if [ -e /usr/local/bin/fbsql ] ; then ldd /usr/local/bin/fbsql; fi
-
-FROM runner

LICENSE

@@ -1,201 +0,0 @@
-                                 Apache License
-                           Version 2.0, January 2004
-                        http://www.apache.org/licenses/
-
-   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
-
-   1. Definitions.
-
-      "License" shall mean the terms and conditions for use, reproduction,
-      and distribution as defined by Sections 1 through 9 of this document.
-
-      "Licensor" shall mean the copyright owner or entity authorized by
-      the copyright owner that is granting the License.
-
-      "Legal Entity" shall mean the union of the acting entity and all
-      other entities that control, are controlled by, or are under common
-      control with that entity. For the purposes of this definition,
-      "control" means (i) the power, direct or indirect, to cause the
-      direction or management of such entity, whether by contract or
-      otherwise, or (ii) ownership of fifty percent (50%) or more of the
-      outstanding shares, or (iii) beneficial ownership of such entity.
-
-      "You" (or "Your") shall mean an individual or Legal Entity
-      exercising permissions granted by this License.
-
-      "Source" form shall mean the preferred form for making modifications,
-      including but not limited to software source code, documentation
-      source, and configuration files.
-
-      "Object" form shall mean any form resulting from mechanical
-      transformation or translation of a Source form, including but
-      not limited to compiled object code, generated documentation,
-      and conversions to other media types.
-
-      "Work" shall mean the work of authorship, whether in Source or
-      Object form, made available under the License, as indicated by a
-      copyright notice that is included in or attached to the work
-      (an example is provided in the Appendix below).
-
-      "Derivative Works" shall mean any work, whether in Source or Object
-      form, that is based on (or derived from) the Work and for which the
-      editorial revisions, annotations, elaborations, or other modifications
-      represent, as a whole, an original work of authorship. For the purposes
-      of this License, Derivative Works shall not include works that remain
-      separable from, or merely link (or bind by name) to the interfaces of,
-      the Work and Derivative Works thereof.
-
-      "Contribution" shall mean any work of authorship, including
-      the original version of the Work and any modifications or additions
-      to that Work or Derivative Works thereof, that is intentionally
-      submitted to Licensor for inclusion in the Work by the copyright owner
-      or by an individual or Legal Entity authorized to submit on behalf of
-      the copyright owner. For the purposes of this definition, "submitted"
-      means any form of electronic, verbal, or written communication sent
-      to the Licensor or its representatives, including but not limited to
-      communication on electronic mailing lists, source code control systems,
-      and issue tracking systems that are managed by, or on behalf of, the
-      Licensor for the purpose of discussing and improving the Work, but
-      excluding communication that is conspicuously marked or otherwise
-      designated in writing by the copyright owner as "Not a Contribution."
-
-      "Contributor" shall mean Licensor and any individual or Legal Entity
-      on behalf of whom a Contribution has been received by Licensor and
-      subsequently incorporated within the Work.
-
-   2. Grant of Copyright License. Subject to the terms and conditions of
-      this License, each Contributor hereby grants to You a perpetual,
-      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-      copyright license to reproduce, prepare Derivative Works of,
-      publicly display, publicly perform, sublicense, and distribute the
-      Work and such Derivative Works in Source or Object form.
-
-   3. Grant of Patent License. Subject to the terms and conditions of
-      this License, each Contributor hereby grants to You a perpetual,
-      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-      (except as stated in this section) patent license to make, have made,
-      use, offer to sell, sell, import, and otherwise transfer the Work,
-      where such license applies only to those patent claims licensable
-      by such Contributor that are necessarily infringed by their
-      Contribution(s) alone or by combination of their Contribution(s)
-      with the Work to which such Contribution(s) was submitted. If You
-      institute patent litigation against any entity (including a
-      cross-claim or counterclaim in a lawsuit) alleging that the Work
-      or a Contribution incorporated within the Work constitutes direct
-      or contributory patent infringement, then any patent licenses
-      granted to You under this License for that Work shall terminate
-      as of the date such litigation is filed.
-
-   4. Redistribution. You may reproduce and distribute copies of the
-      Work or Derivative Works thereof in any medium, with or without
-      modifications, and in Source or Object form, provided that You
-      meet the following conditions:
-
-      (a) You must give any other recipients of the Work or
-          Derivative Works a copy of this License; and
-
-      (b) You must cause any modified files to carry prominent notices
-          stating that You changed the files; and
-
-      (c) You must retain, in the Source form of any Derivative Works
-          that You distribute, all copyright, patent, trademark, and
-          attribution notices from the Source form of the Work,
-          excluding those notices that do not pertain to any part of
-          the Derivative Works; and
-
-      (d) If the Work includes a "NOTICE" text file as part of its
-          distribution, then any Derivative Works that You distribute must
-          include a readable copy of the attribution notices contained
-          within such NOTICE file, excluding those notices that do not
-          pertain to any part of the Derivative Works, in at least one
-          of the following places: within a NOTICE text file distributed
-          as part of the Derivative Works; within the Source form or
-          documentation, if provided along with the Derivative Works; or,
-          within a display generated by the Derivative Works, if and
-          wherever such third-party notices normally appear. The contents
-          of the NOTICE file are for informational purposes only and
-          do not modify the License. You may add Your own attribution
-          notices within Derivative Works that You distribute, alongside
-          or as an addendum to the NOTICE text from the Work, provided
-          that such additional attribution notices cannot be construed
-          as modifying the License.
-
-      You may add Your own copyright statement to Your modifications and
-      may provide additional or different license terms and conditions
-      for use, reproduction, or distribution of Your modifications, or
-      for any such Derivative Works as a whole, provided Your use,
-      reproduction, and distribution of the Work otherwise complies with
-      the conditions stated in this License.
-
-   5. Submission of Contributions. Unless You explicitly state otherwise,
-      any Contribution intentionally submitted for inclusion in the Work
-      by You to the Licensor shall be under the terms and conditions of
-      this License, without any additional terms or conditions.
-      Notwithstanding the above, nothing herein shall supersede or modify
-      the terms of any separate license agreement you may have executed
-      with Licensor regarding such Contributions.
-
-   6. Trademarks. This License does not grant permission to use the trade
-      names, trademarks, service marks, or product names of the Licensor,
-      except as required for reasonable and customary use in describing the
-      origin of the Work and reproducing the content of the NOTICE file.
-
-   7. Disclaimer of Warranty. Unless required by applicable law or
-      agreed to in writing, Licensor provides the Work (and each
-      Contributor provides its Contributions) on an "AS IS" BASIS,
-      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-      implied, including, without limitation, any warranties or conditions
-      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
-      PARTICULAR PURPOSE. You are solely responsible for determining the
-      appropriateness of using or redistributing the Work and assume any
-      risks associated with Your exercise of permissions under this License.
-
-   8. Limitation of Liability. In no event and under no legal theory,
-      whether in tort (including negligence), contract, or otherwise,
-      unless required by applicable law (such as deliberate and grossly
-      negligent acts) or agreed to in writing, shall any Contributor be
-      liable to You for damages, including any direct, indirect, special,
-      incidental, or consequential damages of any character arising as a
-      result of this License or out of the use or inability to use the
-      Work (including but not limited to damages for loss of goodwill,
-      work stoppage, computer failure or malfunction, or any and all
-      other commercial damages or losses), even if such Contributor
-      has been advised of the possibility of such damages.
-
-   9. Accepting Warranty or Additional Liability. While redistributing
-      the Work or Derivative Works thereof, You may choose to offer,
-      and charge a fee for, acceptance of support, warranty, indemnity,
-      or other liability obligations and/or rights consistent with this
-      License. However, in accepting such obligations, You may act only
-      on Your own behalf and on Your sole responsibility, not on behalf
-      of any other Contributor, and only if You agree to indemnify,
-      defend, and hold each Contributor harmless for any liability
-      incurred by, or claims asserted against, such Contributor by reason
-      of your accepting any such warranty or additional liability.
-
-   END OF TERMS AND CONDITIONS
-
-   APPENDIX: How to apply the Apache License to your work.
-
-      To apply the Apache License to your work, attach the following
-      boilerplate notice, with the fields enclosed by brackets "[]"
-      replaced with your own identifying information. (Don't include
-      the brackets!)  The text should be enclosed in the appropriate
-      comment syntax for the file format. We also recommend that a
-      file or class name and description of purpose be included on the
-      same "printed page" as the copyright notice for easier
-      identification within third-party archives.
-
-   Copyright 2023 Molecula Corp. All rights reserved.
-
-   Licensed under the Apache License, Version 2.0 (the "License");
-   you may not use this file except in compliance with the License.
-   You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.

LICENSE-2.0.txt

@@ -1,202 +0,0 @@
-
-                                 Apache License
-                           Version 2.0, January 2004
-                        http://www.apache.org/licenses/
-
-   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
-
-   1. Definitions.
-
-      "License" shall mean the terms and conditions for use, reproduction,
-      and distribution as defined by Sections 1 through 9 of this document.
-
-      "Licensor" shall mean the copyright owner or entity authorized by
-      the copyright owner that is granting the License.
-
-      "Legal Entity" shall mean the union of the acting entity and all
-      other entities that control, are controlled by, or are under common
-      control with that entity. For the purposes of this definition,
-      "control" means (i) the power, direct or indirect, to cause the
-      direction or management of such entity, whether by contract or
-      otherwise, or (ii) ownership of fifty percent (50%) or more of the
-      outstanding shares, or (iii) beneficial ownership of such entity.
-
-      "You" (or "Your") shall mean an individual or Legal Entity
-      exercising permissions granted by this License.
-
-      "Source" form shall mean the preferred form for making modifications,
-      including but not limited to software source code, documentation
-      source, and configuration files.
-
-      "Object" form shall mean any form resulting from mechanical
-      transformation or translation of a Source form, including but
-      not limited to compiled object code, generated documentation,
-      and conversions to other media types.
-
-      "Work" shall mean the work of authorship, whether in Source or
-      Object form, made available under the License, as indicated by a
-      copyright notice that is included in or attached to the work
-      (an example is provided in the Appendix below).
-
-      "Derivative Works" shall mean any work, whether in Source or Object
-      form, that is based on (or derived from) the Work and for which the
-      editorial revisions, annotations, elaborations, or other modifications
-      represent, as a whole, an original work of authorship. For the purposes
-      of this License, Derivative Works shall not include works that remain
-      separable from, or merely link (or bind by name) to the interfaces of,
-      the Work and Derivative Works thereof.
-
-      "Contribution" shall mean any work of authorship, including
-      the original version of the Work and any modifications or additions
-      to that Work or Derivative Works thereof, that is intentionally
-      submitted to Licensor for inclusion in the Work by the copyright owner
-      or by an individual or Legal Entity authorized to submit on behalf of
-      the copyright owner. For the purposes of this definition, "submitted"
-      means any form of electronic, verbal, or written communication sent
-      to the Licensor or its representatives, including but not limited to
-      communication on electronic mailing lists, source code control systems,
-      and issue tracking systems that are managed by, or on behalf of, the
-      Licensor for the purpose of discussing and improving the Work, but
-      excluding communication that is conspicuously marked or otherwise
-      designated in writing by the copyright owner as "Not a Contribution."
-
-      "Contributor" shall mean Licensor and any individual or Legal Entity
-      on behalf of whom a Contribution has been received by Licensor and
-      subsequently incorporated within the Work.
-
-   2. Grant of Copyright License. Subject to the terms and conditions of
-      this License, each Contributor hereby grants to You a perpetual,
-      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-      copyright license to reproduce, prepare Derivative Works of,
-      publicly display, publicly perform, sublicense, and distribute the
-      Work and such Derivative Works in Source or Object form.
-
-   3. Grant of Patent License. Subject to the terms and conditions of
-      this License, each Contributor hereby grants to You a perpetual,
-      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-      (except as stated in this section) patent license to make, have made,
-      use, offer to sell, sell, import, and otherwise transfer the Work,
-      where such license applies only to those patent claims licensable
-      by such Contributor that are necessarily infringed by their
-      Contribution(s) alone or by combination of their Contribution(s)
-      with the Work to which such Contribution(s) was submitted. If You
-      institute patent litigation against any entity (including a
-      cross-claim or counterclaim in a lawsuit) alleging that the Work
-      or a Contribution incorporated within the Work constitutes direct
-      or contributory patent infringement, then any patent licenses
-      granted to You under this License for that Work shall terminate
-      as of the date such litigation is filed.
-
-   4. Redistribution. You may reproduce and distribute copies of the
-      Work or Derivative Works thereof in any medium, with or without
-      modifications, and in Source or Object form, provided that You
-      meet the following conditions:
-
-      (a) You must give any other recipients of the Work or
-          Derivative Works a copy of this License; and
-
-      (b) You must cause any modified files to carry prominent notices
-          stating that You changed the files; and
-
-      (c) You must retain, in the Source form of any Derivative Works
-          that You distribute, all copyright, patent, trademark, and
-          attribution notices from the Source form of the Work,
-          excluding those notices that do not pertain to any part of
-          the Derivative Works; and
-
-      (d) If the Work includes a "NOTICE" text file as part of its
-          distribution, then any Derivative Works that You distribute must
-          include a readable copy of the attribution notices contained
-          within such NOTICE file, excluding those notices that do not
-          pertain to any part of the Derivative Works, in at least one
-          of the following places: within a NOTICE text file distributed
-          as part of the Derivative Works; within the Source form or
-          documentation, if provided along with the Derivative Works; or,
-          within a display generated by the Derivative Works, if and
-          wherever such third-party notices normally appear. The contents
-          of the NOTICE file are for informational purposes only and
-          do not modify the License. You may add Your own attribution
-          notices within Derivative Works that You distribute, alongside
-          or as an addendum to the NOTICE text from the Work, provided
-          that such additional attribution notices cannot be construed
-          as modifying the License.
-
-      You may add Your own copyright statement to Your modifications and
-      may provide additional or different license terms and conditions
-      for use, reproduction, or distribution of Your modifications, or
-      for any such Derivative Works as a whole, provided Your use,
-      reproduction, and distribution of the Work otherwise complies with
-      the conditions stated in this License.
-
-   5. Submission of Contributions. Unless You explicitly state otherwise,
-      any Contribution intentionally submitted for inclusion in the Work
-      by You to the Licensor shall be under the terms and conditions of
-      this License, without any additional terms or conditions.
-      Notwithstanding the above, nothing herein shall supersede or modify
-      the terms of any separate license agreement you may have executed
-      with Licensor regarding such Contributions.
-
-   6. Trademarks. This License does not grant permission to use the trade
-      names, trademarks, service marks, or product names of the Licensor,
-      except as required for reasonable and customary use in describing the
-      origin of the Work and reproducing the content of the NOTICE file.
-
-   7. Disclaimer of Warranty. Unless required by applicable law or
-      agreed to in writing, Licensor provides the Work (and each
-      Contributor provides its Contributions) on an "AS IS" BASIS,
-      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-      implied, including, without limitation, any warranties or conditions
-      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
-      PARTICULAR PURPOSE. You are solely responsible for determining the
-      appropriateness of using or redistributing the Work and assume any
-      risks associated with Your exercise of permissions under this License.
-
-   8. Limitation of Liability. In no event and under no legal theory,
-      whether in tort (including negligence), contract, or otherwise,
-      unless required by applicable law (such as deliberate and grossly
-      negligent acts) or agreed to in writing, shall any Contributor be
-      liable to You for damages, including any direct, indirect, special,
-      incidental, or consequential damages of any character arising as a
-      result of this License or out of the use or inability to use the
-      Work (including but not limited to damages for loss of goodwill,
-      work stoppage, computer failure or malfunction, or any and all
-      other commercial damages or losses), even if such Contributor
-      has been advised of the possibility of such damages.
-
-   9. Accepting Warranty or Additional Liability. While redistributing
-      the Work or Derivative Works thereof, You may choose to offer,
-      and charge a fee for, acceptance of support, warranty, indemnity,
-      or other liability obligations and/or rights consistent with this
-      License. However, in accepting such obligations, You may act only
-      on Your own behalf and on Your sole responsibility, not on behalf
-      of any other Contributor, and only if You agree to indemnify,
-      defend, and hold each Contributor harmless for any liability
-      incurred by, or claims asserted against, such Contributor by reason
-      of your accepting any such warranty or additional liability.
-
-   END OF TERMS AND CONDITIONS
-
-   APPENDIX: How to apply the Apache License to your work.
-
-      To apply the Apache License to your work, attach the following
-      boilerplate notice, with the fields enclosed by brackets "[]"
-      replaced with your own identifying information. (Don't include
-      the brackets!)  The text should be enclosed in the appropriate
-      comment syntax for the file format. We also recommend that a
-      file or class name and description of purpose be included on the
-      same "printed page" as the copyright notice for easier
-      identification within third-party archives.
-
-   Copyright [yyyy] [name of copyright owner]
-
-   Licensed under the Apache License, Version 2.0 (the "License");
-   you may not use this file except in compliance with the License.
-   You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.

Makefile

@@ -1,405 +1,7 @@
-.PHONY: build clean build-lattice cover cover-viz default docker docker-build docker-build-fbsql docker-tag-push generate generate-protoc generate-pql generate-statik generate-stringer install install-protoc-gen-gofast install-protoc install-statik install-peg test docker-login
+.PHONY: vendor
 
-SHELL := /bin/bash
-VERSION := $(shell git describe --tags 2> /dev/null || echo unknown)
-VARIANT = Molecula
-GO=go
-GOOS=$(shell $(GO) env GOOS)
-GOARCH=$(shell $(GO) env GOARCH)
-VERSION_ID=$(if $(TRIAL_DEADLINE),trial-$(TRIAL_DEADLINE)-,)$(VERSION)-$(GOOS)-$(GOARCH)
-DATE_FMT="+%FT%T%z"
-# set SOURCE_DATE_EPOCH like this to use the last git commit timestamp
-# export SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct) instead of the current time from running `date`
-ifdef SOURCE_DATE_EPOCH
-	BUILD_TIME ?= $(shell date -u -d "@$(SOURCE_DATE_EPOCH)" "$(DATE_FMT)" 2>/dev/null || date -u -r "$(SOURCE_DATE_EPOCH)" "$(DATE_FMT)" 2>/dev/null || date -u "$(DATE_FMT)")
-else
-	BUILD_TIME ?= $(shell date -u "$(DATE_FMT)")
-endif
-SHARD_WIDTH = 20
-COMMIT := $(shell git describe --exact-match >/dev/null 2>&1 || git rev-parse --short HEAD)
-LDFLAGS="-X github.com/featurebasedb/featurebase/v3.Version=$(VERSION) -X github.com/featurebasedb/featurebase/v3.BuildTime=$(BUILD_TIME) -X github.com/featurebasedb/featurebase/v3.Variant=$(VARIANT) -X github.com/featurebasedb/featurebase/v3.Commit=$(COMMIT) -X github.com/featurebasedb/featurebase/v3.TrialDeadline=$(TRIAL_DEADLINE)"
-GO_VERSION=1.19
-GO_BUILD_FLAGS=
-DOCKER_BUILD= # set to 1 to use `docker-build` instead of `build` when creating a release
-BUILD_TAGS += 
-TEST_TAGS = roaringparanoia
-TEST_TIMEOUT=10m
-RACE_TEST_TIMEOUT=10m
-# size in GB to use for ramdisk, ?= so you can override it with env
-# 4GB is not enough for `make test`, 8GB usually is.
-RAMDISK_SIZE ?= 8
-
-export GO111MODULE=on
-export GOPRIVATE=github.com/molecula
-export CGO_ENABLED=0
-AWS_ACCOUNTID ?= undefined
-
-# Run tests and compile Pilosa
-default: test build
-
-# Remove build directories
-clean:
-	rm -rf vendor build
-	rm -f *.rpm *.deb
-
-# Set up vendor directory using `go mod vendor`
-vendor: go.mod
-	$(GO) mod vendor
-
-version:
-	@echo $(VERSION)
-
-# We build a list of packages that omits the IDK and batch packages because
-# those packages require fancy environment setup.
-GOPACKAGES := $(shell $(GO) list ./... | grep -v "/v3/idk" | grep -v "/v3/batch")
-
-# Run test suite
-test:
-	$(GO) test $(GOPACKAGES) -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) -v -timeout $(TEST_TIMEOUT) -count=1
-
-# Run test suite with race flag
-test-race:
-	CGO_ENABLED=1 $(GO) test $(GOPACKAGES) -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) -race -timeout $(RACE_TEST_TIMEOUT) -v
-
-testv: testvsub
-
-testv-race: testvsub-race
-
-# testvsub: run go test -v in sub-directories in "local mode" with incremental output,
-#            avoiding go -test ./... "package list mode" which doesn't give output
-#            until the test run finishes. Package list mode makes it hard to
-#            find which test is hung/deadlocked.
-#
-testvsub:
-	@set -e; for pkg in $(GOPACKAGES); do \
-			if [ $${pkg:0:38} == "github.com/featurebasedb/featurebase/v3/idk" ]; then \
-				echo; echo "___ skipping subpkg $$pkg"; \
-				continue; \
-			fi; \
-			echo; echo "___ testing subpkg $$pkg"; \
-			$(GO) test -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) -v -timeout $(RACE_TEST_TIMEOUT) $$pkg || break; \
-			echo; echo "999 done testing subpkg $$pkg"; \
-		done
-
-# make a $(RAMDISK_SIZE)GB RAMDisk. Speed up tests by running
-# them with TMPDIR=/mnt/ramdisk.
-ramdisk-linux:
-	mount -o size=$(RAMDISK__SIZE)G -t tmpfs none /mnt/ramdisk
-
-# make a $(RAMDISK_SIZE)GB RAMDisk. Speed up tests by running
-# them with TMPDIR=/Volumes/RAMDisk. This is more important on
-# OS X than it is on Linux, because there's performance issues
-# with fsync on OS X that can make the SSD slow down to moving-platters
-# drive speeds. Oops.
-ramdisk-osx:
-	diskutil erasevolume HFS+ 'RAMDisk' $$(hdiutil attach -nobrowse -nomount ram://$$(expr 2097152 \* $(RAMDISK_SIZE)))
-
-detach-ramdisk-osx:
-	hdiutil detach /Volumes/RAMDisk
-
-testvsub-race:
-	@set -e; for pkg in $(GOPACKAGES); do \
-           echo; echo "___ testing subpkg $$pkg"; \
-           CGO_ENABLED=1 $(GO) test -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) -v -race -timeout $(RACE_TEST_TIMEOUT) $$pkg || break; \
-           echo; echo "999 done testing subpkg $$pkg"; \
-        done
-
-
-bench:
-	$(GO) test $(GOPACKAGES) -bench=. -run=NoneZ -timeout=127m $(TESTFLAGS)
-
-# Run test suite with coverage enabled
-cover:
-	mkdir -p build
-	$(MAKE) test TESTFLAGS="-coverprofile=build/coverage.out"
-
-# Run test suite with coverage enabled and view coverage results in browser
-cover-viz: cover
-	$(GO) tool cover -html=build/coverage.out
-
-# Build featurebase
-build:
-	$(GO) build -tags='$(BUILD_TAGS)' -ldflags $(LDFLAGS) $(FLAGS) ./cmd/featurebase
-
-package:
-	GOOS=$(GOOS) GOARCH=$(GOARCH) $(MAKE) build
-	GOOS=$(GOOS) GOARCH=$(GOARCH) $(MAKE) build-fbsql
-	GOARCH=$(GOARCH) VERSION=$(VERSION) nfpm package --packager deb --target featurebase.$(VERSION).$(GOARCH).deb
-	GOARCH=$(GOARCH) VERSION=$(VERSION) nfpm package --packager rpm --target featurebase.$(VERSION).$(GOARCH).rpm
-
-# We allow setting a custom docker-compose "project". Multiple of the
-# same docker-compose environment can exist simultaneously as long as
-# they use different projects (the project name is prepended to
-# container names and such). This is useful in a CI environment where
-# we might be running multiple instances of the tests concurrently.
-PROJECT ?= clustertests
-DOCKER_COMPOSE = docker-compose -p $(PROJECT)
-
-# Run cluster integration tests using docker. Requires docker daemon to be
-# running and docker-compose to be installed.
-clustertests: vendor
-	$(DOCKER_COMPOSE) -f internal/clustertests/docker-compose.yml down
-	$(DOCKER_COMPOSE) -f internal/clustertests/docker-compose.yml build
-	$(DOCKER_COMPOSE) -f internal/clustertests/docker-compose.yml up -d pilosa1 pilosa2 pilosa3
-	PROJECT=$(PROJECT) $(DOCKER_COMPOSE) -f internal/clustertests/docker-compose.yml run client1
-	$(DOCKER_COMPOSE) -f internal/clustertests/docker-compose.yml down
-
-# Run the cluster tests with authentication enabled
-AUTH_ARGS="-c /go/src/github.com/featurebasedb/featurebase/internal/clustertests/testdata/featurebase.conf"
-authclustertests: vendor
-	CLUSTERTESTS_FB_ARGS=$(AUTH_ARGS) $(DOCKER_COMPOSE) -f internal/clustertests/docker-compose.yml down
-	CLUSTERTESTS_FB_ARGS=$(AUTH_ARGS) $(DOCKER_COMPOSE) -f internal/clustertests/docker-compose.yml build
-	CLUSTERTESTS_FB_ARGS=$(AUTH_ARGS) $(DOCKER_COMPOSE) -f internal/clustertests/docker-compose.yml up -d pilosa1 pilosa2 pilosa3
-	PROJECT=$(PROJECT) ENABLE_AUTH=1 $(DOCKER_COMPOSE) -f internal/clustertests/docker-compose.yml run client1
-	CLUSTERTESTS_FB_ARGS=$(AUTH_ARGS) $(DOCKER_COMPOSE) -f internal/clustertests/docker-compose.yml down
-
-# Install FeatureBase and IDK
-install: install-featurebase install-idk install-fbsql
-
-install-featurebase:
-	$(GO) install -tags='$(BUILD_TAGS)' -ldflags $(LDFLAGS) $(FLAGS) ./cmd/featurebase
-
-install-idk:
-	$(MAKE) -C ./idk install
-
-install-fbsql:
-	CGO_ENABLED=1 $(GO) install ./cmd/fbsql
-
-# Build the lattice assets
-build-lattice:
-	docker build -t lattice:build ./lattice
-	export LATTICE=`docker create lattice:build`; docker cp $$LATTICE:/lattice/. ./lattice/build && docker rm $$LATTICE
-
-# `go generate` protocol buffers
-generate-protoc: require-protoc require-protoc-gen-gofast
-	$(GO) generate github.com/featurebasedb/featurebase/v3/pb
-
-# `go generate` statik assets (lattice UI)
-generate-statik: build-lattice require-statik
-	$(GO) generate github.com/featurebasedb/featurebase/v3/statik
-
-# `go generate` statik assets (lattice UI) in Docker
-generate-statik-docker: build-lattice
-	docker run --rm -t -v $(PWD):/pilosa golang:1.15.8 sh -c "go get github.com/rakyll/statik && /go/bin/statik -src=/pilosa/lattice/build -dest=/pilosa -f"
-
-# `go generate` stringers
-generate-stringer:
-	$(GO) generate github.com/featurebasedb/featurebase/v3
-
-generate-pql: require-peg
-	cd pql && peg -inline pql.peg && cd ..
-
-generate-proto-grpc: require-protoc require-protoc-gen-go
-	protoc -I proto proto/pilosa.proto --go_out=plugins=grpc:proto
-#	address re-generation here only if we need to	
-#	protoc -I proto proto/vdsm.proto --go_out=plugins=grpc:proto
-
-# `go generate` all needed packages
-generate: generate-protoc generate-statik generate-stringer generate-pql
-
-# Create release using Docker
-docker-release:
-	$(MAKE) docker-build GOOS=linux GOARCH=amd64
-	$(MAKE) docker-build GOOS=linux GOARCH=arm64
-	$(MAKE) docker-build GOOS=darwin GOARCH=amd64
-	$(MAKE) docker-build GOOS=darwin GOARCH=arm64
-
-# Build a release in Docker
-docker-build: vendor
-	docker build \
-	    --build-arg GO_VERSION=$(GO_VERSION) \
-	    --build-arg MAKE_FLAGS="TRIAL_DEADLINE=$(TRIAL_DEADLINE) GOOS=$(GOOS) GOARCH=$(GOARCH)" \
-	    --build-arg SOURCE_DATE_EPOCH=$(SOURCE_DATE_EPOCH) \
-	    --target pilosa-builder \
-	    --tag featurebase:build .
-	docker create --name featurebase-build featurebase:build
-	mkdir -p build/featurebase-$(VERSION_ID)
-	docker cp featurebase-build:/pilosa/build/. ./build/featurebase-$(VERSION_ID)
-	cp NOTICE install/featurebase.conf install/featurebase*.service ./build/featurebase-$(VERSION_ID)
-	docker rm featurebase-build
-	tar -cvz -C build -f build/featurebase-$(VERSION_ID).tar.gz featurebase-$(VERSION_ID)/
-
-# Create Docker image from Dockerfile
-docker-image: vendor
-	docker build \
-	    --build-arg GO_VERSION=$(GO_VERSION) \
-	    --build-arg MAKE_FLAGS="TRIAL_DEADLINE=$(TRIAL_DEADLINE)" \
-	    --tag featurebase:$(VERSION) .
-	@echo Created docker image: featurebase:$(VERSION)
-
-docker-image-featurebase: vendor
-	docker build \
-	    --build-arg GO_VERSION=$(GO_VERSION) \
-		--file Dockerfile-dax \
-		--tag dax/featurebase .
-
-docker-image-featurebase-linux-amd64: vendor
-	docker build \
-	    --build-arg GO_VERSION=$(GO_VERSION) \
-	    --platform linux/amd64 \
-	    --file Dockerfile-dax \
-	    --tag dax/featurebase .
-
-docker-image-featurebase-test: vendor
-	docker build \
-	    --build-arg GO_VERSION=$(GO_VERSION) \
-		--file Dockerfile-clustertests \
-		--tag dax/featurebase-test .
-
-
-# build-for-quick builds a linux featurebase binary outside of docker
-# (which is much faster for some reason), and places it in the .quick
-# subdirectory.
-build-for-quick:
-	GOOS=linux $(MAKE) build FLAGS="-o .quick/fb_linux"
-
-# docker-image-featurebase-quick uses a pre-built featurebase binary
-# to quickly create a fresh docker image without needing to send the
-# context of the featurebase top level directory.
-docker-image-featurebase-quick: build-for-quick
-	docker build \
-	    --build-arg GO_VERSION=$(GO_VERSION) \
-	    --file Dockerfile-dax-quick \
-	    --tag dax/featurebase ./.quick/
-
-
-docker-image-datagen: vendor
-	docker build --tag dax/datagen --file Dockerfile-datagen .
-
-get-account-id:
-	$(eval AWS_ACCOUNTID := $(shell aws sts get-caller-identity --output=json | jq -r .Account))
-
-
-ecr-push-featurebase: docker-login
-	echo "Pushing to account $(AWS_ACCOUNTID), profile $(AWS_PROFILE)"
-	docker tag dax/featurebase:latest $(AWS_ACCOUNTID).dkr.ecr.us-east-2.amazonaws.com/dax/featurebase:latest
-	docker push $(AWS_ACCOUNTID).dkr.ecr.us-east-2.amazonaws.com/dax/featurebase:latest
-
-ecr-push-datagen: docker-login
-	docker tag dax/datagen:latest $(AWS_ACCOUNTID).dkr.ecr.us-east-2.amazonaws.com/dax/datagen:latest
-	docker push $(AWS_ACCOUNTID).dkr.ecr.us-east-2.amazonaws.com/dax/datagen:latest
-
-docker-login: get-account-id
-	aws ecr get-login-password --region us-east-2 | docker login --username AWS --password-stdin $(AWS_ACCOUNTID).dkr.ecr.us-east-2.amazonaws.com
-
-# Create docker image (alias)
-docker: docker-image # alias
-
-# Tag and push a Docker image
-docker-tag-push: vendor
-	docker tag "featurebase:$(VERSION)" $(DOCKER_TARGET)
-	docker push $(DOCKER_TARGET)
-	@echo Pushed docker image: $(DOCKER_TARGET)
-
-# These commands (docker-idk and docker-idk-tag-push)
-# are designed to be used in CI.
-# docker-idk builds idk docker images and tags them - intended for use in CI.
-docker-idk: vendor
-	docker build \
-		-f idk/Dockerfile \
-		--build-arg GO_VERSION=$(GO_VERSION) \
-		--build-arg MAKE_FLAGS="GOOS=$(GOOS) GOARCH=$(GOARCH) BUILD_CGO=$(BUILD_CGO)" \
-		--tag registry.gitlab.com/molecula/featurebase/idk:$(VERSION_ID) .
-	@echo Created docker image: registry.gitlab.com/molecula/featurebase/idk:$(VERSION_ID)
-# docker-idk-tag-push pushes tagged docker images to the GitLab container
-# registry - intended for use in CI.
-docker-idk-tag-push:
-	docker push registry.gitlab.com/molecula/featurebase/idk:$(VERSION_ID)
-	@echo Pushed docker image: registry.gitlab.com/molecula/featurebase/idk:$(VERSION_ID)
-
-# Run golangci-lint
-golangci-lint: require-golangci-lint
-	golangci-lint run --timeout 3m --skip-files '.*\.peg\.go'
-
-# Alias
-linter: golangci-lint
-
-# Better alias
-ocd: golangci-lint
-
-######################
-# Build dependencies #
-######################
-
-# Verifies that needed build dependency is installed. Errors out if not installed.
-require-%:
-	$(if $(shell command -v $* 2>/dev/null),\
-		$(info Verified build dependency "$*" is installed.),\
-		$(error Build dependency "$*" not installed. To install, try `make install-$*`))
-
-install-build-deps: install-protoc-gen-gofast install-protoc install-statik install-peg
-
-install-statik:
-	go install github.com/rakyll/statik@latest
-
-install-protoc-gen-gofast:
-	GO111MODULE=off $(GO) get -u github.com/gogo/protobuf/protoc-gen-gofast
-
-install-protoc:
-	@echo This tool cannot automatically install protoc. Please download and install protoc from https://google.github.io/proto-lens/installing-protoc.html
-	@echo On mac, brew install protobuf seems to work.
-	@echo As of the commit that added this line, protoc-gen-gofast was at 226206f39bd7, and the protoc version in use was:
-	@echo $$ protoc --version
-	@echo libprotoc 3.19.4
-
-install-peg:
-	GO111MODULE=off $(GO) get github.com/pointlander/peg
-
-install-golangci-lint:
-	GO111MODULE=off $(GO) get github.com/golangci/golangci-lint/cmd/golangci-lint
-
-test-external-lookup:
-	$(GO) test . -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) -run ^TestExternalLookup$$ -externalLookupDSN $(EXTERNAL_LOOKUP_DSN)
-
-bnf:
-	ebnf2railroad --no-overview-diagram  --no-optimizations ./sql3/sql3.ebnf
-
-#################################
-# fbsql builds in docker
-#################################
-
-# This allows multiple concurrent builds to happen in CI without
-# creating container name conflicts and such. (different BUILD_NAMEs
-# are passed in from gitlab-ci.yml)
-BUILD_NAME ?= fbsql-build
-
-LDFLAGS_STATIC="-linkmode external -extldflags \"-static\" -X 'github.com/featurebasedb/featurebase/v3/fbsql.Version=$(VERSION)' -X 'github.com/featurebasedb/featurebase/v3/fbsql.BuildTime=$(BUILD_TIME)' "
-
-UNAME_P := $(shell uname -p)
-BUILD_CGO ?= 0
-
-# Build fbsql
-build-fbsql:
-	@echo GOOS=$(GOOS) GOARCH=$(GOARCH) uname -p=$(UNAME_P) build_cgo=$(BUILD_CGO)
-ifeq ($(BUILD_CGO), 0)
-	make build-fbsql-non-cgo
-endif
-ifeq ($(BUILD_CGO), 1)
-	make build-fbsql-cgo
-endif
-
-build-fbsql-non-cgo:
-	CGO_ENABLED=0 $(GO) build -ldflags $(LDFLAGS) $(GO_BUILD_FLAGS) -o fbsql ./cmd/fbsql
-
-build-fbsql-cgo:
-ifeq ($(GOARCH), arm64)
-	CGO_ENABLED=1 $(GO) build -tags dynamic $(GO_BUILD_FLAGS) -o fbsql ./cmd/fbsql
-endif
-ifeq ($(GOARCH), amd64)
-	CC=/usr/bin/musl-gcc CGO_ENABLED=1 $(GO) build -tags "musl static" -ldflags $(LDFLAGS_STATIC) $(GO_BUILD_FLAGS) -o fbsql ./cmd/fbsql
-endif
-
-docker-build-fbsql: vendor
-	DOCKER_BUILDKIT=0 docker build \
-		--file Dockerfile-fbsql \
-	    --build-arg GO_VERSION=$(GO_VERSION) \
-	    --build-arg MAKE_FLAGS="GOOS=$(GOOS) GOARCH=$(GOARCH) BUILD_CGO=$(BUILD_CGO)" \
-	    --build-arg GO_BUILD_FLAGS=$(GO_BUILD_FLAGS) \
-		--build-arg SOURCE_DATE_EPOCH=$(SOURCE_DATE_EPOCH) \
-	    --target builder \
-	    --tag fbsql:$(BUILD_NAME) .
-	mkdir -p build
-	docker create --name $(BUILD_NAME) fbsql:$(BUILD_NAME)
-	docker cp $(BUILD_NAME):/featurebase/fbsql ./build/fbsql_$(GOOS)_$(GOARCH)
-	docker rm $(BUILD_NAME)
+default:
 
+vendor:
+	godep save ./...
+	cp -r $(GOPATH)/src/github.com/gogo/protobuf/proto/testdata vendor/github.com/gogo/protobuf/proto/testdata

NOTES

@@ -0,0 +1,26 @@
+                                                  
+      DB         Profile                          
+        ┌───────────▼────────────────────────────┐
+        │0000000000000000000000000000000000000000│
+        │0000000000000000000000000000000000000000│
+        │0000000000000000000000000000000000000000│
+Bitmap──▶0000000000000000000000000000000000000000│
+        │0000000000000000000000000000000000000000│
+        │────────────────────────────────────────┤
+        │0000000000000000000000000000000000000000│
+        │0000000000000000000000000000000000000000│
+        │0000000000000000000000000000000000000000│
+        │0000000000000000000000000000000000000000│
+        │0000000000000000000000000000000000000000│
+        │────────────────────────────────────────┤
+    F  ▶│0000000000000000000000000000000000000000│
+    r  ││0000000000000000000000000000000000000000│
+    a  ││0000000000000000000000000000000000000000│
+    m  ││0000000000000000000000000000000000000000│
+    e  ▶│0000000000000000000000000000000000000000│
+        └────────────────────────────────────────┘
+                                     ▲───────────▲
+                                         Slice    
+
+
+Fragment=intersection of frame & slice

NOTICE

@@ -1,109 +0,0 @@
-Software license
-================
-
-Copyright (C) 2017-2021 Molecula Corp. All rights reserved.
-
-Third-party software licenses
-=============================
-
-The file /lru/lru.go contains a redistribution of lru
-(github.com/golang/groupcache/lru); the license follows:
-
-    Copyright 2013 Google Inc.
-
-    Licensed under the Apache License, Version 2.0 (the "License");
-    you may not use this file except in compliance with the License.
-    You may obtain a copy of the License at
-
-         http://www.apache.org/licenses/LICENSE-2.0
-
-    Unless required by applicable law or agreed to in writing, software
-    distributed under the License is distributed on an "AS IS" BASIS,
-    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-    See the License for the specific language governing permissions and
-    limitations under the License.
-
-The file /roaring/btree.go contains a modified redistribution of b
-(https://github.com/cznic/b); the license follows:
-
-    Copyright (c) 2014 The b Authors. All rights reserved.
-
-    Redistribution and use in source and binary forms, with or without
-    modification, are permitted provided that the following conditions are
-    met:
-
-       * Redistributions of source code must retain the above copyright
-    notice, this list of conditions and the following disclaimer.
-       * Redistributions in binary form must reproduce the above
-    copyright notice, this list of conditions and the following disclaimer
-    in the documentation and/or other materials provided with the
-    distribution.
-       * Neither the names of the authors nor the names of the
-    contributors may be used to endorse or promote products derived from
-    this software without specific prior written permission.
-
-    THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-    "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-    LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-    A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-    OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-    SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-    LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-    DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-    THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-    (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-    OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-The file /server/tlsconfig.go contains a modified redistribution of bridge 
-(https://github.com/robustirc/bridge); the license follows:
-  
-    Copyright © 2014-2015 The RobustIRC Authors. All rights reserved.
-
-    Redistribution and use in source and binary forms, with or without
-    modification, are permitted provided that the following conditions are met:
-
-        * Redistributions of source code must retain the above copyright
-          notice, this list of conditions and the following disclaimer.
-
-        * Redistributions in binary form must reproduce the above copyright
-          notice, this list of conditions and the following disclaimer in the
-          documentation and/or other materials provided with the distribution.
-
-        * Neither the name of RobustIRC nor the names of contributors may be used
-          to endorse or promote products derived from this software without
-          specific prior written permission.
-
-    THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
-    ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-    WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-    DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE
-    FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
-    DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
-    SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
-    CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
-    OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-    OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-The files /logger/filewriter.go and /logger/filewriter_test.go contain a modified redistribution of reopen (github.com/client9/reopen); the license follows:
-
-    The MIT License (MIT)
-
-    Copyright (c) 2015 Nick Galbreath
-
-    Permission is hereby granted, free of charge, to any person obtaining a copy
-    of this software and associated documentation files (the "Software"), to deal
-    in the Software without restriction, including without limitation the rights
-    to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-    copies of the Software, and to permit persons to whom the Software is
-    furnished to do so, subject to the following conditions:
-
-    The above copyright notice and this permission notice shall be included in all
-    copies or substantial portions of the Software.
-
-    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-    OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-    SOFTWARE.

OPENSOURCE.md

@@ -1,45 +0,0 @@
-## User Contribution Guidelines for FeatureBase
-
-Thank you for your interest in contributing to FeatureBase! We appreciate your support in making this open-source project even better. Here are some guidelines to help you get started with contributing to FeatureBase:
-
-1. Familiarize Yourself with the Project:
-   - Visit the FeatureBase website at www.featurebase.com to understand the project's goals, capabilities, and features.
-   - Read the documentation available on the website, including the installation guide, configuration options, and data modeling concepts.
-   - Explore the codebase by cloning the repository and reviewing the source code.
-
-2. Join the Community:
-   - Visit the FeatureBase community page at https://www.featurebase.com/community to learn more about the project's community and how to get involved.
-   - Join the Discord server at https://discord.gg/FBn2vEp7Na to chat with other contributors and users, ask questions, and share your ideas.
-
-3. Set Up Your Development Environment:
-   - Ensure you have Go installed on your machine. Make sure your shell's search path includes the go/bin directory.
-   - Clone the FeatureBase repository or download it as a zip file from the repository's page.
-   - Follow the "Build FeatureBase Server from source" instructions in the README file to compile the server binary and the ingester binaries.
-
-4. Choose a Contribution Area:
-   - Identify the area you'd like to contribute to, such as bug fixes, new features, performance improvements, documentation updates, or community support.
-   - Check the issue tracker on the repository or the FeatureBase community for open issues or feature requests that align with your interests and skills. Alternatively, propose your own idea by creating a new issue.
-
-5. Create a New Branch:
-   - Before making any changes, create a new branch in the repository's Git repository. This branch will contain your contributions.
-   - Give your branch a descriptive name that reflects the nature of your contribution.
-
-6. Make Your Changes:
-   - Follow the coding style and conventions used in the existing codebase.
-   - Write clear and concise commit messages for each logical change.
-   - If you're introducing new features or modifying existing behavior, make sure to update the documentation to reflect the changes.
-
-7. Test Your Changes:
-   - Run the existing test suite to ensure that your modifications do not introduce any regressions.
-   - If applicable, write additional tests to cover the changes you made.
-   - Document any new testing procedures required for your contribution.
-
-8. Submitting Your Contribution:
-   - Push your branch to the main repository or create a fork and submit a pull request to the main repository.
-   - Provide a detailed description of your changes, including the problem you solved and the approach you took.
-   - Be responsive to any feedback or suggestions provided by the project maintainers or other contributors.
-   - Once your contribution is approved, it will be reviewed and merged into the main codebase.
-
-Please note that by contributing to FeatureBase, you agree that your contributions will be licensed under the Apache 2.0 license, which governs the project.
-
-Thank you for considering contributing to FeatureBase! Your contributions are valuable and help improve the project for everyone.

README-dev.md

@@ -0,0 +1,65 @@
+
+Development Environment
+=======================
+
+Install Go versions 1.6.2+ or 1.7 for your platform.
+
+Fork `github.com/pilosa/pilosa` to your own account. The forked repo will be private.
+
+Make sure `$GOPATH` environment variable points to your Go working directory and `$PATH` incudes `$GOPATH/bin`.
+
+Create a directory (note that we use `github.com/pilosa`, NOT `github.com/USER`) and clone your own Pilosa repo:
+
+```sh
+mkdir -p ${GOPATH}/src/github.com/pilosa && cd $_
+git clone git@github.com:${USER}/pilosa.git
+```
+
+`cd` to your pilosa directory:
+
+```sh
+cd ${GOPATH}/src/github.com/pilosa/pilosa
+```
+
+Install [Glide][] to manage dependencies.
+
+Install Pilosa command line tools:
+
+```sh
+go install github.com/pilosa/pilosa/cmd/...
+```
+
+Running `pilosa` should now run a Pilosa instance.
+
+In order to sync your fork with upstream Pilosa repo, add an *upstream* to your repo:
+
+```sh
+cd ${GOPATH}/src/github.com/pilosa/pilosa
+git remote add upstream git@github.com:pilosa/pilosa.git
+```
+
+Before starting to work on a task, sync your branch with the upstream:
+
+```sh
+git fetch upstream
+git checkout master
+git merge upstream/master
+```
+
+Create a branch for the task:
+
+```sh
+git checkout -b a-branch-for-the-task
+```
+
+Update the code in the branch, and commit it.
+
+Push it to your own repo:
+
+```sh
+git push --set-upstream origin a-branch-for-the-task
+```
+
+All left to do is creating a pull request on github.com.
+
+[Glide]: http://glide.sh/

README.md

@@ -1,72 +1,304 @@
-# FeatureBase Community
+# pilosa
 
-FeatureBase Community is now archived and no longer maintained.
+Pilosa is a bitmap index database.
 
-* [FeatureBase Community Help](https://github.com/FeatureBaseDB/FB-community-help)
-
-
-
-## Pilosa is now FeatureBase
-
-As of September 7, 2022, the Pilosa project is now FeatureBase. The core of the project remains the same: FeatureBase is the first real-time distributed database built entirely on bitmaps. (More information about updated capabilities and improvements below.)
-
-FeatureBase delivers low-latency query results, regardless of throughput or query volumes, on fresh data with extreme efficiency. It works because bitmaps are faster, simpler, and far more I/O efficient than traditional column-oriented data formats. With FeatureBase, you can ingest data from batch data sources (e.g. S3, CSV, Snowflake, BigQuery, etc.) and/or streaming data sources (e.g. Kafka/Confluent, Kinesis, Pulsar).
-
-For more information about FeatureBase, please visit [www.featurebase.com][HomePage].
 
 ## Getting Started
 
-* [Learn how to install FeatureBase Community](https://github.com/FeatureBaseDB/FB-community-help/blob/main/docs/community/com-getstart/com-getstart-home.md)
+Pilosa requires Go 1.7 or greater.
 
-### Build FeatureBase Server from source
+You can download the source by running `go get`:
 
-0. Install go. Ensure that your shell's search path includes the go/bin directory.
-1. Clone the FeatureBase repository (or download as zip).
-2. In the featurebase directory, run `make install` to compile the FeatureBase server binary. By default, it will be installed in the go/bin directory.
-3. In the idk directory, run `make install` to compile the ingester binaries. By default, they will be installed in the go/bin directory.
-4. Run `featurebase server --handler.allowed-origins=http://localhost:3000` to run FeatureBase server with default settings (learn more about configuring FeatureBase at the link below). The `--handler.allowed-origins` parameter allows the standalone web UI to talk to the server; this can be omitted if the web UI is not needed.
-5. Run `curl localhost:10101/status` to verify the server is running and accessible.
+```sh
+$ go get github.com/pilosa/pilosa
+```
 
-### Data Model
+Now you can install the `pilosa` binary:
 
-Because FeatureBase is built on bitmaps, there is bit of a learning curve to grasp how your data is represented. 
+```sh
+$ go install github.com/pilosa/pilosa/cmd/...
+```
 
-* [Learn about Data Modeling](https://github.com/FeatureBaseDB/FB-community-help/blob/main/docs/concepts/concepts-home.md)
+Now run a single pilosa node with the default configuration:
+
+```sh
+pilosa
+```
+
+If you would like to quickly create a multi-node pilosa cluster, see the `pilosactl create` documentation.
+
+## Configuration
+
+You can specify a configuration by setting the `-config` flag when running `pilosa`.
 
 
-### Ingest Data and Query
+```sh
+pilosa -config custom-config-file.cfg
+```
 
-* [Learn how to ingest data from multiple data sources](https://github.com/FeatureBaseDB/FB-community-help/blob/main/docs/community/com-ingest/com-ingest-manage.md)
+The config file uses the [TOML](https://github.com/toml-lang/toml) configuration file format,
+and should look like:
 
-## Community
+```
+data-dir = "/tmp/pil0"
+host = "127.0.0.1:15000"
 
-You can email us at community@featurebase.com and [learn more about contributing](https://github.com/FeatureBaseDB/featurebase/blob/master/OPENSOURCE.md).
+[cluster]
+replicas = 2
 
-Chat with us: [https://discord.gg/FBn2vEp7Na][Discord]
+[[cluster.node]]
+host = "127.0.0.1:15000"
 
-## What's Changed Since the Pilosa Days? 
+[[cluster.node]]
+host = "127.0.0.1:15001"
+```
 
-A lot has changed since the days of Pilosa. This list highlights some new capabilites included in FeatureBase. We have also made signficant improvements to the performance, scalability, and stability of the FeatureBase product. 
+The first two configuration options will be unique to each node in the cluster:
 
-* Query Languages: FeatureBase supports Pilosa Query Language (PQL), as well as SQL
-* Stream and Batch Ingest: Combine real-time data streams with batch historical data and act on it within milliseconds.
-* Mutable: Perform inserts, updates, and deletes at scale, in real time and on-the-fly. This is key for meeting data compliance requirements, and for reflecting the constantly-changing nature of high-volume data.
-* Multi-Valued Set Fields: Store multiple comma-delimited values within a single field while *increasing* query performance of counts, TopKs, etc.
-* Time Quantums: Setting a time quantum on a field creates extra views which allow ranged Row queries down to the time interval specified. For example, if the time quantum is set to YMD, ranged Row queries down to the granularity of a day are supported.
-* RBF storage backend: this is a new compressed bitmap format which improves performance in a number of ways: ACID support on a per shard basis, prevents issues with the number of open files, reduces memory allocation and lock contention for reads, provides more consistent garbage collection, and allows backups to run concurrently with writes. However, because of this change, Pilosa backup files cannot be restored into FeatureBase.
+`data-dir`: directory in which data is stored to disk
 
-## License
+`host`: IP and port of the pilosa node
 
-FeatureBase is licensed under the [Apache License, Version 2.0][License]
+The remaining configuration options should be the same on every node in the cluster.
 
-[Community]: https://github.com/FeatureBaseDB/FB-community-help/tree/main
-[Install]:https://github.com/FeatureBaseDB/FB-community-help/blob/main/docs/community/com-getstart/com-getstart-home.md
-[Config]: https://github.com/FeatureBaseDB/FB-community-help/tree/main/docs/community/com-config
-[DataModel]: https://github.com/FeatureBaseDB/FB-community-help/blob/main/docs/concepts/concepts-home.md
-[Discord]: https://discord.gg/FBn2vEp7Na
-[HomePage]: http://featurebase.com?utm_campaign=Open%20Source&utm_source=GitHub
-[Ingest]: https://github.com/FeatureBaseDB/FB-community-help/blob/main/docs/community/com-ingest/com-ingest-manage.md
- 
-[License]: http://www.apache.org/licenses/LICENSE-2.0
-[PQL]: https://docs.featurebase.com/docs/pql-guide/pql-home/?utm_campaign=Open%20Source&utm_source=GitHub
-[SQL]: https://docs.featurebase.com/docs/sql-guide/sql-guide-home/?utm_campaign=Open%20Source&utm_source=GitHub
+`replicas`: the number of replicas within the cluster
+
+`[[cluster.node]]`: specifies each node within the cluster
+
+## Usage
+
+You can interact with Pilosa via HTTP requests to the host:port on which you have Pilosa running.
+The following examples illustrate how to do this using `curl` with a Pilosa cluster running on
+127.0.0.1 port 15000.
+
+Return the version of Pilosa:
+```sh
+$ curl "http://127.0.0.1:15000/version"
+```
+
+Return a list of all databases and frames in the index:
+```sh
+$ curl "http://127.0.0.1:15000/schema"
+```
+
+
+### Queries
+
+Queries to Pilosa require sending a POST request where the query itself is sent as POST data.
+You specify the database on which to perform the query with a URL argument `db=database-name`.
+
+A query sent to database `exampleDB` will have the following format:
+
+```sh
+$ curl -X POST "http://127.0.0.1:15000/query?db=exampleDB" -d 'Query()'
+```
+
+The `Query()` object referenced above should be made up of one or more of the query types listed below.
+So for example, a SetBit() query would look like this:
+```sh
+$ curl -X POST "http://127.0.0.1:15000/query?db=exampleDB" -d 'SetBit(id=10, frame="foo", profileID=1)'
+```
+
+Query results have the format `{"results":[]}`, where `results` is a list of results for each `Query()`. This
+means that you can provide multiple `Query()` objects with each HTTP request and `results` will contain
+the results of all of the queries.
+
+```sh
+$ curl -X POST "http://127.0.0.1:15000/query?db=exampleDB" -d 'Query() Query() Query()'
+```
+
+---
+#### SetBit()
+```
+SetBit(id=10, frame="foo", profileID=1)
+```
+A return value of `{"results":[true]}` indicates that the bit was toggled from 0 to 1.
+A return value of `{"results":[false]}` indicates that the bit was already set to 1 and therefore nothing changed.
+
+---
+#### ClearBit()
+```
+ClearBit(id=10, frame="foo", profileID=1)
+```
+A return value of `{"results":[true]}` indicates that the bit was toggled from 1 to 0.
+A return value of `{"results":[false]}` indicates that the bit was already set to 0 and therefore nothing changed.
+
+---
+#### SetBitmapAttrs()
+```
+SetBitmapAttrs(id=10, frame="foo", category=123, color="blue", happy=true)
+```
+Returns `{"results":[null]}`
+
+---
+#### Bitmap()
+```
+Bitmap(id=10, frame="foo")
+```
+Returns `{"results":[{"attrs":{"category":123,"color":"blue","happy":true},"bits":[1,2]}]}` where `attrs` are the
+attributes set using `SetBitmapAttrs()` and `bits` are the bits set using `SetBit()`.
+
+---
+#### Union()
+```
+Union(Bitmap(id=10, frame="foo"), Bitmap(id=20, frame="foo")))
+```
+Returns a result set similar to that of a `Bitmap()` query, only the `attrs` dictionary will be empty: `{"results":[{"attrs":{},"bits":[1,2]}]}`.
+Note that a `Union()` query can be nested within other queries anywhere that you would otherwise provide a `Bitmap()`.
+
+---
+#### Intersect()
+```
+Intersect(Bitmap(id=10, frame="foo"), Bitmap(id=20, frame="foo")))
+```
+Returns a result set similar to that of a `Bitmap()` query, only the `attrs` dictionary will be empty: `{"results":[{"attrs":{},"bits":[1]}]}`.
+Note that an `Intersect()` query can be nested within other queries anywhere that you would otherwise provide a `Bitmap()`.
+
+---
+#### Difference()
+```
+Difference(Bitmap(id=10, frame="foo"), Bitmap(id=20, frame="foo")))
+```
+`Difference()` represents all of the bits that are set in the first `Bitmap()` but are not set in the second `Bitmap()`.  It returns a result set similar to that of a `Bitmap()` query, only the `attrs` dictionary will be empty: `{"results":[{"attrs":{},"bits":[2]}]}`.
+Note that a `Difference()` query can be nested within other queries anywhere that you would otherwise provide a `Bitmap()`.
+
+---
+#### Count()
+```
+Count(Bitmap(id=10, frame="foo"))
+```
+Returns the count of the number of bits set in `Bitmap()`: `{"results":[28]}`
+
+---
+#### Range()
+```
+Range(id=10, frame="foo", start="1970-01-01T00:00", end="2000-01-02T03:04")
+```
+
+---
+#### TopN()
+```
+TopN(frame="bar", n=20)
+```
+Returns the top 20 Bitmaps from frame `bar`.
+
+```
+TopN(Bitmap(id=10, frame="foo"), frame="bar", n=20)
+```
+Returns the top 20 Bitmaps from `bar` sorted by the count of bits in the intersection with `Bitmap(id=10)`.
+
+
+```
+TopN(Bitmap(id=10, frame="foo"), frame="bar", n=20, field="category", [81,82])
+```
+
+Returns the top 20 Bitmaps from `bar`in attribute `category` with values `81 or
+82` sorted by the count of bits in the intersection with `Bitmap(id=10)`.
+
+## Development
+
+### Updating dependencies
+
+To update dependencies, you'll need to install [Glide][].
+
+Then add the new dependencies in your project:
+
+```sh
+$ glide get github.com/foo/bar
+```
+
+### Protobuf
+
+If you update protobuf (pilosa/internal/internal.proto), then you need to run `go generate`
+```sh
+$ go generate
+```
+
+### Version
+
+In order to set the version number, compile Pilosa with the following argument:
+```sh
+$ go install --ldflags="-X main.Version=1.0.0"
+```
+
+[Glide]: http://glide.sh/
+
+## Pilosactl
+
+Pilosactl contains a suite of tools for interacting with pilosa. Run `pilosactl` for an overview of commands, and `pilosactl <command> -h` for specific information on that command.
+
+### Create
+
+`pilosactl create` is used to create pilosa clusters. It has a number of options for controlling how the cluster is configured, what hosts it is on, and even the ability to build the pilosa binary locally and copy it to each cluster node automatically. To start pilosa on remote hosts, you only need `ssh` access to those hosts. See `pilosactl create -h` for a full list of options.
+
+Examples:
+
+Create a 5 node cluster locally (using 5 different ports), with a replication factor of 2.
+```
+pilosactl create \
+  -serverN 5 \
+  -replicaN 2
+```
+
+Create a cluster on 3 remote hosts - all logs will come to local stderr, pilosa binary must be available on remote hosts. The ssh user on the remote hosts needs to be the same as your local user. Otherwise use the `ssh-user` option.
+```
+pilosactl create \
+  -hosts="node1.example.com:15000,node2.example.com:15000,node3.example.com:15000"
+```
+
+Create a cluster on 3 remote hosts running OSX, but build the binary locally and copy it up. Stream the stderr of each node to a separate local log file.
+```
+pilosactl create \
+  -hosts="mac1.example.com:15000,mac2.example.com:15000,mac3.example.com:15000" \
+  -copy-binary \
+  -goos=darwin \
+  -goarch=amd64 \
+  -log-file-prefix=clusterlogs
+```
+
+### Bagent
+
+`pilosactl bagent` is what you want if you just want to run a simple benchmark against an existing cluster. Running it with no arguments will print some help, including the set of subcommands that it may be passed. Calling a subcommand with `-h'` will print the options for that subcommand. The `agent-num` flag can be passed an integer which can change the behavior the benchmarks that are run. This is useful when multiple invocations of the same benchmark are made by the `bspawn` command - they can each (for example) set different bits even though they all have the same arguments.
+
+E.G.
+```
+pilosactl bagent \
+  -hosts="localhost:15000,localhost:15001" \
+  import -h
+```
+
+Multiple subcommands and their arguments may be concatenated at the command line and they will be run serially. This is useful (i.e.) for importing a bunch of data, and then executing queries against it.
+
+This will generate and import a bunch of data, and then execute random queries against it.
+
+```
+pilosactl bagent \
+  -hosts="localhost:15000,localhost:15001" \
+  import -max-bits-per-map=10000 \
+  random-query -iterations 100
+```
+
+### Bspawn
+`pilosactl bspawn` allows you to automate the creation of clusters and the running of complex benchmarks which span multiple benchmark agents against them. It has a number of options which are described by `pilosactl bspawn` with no arguments, and also takes a config file which describes the Benchmark itself - this file is described below.
+
+#### Configuration Format
+
+The configuration file is a json object with the top level key `benchmarks`. This contains a list of objects each of which represents a `bagent` command (the `args` key) that will be run some number of times concurrently (the `num` key), and a `name` which should describe the overall effect that command. An example is below.
+```json
+{
+    "benchmarks": [
+        {
+            "num": 3,
+            "name": "set-diags",
+            "args": ["diagonal-set-bits", "-iterations", "30000", "-client-type", "round_robin"]
+        },
+        {
+            "num": 2,
+            "name": "rand-plus-zipf",
+            "args": ["random-set-bits", "-iterations", "20000", "zipf", "-iterations", "100"]
+        }
+    ]
+}
+```
+
+All of the benchmarks, and agents are run concurrently. Each agent will be passed an `agent-num` which can modify the behavior in a way that is benchmark specific. See the documentation for each benchmark to see how `agent-num` changes its behavior.

api/client/grpc.go

@@ -1,204 +0,0 @@
-// Copyright 2022 Molecula Corp. (DBA FeatureBase).
-// SPDX-License-Identifier: Apache-2.0
-package client
-
-import (
-	"context"
-	"crypto/tls"
-	"sync"
-
-	"github.com/featurebasedb/featurebase/v3/logger"
-	pb "github.com/featurebasedb/featurebase/v3/proto"
-	"github.com/pkg/errors"
-	"google.golang.org/grpc"
-	"google.golang.org/grpc/connectivity"
-	"google.golang.org/grpc/credentials"
-	"google.golang.org/grpc/credentials/insecure"
-)
-
-const maxMsgSize = 1024 * 1024 * 100 // 100 megs ought to be enough for anybody!
-
-// GRPCClient is a client for working with the gRPC server.
-type GRPCClient struct {
-	dialTargets []string
-	tlsConfig   *tls.Config
-	logger      logger.Logger
-
-	mu          sync.RWMutex
-	conn        *grpc.ClientConn
-	targetIndex int
-}
-
-// NewGRPCClient returns a new instance of GRPCClient.
-func NewGRPCClient(dialTargets []string, tlsConfig *tls.Config, logger logger.Logger) (*GRPCClient, error) {
-	c := &GRPCClient{
-		dialTargets: dialTargets,
-		tlsConfig:   tlsConfig,
-		logger:      logger,
-	}
-	// resetConn sets GRPCClient.conn when it doesn't
-	// exist yet.
-	if err := c.resetConn(); err != nil {
-		return nil, errors.Wrap(err, "setting connection")
-	}
-
-	return c, nil
-}
-
-// resetConn resets the gRPC client connection. This method
-// can also be used to initially set the client connection
-// because it only tries to first close the connection if
-// the connection already exists.
-func (c *GRPCClient) resetConn() error {
-	c.mu.Lock()
-	defer c.mu.Unlock()
-
-	// If an existing connection exists, close it first.
-	if c.conn != nil {
-		if err := c.conn.Close(); err != nil {
-			return errors.Wrap(err, "closing existing connection")
-		}
-	}
-
-	var opts []grpc.DialOption
-	if c.tlsConfig != nil {
-		creds := credentials.NewTLS(c.tlsConfig)
-		opts = append(opts, grpc.WithTransportCredentials(creds))
-	} else {
-		opts = append(opts, grpc.WithTransportCredentials(insecure.NewCredentials()))
-	}
-
-	opts = append(opts, grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(maxMsgSize)))
-
-	var err error
-	if c.conn, err = grpc.Dial(c.dialTargets[c.getTargetIndex()], opts...); err != nil {
-		return errors.Wrap(err, "creating new grpc client")
-	}
-
-	return nil
-}
-
-// getTargetIndex gets the current target index, then increments it for
-// next time. Unprotected.
-func (c *GRPCClient) getTargetIndex() int {
-	if len(c.dialTargets) == 0 {
-		return 0
-	}
-	ret := c.targetIndex
-	c.targetIndex = (c.targetIndex + 1) % len(c.dialTargets) // cycle through dialTargets
-	return ret
-}
-
-// Close closes any connections the client has opened.
-func (c *GRPCClient) Close() error {
-	c.mu.RLock()
-	defer c.mu.RUnlock()
-
-	if c.conn != nil {
-		return c.conn.Close()
-	}
-	return nil
-}
-
-// Conn returns the gRPC client connection. If the connection
-// has gone into state `TransientFailure`, this method tries
-// to reset the connection and return that new connection.
-func (c *GRPCClient) Conn() *grpc.ClientConn {
-	c.mu.RLock()
-	if c.conn == nil {
-		c.mu.RUnlock()
-		return nil
-	} else if c.conn.GetState() != connectivity.TransientFailure {
-		defer c.mu.RUnlock()
-		return c.conn
-	}
-	c.mu.RUnlock()
-
-	if err := c.resetConn(); err != nil {
-		c.logger.Errorf("error resetting connection: %s", err)
-	}
-
-	c.mu.RLock()
-	defer c.mu.RUnlock()
-	return c.conn
-}
-
-// Query returns a stream of RowResponse for the given index and PQL string.
-func (c *GRPCClient) Query(ctx context.Context, index string, pql string) (pb.StreamClient, error) {
-	conn := c.Conn()
-
-	if conn == nil {
-		return nil, errors.New("client has not established a grpc connection")
-	}
-
-	grpcClient := pb.NewPilosaClient(conn)
-
-	stream, err := grpcClient.QueryPQL(ctx, &pb.QueryPQLRequest{
-		Index: index,
-		Pql:   pql,
-	})
-	if err != nil {
-		return nil, errors.Wrap(err, "getting stream")
-	} else if stream == nil {
-		return nil, errors.New("could not create stream")
-	}
-
-	return stream, err
-}
-
-// QueryUnary returns a TableResponse for the given index and PQL string.
-func (c *GRPCClient) QueryUnary(ctx context.Context, index string, pql string) (*pb.TableResponse, error) {
-	conn := c.Conn()
-
-	if conn == nil {
-		return nil, errors.New("client has not established a grpc connection")
-	}
-
-	grpcClient := pb.NewPilosaClient(conn)
-
-	return grpcClient.QueryPQLUnary(ctx, &pb.QueryPQLRequest{
-		Index: index,
-		Pql:   pql,
-	})
-}
-
-// Inspect returns a stream of RowResponse for the given index, columns, and filters.
-// It is intended to mimic something like "select [fields] from table where recordID IN (...)".
-func (c *GRPCClient) Inspect(ctx context.Context, index string, columnIDs []uint64, columnKeys []string, query string, fieldFilters []string, limit, offset uint64) (pb.StreamClient, error) {
-	conn := c.Conn()
-
-	if conn == nil {
-		return nil, errors.New("client has not established a grpc connection")
-	}
-
-	if len(columnIDs) > 0 && len(columnKeys) > 0 {
-		return nil, errors.New("only provide column ids or keys, not both")
-	}
-
-	// Convert columns to proto type IdsOrKeys.
-	idsOrKeys := &pb.IdsOrKeys{}
-	if len(columnKeys) > 0 {
-		idsOrKeys.Type = &pb.IdsOrKeys_Keys{Keys: &pb.StringArray{Vals: columnKeys}}
-	} else {
-		idsOrKeys.Type = &pb.IdsOrKeys_Ids{Ids: &pb.Uint64Array{Vals: columnIDs}}
-	}
-
-	grpcClient := pb.NewPilosaClient(conn)
-
-	stream, err := grpcClient.Inspect(ctx, &pb.InspectRequest{
-		Index:        index,
-		Columns:      idsOrKeys,
-		FilterFields: fieldFilters,
-		Limit:        limit,
-		Offset:       offset,
-		Query:        query,
-	})
-
-	if err != nil {
-		return nil, errors.Wrap(err, "getting stream")
-	} else if stream == nil {
-		return nil, errors.New("could not create stream")
-	}
-
-	return stream, err
-}

api_directive.go

@@ -1,987 +0,0 @@
-// Copyright 2021 Molecula Corp. All rights reserved.
-package pilosa
-
-import (
-	"context"
-	"io"
-	"log"
-	"sync"
-
-	"github.com/featurebasedb/featurebase/v3/dax"
-	"github.com/featurebasedb/featurebase/v3/dax/computer"
-	"github.com/featurebasedb/featurebase/v3/dax/storage"
-	"github.com/featurebasedb/featurebase/v3/disco"
-	"github.com/pkg/errors"
-)
-
-// ApplyDirective applies a Directive received, from the Controller, at the
-// /directive endpoint.
-func (api *API) ApplyDirective(ctx context.Context, d *dax.Directive) error {
-	// Get the current directive for comparison.
-	previousDirective := api.holder.Directive()
-
-	// Check that incoming version is newer.
-	// Note: 0 is an invalid Directive version. This decision was made because
-	// previousDirective is not a pointer to a directive, but a concrete
-	// Directive. Which means we can't check for nil, and by default it has a
-	// version of 0. So in order to ensure the version has increased, we need to
-	// require that incoming directive versions are greater than 0.
-	if d.Version == 0 {
-		return errors.Errorf("directive version cannot be 0")
-	} else if previousDirective.Version >= d.Version {
-		return errors.Errorf("directive version mismatch, got %d, but already have %d", d.Version, previousDirective.Version)
-	}
-
-	// Handle the operations based on the directive method.
-	switch d.Method {
-	case dax.DirectiveMethodDiff:
-		// In order to prevent adding too much code specific to handling a diff
-		// directive (e.g. adding something like an `enactDirectiveDiff()`
-		// method), we are instead going to build a full Directive based on the
-		// diff, and then proceed normally as if we had received a full
-		// Directive. We do that by copying the previous Directive and then
-		// applying the diffs to the copy.
-		newD := previousDirective.Copy()
-
-		// Apply the diffs from the incoming Directive to the new, copied
-		// Directive.
-		newD.ApplyDiff(d)
-
-		// Now proceed with the new diff as if we had received it as a full diff.
-		d = newD
-
-	case dax.DirectiveMethodFull:
-		// pass: normal operation
-
-	case dax.DirectiveMethodReset:
-		// Delete all tables.
-		if err := api.deleteAllIndexes(ctx); err != nil {
-			return errors.Wrap(err, "deleting all indexes")
-		}
-		// Set previousDirective to empty so the diff handles everything as new.
-		previousDirective = dax.Directive{}
-
-	case dax.DirectiveMethodSnapshot:
-		// TODO(tlt): this was the existing logic, but we should really diff the
-		// directive and ensure that overwriting the value in the cache doesn't
-		// have a negative effect.
-		api.holder.SetDirective(d)
-		return nil
-
-	default:
-		return errors.Errorf("invalid directive method: %s", d.Method)
-	}
-
-	// Cache this directive as the latest applied. There is functionality within
-	// the "enactDirective" stage of ApplyDirective which validates against this
-	// cached Directive, so it's important that it be set before calling
-	// enactDirective(). An example: when loading partition data from the
-	// Writelogger, there are validations to ensure that the partition being
-	// loaded is meant to be handled by this node; that validation is done
-	// against the cached Directive.
-	// TODO(tlt): despite what this comment says, this logic is not sound; we
-	// shouldn't be setting the directive until enactiveDirective() succeeds.
-	api.holder.SetDirective(d)
-	defer api.holder.SetDirectiveApplied(true)
-
-	return api.enactDirective(ctx, &previousDirective, d)
-}
-
-// deleteAllIndexes deletes all indexes handled by this node.
-func (api *API) deleteAllIndexes(ctx context.Context) error {
-	indexes, err := api.Schema(ctx, false)
-	if err != nil {
-		return errors.Wrap(err, "getting schema")
-	}
-
-	for i := range indexes {
-		if err := api.DeleteIndex(ctx, indexes[i].Name); err != nil {
-			return errors.Wrapf(err, "deleting index: %s", indexes[i].Name)
-		}
-	}
-
-	return nil
-}
-
-// directiveJobType allows us to switch on jobType in the directiveWorker in
-// order to use a single worker pool for all job types (as opposed to having a
-// separate worker pool for each job type).
-type directiveJobType interface {
-	// We have this method just to prevent *any* struct from implementing this
-	// interface automatically. But, interestingly enough, we don't actually
-	// have to have this method on the implementation because we embed the
-	// interface.
-	isJobType() bool
-}
-
-type directiveJobTableKeys struct {
-	directiveJobType
-	idx       *Index
-	tkey      dax.TableKey
-	partition dax.PartitionNum
-}
-
-type directiveJobFieldKeys struct {
-	directiveJobType
-	tkey  dax.TableKey
-	field dax.FieldName
-}
-
-type directiveJobShards struct {
-	directiveJobType
-	tkey  dax.TableKey
-	shard dax.ShardNum
-}
-
-// directiveWorker is a worker in a worker pool which handles portions of a
-// directive. Multiple instances of directiveWorker run in goroutines in order
-// to load data from snapshotter and writelogger concurrently. Note: unlike the
-// api.ingestWorkerPool, of which one pool is always running, the
-// directiveWorker pool is only running during the life of the
-// api.ApplyDirective call. Technically, this means that multiple
-// directiveWorker pools could be active at the same time, but we should never
-// be running more than once instance of ApplyDirective concurrently.
-func (api *API) directiveWorker(ctx context.Context, jobs <-chan directiveJobType, errs chan<- error) {
-	for j := range jobs {
-		switch job := j.(type) {
-		case directiveJobTableKeys:
-			if err := api.loadTableKeys(ctx, job.idx, job.tkey, job.partition); err != nil {
-				errs <- errors.Wrapf(err, "loading table keys: %s, %s", job.tkey, job.partition)
-			}
-		case directiveJobFieldKeys:
-			if err := api.loadFieldKeys(ctx, job.tkey, job.field); err != nil {
-				errs <- errors.Wrapf(err, "loading field keys: %s, %s", job.tkey, job.field)
-			}
-		case directiveJobShards:
-			if err := api.loadShard(ctx, job.tkey, job.shard); err != nil {
-				errs <- errors.Wrapf(err, "loading shard: %s, %s", job.tkey, job.shard)
-			}
-		default:
-			errs <- errors.Errorf("unsupported job type: %T %[1]v", job)
-		}
-
-		select {
-		case <-ctx.Done():
-			return
-		default:
-			// continue pulling jobs off the channel
-		}
-	}
-}
-
-func (api *API) enactDirective(ctx context.Context, fromD, toD *dax.Directive) error {
-	// enactTables is called before the jobs that run in the worker pool because
-	// it probably makes sense to apply the schema before trying to load data
-	// concurrently.
-	if err := api.enactTables(ctx, fromD, toD); err != nil {
-		return errors.Wrap(err, "enactTables")
-	}
-
-	// The following types use a shared pool of workers to run each
-	// directiveJobType.
-
-	var wg sync.WaitGroup
-
-	// open job channel
-	jobs := make(chan directiveJobType, api.directiveWorkerPoolSize)
-	errs := make(chan error)
-	done := make(chan struct{})
-
-	// Spin up n workers in goroutines that pull jobs from the jobs channel.
-	for i := 0; i < api.directiveWorkerPoolSize; i++ {
-		wg.Add(1)
-		go func() {
-			api.directiveWorker(ctx, jobs, errs)
-			defer wg.Done()
-		}()
-	}
-
-	// Wait for the WaitGroup counter to reach 0. When it has, indicate that
-	// we're done processing all jobs by closing the done channel.
-	go func() {
-		wg.Wait()
-		close(done)
-	}()
-
-	// Run through all the "enact" methods. These push jobs onto the jobs
-	// channel. Once all the jobs have been queued to the channel, we close the
-	// jobs channel. This allows the directiveWorkers to exit out of the
-	// function, which will then decrement the WaitGroup counter.
-	go func() {
-		api.pushJobsTableKeys(ctx, jobs, fromD, toD)
-		api.pushJobsFieldKeys(ctx, jobs, fromD, toD)
-		api.pushJobsShards(ctx, jobs, fromD, toD)
-		close(jobs)
-	}()
-
-	// Keep running until we get an error or until the done channel is closed.
-	// Note: the code is written such that only non-nil errors are pushed to the
-	// errs channel.
-	for {
-		select {
-		case err := <-errs:
-			return err
-		case <-done:
-			return nil
-		}
-	}
-}
-
-func (api *API) enactTables(ctx context.Context, fromD, toD *dax.Directive) error {
-	currentIndexes := api.holder.Indexes()
-
-	// Make a list of indexes that currently exist (from).
-	from := make(dax.TableKeys, 0, len(currentIndexes))
-	for _, idx := range currentIndexes {
-		qtid, err := dax.QualifiedTableIDFromKey(idx.Name())
-		if err != nil {
-			return errors.Wrap(err, "converting index name to qualified table id")
-		}
-		from = append(from, qtid.Key())
-	}
-
-	// TODO sanity check holder against fromD. We're getting existing
-	// indexes from holder, but in theory fromD should be
-	// identical. If we have an error in our directive-caching logic
-	// (it has happened before (just now, in fact!) and we'd be
-	// foolish to think it won't happen again), or we have schema
-	// mutations that are not going through the directive path, we
-	// could potentially catch them here.
-
-	// Make a list of tables that are in the directive (to) along with a map of
-	// tableKey to table (m).
-	m := make(map[dax.TableKey]*dax.QualifiedTable, len(toD.Tables))
-	to := make(dax.TableKeys, 0, len(toD.Tables))
-	for _, t := range toD.Tables {
-		m[t.Key()] = t
-		to = append(to, t.Key())
-	}
-
-	sc := newSliceComparer(from, to)
-
-	// Remove all indexes that are no longer part of the directive.
-	for _, tkey := range sc.removed() {
-		idx := string(tkey)
-		if err := api.DeleteIndex(ctx, idx); err != nil {
-			return errors.Wrapf(err, "deleting index: %s", tkey)
-		}
-	}
-
-	// Put partitions into a map by table.
-	partitionMap := toD.TranslatePartitionsMap()
-
-	// Add all indexes that weren't previously (but now are) a part of the
-	// directive.
-	for _, tkey := range sc.added() {
-		if qtbl, found := m[tkey]; !found {
-			return errors.Errorf("table '%s' was not in map", tkey)
-		} else if err := api.createTableAndFields(qtbl, partitionMap[tkey]); err != nil {
-			return err
-		}
-	}
-
-	// Check fields on all indexes present in both from and to.
-	for _, tkey := range sc.same() {
-		if err := api.enactFieldsForTable(ctx, tkey, fromD, toD); err != nil {
-			return errors.Wrapf(err, "enacting fields for table: '%s'", tkey)
-		}
-	}
-
-	return nil
-}
-
-func (api *API) enactFieldsForTable(ctx context.Context, tkey dax.TableKey, fromD, toD *dax.Directive) error {
-	qtid := tkey.QualifiedTableID()
-
-	fromT, err := fromD.Table(qtid)
-	if err != nil {
-		return errors.Wrap(err, "getting from table")
-	}
-	toT, err := toD.Table(qtid)
-	if err != nil {
-		return errors.Wrap(err, "getting to table")
-	}
-
-	// Get the index for tkey.
-	idx := api.holder.Index(string(tkey))
-	if idx == nil {
-		return errors.Errorf("index not found: %s", tkey)
-	}
-
-	sc := newSliceComparer(fromT.FieldNames(), toT.FieldNames())
-
-	// Add fields new to toT.
-	for _, fldName := range sc.added() {
-		if field, found := toT.Field(fldName); !found {
-			return dax.NewErrFieldDoesNotExist(fldName)
-		} else if err := createField(idx, field); err != nil {
-			return errors.Wrapf(err, "creating field: %s/%s", tkey, fldName)
-		}
-	}
-
-	// Remove fields which don't exist in toT.
-	for _, fldName := range sc.removed() {
-		if err := api.DeleteField(ctx, string(tkey), string(fldName)); err != nil {
-			return errors.Wrapf(err, "deleting field: %s/%s", tkey, fldName)
-		}
-	}
-
-	// // Update any field options which have changed for existing fields.
-	// for _, fldName := range sc.same() {
-	// 	// handle changed field options??
-	// }
-
-	return nil
-}
-
-func (api *API) pushJobsTableKeys(ctx context.Context, jobs chan<- directiveJobType, fromD, toD *dax.Directive) {
-	toPartitionsMap := toD.TranslatePartitionsMap()
-
-	// Get the diff between from/to directive.partitions.
-	partComp := newPartitionsComparer(fromD.TranslatePartitionsMap(), toPartitionsMap)
-
-	// Remove any partitions which are no longer assigned to this worker.
-	// TODO(tlt): currently, this is just removing the file lock on the
-	// resource; it's not actually removing the resource from the local
-	// computer. We should do that.
-	for tkey, partitions := range partComp.removed() {
-		qtid := tkey.QualifiedTableID()
-		for _, partition := range partitions {
-			api.serverlessStorage.RemoveTableKeyResource(qtid, partition)
-		}
-	}
-
-	// Loop over the partition map and load from Writelogger.
-	for tkey, partitions := range partComp.added() {
-		// Get index in order to find the translate stores (by partition) for
-		// the table.
-		idx := api.holder.Index(string(tkey))
-		if idx == nil {
-			log.Printf("index not found in holder: %s", tkey)
-			continue
-		}
-
-		// Update the cached version of translate partitions that we keep on the
-		// Index.
-		idx.SetTranslatePartitions(toPartitionsMap[tkey])
-
-		for _, partition := range partitions {
-			jobs <- directiveJobTableKeys{
-				idx:       idx,
-				tkey:      tkey,
-				partition: partition,
-			}
-		}
-	}
-}
-
-func (api *API) loadTableKeys(ctx context.Context, idx *Index, tkey dax.TableKey, partition dax.PartitionNum) error {
-	qtid := tkey.QualifiedTableID()
-
-	resource := api.serverlessStorage.GetTableKeyResource(qtid, partition)
-	if resource.IsLocked() {
-		api.logger().Warnf("skipping loadTableKeys (already held) %s %d", tkey, partition)
-		return nil
-	}
-
-	// load latest snapshot
-	if rc, err := resource.LoadLatestSnapshot(); err != nil {
-		return errors.Wrap(err, "loading table key snapshot")
-	} else if rc != nil {
-		defer rc.Close()
-		if err := api.TranslateIndexDB(ctx, string(tkey), int(partition), rc); err != nil {
-			return errors.Wrap(err, "restoring table keys")
-		}
-	}
-
-	// define write log loading in a function since we have to do it
-	// before and after locking
-	loadWriteLog := func() error {
-		writelog, err := resource.LoadWriteLog()
-		if err != nil {
-			return errors.Wrap(err, "getting write log reader for table keys")
-		}
-		if writelog == nil {
-			return nil
-		}
-		reader := storage.NewTableKeyReader(qtid, partition, writelog)
-		defer reader.Close()
-		store := idx.TranslateStore(int(partition))
-		for msg, err := reader.Read(); err != io.EOF; msg, err = reader.Read() {
-			if err != nil {
-				return errors.Wrap(err, "reading from log reader")
-			}
-			for key, id := range msg.StringToID {
-				if err := store.ForceSet(id, key); err != nil {
-					return errors.Wrapf(err, "forcing set id, key: %d, %s", id, key)
-				}
-			}
-		}
-		return nil
-	}
-	// 1st write log load
-	if err := loadWriteLog(); err != nil {
-		return err
-	}
-
-	// acquire lock on this partition's keys
-	if err := resource.Lock(); err != nil {
-		return errors.Wrap(err, "locking table key partition")
-	}
-
-	// reload writelog in case of changes between last load and
-	// lock. The resource object takes care of only loading new data.
-	return loadWriteLog()
-}
-
-func (api *API) pushJobsFieldKeys(ctx context.Context, jobs chan<- directiveJobType, fromD, toD *dax.Directive) {
-	// Get the diff between from/to directive.fields.
-	fieldComp := newFieldsComparer(fromD.TranslateFieldsMap(), toD.TranslateFieldsMap())
-
-	// Remove any field keys which are no longer assigned to this worker.
-	// TODO(tlt): currently, this is just removing the file lock on the
-	// resource; it's not actually removing the resource from the local
-	// computer. We should do that.
-	for tkey, fields := range fieldComp.removed() {
-		qtid := tkey.QualifiedTableID()
-		for _, field := range fields {
-			api.serverlessStorage.RemoveFieldKeyResource(qtid, field)
-		}
-	}
-
-	// Loop over the field map and load from Writelogger.
-	for tkey, fields := range fieldComp.added() {
-		for _, field := range fields {
-			jobs <- directiveJobFieldKeys{
-				tkey:  tkey,
-				field: field,
-			}
-		}
-	}
-}
-
-func (api *API) loadFieldKeys(ctx context.Context, tkey dax.TableKey, field dax.FieldName) error {
-	qtid := tkey.QualifiedTableID()
-
-	resource := api.serverlessStorage.GetFieldKeyResource(qtid, field)
-	if resource.IsLocked() {
-		api.logger().Warnf("skipping loadFieldKeys (already held) %s %s", tkey, field)
-		return nil
-	}
-
-	// load latest snapshot
-	if rc, err := resource.LoadLatestSnapshot(); err != nil {
-		return errors.Wrap(err, "loading field key snapshot")
-	} else if rc != nil {
-		defer rc.Close()
-		if err := api.TranslateFieldDB(ctx, string(tkey), string(field), rc); err != nil {
-			return errors.Wrap(err, "restoring field keys")
-		}
-	}
-
-	// define write log loading in a function since we have to do it
-	// before and after locking
-	loadWriteLog := func() error {
-		writelog, err := resource.LoadWriteLog()
-		if err != nil {
-			return errors.Wrap(err, "getting write log reader for field keys")
-		}
-		if writelog == nil {
-			return nil
-		}
-		reader := storage.NewFieldKeyReader(qtid, field, writelog)
-		defer reader.Close()
-		// Get field in order to find the translate store.
-		fld := api.holder.Field(string(tkey), string(field))
-		if fld == nil {
-			log.Printf("field not found in holder: %s", field)
-			return nil
-		}
-		store := fld.TranslateStore()
-
-		for msg, err := reader.Read(); err != io.EOF; msg, err = reader.Read() {
-			if err != nil {
-				return errors.Wrap(err, "reading from log reader")
-			}
-			for key, id := range msg.StringToID {
-				if err := store.ForceSet(id, key); err != nil {
-					return errors.Wrapf(err, "forcing set id, key: %d, %s", id, key)
-				}
-			}
-		}
-		return nil
-	}
-	// 1st write log load
-	if err := loadWriteLog(); err != nil {
-		return err
-	}
-
-	// acquire lock on this partition's keys
-	if err := resource.Lock(); err != nil {
-		return errors.Wrap(err, "locking field key partition")
-	}
-
-	// reload writelog in case of changes between last load and
-	// lock. The resource object takes care of only loading new data.
-	return loadWriteLog()
-}
-
-func (api *API) pushJobsShards(ctx context.Context, jobs chan<- directiveJobType, fromD, toD *dax.Directive) {
-	// Put shards into a map by table.
-	shardMap := toD.ComputeShardsMap()
-
-	// Get the diff between from/to directive shards.
-	shardComp := newShardsComparer(fromD.ComputeShardsMap(), shardMap)
-
-	// Remove any shards which are no longer assigned to this worker.
-	// TODO(tlt): currently, this is just removing the file lock on the
-	// resource; it's not actually removing the resource from the local
-	// computer. We should do that.
-	for tkey, shards := range shardComp.removed() {
-		qtid := tkey.QualifiedTableID()
-		for _, shard := range shards {
-			partition := dax.PartitionNum(disco.ShardToShardPartition(string(tkey), uint64(shard), disco.DefaultPartitionN))
-			api.serverlessStorage.RemoveShardResource(qtid, partition, shard)
-		}
-	}
-
-	// Loop over the shard map and load from Writelogger.
-	for tkey, shards := range shardComp.added() {
-		for _, shard := range shards {
-			jobs <- directiveJobShards{
-				tkey:  tkey,
-				shard: shard,
-			}
-		}
-	}
-}
-
-func (api *API) loadShard(ctx context.Context, tkey dax.TableKey, shard dax.ShardNum) error {
-	qtid := tkey.QualifiedTableID()
-
-	partition := dax.PartitionNum(disco.ShardToShardPartition(string(tkey), uint64(shard), disco.DefaultPartitionN))
-
-	resource := api.serverlessStorage.GetShardResource(qtid, partition, shard)
-	if resource.IsLocked() {
-		api.logger().Warnf("skipping loadShard (already held) %s %d", tkey, shard)
-		return nil
-	}
-
-	if rc, err := resource.LoadLatestSnapshot(); err != nil {
-		return errors.Wrap(err, "reading latest snapshot for shard")
-	} else if rc != nil {
-		defer rc.Close()
-		if err := api.RestoreShard(ctx, string(tkey), uint64(shard), rc); err != nil {
-			return errors.Wrap(err, "restoring shard data")
-		}
-	}
-
-	// define write log loading in a func because we do it twice.
-	loadWriteLog := func() error {
-		writelog, err := resource.LoadWriteLog()
-		if err != nil {
-			return errors.Wrap(err, "")
-		}
-		if writelog == nil {
-			return nil
-		}
-
-		reader := storage.NewShardReader(qtid, partition, shard, writelog)
-		defer reader.Close()
-		for logMsg, err := reader.Read(); err != io.EOF; logMsg, err = reader.Read() {
-			if err != nil {
-				return errors.Wrap(err, "reading from log reader")
-			}
-
-			switch msg := logMsg.(type) {
-			case *computer.ImportRoaringMessage:
-				req := &ImportRoaringRequest{
-					Clear:           msg.Clear,
-					Action:          msg.Action,
-					Block:           msg.Block,
-					Views:           msg.Views,
-					UpdateExistence: msg.UpdateExistence,
-					SuppressLog:     true,
-				}
-				if err := api.ImportRoaring(ctx, msg.Table, msg.Field, msg.Shard, true, req); err != nil {
-					return errors.Wrapf(err, "import roaring, table: %s, field: %s, shard: %d", msg.Table, msg.Field, msg.Shard)
-				}
-
-			case *computer.ImportMessage:
-				req := &ImportRequest{
-					Index:      msg.Table,
-					Field:      msg.Field,
-					Shard:      msg.Shard,
-					RowIDs:     msg.RowIDs,
-					ColumnIDs:  msg.ColumnIDs,
-					RowKeys:    msg.RowKeys,
-					ColumnKeys: msg.ColumnKeys,
-					Timestamps: msg.Timestamps,
-					Clear:      msg.Clear,
-				}
-
-				qcx := api.Txf().NewQcx()
-				defer qcx.Abort()
-
-				opts := []ImportOption{
-					OptImportOptionsClear(msg.Clear),
-					OptImportOptionsIgnoreKeyCheck(msg.IgnoreKeyCheck),
-					OptImportOptionsPresorted(msg.Presorted),
-					OptImportOptionsSuppressLog(true),
-				}
-				if err := api.Import(ctx, qcx, req, opts...); err != nil {
-					return errors.Wrapf(err, "import, table: %s, field: %s, shard: %d", msg.Table, msg.Field, msg.Shard)
-				}
-
-			case *computer.ImportValueMessage:
-				req := &ImportValueRequest{
-					Index:           msg.Table,
-					Field:           msg.Field,
-					Shard:           msg.Shard,
-					ColumnIDs:       msg.ColumnIDs,
-					ColumnKeys:      msg.ColumnKeys,
-					Values:          msg.Values,
-					FloatValues:     msg.FloatValues,
-					TimestampValues: msg.TimestampValues,
-					StringValues:    msg.StringValues,
-					Clear:           msg.Clear,
-				}
-
-				qcx := api.Txf().NewQcx()
-				defer qcx.Abort()
-
-				opts := []ImportOption{
-					OptImportOptionsClear(msg.Clear),
-					OptImportOptionsIgnoreKeyCheck(msg.IgnoreKeyCheck),
-					OptImportOptionsPresorted(msg.Presorted),
-					OptImportOptionsSuppressLog(true),
-				}
-				if err := api.ImportValue(ctx, qcx, req, opts...); err != nil {
-					return errors.Wrapf(err, "import value, table: %s, field: %s, shard: %d", msg.Table, msg.Field, msg.Shard)
-				}
-			case *computer.ImportRoaringShardMessage:
-				req := &ImportRoaringShardRequest{
-					Remote:      true,
-					Views:       make([]RoaringUpdate, len(msg.Views)),
-					SuppressLog: true,
-				}
-				for i, view := range msg.Views {
-					req.Views[i] = RoaringUpdate{
-						Field:        view.Field,
-						View:         view.View,
-						Clear:        view.Clear,
-						Set:          view.Set,
-						ClearRecords: view.ClearRecords,
-					}
-				}
-				if err := api.ImportRoaringShard(ctx, msg.Table, msg.Shard, req); err != nil {
-					return errors.Wrapf(err, "import roaring shard table: %s, shard: %d", msg.Table, msg.Shard)
-				}
-			}
-		}
-		return nil
-	}
-	// 1st write log load
-	if err := loadWriteLog(); err != nil {
-		return err
-	}
-
-	// acquire lock on this partition's keys
-	if err := resource.Lock(); err != nil {
-		return errors.Wrap(err, "locking field key partition")
-	}
-
-	// reload writelog in case of changes between last load and
-	// lock. The resource object takes care of only loading new data.
-	return loadWriteLog()
-}
-
-//////////////////////////////////////////////////////////////
-
-// sliceComparer is used to compare the differences between two slices of comparables.
-type sliceComparer[K comparable] struct {
-	from []K
-	to   []K
-}
-
-func newSliceComparer[K comparable](from []K, to []K) *sliceComparer[K] {
-	return &sliceComparer[K]{
-		from: from,
-		to:   to,
-	}
-}
-
-// added returns the items which are present in `to` but not in `from`.
-func (s *sliceComparer[K]) added() []K {
-	return thingsAdded(s.from, s.to)
-}
-
-// removed returns the items which are present in `from` but not in `to`.
-func (s *sliceComparer[K]) removed() []K {
-	return thingsAdded(s.to, s.from)
-}
-
-// same returns the items which are in both `to` and `from`.
-func (s *sliceComparer[K]) same() []K {
-	var same []K
-	for _, fromThing := range s.from {
-		for _, toThing := range s.to {
-			if fromThing == toThing {
-				same = append(same, fromThing)
-				break
-			}
-		}
-	}
-	return same
-}
-
-// thingsAdded returns the comparable things which are present in `to` but not
-// in `from`.
-func thingsAdded[K comparable](from []K, to []K) []K {
-	var added []K
-	for i := range to {
-		var found bool
-		for j := range from {
-			if from[j] == to[i] {
-				found = true
-				break
-			}
-		}
-		if !found {
-			added = append(added, to[i])
-		}
-	}
-	return added
-}
-
-// partitionsComparer is used to compare the differences between two maps of
-// table:[]partition.
-type partitionsComparer struct {
-	from map[dax.TableKey]dax.PartitionNums
-	to   map[dax.TableKey]dax.PartitionNums
-}
-
-func newPartitionsComparer(from map[dax.TableKey]dax.PartitionNums, to map[dax.TableKey]dax.PartitionNums) *partitionsComparer {
-	return &partitionsComparer{
-		from: from,
-		to:   to,
-	}
-}
-
-// added returns the partitions which are present in `to` but not in `from`. The
-// results remain in the format of a map of table:[]partition.
-func (p *partitionsComparer) added() map[dax.TableKey]dax.PartitionNums {
-	return partitionsAdded(p.from, p.to)
-}
-
-// removed returns the partitions which are present in `from` but not in `to`.
-// The results remain in the format of a map of table:[]partition.
-func (p *partitionsComparer) removed() map[dax.TableKey]dax.PartitionNums {
-	return partitionsAdded(p.to, p.from)
-}
-
-// partitionsAdded returns the partitions which are present in `to` but not in `from`.
-func partitionsAdded(from map[dax.TableKey]dax.PartitionNums, to map[dax.TableKey]dax.PartitionNums) map[dax.TableKey]dax.PartitionNums {
-	if from == nil {
-		return to
-	}
-
-	added := make(map[dax.TableKey]dax.PartitionNums)
-	for tt, tps := range to {
-		fps, found := from[tt]
-		if !found {
-			added[tt] = tps
-			continue
-		}
-
-		addedPartitions := dax.PartitionNums{}
-		for i := range tps {
-			var found bool
-			for j := range fps {
-				if fps[j] == tps[i] {
-					found = true
-					break
-				}
-			}
-			if !found {
-				addedPartitions = append(addedPartitions, tps[i])
-			}
-		}
-
-		if len(addedPartitions) > 0 {
-			added[tt] = addedPartitions
-		}
-	}
-	return added
-}
-
-// fieldsComparer is used to compare the differences between two maps of
-// table:[]fieldVersion.
-type fieldsComparer struct {
-	from map[dax.TableKey][]dax.FieldName
-	to   map[dax.TableKey][]dax.FieldName
-}
-
-func newFieldsComparer(from map[dax.TableKey][]dax.FieldName, to map[dax.TableKey][]dax.FieldName) *fieldsComparer {
-	return &fieldsComparer{
-		from: from,
-		to:   to,
-	}
-}
-
-// added returns the fields which are present in `to` but not in `from`. The
-// results remain in the format of a map of table:[]field.
-func (f *fieldsComparer) added() map[dax.TableKey][]dax.FieldName {
-	return fieldsAdded(f.from, f.to)
-}
-
-// removed returns the fields which are present in `from` but not in `to`.
-// The results remain in the format of a map of table:[]field.
-func (f *fieldsComparer) removed() map[dax.TableKey][]dax.FieldName {
-	return fieldsAdded(f.to, f.from)
-}
-
-// fieldsAdded returns the fields which are present in `to` but not in `from`.
-func fieldsAdded(from map[dax.TableKey][]dax.FieldName, to map[dax.TableKey][]dax.FieldName) map[dax.TableKey][]dax.FieldName {
-	if from == nil {
-		return to
-	}
-
-	added := make(map[dax.TableKey][]dax.FieldName)
-	for tt, tps := range to {
-		fps, found := from[tt]
-		if !found {
-			added[tt] = tps
-			continue
-		}
-
-		addedFieldVersions := []dax.FieldName{}
-		for i := range tps {
-			var found bool
-			for j := range fps {
-				if fps[j] == tps[i] {
-					found = true
-					break
-				}
-			}
-			if !found {
-				addedFieldVersions = append(addedFieldVersions, tps[i])
-			}
-		}
-
-		if len(addedFieldVersions) > 0 {
-			added[tt] = addedFieldVersions
-		}
-	}
-	return added
-}
-
-// shardsComparer is used to compare the differences between two maps of
-// table:[]shardV.
-type shardsComparer struct {
-	from map[dax.TableKey]dax.ShardNums
-	to   map[dax.TableKey]dax.ShardNums
-}
-
-func newShardsComparer(from map[dax.TableKey]dax.ShardNums, to map[dax.TableKey]dax.ShardNums) *shardsComparer {
-	return &shardsComparer{
-		from: from,
-		to:   to,
-	}
-}
-
-// added returns the shards which are present in `to` but not in `from`. The
-// results remain in the format of a map of table:[]shard.
-func (s *shardsComparer) added() map[dax.TableKey]dax.ShardNums {
-	return shardsAdded(s.from, s.to)
-}
-
-// removed returns the shards which are present in `from` but not in `to`. The
-// results remain in the format of a map of table:[]shard.
-func (s *shardsComparer) removed() map[dax.TableKey]dax.ShardNums {
-	return shardsAdded(s.to, s.from)
-}
-
-// shardsAdded returns the shards which are present in `to` but not in `from`.
-func shardsAdded(from map[dax.TableKey]dax.ShardNums, to map[dax.TableKey]dax.ShardNums) map[dax.TableKey]dax.ShardNums {
-	if from == nil {
-		return to
-	}
-
-	added := make(map[dax.TableKey]dax.ShardNums)
-	for tt, tss := range to {
-		fss, found := from[tt]
-		if !found {
-			added[tt] = tss
-			continue
-		}
-
-		addedShards := dax.ShardNums{}
-		for i := range tss {
-			var found bool
-			for j := range fss {
-				if fss[j] == tss[i] {
-					found = true
-					break
-				}
-			}
-			if !found {
-				addedShards = append(addedShards, tss[i])
-			}
-		}
-
-		if len(addedShards) > 0 {
-			added[tt] = addedShards
-		}
-	}
-	return added
-}
-
-// createTableAndFields creates the FeatureBase Tables and Fields provided in
-// the dax.Directive format.
-func (api *API) createTableAndFields(tbl *dax.QualifiedTable, partitions dax.PartitionNums) error {
-	cim := &CreateIndexMessage{
-		Index:     string(tbl.Key()),
-		CreatedAt: 0,
-		Meta: IndexOptions{
-			Keys:           tbl.StringKeys(),
-			TrackExistence: true,
-		},
-	}
-
-	// Create the index in etcd as the system of record.
-	if err := api.holder.persistIndex(context.Background(), cim); err != nil {
-		return errors.Wrap(err, "persisting index")
-	}
-
-	idx, err := api.holder.createIndexWithPartitions(cim, partitions)
-	if err != nil {
-		return errors.Wrapf(err, "adding index: %s", tbl.Name)
-	}
-
-	// Add the fields
-	for _, fld := range tbl.Fields {
-		if fld.IsPrimaryKey() {
-			continue
-		}
-		if err := createField(idx, fld); err != nil {
-			return errors.Wrapf(err, "creating field: %s", fld.Name)
-		}
-	}
-
-	return nil
-}
-
-// createField creates a FeatureBase Field in the provided FeatureBase Index
-// based on the provided field's type.
-func createField(idx *Index, fld *dax.Field) error {
-	opts, err := FieldOptionsFromField(fld)
-	if err != nil {
-		return errors.Wrapf(err, "creating field options from field: %s", fld.Name)
-	}
-
-	if _, err := idx.createNullableField(string(fld.Name), "", opts...); err != nil {
-		return errors.Wrapf(err, "creating field on index: %s", fld.Name)
-	}
-	return nil
-}

api_directive_internal_test.go

@@ -1,25 +0,0 @@
-package pilosa
-
-import (
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-)
-
-func TestThingsAddedGeneric(t *testing.T) {
-	from := []string{"a", "b", "c"}
-	to := []string{"b", "c", "d"}
-
-	added := thingsAdded(from, to)
-	assert.Equal(t, added, []string{"d"})
-}
-
-func TestSliceComparer(t *testing.T) {
-	from := []string{"a", "b", "c"}
-	to := []string{"b", "c", "d"}
-
-	sc := newSliceComparer(from, to)
-
-	added := sc.added()
-	assert.Equal(t, added, []string{"d"})
-}

api_directive_test.go

@@ -1,97 +0,0 @@
-package pilosa_test
-
-import (
-	"context"
-	"testing"
-
-	pilosa "github.com/featurebasedb/featurebase/v3"
-	"github.com/featurebasedb/featurebase/v3/dax"
-	daxtest "github.com/featurebasedb/featurebase/v3/dax/test"
-	"github.com/featurebasedb/featurebase/v3/test"
-	"github.com/stretchr/testify/assert"
-)
-
-// Ensure holder can handle an incoming directive.
-func TestAPI_Directive(t *testing.T) {
-
-	c := test.MustRunCluster(t, 1)
-	defer c.Close()
-
-	api := c.GetPrimary().API
-	ctx := context.Background()
-
-	qdbid := dax.NewQualifiedDatabaseID("acme", "db1")
-	tbl1 := daxtest.TestQualifiedTableWithID(t, qdbid, "1", "tbl1", 12, false)
-	tbl2 := daxtest.TestQualifiedTableWithID(t, qdbid, "2", "tbl2", 12, false)
-	tbl3 := daxtest.TestQualifiedTableWithID(t, qdbid, "3", "tbl3", 12, false)
-
-	t.Run("Schema", func(t *testing.T) {
-
-		// Empty directive (and empty holder).
-		{
-			d := &dax.Directive{
-				Method:  dax.DirectiveMethodFull,
-				Version: 1,
-			}
-			err := api.ApplyDirective(ctx, d)
-			assert.NoError(t, err)
-			assertTablesMatch(t, []string{}, api.Holder().Indexes())
-		}
-
-		// Add a new table.
-		{
-			d := &dax.Directive{
-				Method: dax.DirectiveMethodFull,
-				Tables: []*dax.QualifiedTable{
-					tbl1,
-				},
-				Version: 2,
-			}
-			err := api.ApplyDirective(ctx, d)
-			assert.NoError(t, err)
-			assertTablesMatch(t, []string{"tbl__acme__db1__1"}, api.Holder().Indexes())
-		}
-
-		// Add a new table, and keep the existing table.
-		{
-			d := &dax.Directive{
-				Method: dax.DirectiveMethodFull,
-				Tables: []*dax.QualifiedTable{
-					tbl1,
-					tbl2,
-				},
-				Version: 3,
-			}
-			err := api.ApplyDirective(ctx, d)
-			assert.NoError(t, err)
-			assertTablesMatch(t, []string{"tbl__acme__db1__1", "tbl__acme__db1__2"}, api.Holder().Indexes())
-		}
-
-		// Add a new table and remove one of the existing tables.
-		{
-			d := &dax.Directive{
-				Method: dax.DirectiveMethodFull,
-				Tables: []*dax.QualifiedTable{
-					tbl2,
-					tbl3,
-				},
-				Version: 4,
-			}
-			err := api.ApplyDirective(ctx, d)
-			assert.NoError(t, err)
-			assertTablesMatch(t, []string{"tbl__acme__db1__2", "tbl__acme__db1__3"}, api.Holder().Indexes())
-		}
-	})
-}
-
-// assertTablesMatch is a helper function which asserts that the list of index
-// names in `actual` match those provided in `expected`.
-func assertTablesMatch(t *testing.T, expected []string, actual []*pilosa.Index) {
-	t.Helper()
-
-	act := make([]string, len(actual))
-	for i := range actual {
-		act[i] = actual[i].Name()
-	}
-	assert.ElementsMatch(t, expected, act)
-}

apimethod_string.go

@@ -1,57 +0,0 @@
-// Code generated by "stringer -type=apiMethod"; DO NOT EDIT.
-
-package pilosa
-
-import "strconv"
-
-func _() {
-	// An "invalid array index" compiler error signifies that the constant values have changed.
-	// Re-run the stringer command to generate them again.
-	var x [1]struct{}
-	_ = x[apiClusterMessage-0]
-	_ = x[apiCreateField-1]
-	_ = x[apiCreateIndex-2]
-	_ = x[apiDeleteField-3]
-	_ = x[apiDeleteAvailableShard-4]
-	_ = x[apiDeleteIndex-5]
-	_ = x[apiDeleteView-6]
-	_ = x[apiExportCSV-7]
-	_ = x[apiFragmentBlockData-8]
-	_ = x[apiFragmentBlocks-9]
-	_ = x[apiFragmentData-10]
-	_ = x[apiTranslateData-11]
-	_ = x[apiFieldTranslateData-12]
-	_ = x[apiField-13]
-	_ = x[apiImport-14]
-	_ = x[apiImportValue-15]
-	_ = x[apiIndex-16]
-	_ = x[apiQuery-17]
-	_ = x[apiRecalculateCaches-18]
-	_ = x[apiSchema-19]
-	_ = x[apiShardNodes-20]
-	_ = x[apiState-21]
-	_ = x[apiViews-22]
-	_ = x[apiApplySchema-23]
-	_ = x[apiStartTransaction-24]
-	_ = x[apiFinishTransaction-25]
-	_ = x[apiTransactions-26]
-	_ = x[apiGetTransaction-27]
-	_ = x[apiActiveQueries-28]
-	_ = x[apiPastQueries-29]
-	_ = x[apiIDReserve-30]
-	_ = x[apiIDCommit-31]
-	_ = x[apiIDReset-32]
-	_ = x[apiPartitionNodes-33]
-	_ = x[apiMutexCheck-34]
-}
-
-const _apiMethod_name = "apiClusterMessageapiCreateFieldapiCreateIndexapiDeleteFieldapiDeleteAvailableShardapiDeleteIndexapiDeleteViewapiExportCSVapiFragmentBlockDataapiFragmentBlocksapiFragmentDataapiTranslateDataapiFieldTranslateDataapiFieldapiImportapiImportValueapiIndexapiQueryapiRecalculateCachesapiSchemaapiShardNodesapiStateapiViewsapiApplySchemaapiStartTransactionapiFinishTransactionapiTransactionsapiGetTransactionapiActiveQueriesapiPastQueriesapiIDReserveapiIDCommitapiIDResetapiPartitionNodesapiMutexCheck"
-
-var _apiMethod_index = [...]uint16{0, 17, 31, 45, 59, 82, 96, 109, 121, 141, 158, 173, 189, 210, 218, 227, 241, 249, 257, 277, 286, 299, 307, 315, 329, 348, 368, 383, 400, 416, 430, 442, 453, 463, 480, 493}
-
-func (i apiMethod) String() string {
-	if i < 0 || i >= apiMethod(len(_apiMethod_index)-1) {
-		return "apiMethod(" + strconv.FormatInt(int64(i), 10) + ")"
-	}
-	return _apiMethod_name[_apiMethod_index[i]:_apiMethod_index[i+1]]
-}

apply.go

@@ -1,686 +0,0 @@
-// Copyright 2021 Molecula Corp. All rights reserved.
-package pilosa
-
-import (
-	"context"
-	"fmt"
-	"io"
-	"os"
-	"path/filepath"
-	"sort"
-	"strings"
-	"sync"
-
-	"github.com/apache/arrow/go/v10/arrow"
-	"github.com/apache/arrow/go/v10/arrow/array"
-	"github.com/apache/arrow/go/v10/arrow/memory"
-	"github.com/featurebasedb/featurebase/v3/pql"
-	"github.com/featurebasedb/featurebase/v3/tracing"
-	"github.com/featurebasedb/featurebase/v3/vprint"
-	"github.com/gomem/gomem/pkg/dataframe"
-	"github.com/pkg/errors"
-
-	ivy "robpike.io/ivy/arrow"
-	config "robpike.io/ivy/config"
-	"robpike.io/ivy/exec"
-	"robpike.io/ivy/parse"
-	"robpike.io/ivy/run"
-	"robpike.io/ivy/scan"
-	"robpike.io/ivy/value"
-)
-
-type (
-	ApplyResult *arrow.Column
-)
-
-func runIvyString(context value.Context, str string) (ok bool, err error) {
-	defer func() {
-		if r := recover(); r != nil {
-			err = r.(value.Error)
-		}
-	}()
-	scanner := scan.New(context, "<args>", strings.NewReader(str))
-	parser := parse.NewParser("<args>", scanner, context)
-	ok = run.Run(parser, context, false)
-	return
-}
-
-// Possibly combine all arrays together then apply some interesting
-// computation at the end?
-func IvyReduce(reduceCode string, opCode string, opt *ExecOptions) (func(ctx context.Context, prev, v interface{}) interface{}, func() (*dataframe.DataFrame, error)) {
-	var accumulator value.Value
-	mu := &sync.Mutex{}
-	concat := value.BinaryOps[opCode]
-	conf := getDefaultConfig()
-	ctxIvy := exec.NewContext(&conf)
-	// concat returned results at coordinating node.
-	reduceFn := func(ctx context.Context, prev, v interface{}) interface{} {
-		if v == nil {
-			return prev
-		}
-		if accumulator == nil {
-			switch val := v.(type) {
-			case *dataframe.DataFrame:
-				col := val.ColumnAt(0)
-				resolver := dataframe.NewChunkResolver(col)
-				accumulator = value.NewArrowVector(col, &conf, &resolver)
-			case value.Value:
-				accumulator = v.(value.Value)
-			default:
-				return errors.New(fmt.Sprintf("ivy reduction failed first unexpected type %T", v))
-			}
-			return nil
-		}
-		switch val := v.(type) {
-		case *dataframe.DataFrame:
-			col := val.ColumnAt(0)
-			resolver := dataframe.NewChunkResolver(col)
-			x := value.NewArrowVector(col, &conf, &resolver)
-			mu.Lock() // i'm being overyerly cautious..need to confirm this can be concurrent
-			accumulator = concat.EvalBinary(ctxIvy, accumulator, x)
-			mu.Unlock()
-		case value.Value:
-			mu.Lock()
-			accumulator = concat.EvalBinary(ctxIvy, accumulator, val)
-			mu.Unlock()
-		default:
-			return errors.New(fmt.Sprintf("ivy reduction failed unexpected type %T", v))
-		}
-
-		return nil
-	}
-	tablerFn := func() (*dataframe.DataFrame, error) {
-		pool := memory.NewGoAllocator() // TODO(twg) 2022/09/01 singledton?
-		if opt.Remote {
-			col := value.ToArrowColumn(accumulator, pool)
-			return dataframe.NewDataFrameFromColumns(pool, []arrow.Column{*col})
-		}
-		// only actually reduce on the initiating node i hate the network
-		// over head but oh well
-		ctxIvy.AssignGlobal("_", accumulator)
-		ok, err := runIvyString(ctxIvy, reduceCode)
-		if err != nil {
-			return nil, err
-		}
-		if ok {
-			v := ctxIvy.Global("_")
-			if v == nil {
-				return nil, errors.New("ivy reduction no result ")
-			}
-			col := value.ToArrowColumn(ctxIvy.Global("_"), pool)
-
-			return dataframe.NewDataFrameFromColumns(pool, []arrow.Column{*col})
-		}
-		return nil, errors.New("ivy reduction failed ")
-	}
-
-	return reduceFn, tablerFn
-}
-
-// executeApply executes a Apply() call.
-func (e *executor) executeApply(ctx context.Context, qcx *Qcx, index string, c *pql.Call, shards []uint64, opt *ExecOptions) (*dataframe.DataFrame, error) {
-	if !e.dataframeEnabled {
-		return nil, errors.New("Dataframe support not enabled")
-	}
-	span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeMax")
-	defer span.Finish()
-
-	if _, err := c.FirstStringArg("_ivy"); err != nil {
-		return nil, errors.Wrap(err, " no ivy program supplied")
-	}
-
-	if len(c.Children) > 1 {
-		return nil, errors.New("Apply() only accepts a single bitmap input filter")
-	}
-
-	// Execute calls in bulk on each remote node and merge.
-	mapFn := func(ctx context.Context, shard uint64, mopt *mapOptions) (_ interface{}, err error) {
-		return e.executeApplyShard(ctx, qcx, index, c, shard)
-	}
-	ivyReduce, ok, err := c.StringArg("_ivyReduce")
-	if err != nil {
-		return nil, err
-	}
-	reduceFn, tablerFn := IvyReduce("_", ",", opt)
-	if ok {
-		reduceFn, tablerFn = IvyReduce(ivyReduce, ",", opt)
-	}
-
-	_, err = e.mapReduce(ctx, index, shards, c, opt, mapFn, reduceFn)
-	if err != nil {
-		return nil, err
-	}
-	return tablerFn()
-}
-
-func getDefaultConfig() config.Config {
-	maxbits := uint(1e9)   // "maximum size of an integer, in bits; 0 means no limit")
-	maxdigits := uint(1e4) // "above this many `digits`, integers print as floating point; 0 disables")
-	maxstack := uint(100000)
-	origin := 1  // "set index origin to `n` (must be 0 or 1)")
-	prompt := "" // flag.String("prompt", "", "command `prompt`")
-	format := ""
-	//      debugFlag := "" // flag.String("debug", "", "comma-separated `names` of debug settings to enable")
-	conf := config.Config{}
-	conf.SetFormat(format)
-	conf.SetMaxBits(maxbits)
-	conf.SetMaxDigits(maxdigits)
-	conf.SetMaxStack(maxstack)
-	conf.SetOrigin(origin)
-	conf.SetPrompt(prompt)
-	conf.SetOutput(io.Discard)
-	conf.SetErrOutput(io.Discard)
-	conf.SetEmbedded(true) // needed to propagate panic
-	return conf
-}
-
-func filterDataframe(resolver dataframe.Resolver, pool memory.Allocator, filter []int64) (*dataframe.IndexResolver, error) {
-	if resolver.NumRows() == 0 {
-		return nil, errors.New("No data")
-	}
-	indexResolver := dataframe.NewIndexResolver(len(filter), uint32(ShardWidth-1))
-	for i, id := range filter {
-		if int(id) >= resolver.NumRows() {
-			continue
-		}
-		c, o := resolver.Resolve(int(id))
-		indexResolver.Set(i, c, o)
-
-	}
-	return indexResolver, nil
-}
-
-func (e *executor) executeApplyShard(ctx context.Context, qcx *Qcx, index string, c *pql.Call, shard uint64) (value.Value, error) {
-	span, _ := tracing.StartSpanFromContext(ctx, "Executor.executeApplyShard")
-	defer span.Finish()
-
-	ivyProgram, ok, err := c.StringArg("_ivy")
-	if err != nil || !ok {
-		return nil, errors.Wrap(err, "finding ivy program")
-	}
-	var filter *Row
-	if len(c.Children) == 1 {
-		row, err := e.executeBitmapCallShard(ctx, qcx, index, c.Children[0], shard)
-		if err != nil {
-			return nil, err
-		}
-		filter = row
-		if !filter.Any() {
-			// no need to actuall run the query for its not operating against any values
-			return value.NewVector([]value.Value{}), nil
-		}
-	}
-	//
-	pool := memory.NewGoAllocator() // TODO(twg) 2022/09/01 singledton?
-
-	ids := filter.ShardColumns() // needs to be shard columns
-	// Fetch index.
-	idx := e.Holder.Index(index)
-	if idx == nil {
-		return nil, newNotFoundError(ErrIndexNotFound, index)
-	}
-
-	fname := idx.GetDataFramePath(shard)
-
-	if !e.dataFrameExists(fname) {
-		return value.NewVector([]value.Value{}), nil
-	}
-
-	table, err := e.getDataTable(ctx, fname, pool)
-	if err != nil {
-		return nil, err
-	}
-	defer table.Release()
-	df, err := dataframe.NewDataFrameFromTable(pool, table)
-	if err != nil {
-		return nil, err
-	}
-	p := dataframe.NewChunkResolver(df.ColumnAt(0))
-	var resolver dataframe.Resolver
-	resolver = &p
-	if filter != nil {
-		if len(ids) == 0 {
-			return value.NewVector([]value.Value{}), nil
-		}
-		resolver, err = filterDataframe(resolver, pool, ids)
-		if err != nil {
-			return nil, err
-		}
-	}
-	conf := getDefaultConfig()
-	context, err := ivy.RunArrow(dataframe.NewTableFacade(df), ivyProgram, conf, resolver)
-	if err != nil {
-		return nil, fmt.Errorf("ivy map error: %w", err)
-	}
-	return context.Global("_"), nil
-}
-
-// ///////////////////////////////////////////////////////
-// all the ingest supporting functions
-// ///////////////////////////////////////////////////////
-
-func NewShardFile(ctx context.Context, name string, mem memory.Allocator, e *executor) (*ShardFile, error) {
-	if !e.dataFrameExists(name) {
-		return &ShardFile{dest: name, executor: e, strings: make(map[key][]string)}, nil
-	}
-	// else read in existing
-	table, err := e.getDataTable(ctx, name, mem)
-	if err != nil {
-		return nil, err
-	}
-	return &ShardFile{table: table, schema: table.Schema(), dest: name, executor: e, strings: make(map[key][]string)}, nil
-}
-
-type NameType struct {
-	Name     string
-	DataType arrow.DataType
-}
-type ChangesetRequest struct {
-	ShardIds     []int64 // only shardwidth bits to provide 0 indexing inside shard file
-	Columns      []interface{}
-	SimpleSchema []NameType
-}
-
-// TODO(twg) 2022/09/30 Needs a refactor
-func cast(v interface{}) arrow.DataType {
-	switch v.(type) {
-	case *arrow.Int64Type:
-		return arrow.PrimitiveTypes.Int64
-	case int64:
-		return arrow.PrimitiveTypes.Int64
-	case *arrow.Float64Type:
-		return arrow.PrimitiveTypes.Float64
-	case float64:
-		return arrow.PrimitiveTypes.Float64
-	case *arrow.StringType:
-		return arrow.BinaryTypes.String
-	default:
-		vprint.VV("%T .... %v", v, v)
-	}
-	return arrow.PrimitiveTypes.Int64
-}
-
-func (cr *ChangesetRequest) ArrowSchema() *arrow.Schema {
-	fields := make([]arrow.Field, len(cr.SimpleSchema))
-	for i := range cr.SimpleSchema {
-		fields[i] = arrow.Field{Name: cr.SimpleSchema[i].Name, Type: cast(cr.SimpleSchema[i].DataType)}
-	}
-	return arrow.NewSchema(fields, nil)
-}
-
-type key struct {
-	col   int
-	chunk int
-}
-
-type ShardFile struct {
-	table      arrow.Table
-	schema     *arrow.Schema
-	beforeRows int64
-	added      int64
-	columns    []interface{}
-	dest       string
-	executor   *executor
-	strings    map[key][]string
-}
-
-func compareSchema(s1, s2 *arrow.Schema) bool {
-	if s1 == nil || s2 == nil {
-		return false
-	}
-	if len(s1.Fields()) != len(s2.Fields()) {
-		return false
-	}
-	for i := 0; i < len(s1.Fields()); i++ {
-		f1 := s1.Field(i)
-		f2 := s2.Field(i)
-		if f1.Name != f2.Name {
-			return false
-		}
-		if f1.Type != f2.Type {
-			return false
-		}
-	}
-	return true
-}
-
-func (sf *ShardFile) EnsureSchema(cs *ChangesetRequest) error {
-	schema := cs.ArrowSchema()
-	if sf.schema == nil {
-		sf.schema = schema
-	} else {
-		if !compareSchema(sf.schema, schema) {
-			vprint.VV("incomeing schema", schema)
-			vprint.VV("existing schema", sf.schema)
-			return errors.New("dataframe schema's don't match")
-		}
-	}
-	sf.columns = make([]interface{}, len(sf.schema.Fields()))
-	return nil
-}
-
-func (sf *ShardFile) buildAppenders(maxid int64) {
-	if sf.table != nil {
-		sf.beforeRows = sf.table.NumRows()
-	}
-	if maxid < sf.beforeRows {
-		// no need to add new rows
-		return
-	}
-	newSize := maxid - sf.beforeRows + 1
-	for i := 0; i < len(sf.schema.Fields()); i++ {
-		switch sf.schema.Field(i).Type {
-		case arrow.PrimitiveTypes.Int64:
-			sf.columns[i] = make([]int64, newSize)
-		case arrow.PrimitiveTypes.Float64:
-			sf.columns[i] = make([]float64, newSize)
-		case arrow.BinaryTypes.String:
-			sf.columns[i] = make([]string, newSize)
-		}
-	}
-	sf.added = newSize
-}
-
-// the row offset must be reset to 0 for the slices being appended
-func (sf *ShardFile) SetIntValue(col int, row int64, val int64) {
-	v := sf.columns[col].([]int64)
-	v[row-sf.beforeRows] = val
-}
-
-func (sf *ShardFile) SetFloatValue(col int, row int64, val float64) {
-	v := sf.columns[col].([]float64)
-	v[row-sf.beforeRows] = val
-}
-
-func (sf *ShardFile) SetStringValue(col int, row int64, val string) {
-	v := sf.columns[col].([]string)
-	v[row-sf.beforeRows] = val
-}
-
-func (sf *ShardFile) Process(cs *ChangesetRequest) error {
-	err := sf.process(cs)
-	if err != nil {
-		return err
-	}
-	rtemp := sf.dest + ".temp"
-	err = sf.Save(rtemp)
-	if err != nil {
-		return err
-	}
-	return os.Rename(rtemp+sf.executor.TableExtension(), sf.dest+sf.executor.TableExtension())
-}
-
-func (sf *ShardFile) LoadBlobs() error {
-	for col := 0; col < len(sf.schema.Fields()); col++ {
-		column := sf.table.Column(col)
-		switch column.DataType() {
-		case arrow.BinaryTypes.String:
-			for i, chunk := range column.Data().Chunks() {
-				stringData := chunk.(*array.String)
-				k := key{col: col, chunk: i}
-				for j := 0; j < stringData.Len(); j++ {
-					v := stringData.Value(j)
-					sf.strings[k] = append(sf.strings[k], v)
-
-				}
-			}
-		}
-	}
-	return nil
-}
-
-func (sf *ShardFile) ReplaceString(col, chunk, l int, s string) {
-	sf.strings[key{col: col, chunk: chunk}][l] = s
-}
-
-func (sf *ShardFile) process(cs *ChangesetRequest) error {
-	offset := 0
-	if sf.table != nil {
-		// need to load blobs prior
-		sf.LoadBlobs()
-		column := sf.table.Column(0)
-		resolver := dataframe.NewChunkResolver(column)
-		for i, rowid := range cs.ShardIds {
-			offset = i
-			if rowid >= sf.table.NumRows() {
-				break
-			}
-			chunk, l := resolver.Resolve(int(rowid))
-			for col := 0; col < len(sf.schema.Fields()); col++ {
-				column := sf.table.Column(col)
-				switch column.DataType() {
-				case arrow.PrimitiveTypes.Int64:
-					v := column.Data().Chunk(chunk).(*array.Int64).Int64Values()
-					v[l] = cs.Columns[col].([]int64)[i]
-				case arrow.PrimitiveTypes.Float64:
-					v := column.Data().Chunk(chunk).(*array.Float64).Float64Values()
-					v[l] = cs.Columns[col].([]float64)[i]
-				case arrow.BinaryTypes.String:
-					// TODO(twg) 2023/01/09 How to update existing?
-					new := cs.Columns[col].([]string)[i]
-					sf.ReplaceString(col, chunk, l, new)
-				default:
-					panic(fmt.Sprintf("Unknown Type %v", column.DataType()))
-				}
-			}
-		}
-	}
-	max := cs.ShardIds[len(cs.ShardIds)-1]
-	sf.buildAppenders(max)
-	// need to check if only replace and no apend
-	if sf.added > 0 {
-		for i, rowid := range cs.ShardIds[offset:] {
-			i += offset
-
-			for col := 0; col < len(sf.schema.Fields()); col++ {
-				switch sf.schema.Field(col).Type {
-				case arrow.PrimitiveTypes.Int64:
-					sf.SetIntValue(col, rowid, cs.Columns[col].([]int64)[i])
-				case arrow.PrimitiveTypes.Float64:
-					sf.SetFloatValue(col, rowid, cs.Columns[col].([]float64)[i])
-				case arrow.BinaryTypes.String:
-					sf.SetStringValue(col, rowid, cs.Columns[col].([]string)[i])
-				default:
-					panic(fmt.Sprintf("2 Unknown Type %v", sf.schema.Field(col).Type))
-				}
-			}
-		}
-	}
-
-	return nil
-}
-
-type twoSlices struct {
-	id_slice    []int
-	lists_slice [][]string
-}
-
-type SortByOther twoSlices
-
-func (sbo SortByOther) Len() int {
-	return len(sbo.id_slice)
-}
-
-func (sbo SortByOther) Swap(i, j int) {
-	sbo.id_slice[i], sbo.id_slice[j] = sbo.id_slice[j], sbo.id_slice[i]
-	sbo.lists_slice[i], sbo.lists_slice[j] = sbo.lists_slice[j], sbo.lists_slice[i]
-}
-
-func (sbo SortByOther) Less(i, j int) bool {
-	return sbo.id_slice[i] < sbo.id_slice[j]
-}
-
-func (sf *ShardFile) buildFromStrings(idx int, mem memory.Allocator) []arrow.Array {
-	ids := make([]int, 0)
-	lists := make([][]string, 0)
-	for k, v := range sf.strings {
-		if k.col == idx { // ugh not ordered :(
-			ids = append(ids, k.chunk)
-			lists = append(lists, v)
-		}
-	}
-	// sort ids/lists
-	parts := twoSlices{id_slice: ids, lists_slice: lists}
-	sort.Sort(SortByOther(parts))
-
-	builder := array.NewStringBuilder(mem)
-	chunks := make([]arrow.Array, 0)
-	for _, v := range parts.lists_slice {
-		builder.AppendValues(v, nil)
-		newChunk := builder.NewArray()
-		chunks = append(chunks, newChunk)
-	}
-	return chunks
-}
-
-func (sf *ShardFile) Save(name string) error {
-	parts := make([]arrow.Array, 0)
-	mem := memory.NewGoAllocator()
-	for col := 0; col < len(sf.schema.Fields()); col++ {
-		chunks := make([]arrow.Array, 0)
-		if sf.table != nil {
-			// we append if there was existing file
-			column := sf.table.Column(col)
-			// if primitive type
-			switch column.DataType() {
-			case arrow.BinaryTypes.String:
-				chunks = sf.buildFromStrings(col, mem)
-			default:
-				chunks = append(chunks, column.Data().Chunks()...)
-			}
-			// else binary type
-		}
-		switch sf.schema.Field(col).Type {
-		case arrow.PrimitiveTypes.Int64:
-			// case *arrow.Int64Type:
-			if sf.added > 0 {
-				ibuild := array.NewInt64Builder(mem)
-				ibuild.AppendValues(sf.columns[col].([]int64), nil) // TODO(twg) 2022/09/28 need to handle null
-				newChunk := ibuild.NewArray()
-				chunks = append(chunks, newChunk)
-			}
-			record, err := array.Concatenate(chunks, mem)
-			if err != nil {
-				return err
-			}
-			parts = append(parts, record)
-		case arrow.PrimitiveTypes.Float64:
-			// case *arrow.Float64Type:
-			if sf.added > 0 {
-				fbuild := array.NewFloat64Builder(mem)
-				fbuild.AppendValues(sf.columns[col].([]float64), nil) // TODO(twg) 2022/09/28 need to handle null
-				newChunk := fbuild.NewArray()
-				chunks = append(chunks, newChunk)
-			}
-			record, err := array.Concatenate(chunks, mem)
-			if err != nil {
-				return err
-			}
-			parts = append(parts, record)
-		case arrow.BinaryTypes.String:
-			if sf.added > 0 {
-				fbuild := array.NewStringBuilder(mem)
-				fbuild.AppendValues(sf.columns[col].([]string), nil) // TODO(twg) 2022/09/28 need to handle null
-				newChunk := fbuild.NewArray()
-				chunks = append(chunks, newChunk)
-			}
-			record, err := array.Concatenate(chunks, mem)
-			if err != nil {
-				return err
-			}
-			parts = append(parts, record)
-		default:
-			vprint.VV("UNKNOWN %T", sf.schema.Field(col).Type)
-		}
-	}
-	rec := array.NewRecord(sf.schema, parts, sf.beforeRows+sf.added)
-	table := array.NewTableFromRecords(sf.schema, []arrow.Record{rec})
-
-	return sf.executor.SaveTable(name, table, mem)
-}
-
-// TODO(twg) 2022/10/03 Not a huge fan of the global variable will look at adding to executor structure
-// when dataframe is fully integrated
-var (
-	dataframeShardLocks map[uint64]*sync.Mutex
-	muWriteDataframe    sync.Mutex
-)
-
-func init() {
-	dataframeShardLocks = make(map[uint64]*sync.Mutex)
-}
-
-func getDataframeWritelock(shard uint64) *sync.Mutex {
-	muWriteDataframe.Lock()
-	defer muWriteDataframe.Unlock()
-	lock, ok := dataframeShardLocks[shard]
-	if ok {
-		return lock
-	}
-	newLock := sync.Mutex{}
-	dataframeShardLocks[shard] = &newLock
-	return &newLock
-}
-
-func (api *API) ApplyDataframeChangeset(ctx context.Context, index string, cs *ChangesetRequest, shard uint64) error {
-	// TODO(twg) 2022/09/29 need to validate api call
-	idx := api.Holder().Index(index)
-
-	// check if dataframe exists
-	fname := idx.GetDataFramePath(shard)
-
-	// only 1 shard writer allowed at at time so wait for it to be available
-	mu := getDataframeWritelock(shard)
-	mu.Lock()
-	defer mu.Unlock()
-	mem := memory.NewGoAllocator()
-	shardFile, err := NewShardFile(ctx, fname, mem, api.server.executor)
-	if err != nil {
-		return err
-	}
-
-	err = shardFile.EnsureSchema(cs)
-	if err != nil {
-		return err
-	}
-
-	return shardFile.Process(cs)
-}
-
-type column struct {
-	Name string
-	Type string
-}
-
-func (api *API) GetDataframeSchema(ctx context.Context, indexName string) (interface{}, error) {
-	idx, err := api.Index(ctx, indexName)
-	if err != nil {
-		return nil, err
-	}
-	base := idx.DataframesPath()
-	dir, _ := os.Open(base)
-	files, _ := dir.Readdir(0)
-	parts := make([]column, 0)
-	mem := memory.NewGoAllocator()
-	for i := range files {
-		file := files[i]
-		name := file.Name()
-		if api.server.executor.IsDataframeFile(name) {
-			// strip off the parquet extenison
-			name = strings.TrimSuffix(name, filepath.Ext(name))
-			// read the parquet file and extract the schema
-			fname := filepath.Join(base, name)
-			table, err := api.server.executor.getDataTable(ctx, fname, mem)
-			if err != nil {
-				return nil, err
-			}
-			for i := 0; i < int(table.NumCols()); i++ {
-				col := table.Column(i)
-				part := column{Name: col.Name(), Type: col.DataType().String()}
-				parts = append(parts, part)
-			}
-			break // only go on first file
-		}
-	}
-	return parts, nil
-}

arrow.go

@@ -1,562 +0,0 @@
-// Copyright 2021 Molecula Corp. All rights reserved.
-package pilosa
-
-import (
-	"context"
-	"encoding/json"
-	"fmt"
-	"io"
-	"os"
-	"strings"
-	"sync"
-
-	"github.com/apache/arrow/go/v10/arrow"
-	"github.com/apache/arrow/go/v10/arrow/array"
-	"github.com/apache/arrow/go/v10/arrow/ipc"
-	"github.com/apache/arrow/go/v10/arrow/memory"
-	"github.com/apache/arrow/go/v10/parquet"
-	"github.com/apache/arrow/go/v10/parquet/file"
-	"github.com/apache/arrow/go/v10/parquet/pqarrow"
-	"github.com/featurebasedb/featurebase/v3/pql"
-	"github.com/featurebasedb/featurebase/v3/tracing"
-	"github.com/gomem/gomem/pkg/dataframe"
-	"github.com/pkg/errors"
-)
-
-/*
-The function Arrow provides filtered access to the raw values stored in the dataframe.
-If Arrow is just provided a bitmap filter, such as ConstRow or any Bitmap Operation,
-all the values associated with each column are returned.  This set can be limited with
-the addition of the header parameter
-Example:
-Arrow(ConstRow(columns=[2,4,6]),header=["fval"])
-*/
-
-// executeApply executes a Arrow() call.
-func (e *executor) executeArrow(ctx context.Context, qcx *Qcx, index string, c *pql.Call, shards []uint64, opt *ExecOptions) (arrow.Table, error) {
-	if !e.dataframeEnabled {
-		return nil, errors.New("Dataframe support not enabled")
-	}
-	span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeArrow")
-	defer span.Finish()
-	if len(c.Children) > 1 {
-		return nil, errors.New("Apply() only accepts a single bitmap input filter")
-	}
-	var columnFilter []string
-	if cols, ok := c.Args["header"].([]interface{}); ok {
-		columnFilter = make([]string, 0, len(cols))
-		for _, v := range cols {
-			columnFilter = append(columnFilter, v.(string))
-		}
-	}
-	mapcounter := 0
-	reducecounter := 0
-	pool := memory.NewGoAllocator() // TODO(twg) 2022/09/01 singledton?
-	// Execute calls in bulk on each remote node and merge.
-	mu := &sync.Mutex{}
-	mapFn := func(ctx context.Context, shard uint64, mopt *mapOptions) (_ interface{}, err error) {
-		mu.Lock()
-		mapcounter++
-		mu.Unlock()
-		return e.executeArrowShard(ctx, qcx, index, c, shard, pool, columnFilter)
-	}
-	tables := make([]*BasicTable, 0)
-
-	reduceFn := func(ctx context.Context, prev, v interface{}) interface{} {
-		mu.Lock()
-		reducecounter++
-		mu.Unlock()
-		if v == nil {
-			return prev
-		}
-		switch t := v.(type) {
-		case *BasicTable:
-
-			if t.resolver != nil {
-				mu.Lock()
-				tables = append(tables, t)
-				mu.Unlock()
-			}
-		case arrow.Table:
-			if t.NumRows() > 0 {
-				bt := BasicTableFromArrow(t, pool)
-				mu.Lock()
-				tables = append(tables, bt)
-				mu.Unlock()
-			}
-		}
-		return nil
-	}
-
-	_, err := e.mapReduce(ctx, index, shards, c, opt, mapFn, reduceFn)
-	if err != nil {
-		return nil, err
-	}
-	if len(tables) == 0 {
-		return &BasicTable{name: "empty"}, nil
-	}
-	tbl := Concat(tables[0].Schema(), tables, pool)
-	r := dataframe.NewChunkResolver(tbl.Column(0))
-	return &BasicTable{resolver: &r, table: tbl}, nil
-}
-
-type BasicTable struct {
-	resolver dataframe.Resolver
-	table    arrow.Table
-	filtered bool
-	name     string
-}
-
-func (st *BasicTable) Name() string {
-	return st.name
-}
-
-func (st *BasicTable) Schema() *arrow.Schema {
-	if st.table != nil {
-		return st.table.Schema()
-	}
-	return &arrow.Schema{}
-}
-
-func (st *BasicTable) IsFiltered() bool {
-	return st.filtered
-}
-
-func (st *BasicTable) NumRows() int64 {
-	if st.resolver == nil {
-		return 0
-	}
-	return int64(st.resolver.NumRows())
-}
-
-func (st *BasicTable) NumCols() int64 {
-	if st.table != nil {
-		return st.table.NumCols()
-	}
-	return 0
-}
-
-func (st *BasicTable) Column(i int) *arrow.Column {
-	if st.table != nil {
-		return st.table.Column(i)
-	}
-	return nil
-}
-
-func (st *BasicTable) Retain() {
-	if st.table != nil {
-		st.table.Retain()
-	}
-}
-
-func (st *BasicTable) Release() {
-	if st.table != nil {
-		st.table.Retain()
-	}
-}
-
-func (st *BasicTable) Get(column, row int) interface{} {
-	field := st.Schema().Field(column)
-	c, i := st.resolver.Resolve(row)
-	nullable := field.Nullable
-
-	chunk := st.Column(column).Data().Chunk(c)
-	// TODO(twg) 2023/01/26 potential NULL support?
-	if nullable && chunk.IsNull(i) {
-		return nil
-	}
-	switch field.Type.(type) {
-	case *arrow.BooleanType:
-		return chunk.(*array.Boolean).Value(i)
-	case *arrow.Int8Type:
-		v := chunk.(*array.Int8).Int8Values()
-		return int64(v[i])
-	case *arrow.Int16Type:
-		v := chunk.(*array.Int16).Int16Values()
-		return int64(v[i])
-	case *arrow.Int32Type:
-		v := chunk.(*array.Int32).Int32Values()
-		return int64(v[i])
-	case *arrow.Int64Type:
-		v := chunk.(*array.Int64).Int64Values()
-		return int64(v[i])
-	case *arrow.Uint8Type:
-		v := chunk.(*array.Uint8).Uint8Values()
-		return uint64(v[i])
-	case *arrow.Uint16Type:
-		v := chunk.(*array.Uint16).Uint16Values()
-		return uint64(v[i])
-	case *arrow.Uint32Type:
-		v := chunk.(*array.Uint32).Uint32Values()
-		return uint64(v[i])
-	case *arrow.Uint64Type:
-		v := chunk.(*array.Uint64).Uint64Values()
-		return v[i]
-	case *arrow.Float32Type:
-		v := chunk.(*array.Float32).Float32Values()
-		return float64(v[i])
-	case *arrow.Float64Type:
-		v := chunk.(*array.Float64).Float64Values()
-		return v[i]
-	case *arrow.StringType:
-		return chunk.(*array.String).Value(i)
-	}
-	return 0
-}
-
-func builderFrom(mem memory.Allocator, dt arrow.DataType, size int64) array.Builder {
-	var bldr array.Builder
-	switch dt := dt.(type) {
-	case *arrow.BooleanType:
-		bldr = array.NewBooleanBuilder(mem)
-	case *arrow.Int8Type:
-		bldr = array.NewInt8Builder(mem)
-	case *arrow.Int16Type:
-		bldr = array.NewInt16Builder(mem)
-	case *arrow.Int32Type:
-		bldr = array.NewInt32Builder(mem)
-	case *arrow.Int64Type:
-		bldr = array.NewInt64Builder(mem)
-	case *arrow.Uint8Type:
-		bldr = array.NewUint8Builder(mem)
-	case *arrow.Uint16Type:
-		bldr = array.NewUint16Builder(mem)
-	case *arrow.Uint32Type:
-		bldr = array.NewUint32Builder(mem)
-	case *arrow.Uint64Type:
-		bldr = array.NewUint64Builder(mem)
-	case *arrow.Float32Type:
-		bldr = array.NewFloat32Builder(mem)
-	case *arrow.Float64Type:
-		bldr = array.NewFloat64Builder(mem)
-	case *arrow.StringType:
-		bldr = array.NewStringBuilder(mem)
-	default:
-		panic(fmt.Errorf("builderFrom: invalid Arrow type %v", dt))
-	}
-	bldr.Reserve(int(size))
-	return bldr
-}
-
-func appendData(bldr array.Builder, v interface{}) {
-	switch bldr := bldr.(type) {
-	case *array.BooleanBuilder:
-		bldr.Append(v.(bool))
-	case *array.Int8Builder:
-		bldr.Append(v.(int8))
-	case *array.Int16Builder:
-		bldr.Append(v.(int16))
-	case *array.Int32Builder:
-		bldr.Append(v.(int32))
-	case *array.Int64Builder:
-		bldr.Append(v.(int64))
-	case *array.Uint8Builder:
-		bldr.Append(v.(uint8))
-	case *array.Uint16Builder:
-		bldr.Append(v.(uint16))
-	case *array.Uint32Builder:
-		bldr.Append(v.(uint32))
-	case *array.Uint64Builder:
-		bldr.Append(v.(uint64))
-	case *array.Float32Builder:
-		bldr.Append(v.(float32))
-	case *array.Float64Builder:
-		bldr.Append(v.(float64))
-	case *array.StringBuilder:
-		bldr.Append(v.(string))
-	default:
-		panic(fmt.Errorf("appendData: invalid Arrow builder type %T", bldr))
-	}
-}
-
-func Concat(schema *arrow.Schema, tables []*BasicTable, mem memory.Allocator) arrow.Table {
-	if len(tables) == 1 {
-		if !tables[0].IsFiltered() {
-			return tables[0]
-		}
-	}
-	cols := make([]arrow.Column, len(schema.Fields()))
-
-	defer func(cols []arrow.Column) {
-		for i := range cols {
-			cols[i].Release()
-		}
-	}(cols)
-	sz := 0
-	for i := range tables {
-		sz += int(tables[i].NumRows())
-	}
-	for i := range cols {
-		field := schema.Field(i)
-		arrs := make([]arrow.Array, 0)
-		builder := builderFrom(mem, field.Type, int64(sz))
-		for t := range tables {
-			table := tables[t]
-			if table.IsFiltered() {
-				for row := 0; row < int(table.NumRows()); row++ {
-					v := table.Get(i, row)
-					appendData(builder, v)
-				}
-				arrs = append(arrs, builder.NewArray())
-
-			} else {
-				parts := table.Column(i).Data()
-				arrs = append(arrs, parts.Chunks()...)
-			}
-		}
-		chunk := arrow.NewChunked(field.Type, arrs)
-		cols[i] = *arrow.NewColumn(field, chunk)
-		chunk.Release()
-	}
-	return array.NewTable(schema, cols, -1)
-}
-
-func (st *BasicTable) MarshalJSON() ([]byte, error) {
-	results := make(map[string]interface{})
-	n := 0
-	if st.table != nil {
-		n = int(st.table.NumCols())
-	}
-	for b := 0; b < n; b++ {
-		col := st.table.Column(b)
-		result := make([]interface{}, st.resolver.NumRows())
-		for n := st.resolver.NumRows() - 1; n >= 0; n-- {
-			v := st.Get(b, n)
-			result[n] = v
-
-		}
-		results[col.Name()] = result
-	}
-	return json.Marshal(results)
-}
-
-func BasicTableFromArrow(table arrow.Table, mem memory.Allocator) *BasicTable {
-	col := table.Column(0)
-	r := dataframe.NewChunkResolver(col)
-	return &BasicTable{resolver: &r, table: table}
-}
-
-func filterColumns(filters []string, table arrow.Table) arrow.Table {
-	filters = append(filters, "_ID")
-	schema := table.Schema()
-	// TODO(twg) 2022/11/09 add glob support
-	allFields := schema.Fields()
-	in := func(key string) bool {
-		for _, v := range filters {
-			if v == key {
-				return true
-			}
-		}
-		return false
-	}
-	cols := make([]arrow.Column, 0)
-	fields := make([]arrow.Field, 0)
-	for i := range allFields {
-		field := allFields[i]
-		if in(field.Name) {
-			cols = append(cols, *table.Column(i))
-			fields = append(fields, field)
-		}
-	}
-
-	filterdSchema := arrow.NewSchema(fields, nil) // TODO(twg) 2022/11/09 handle meta:w
-	return array.NewTable(filterdSchema, cols, table.NumRows())
-}
-
-func (e *executor) executeArrowShard(ctx context.Context, qcx *Qcx, index string, c *pql.Call, shard uint64, pool memory.Allocator, columnFilter []string) (*BasicTable, error) {
-	name := fmt.Sprintf("a. %v", shard)
-	span, _ := tracing.StartSpanFromContext(ctx, "Executor.executeArrowShard")
-	defer span.Finish()
-
-	var filter *Row
-	if len(c.Children) == 1 {
-		row, err := e.executeBitmapCallShard(ctx, qcx, index, c.Children[0], shard)
-		if err != nil {
-			return nil, err
-		}
-		filter = row
-		if !filter.Any() {
-			// no need to actuall run the query for its not operating against any values
-			return &BasicTable{name: name}, nil
-		}
-	}
-	//
-	ids := filter.ShardColumns() // needs to be shard columns
-	// Fetch index.
-	idx := e.Holder.Index(index)
-	if idx == nil {
-		return nil, newNotFoundError(ErrIndexNotFound, index)
-	}
-
-	fname := idx.GetDataFramePath(shard)
-
-	if !e.dataFrameExists(fname) {
-		return &BasicTable{name: name}, nil
-	}
-
-	table, err := e.getDataTable(ctx, fname, pool)
-	if err != nil {
-		return nil, errors.Wrap(err, "arrow readTableParquet")
-	}
-	defer table.Release()
-	if len(columnFilter) > 0 {
-		table = filterColumns(columnFilter, table)
-	}
-	df, err := dataframe.NewDataFrameFromTable(pool, table)
-	if err != nil {
-		return nil, errors.Wrap(err, "arrow NewDataFromTable")
-	}
-	p := dataframe.NewChunkResolver(df.ColumnAt(0))
-	var resolver dataframe.Resolver
-	resolver = &p
-	if filter != nil {
-		if len(ids) == 0 {
-			return &BasicTable{name: name}, nil
-		}
-		resolver, err = filterDataframe(resolver, pool, ids)
-		if err != nil {
-			return nil, errors.Wrap(err, "filtering dataframe")
-		}
-	}
-	table.Retain()
-	return &BasicTable{resolver: resolver, table: table, filtered: filter != nil, name: name}, nil
-}
-
-func (e *executor) dataFrameExists(fname string) bool {
-	if e.typeIsParquet() {
-		if _, err := os.Stat(fname + ".parquet"); os.IsNotExist(err) {
-			return false
-		}
-		return true
-	}
-	if _, err := os.Stat(fname + ".arrow"); os.IsNotExist(err) {
-		return false
-	}
-	return true
-}
-
-func (e *executor) getDataTable(ctx context.Context, fname string, mem memory.Allocator) (arrow.Table, error) {
-	if e.typeIsParquet() {
-		table, err := readTableParquetCtx(ctx, fname, mem)
-		return table, err
-	}
-	return readTableArrow(fname, mem)
-}
-
-func (e *executor) typeIsParquet() bool {
-	return e.datafameUseParquet
-}
-
-func (e *executor) IsDataframeFile(name string) bool {
-	if e.typeIsParquet() {
-		return strings.HasSuffix(name, ".parquet")
-	}
-	return strings.HasSuffix(name, ".arrow")
-}
-
-func (e *executor) SaveTable(name string, table arrow.Table, mem memory.Allocator) error {
-	if e.typeIsParquet() {
-		return writeTableParquet(table, name)
-	}
-	return writeTableArrow(table, name, mem)
-}
-
-func (e *executor) TableExtension() string {
-	if e.typeIsParquet() {
-		return ".parquet"
-	}
-	return ".arrow"
-}
-
-func readTableArrow(filename string, mem memory.Allocator) (arrow.Table, error) {
-	r, err := os.Open(filename + ".arrow")
-	if err != nil {
-		return nil, err
-	}
-	rr, err := ipc.NewFileReader(r, ipc.WithAllocator(mem))
-	if err != nil {
-		return nil, err
-	}
-	defer rr.Close()
-	records := make([]arrow.Record, rr.NumRecords())
-	i := 0
-	for {
-		rec, err := rr.Read()
-		if err == io.EOF {
-			break
-		} else if err != nil {
-			return nil, err
-		}
-		records[i] = rec
-		i++
-	}
-	records = records[:i]
-	table := array.NewTableFromRecords(rr.Schema(), records)
-	return table, nil
-}
-
-func readTableParquetCtx(ctx context.Context, filename string, mem memory.Allocator) (arrow.Table, error) {
-	r, err := os.Open(filename + ".parquet")
-	if err != nil {
-		return nil, err
-	}
-	defer r.Close()
-
-	pf, err := file.NewParquetReader(r)
-	if err != nil {
-		return nil, err
-	}
-
-	reader, err := pqarrow.NewFileReader(pf, pqarrow.ArrowReadProperties{}, mem)
-	if err != nil {
-		return nil, err
-	}
-	return reader.ReadTable(ctx)
-}
-
-func writeTableParquet(table arrow.Table, filename string) error {
-	f, err := os.Create(filename + ".parquet")
-	if err != nil {
-		return err
-	}
-	defer f.Close()
-	props := parquet.NewWriterProperties(parquet.WithDictionaryDefault(false))
-	arrProps := pqarrow.DefaultWriterProps()
-	chunkSize := 10 * 1024 * 1024
-	err = pqarrow.WriteTable(table, f, int64(chunkSize), props, arrProps)
-	if err != nil {
-		return err
-	}
-	f.Sync()
-	return nil
-}
-
-func writeTableArrow(table arrow.Table, filename string, mem memory.Allocator) error {
-	f, err := os.Create(filename + ".arrow")
-	if err != nil {
-		return err
-	}
-	defer f.Close()
-	writer, err := ipc.NewFileWriter(f, ipc.WithAllocator(mem), ipc.WithSchema(table.Schema()))
-	if err != nil {
-		panic(err)
-	}
-	chunkSize := int64(0)
-	tr := array.NewTableReader(table, chunkSize)
-	defer tr.Release()
-	n := 0
-	for tr.Next() {
-		arec := tr.Record()
-		err = writer.Write(arec)
-		if err != nil {
-			panic(err)
-		}
-		n++
-	}
-	err = writer.Close()
-	if err != nil {
-		panic(err)
-	}
-	f.Sync()
-	return nil
-}

arrow_test.go

@@ -1,53 +0,0 @@
-// Copyright 2021 Molecula Corp. All rights reserved.
-package pilosa
-
-import (
-	"context"
-	"encoding/hex"
-	"math/rand"
-	"os"
-	"path/filepath"
-	"testing"
-
-	"github.com/apache/arrow/go/v10/arrow"
-	"github.com/apache/arrow/go/v10/arrow/array"
-	"github.com/apache/arrow/go/v10/arrow/memory"
-)
-
-func TempFileName(prefix string) string {
-	randBytes := make([]byte, 16)
-	rand.Read(randBytes)
-	return filepath.Join(os.TempDir(), prefix+hex.EncodeToString(randBytes))
-}
-
-func Test_TableParquet(t *testing.T) {
-	// create a arrow table
-	schema := arrow.NewSchema(
-		[]arrow.Field{
-			{Name: "num", Type: arrow.PrimitiveTypes.Float64},
-		},
-		nil, // no metadata
-	)
-	mem := memory.NewGoAllocator()
-	b := array.NewRecordBuilder(mem, schema)
-	defer b.Release()
-	b.Field(0).(*array.Float64Builder).AppendValues([]float64{1.0, 1.5, 2.0}, nil)
-	table := array.NewTableFromRecords(schema, []arrow.Record{b.NewRecord()})
-	defer table.Release()
-	fileName := TempFileName("pq-")
-	// save it as  a parquet file
-	err := writeTableParquet(table, fileName)
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer os.Remove(fileName)
-
-	// read it back in and compare the result
-	got, err := readTableParquetCtx(context.Background(), fileName, mem)
-	if err != nil {
-		t.Fatalf("readTableParquetCtx() error = %v", err)
-	}
-	if got.NumCols() != table.NumCols() {
-		t.Errorf("got:%v expected:%v", got.NumCols(), table.NumCols())
-	}
-}

attr.go

@@ -0,0 +1,478 @@
+package pilosa
+
+import (
+	"bytes"
+	"crypto/sha1"
+	"encoding/binary"
+	"fmt"
+	"sort"
+	"sync"
+	"time"
+
+	"github.com/boltdb/bolt"
+	"github.com/gogo/protobuf/proto"
+	"github.com/pilosa/pilosa/internal"
+)
+
+// AttrBlockSize is the size of attribute blocks for anti-entropy.
+const AttrBlockSize = 100
+
+// Attribute data type enum.
+const (
+	AttrTypeString = 1
+	AttrTypeUint   = 2
+	AttrTypeBool   = 3
+)
+
+// AttrStore represents a storage layer for attributes.
+type AttrStore struct {
+	mu   sync.Mutex
+	path string
+	db   *bolt.DB
+
+	// in-memory cache
+	attrs map[uint64]map[string]interface{}
+}
+
+// NewAttrStore returns a new instance of AttrStore.
+func NewAttrStore(path string) *AttrStore {
+	return &AttrStore{
+		path:  path,
+		attrs: make(map[uint64]map[string]interface{}),
+	}
+}
+
+// Path returns path to the store's data file.
+func (s *AttrStore) Path() string { return s.path }
+
+// Open opens and initializes the store.
+func (s *AttrStore) Open() error {
+	// Open storage.
+	db, err := bolt.Open(s.path, 0666, &bolt.Options{Timeout: 1 * time.Second})
+	if err != nil {
+		return err
+	}
+	s.db = db
+
+	// Initialize database.
+	if err := s.db.Update(func(tx *bolt.Tx) error {
+		if _, err := tx.CreateBucketIfNotExists([]byte("attrs")); err != nil {
+			return err
+		}
+		return nil
+	}); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// Close closes the store.
+func (s *AttrStore) Close() error {
+	if s.db != nil {
+		s.db.Close()
+	}
+	return nil
+}
+
+// Attrs returns a set of attributes by ID.
+func (s *AttrStore) Attrs(id uint64) (m map[string]interface{}, err error) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+
+	// Check cache for map.
+	if m = s.attrs[id]; m != nil {
+		return m, nil
+	}
+
+	// Find attributes from storage.
+	if err = s.db.View(func(tx *bolt.Tx) error {
+		m, err = txAttrs(tx, id)
+		if err != nil {
+			return err
+		}
+		return nil
+	}); err != nil {
+		return nil, err
+	}
+
+	// Add to cache.
+	s.attrs[id] = m
+
+	return
+}
+
+// SetAttrs sets attribute values for a given ID.
+func (s *AttrStore) SetAttrs(id uint64, m map[string]interface{}) error {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+
+	var attr map[string]interface{}
+	if err := s.db.Update(func(tx *bolt.Tx) error {
+		tmp, err := txUpdateAttrs(tx, id, m)
+		if err != nil {
+			return err
+		}
+		attr = tmp
+
+		return nil
+	}); err != nil {
+		return err
+	}
+
+	// Swap attributes map in cache.
+	s.attrs[id] = attr
+
+	return nil
+}
+
+// SetBulkAttrs sets attribute values for a set of ids.
+func (s *AttrStore) SetBulkAttrs(m map[uint64]map[string]interface{}) error {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+
+	attrs := make(map[uint64]map[string]interface{})
+	if err := s.db.Update(func(tx *bolt.Tx) error {
+		// Collect and sort keys.
+		ids := make([]uint64, 0, len(m))
+		for id := range m {
+			ids = append(ids, id)
+		}
+		sort.Sort(uint64Slice(ids))
+
+		// Update attributes for each id.
+		for _, id := range ids {
+			attr, err := txUpdateAttrs(tx, id, m[id])
+			if err != nil {
+				return err
+			}
+			attrs[id] = attr
+		}
+
+		return nil
+	}); err != nil {
+		return err
+	}
+
+	// Swap attributes map in cache.
+	for id, attr := range attrs {
+		s.attrs[id] = attr
+	}
+
+	return nil
+}
+
+// Blocks returns a list of all blocks in the store.
+func (s *AttrStore) Blocks() ([]AttrBlock, error) {
+	tx, err := s.db.Begin(false)
+	if err != nil {
+		return nil, err
+	}
+	defer tx.Rollback()
+
+	// Wrap cursor to segment by block.
+	cur := newBlockCursor(tx.Bucket([]byte("attrs")).Cursor(), AttrBlockSize)
+
+	// Iterate over each block.
+	var blocks []AttrBlock
+	for cur.nextBlock() {
+		block := AttrBlock{ID: cur.blockID()}
+
+		// Compute checksum of every key/value in block.
+		h := sha1.New()
+		for k, v := cur.next(); k != nil; k, v = cur.next() {
+			h.Write(k)
+			h.Write(v)
+		}
+		block.Checksum = h.Sum(nil)
+
+		// Append block.
+		blocks = append(blocks, block)
+	}
+
+	return blocks, nil
+}
+
+// BlockData returns all data for a single block.
+func (s *AttrStore) BlockData(i uint64) (map[uint64]map[string]interface{}, error) {
+	m := make(map[uint64]map[string]interface{})
+
+	// Start read-only transaction.
+	tx, err := s.db.Begin(false)
+	if err != nil {
+		return nil, err
+	}
+	defer tx.Rollback()
+
+	// Move to the start of the block.
+	min := u64tob(uint64(i) * AttrBlockSize)
+	max := u64tob(uint64(i+1) * AttrBlockSize)
+	cur := tx.Bucket([]byte("attrs")).Cursor()
+	for k, v := cur.Seek(min); k != nil; k, v = cur.Next() {
+		// Exit if we're past the end of the block.
+		if bytes.Compare(k, max) != -1 {
+			break
+		}
+
+		// Decode attribute map and associate with id.
+		var pb internal.AttrMap
+		if err := proto.Unmarshal(v, &pb); err != nil {
+			return nil, err
+		}
+		m[btou64(k)] = decodeAttrs(pb.GetAttrs())
+	}
+
+	return m, nil
+}
+
+// txAttrs returns a map of attributes for a bitmap.
+func txAttrs(tx *bolt.Tx, id uint64) (map[string]interface{}, error) {
+	v := tx.Bucket([]byte("attrs")).Get(u64tob(id))
+	if v == nil {
+		return emptyMap, nil
+	}
+
+	var pb internal.AttrMap
+	if err := proto.Unmarshal(v, &pb); err != nil {
+		return nil, err
+	}
+	return decodeAttrs(pb.GetAttrs()), nil
+}
+
+// txUpdateAttrs updates the attributes for an id.
+// Returns the new combined set of attributes for the id.
+func txUpdateAttrs(tx *bolt.Tx, id uint64, m map[string]interface{}) (map[string]interface{}, error) {
+	attr, err := txAttrs(tx, id)
+	if err != nil {
+		return nil, err
+	}
+
+	// Create a new map if it is empty so we don't update emptyMap.
+	if len(attr) == 0 {
+		attr = make(map[string]interface{}, len(m))
+	}
+
+	// Merge attributes with original values.
+	// Nil values should delete keys.
+	for k, v := range m {
+		if v == nil {
+			delete(attr, k)
+			continue
+		}
+
+		switch v := v.(type) {
+		case int:
+			attr[k] = uint64(v)
+		case uint:
+			attr[k] = uint64(v)
+		case int64:
+			attr[k] = uint64(v)
+		case string, uint64, bool:
+			attr[k] = v
+		default:
+			return nil, fmt.Errorf("invalid attr type: %T", v)
+		}
+	}
+
+	// Marshal and save new values.
+	buf, err := proto.Marshal(&internal.AttrMap{Attrs: encodeAttrs(attr)})
+	if err != nil {
+		return nil, err
+	}
+	if err := tx.Bucket([]byte("attrs")).Put(u64tob(id), buf); err != nil {
+		return nil, err
+	}
+	return attr, nil
+}
+
+func encodeAttrs(m map[string]interface{}) []*internal.Attr {
+	keys := make([]string, 0, len(m))
+	for k := range m {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+
+	a := make([]*internal.Attr, len(keys))
+	for i := range keys {
+		a[i] = encodeAttr(keys[i], m[keys[i]])
+	}
+	return a
+}
+
+func decodeAttrs(pb []*internal.Attr) map[string]interface{} {
+	m := make(map[string]interface{}, len(pb))
+	for i := range pb {
+		key, value := decodeAttr(pb[i])
+		m[key] = value
+	}
+	return m
+}
+
+// encodeAttr converts a key/value pair into an Attr internal representation.
+func encodeAttr(key string, value interface{}) *internal.Attr {
+	pb := &internal.Attr{Key: key}
+	switch value := value.(type) {
+	case string:
+		pb.Type = AttrTypeString
+		pb.StringValue = value
+	case float64:
+		pb.Type = AttrTypeUint
+		pb.UintValue = uint64(value)
+	case uint64:
+		pb.Type = AttrTypeUint
+		pb.UintValue = value
+	case int64:
+		pb.Type = AttrTypeUint
+		pb.UintValue = uint64(value)
+	case bool:
+		pb.Type = AttrTypeBool
+		pb.BoolValue = value
+	}
+	return pb
+}
+
+// decodeAttr converts from an Attr internal representation to a key/value pair.
+func decodeAttr(attr *internal.Attr) (key string, value interface{}) {
+	switch attr.Type {
+	case AttrTypeString:
+		return attr.Key, attr.StringValue
+	case AttrTypeUint:
+		return attr.Key, attr.UintValue
+	case AttrTypeBool:
+		return attr.Key, attr.BoolValue
+	default:
+		return attr.Key, nil
+	}
+}
+
+// cloneAttrs returns a shallow clone of m.
+func cloneAttrs(m map[string]interface{}) map[string]interface{} {
+	other := make(map[string]interface{}, len(m))
+	for k, v := range m {
+		other[k] = v
+	}
+	return other
+}
+
+// u64tob encodes v to big endian encoding.
+func u64tob(v uint64) []byte {
+	b := make([]byte, 8)
+	binary.BigEndian.PutUint64(b, v)
+	return b
+}
+
+// btou64 decodes b from big endian encoding.
+func btou64(b []byte) uint64 { return binary.BigEndian.Uint64(b) }
+
+// emptyMap is a reusable map that contains no keys.
+var emptyMap = make(map[string]interface{})
+
+// AttrBlock represents a checksummed block of the attribute store.
+type AttrBlock struct {
+	ID       uint64 `json:"id"`
+	Checksum []byte `json:"checksum"`
+}
+
+// AttrBlocks represents a list of blocks.
+type AttrBlocks []AttrBlock
+
+// Diff returns a list of block ids that are different or are new in other.
+// Block lists must be in sorted order.
+func (a AttrBlocks) Diff(other []AttrBlock) []uint64 {
+	var ids []uint64
+	for {
+		// Read next block from each list.
+		var blk0, blk1 *AttrBlock
+		if len(a) > 0 {
+			blk0 = &a[0]
+		}
+		if len(other) > 0 {
+			blk1 = &other[0]
+		}
+
+		// Exit if "a" contains no more blocks.
+		if blk0 == nil {
+			return ids
+		}
+
+		// Add block ID if it's different or if it's only in "a".
+		if blk1 == nil || blk0.ID < blk1.ID {
+			ids = append(ids, blk0.ID)
+			a = a[1:]
+		} else if blk1.ID < blk0.ID {
+			other = other[1:]
+		} else {
+			if !bytes.Equal(blk0.Checksum, blk1.Checksum) {
+				ids = append(ids, blk0.ID)
+			}
+			a, other = a[1:], other[1:]
+		}
+	}
+}
+
+// blockCursor represents a cursor for iterating over blocks of a bolt bucket.
+type blockCursor struct {
+	cur  *bolt.Cursor
+	base uint64
+	n    uint64
+
+	buf struct {
+		key    []byte
+		value  []byte
+		filled bool
+	}
+}
+
+// newBlockCursor returns a new block cursor that wraps cur using n sized blocks.
+func newBlockCursor(c *bolt.Cursor, n int) blockCursor {
+	cur := blockCursor{
+		cur: c,
+		n:   uint64(n),
+	}
+	cur.buf.key, cur.buf.value = c.First()
+	cur.buf.filled = true
+	return cur
+}
+
+// blockID returns the current block ID. Only valid after call to nextBlock().
+func (cur *blockCursor) blockID() uint64 { return cur.base }
+
+// nextBlock moves the cursor to the next block.
+// Returns true if another block exists, otherwise returns false.
+func (cur *blockCursor) nextBlock() bool {
+	if cur.buf.key == nil {
+		return false
+	}
+
+	cur.base = binary.BigEndian.Uint64(cur.buf.key) / cur.n
+	return true
+}
+
+// next returns the next key/value within the block.
+// Returns nils at the end of the block.
+func (cur *blockCursor) next() (key, value []byte) {
+	// Use buffered value, if set.
+	if cur.buf.filled {
+		key, value = cur.buf.key, cur.buf.value
+		cur.buf.filled = false
+		return key, value
+	}
+
+	// Read next key.
+	key, value = cur.cur.Next()
+
+	// Fill buffer for EOF.
+	if key == nil {
+		cur.buf.key, cur.buf.value, cur.buf.filled = key, value, false
+		return nil, nil
+	}
+
+	// Parse key and buffer if outside of block.
+	id := binary.BigEndian.Uint64(key)
+	if id/cur.n > cur.base {
+		cur.buf.key, cur.buf.value, cur.buf.filled = key, value, true
+		return nil, nil
+	}
+
+	return key, value
+}

attr_test.go

@@ -0,0 +1,145 @@
+package pilosa_test
+
+import (
+	"io/ioutil"
+	"os"
+	"reflect"
+	"testing"
+
+	"github.com/pilosa/pilosa"
+)
+
+// Ensure database can set and retrieve profile attributes.
+func TestAttrStore_Attrs(t *testing.T) {
+	s := MustOpenAttrStore()
+	defer s.Close()
+
+	// Set attributes.
+	if err := s.SetAttrs(1, map[string]interface{}{"A": uint64(100)}); err != nil {
+		t.Fatal(err)
+	} else if err := s.SetAttrs(2, map[string]interface{}{"A": uint64(200)}); err != nil {
+		t.Fatal(err)
+	} else if err := s.SetAttrs(1, map[string]interface{}{"B": "VALUE"}); err != nil {
+		t.Fatal(err)
+	}
+
+	// Retrieve attributes for profile #1.
+	if m, err := s.Attrs(1); err != nil {
+		t.Fatal(err)
+	} else if !reflect.DeepEqual(m, map[string]interface{}{"A": uint64(100), "B": "VALUE"}) {
+		t.Fatalf("unexpected attrs(1): %#v", m)
+	}
+
+	// Retrieve attributes for profile #2.
+	if m, err := s.Attrs(2); err != nil {
+		t.Fatal(err)
+	} else if !reflect.DeepEqual(m, map[string]interface{}{"A": uint64(200)}) {
+		t.Fatalf("unexpected attrs(2): %#v", m)
+	}
+}
+
+// Ensure database returns a non-nil empty map if unset.
+func TestAttrStore_Attrs_Empty(t *testing.T) {
+	s := MustOpenAttrStore()
+	defer s.Close()
+
+	if m, err := s.Attrs(100); err != nil {
+		t.Fatal(err)
+	} else if m == nil || len(m) > 0 {
+		t.Fatalf("unexpected attrs: %#v", m)
+	}
+}
+
+// Ensure database can unset attributes if explicitly set to nil.
+func TestAttrStore_Attrs_Unset(t *testing.T) {
+	s := MustOpenAttrStore()
+	defer s.Close()
+
+	// Set attributes.
+	if err := s.SetAttrs(1, map[string]interface{}{"A": "X", "B": "Y"}); err != nil {
+		t.Fatal(err)
+	} else if err := s.SetAttrs(1, map[string]interface{}{"B": nil}); err != nil {
+		t.Fatal(err)
+	}
+
+	// Verify attributes.
+	if m, err := s.Attrs(1); err != nil {
+		t.Fatal(err)
+	} else if !reflect.DeepEqual(m, map[string]interface{}{"A": "X"}) {
+		t.Fatalf("unexpected attrs: %#v", m)
+	}
+}
+
+// Ensure attribute block checksums can be returned.
+func TestAttrStore_Blocks(t *testing.T) {
+	s := MustOpenAttrStore()
+	defer s.Close()
+
+	// Set attributes.
+	if err := s.SetAttrs(1, map[string]interface{}{"A": uint64(100)}); err != nil {
+		t.Fatal(err)
+	} else if err := s.SetAttrs(2, map[string]interface{}{"A": uint64(200)}); err != nil {
+		t.Fatal(err)
+	} else if err := s.SetAttrs(100, map[string]interface{}{"B": "VALUE"}); err != nil {
+		t.Fatal(err)
+	} else if err := s.SetAttrs(350, map[string]interface{}{"C": "FOO"}); err != nil {
+		t.Fatal(err)
+	}
+
+	// Retrieve blocks.
+	blks0, err := s.Blocks()
+	if err != nil {
+		t.Fatal(err)
+	} else if len(blks0) != 3 || blks0[0].ID != 0 || blks0[1].ID != 1 || blks0[2].ID != 3 {
+		t.Fatalf("unexpected blocks: %#v", blks0)
+	}
+
+	// Change second block.
+	if err := s.SetAttrs(100, map[string]interface{}{"X": 12}); err != nil {
+		t.Fatal(err)
+	}
+
+	// Ensure second block changed.
+	blks1, err := s.Blocks()
+	if err != nil {
+		t.Fatal(err)
+	} else if !reflect.DeepEqual(blks0[0], blks1[0]) {
+		t.Fatalf("block 0 mismatch: %#v != %#v", blks0[0], blks1[0])
+	} else if reflect.DeepEqual(blks0[1], blks1[1]) {
+		t.Fatalf("block 1 match: %#v ", blks0[0])
+	} else if !reflect.DeepEqual(blks0[2], blks1[2]) {
+		t.Fatalf("block 2 mismatch: %#v != %#v", blks0[2], blks1[2])
+	}
+}
+
+// AttrStore represents a test wrapper for pilosa.AttrStore.
+type AttrStore struct {
+	*pilosa.AttrStore
+}
+
+// NewAttrStore returns a new instance of AttrStore.
+func NewAttrStore() *AttrStore {
+	f, err := ioutil.TempFile("", "pilosa-attr-")
+	if err != nil {
+		panic(err)
+	}
+	f.Close()
+	os.Remove(f.Name())
+
+	return &AttrStore{AttrStore: pilosa.NewAttrStore(f.Name())}
+}
+
+// MustOpenAttrStore returns a new, opened attribute store at a temporary path. Panic on error.
+func MustOpenAttrStore() *AttrStore {
+	s := NewAttrStore()
+	if err := s.Open(); err != nil {
+		panic(err)
+	}
+	return s
+}
+
+// Close closes the database and removes the underlying data.
+func (s *AttrStore) Close() error {
+	defer os.RemoveAll(s.Path())
+	return s.AttrStore.Close()
+}

audit.go

@@ -1,13 +0,0 @@
-// Copyright 2022 Molecula Corp. (DBA FeatureBase).
-// SPDX-License-Identifier: Apache-2.0
-package pilosa
-
-import (
-	"github.com/featurebasedb/featurebase/v3/testhook"
-)
-
-var NewAuditor func() testhook.Auditor = NewNopAuditor
-
-func NewNopAuditor() testhook.Auditor {
-	return testhook.NewNopAuditor()
-}

audit_internal_test.go

@@ -1,40 +0,0 @@
-// Copyright 2022 Molecula Corp. (DBA FeatureBase).
-// SPDX-License-Identifier: Apache-2.0
-package pilosa
-
-import (
-	"fmt"
-	"reflect"
-
-	"github.com/featurebasedb/featurebase/v3/testhook"
-)
-
-// These audit hooks are desireable during testing, but not in
-// production.
-type auditorViewHooks struct{}
-type auditorFragmentHooks struct{}
-
-// static type checks
-var _ testhook.RegistryHookLive = &auditorViewHooks{}
-var _ testhook.RegistryHookLive = &auditorFragmentHooks{}
-
-func (*auditorViewHooks) Live(o interface{}, entry *testhook.RegistryEntry) error {
-	if entry != nil && entry.OpenCount != 0 {
-		return fmt.Errorf("view %s still open", o.(*view).name)
-	}
-	return nil
-}
-
-func (*auditorFragmentHooks) Live(o interface{}, entry *testhook.RegistryEntry) error {
-	if entry != nil && entry.OpenCount != 0 {
-		return fmt.Errorf("fragment %s still open", o.(*fragment).path())
-	}
-	return nil
-}
-
-func GetInternalTestHooks() testhook.RegistryHooks {
-	return map[reflect.Type]testhook.RegistryHook{
-		reflect.TypeOf((*view)(nil)):     &auditorViewHooks{},
-		reflect.TypeOf((*fragment)(nil)): &auditorFragmentHooks{},
-	}
-}

audit_test.go

@@ -1,95 +0,0 @@
-// Copyright 2022 Molecula Corp. (DBA FeatureBase).
-// SPDX-License-Identifier: Apache-2.0
-package pilosa_test
-
-import (
-	"fmt"
-	"os"
-	"reflect"
-
-	"github.com/featurebasedb/featurebase/v3"
-	"github.com/featurebasedb/featurebase/v3/testhook"
-)
-
-// AuditLeaksOn is a global switch to turn on resource
-// leak checking at the end of a test run.
-var AuditLeaksOn = true
-
-// for tests, we use a single shared auditor used by all of the holders.
-var globalTestAuditor = testhook.NewVerifyCloseAuditor(testHooks)
-
-// These audit hooks are desireable during testing, but not in
-// production.
-type auditorIndexHooks struct{}
-type auditorFieldHooks struct{}
-type auditorHolderHooks struct{}
-
-// static type checking
-var _ testhook.RegistryHookLive = &auditorIndexHooks{}
-var _ testhook.RegistryHookLive = &auditorFieldHooks{}
-var _ testhook.RegistryHookPostDestroy = &auditorHolderHooks{}
-var _ testhook.RegistryHookLive = &auditorHolderHooks{}
-
-var testHooks = map[reflect.Type]testhook.RegistryHook{
-	reflect.TypeOf((*pilosa.Index)(nil)):  &auditorIndexHooks{},
-	reflect.TypeOf((*pilosa.Field)(nil)):  &auditorFieldHooks{},
-	reflect.TypeOf((*pilosa.Holder)(nil)): &auditorHolderHooks{},
-}
-
-func init() {
-	if !AuditLeaksOn {
-		return
-	}
-	for k, v := range pilosa.GetInternalTestHooks() {
-		testHooks[k] = v
-	}
-	testhook.RegisterPreTestHook(func() error {
-		pilosa.NewAuditor = NewTestAuditor
-		return nil
-	})
-	testhook.RegisterPostTestHook(func() error {
-		err, errs := globalTestAuditor.FinalCheck()
-		if err != nil {
-			for i, e := range errs {
-				fmt.Fprintf(os.Stderr, "[%d]: %v\n", i, e)
-			}
-		}
-		return err
-	})
-}
-
-func NewTestAuditor() testhook.Auditor {
-	return globalTestAuditor
-}
-
-func (*auditorIndexHooks) Live(o interface{}, entry *testhook.RegistryEntry) error {
-	if entry != nil && entry.OpenCount != 0 {
-		return fmt.Errorf("index %s still open", o.(*pilosa.Index).Name())
-	}
-	return nil
-}
-
-func (*auditorFieldHooks) Live(o interface{}, entry *testhook.RegistryEntry) error {
-	if entry != nil && entry.OpenCount != 0 {
-		return fmt.Errorf("field %s still open", o.(*pilosa.Field).Name())
-	}
-	return nil
-}
-
-func (*auditorHolderHooks) WasDestroyed(o interface{}, kv testhook.KV, ent *testhook.RegistryEntry, err error) error {
-	path := o.(*pilosa.Holder).Path()
-	if path == "" {
-		fmt.Fprintf(os.Stderr, "OOPS: trying to destroy a holder with no path! created: %s\n",
-			ent.Stack)
-	} else {
-		os.RemoveAll(o.(*pilosa.Holder).Path())
-	}
-	return err
-}
-
-func (*auditorHolderHooks) Live(o interface{}, entry *testhook.RegistryEntry) error {
-	if entry != nil && entry.OpenCount != 0 {
-		return fmt.Errorf("holder %s still open", o.(*pilosa.Holder).Path())
-	}
-	return nil
-}

authn/authenticate.go

@@ -1,445 +0,0 @@
-// Copyright 2022 Molecula Corp. (DBA FeatureBase).
-// SPDX-License-Identifier: Apache-2.0
-
-// Package authn handles authentication
-package authn
-
-import (
-	"context"
-	"encoding/hex"
-	"encoding/json"
-	"fmt"
-	"net"
-	"net/http"
-	"net/url"
-	"strconv"
-	"strings"
-	"time"
-
-	"github.com/golang-jwt/jwt"
-	"google.golang.org/grpc"
-	"google.golang.org/grpc/metadata"
-
-	"github.com/featurebasedb/featurebase/v3/logger"
-	"github.com/pkg/errors"
-	"golang.org/x/oauth2"
-)
-
-// AuthContextKey is a unique type to prevent collisions when using context.WithValue()
-type AuthContextKey string
-
-const (
-	// AccessCookieName is the name of the cookie that holds the access token.
-	AccessCookieName = "molecula-chip"
-
-	// RefreshCookieName is the name of the cookie that holds the refresh token.
-	RefreshCookieName = "refresh-molecula-chip"
-
-	// RefreshHeaderName is the name of the header that holds the refresh token.
-	RefreshHeaderName = "X-Molecula-Refresh-Token"
-
-	// ContextValueAccessToken is the key used to set AccessTokens in a ctx.
-	ContextValueAccessToken = AuthContextKey("Access")
-
-	// ContextValueRefreshToken is the key used to set RefreshTokens in a ctx.
-	ContextValueRefreshToken = AuthContextKey("Refresh")
-)
-
-// cachedGroups is used to hold groups and when they were last cached
-type cachedGroups struct {
-	cacheTime time.Time
-	groups    []Group
-}
-
-// UserInfo holds the information about the user from the token
-type UserInfo struct {
-	UserID       string    `json:"userid"`
-	UserName     string    `json:"username"`
-	Groups       []Group   `json:"groups"`
-	Expiry       time.Time `json:"expiry"`
-	Token        string    `json:"token"`
-	RefreshToken string    `json:"refreshtoken"`
-}
-
-// Group holds group information for an authenticated user
-type Group struct {
-	GroupID   string `json:"id"`
-	GroupName string `json:"displayName"`
-}
-
-// Groups holds a slice of Group for marshalling from JSON
-type Groups struct {
-	NextLink string  `json:"@odata.nextLink"`
-	Groups   []Group `json:"value"`
-}
-
-// Auth holds state, configuration, and utilities needed for authentication.
-type Auth struct {
-	logger            logger.Logger
-	accessCookieName  string
-	refreshCookieName string
-	secretKey         []byte
-	groupEndpoint     string
-	logoutEndpoint    string
-	fbURL             string // fbURL is the domain featurebase is hosted on, used for post logout redirection
-	oAuthConfig       *oauth2.Config
-	cacheTTL          time.Duration           // cacheTTL is used to determine if a cached item should be refreshed or not
-	groupsCache       map[string]cachedGroups // groupsCache is a map of accessToken -> group memberships
-	lastCacheClean    time.Time               // last cache clean is the time that the cache was last cleaned
-	allowedNetworks   []net.IPNet             // list of allowed networks for ingest
-}
-
-// NewAuth instantiates and returns a new Auth struct
-func NewAuth(logger logger.Logger, url string, scopes []string, authURL, tokenURL, groupEndpoint, logout, clientID, clientSecret, secretKey string, configuredIPs []string) (auth *Auth, err error) {
-	auth = &Auth{
-		logger:            logger,
-		accessCookieName:  AccessCookieName,
-		refreshCookieName: RefreshCookieName,
-		groupEndpoint:     groupEndpoint,
-		logoutEndpoint:    logout,
-		fbURL:             url,
-		oAuthConfig: &oauth2.Config{
-			RedirectURL:  fmt.Sprintf("%s/redirect", url),
-			ClientID:     clientID,
-			ClientSecret: clientSecret,
-			Scopes:       scopes,
-			Endpoint: oauth2.Endpoint{
-				AuthURL:  authURL,
-				TokenURL: tokenURL,
-			},
-		},
-		groupsCache:    map[string]cachedGroups{},
-		cacheTTL:       10 * time.Minute,
-		lastCacheClean: time.Now(),
-	}
-
-	if auth.secretKey, err = decodeHex(secretKey); err != nil {
-		return nil, errors.Wrap(err, "decoding secret key")
-	}
-
-	// convert IPs and add them to allowed networks
-	err = auth.convertIP(configuredIPs)
-	if err != nil {
-		return nil, err
-	}
-
-	return auth, nil
-}
-
-// CleanOAuthConfig returns a's oauthConfig without the client secret
-func (a Auth) CleanOAuthConfig() oauth2.Config {
-	b := *a.oAuthConfig
-	b.ClientSecret = ""
-	return b
-}
-
-// SecretKey is a convenient function to get the SecretKey from an Auth struct
-func (a Auth) SecretKey() []byte {
-	return a.secretKey
-}
-
-// refreshToken refreshes a given access/refresh token pair
-func (a *Auth) refreshToken(access, refresh string) (string, string, error) {
-	resp, err := http.PostForm(a.oAuthConfig.Endpoint.TokenURL,
-		url.Values{
-			"grant_type":    {"refresh_token"},
-			"refresh_token": {refresh},
-			"client_id":     {a.oAuthConfig.ClientID},
-			"client_secret": {a.oAuthConfig.ClientSecret},
-		},
-	)
-	if err != nil {
-		return "", "", errors.Wrap(err, "refreshing token")
-	}
-
-	if resp.StatusCode != http.StatusOK {
-		return "", "", fmt.Errorf("refreshing token: %s", resp.Status)
-	}
-
-	defer resp.Body.Close()
-
-	var t oauth2.Token
-	if err := json.NewDecoder(resp.Body).Decode(&t); err != nil {
-		return "", "", errors.Wrap(err, "decoding refreshed token")
-	}
-
-	// remove the old groups from the groups cache
-	delete(a.groupsCache, access)
-
-	return t.AccessToken, t.RefreshToken, nil
-}
-
-// Authenticate takes in a auth token `access` and returns UserInfo from that token
-// it is caller's responsibility to inform the user that the access token has been refreshed
-func (a *Auth) Authenticate(access, refresh string) (*UserInfo, error) {
-	// clean up the cache every 30 minutes or so
-	if time.Since(a.lastCacheClean) >= 30*time.Minute {
-		a.cleanCache()
-	}
-
-	if len(access) == 0 {
-		return nil, fmt.Errorf("auth token is empty")
-	}
-
-	// NOTE: we are using ParseUnverified here because the IDP validates the
-	// token's signature when we get the user's groups, we just need to make
-	// sure it's not expired and is well-formed
-	token, _, err := new(jwt.Parser).ParseUnverified(access, &jwt.MapClaims{})
-	// well-formed-ness check
-	if token == nil || token.Claims == nil || err != nil {
-		return nil, fmt.Errorf("parsing auth token: %v", err)
-	}
-
-	claims := *token.Claims.(*jwt.MapClaims)
-
-	// expiry check
-	if exp, ok := claims["exp"]; ok {
-		var expiry int64
-		switch v := exp.(type) {
-		case string:
-			expiry, err = strconv.ParseInt(v, 10, 64)
-			if err != nil {
-				return nil, fmt.Errorf("parsing exp string: %v", err)
-			}
-		case float64:
-			expiry = int64(v)
-		case int64:
-			expiry = v
-		}
-
-		if expiry < time.Now().UTC().Unix() {
-			access, refresh, err = a.refreshToken(access, refresh)
-			if err != nil {
-				return nil, fmt.Errorf("token is expired: %w", err)
-			}
-		}
-	}
-
-	userInfo := UserInfo{
-		Token:        access,
-		RefreshToken: refresh,
-		Groups:       []Group{},
-	}
-
-	if uid, ok := claims["oid"].(string); ok {
-		userInfo.UserID = uid
-	}
-	if name, ok := claims["name"].(string); ok {
-		userInfo.UserName = name
-	}
-
-	if userInfo.Groups, err = a.getGroups(access); err != nil {
-		return nil, errors.Wrap(err, "getting groups")
-	}
-
-	return &userInfo, nil
-}
-
-// cleanCache removes old items from our cache
-func (a *Auth) cleanCache() {
-	for access, tkn := range a.groupsCache {
-		// if it's been more than 24 hours since the groups were cached
-		if time.Since(tkn.cacheTime) >= 24*time.Hour {
-			// remove it from our cache
-			delete(a.groupsCache, access)
-		}
-	}
-	a.lastCacheClean = time.Now()
-}
-
-// Login redirects a user to login to their configured oAuth authorize endpoint
-func (a *Auth) Login(w http.ResponseWriter, r *http.Request) {
-	authURL := a.oAuthConfig.AuthCodeURL(a.oAuthConfig.Endpoint.AuthURL)
-	http.Redirect(w, r, authURL, http.StatusTemporaryRedirect)
-}
-
-// Logout clears out the user's cookie, removes the token from our cache, and
-// redirects user to IdP's logout endpoint
-func (a *Auth) Logout(w http.ResponseWriter, r *http.Request) {
-	// remove the access token from a.groupsCache
-	if access, err := r.Cookie(a.accessCookieName); err == nil {
-		delete(a.groupsCache, access.Value)
-	}
-	// clear cookie
-	http.SetCookie(w, &http.Cookie{
-		Name:     a.accessCookieName,
-		Value:    "",
-		Path:     "/",
-		Secure:   true,
-		HttpOnly: true,
-		SameSite: http.SameSiteStrictMode,
-		Expires:  time.Unix(0, 0),
-	})
-	http.SetCookie(w, &http.Cookie{
-		Name:     a.refreshCookieName,
-		Value:    "",
-		Path:     "/",
-		Secure:   true,
-		HttpOnly: true,
-		SameSite: http.SameSiteStrictMode,
-		Expires:  time.Unix(0, 0),
-	})
-
-	http.Redirect(w, r, fmt.Sprintf("%s?post_logout_redirect_uri=%s/", a.logoutEndpoint, a.fbURL), http.StatusTemporaryRedirect)
-}
-
-// Redirect handles the oAuth /redirect endpoint. It gets an access token and
-// returns it to the user in the form of a cookie
-func (a *Auth) Redirect(w http.ResponseWriter, r *http.Request) {
-	token, err := a.oAuthConfig.Exchange(r.Context(), r.FormValue("code"), oauth2.AccessTypeOffline)
-	if err != nil {
-		a.logger.Warnf("getting token from IdP: %+v", err)
-		http.Error(w, "Bad Request", http.StatusBadRequest)
-		return
-	}
-
-	a.SetCookie(w, token.AccessToken, token.RefreshToken, token.Expiry)
-	http.Redirect(w, r, "/", http.StatusTemporaryRedirect)
-}
-
-// getGroups gets the group membership for a given token from configured IdP
-func (a *Auth) getGroups(token string) ([]Group, error) {
-	var groups Groups
-
-	gc, ok := a.groupsCache[token]
-	if ok && (time.Since(gc.cacheTime) < a.cacheTTL) && len(gc.groups) > 0 {
-		return gc.groups, nil
-	}
-
-	nextLink := a.groupEndpoint
-	for nextLink != "" {
-		req, err := http.NewRequest("GET", nextLink, nil)
-		if err != nil {
-			return nil, errors.Wrap(err, "creating new request to group endpoint")
-		}
-
-		req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", token))
-		response, err := http.DefaultClient.Do(req)
-		if err != nil {
-			return nil, errors.Wrap(err, "getting group membership info")
-		}
-		if response.StatusCode != http.StatusOK {
-			return nil, fmt.Errorf("getting group membership info: %s", response.Status)
-		}
-
-		var g Groups
-		if err = json.NewDecoder(response.Body).Decode(&g); err != nil {
-			return groups.Groups, errors.Wrap(err, "failed unmarshalling group membership response")
-		}
-
-		response.Body.Close()
-		groups.Groups = append(groups.Groups, g.Groups...)
-		nextLink = g.NextLink
-	}
-
-	if len(groups.Groups) == 0 {
-		return nil, fmt.Errorf("no groups found")
-	}
-
-	a.groupsCache[token] = cachedGroups{
-		cacheTime: time.Now(),
-		groups:    groups.Groups,
-	}
-	return groups.Groups, nil
-}
-
-func (a *Auth) SetCookie(w http.ResponseWriter, access, refresh string, expiry time.Time) error {
-	http.SetCookie(w, &http.Cookie{
-		Name:     a.refreshCookieName,
-		Value:    refresh,
-		Path:     "/",
-		Secure:   true,
-		HttpOnly: true,
-		SameSite: http.SameSiteStrictMode,
-		Expires:  expiry,
-	})
-
-	http.SetCookie(w, &http.Cookie{
-		Name:     a.accessCookieName,
-		Value:    access,
-		Path:     "/",
-		Secure:   true,
-		HttpOnly: true,
-		SameSite: http.SameSiteStrictMode,
-		Expires:  expiry,
-	})
-	return nil
-}
-
-func (a *Auth) SetGRPCMetadata(ctx context.Context, md metadata.MD, access, refresh string) (context.Context, error) {
-	mCookies := map[string]string{}
-	if c, ok := md["cookie"]; ok {
-		for _, cookie := range c {
-			name, val := parseCookie(cookie)
-			mCookies[name] = val
-		}
-	}
-
-	mCookies[a.accessCookieName] = access
-	mCookies[a.refreshCookieName] = refresh
-
-	cookies := []string{}
-	for name, val := range mCookies {
-		cookies = append(cookies, name+"="+val)
-	}
-
-	md["cookie"] = cookies
-	return metadata.NewIncomingContext(ctx, md), grpc.SetHeader(ctx, md)
-}
-
-func decodeHex(hexstr string) ([]byte, error) {
-	data, err := hex.DecodeString(hexstr)
-	if err != nil {
-		return nil, errors.Wrap(err, "decoding hex string to byte slice")
-	}
-	if len(data) != 32 {
-		return nil, fmt.Errorf("invalid key length")
-	}
-	return data, nil
-}
-
-func (a *Auth) convertIP(configuredIPs []string) error {
-	sz := len(configuredIPs)
-	nets := make([]net.IPNet, sz)
-	for i, ip := range configuredIPs {
-		// skip empty strings
-		if ip == "" {
-			sz--
-			continue
-		}
-		// for IPs passed without a subnet, append /32 to only allow 1 IP
-		// this step is needed because ParseCIDR method assumes a CIDR address
-		if !strings.Contains(ip, "/") {
-			ip = ip + "/32"
-		}
-		_, subnet, err := net.ParseCIDR(ip)
-		if err != nil {
-			return errors.Wrapf(err, "parsing CIDR for %v", ip)
-		}
-		nets[i] = *subnet
-	}
-	a.allowedNetworks = nets[:sz]
-	return nil
-}
-
-// if IP is in allowed networks, then return true to grant admin permissions
-func (a *Auth) CheckAllowedNetworks(clientIP string) bool {
-	clientIP = strings.Split(clientIP, ":")[0]
-	convertedIP := net.ParseIP(clientIP)
-	for _, network := range a.allowedNetworks {
-		if network.Contains(convertedIP) {
-			return true
-		}
-	}
-	return false
-}
-
-func parseCookie(cookie string) (name, data string) {
-	vals := strings.Split(cookie, "=")
-	if len(vals) == 0 {
-		vals = []string{"", ""}
-	} else if len(vals) < 2 {
-		vals = append(vals, "")
-	}
-	return vals[0], vals[1]
-}

authn/authenticate_internal_test.go

@@ -1,760 +0,0 @@
-package authn
-
-import (
-	"bytes"
-	"context"
-	"encoding/hex"
-	"encoding/json"
-	"fmt"
-	"net"
-	"net/http"
-	"net/http/httptest"
-	"os"
-	"reflect"
-	"strconv"
-	"strings"
-	"testing"
-	"time"
-
-	"github.com/featurebasedb/featurebase/v3/logger"
-	"github.com/golang-jwt/jwt"
-	"google.golang.org/grpc"
-	"google.golang.org/grpc/metadata"
-)
-
-func NewTestAuth(t *testing.T) *Auth {
-	t.Helper()
-	var (
-		ClientID         = "e9088663-eb08-41d7-8f65-efb5f54bbb71"
-		ClientSecret     = "DEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEF"
-		AuthorizeURL     = "https://login.microsoftonline.com/4a137d66-d161-4ae4-b1e6-07e9920874b8/oauth2/v2.0/authorize"
-		TokenURL         = "https://login.microsoftonline.com/4a137d66-d161-4ae4-b1e6-07e9920874b8/oauth2/v2.0/token"
-		GroupEndpointURL = "https://graph.microsoft.com/v1.0/me/transitiveMemberOf/microsoft.graph.group?$count=true"
-		LogoutURL        = "https://login.microsoftonline.com/common/oauth2/v2.0/logout"
-		Scopes           = []string{"https://graph.microsoft.com/.default", "offline_access"}
-		Key              = "DEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEF"
-		configuredIPs    = []string{}
-	)
-
-	a, err := NewAuth(
-		logger.NewStandardLogger(os.Stdout),
-		"http://localhost:10101/",
-		Scopes,
-		AuthorizeURL,
-		TokenURL,
-		GroupEndpointURL,
-		LogoutURL,
-		ClientID,
-		ClientSecret,
-		Key,
-		configuredIPs,
-	)
-	if err != nil {
-		t.Fatalf("building auth object%s", err)
-	}
-	return a
-}
-
-func TestSetGRPCMetadata(t *testing.T) {
-	a := NewTestAuth(t)
-	for name, md := range map[string]metadata.MD{
-		"empty":     {},
-		"something": {"cookie": []string{a.accessCookieName + "=something"}},
-		"somethingElse": {"cookie": []string{
-			a.accessCookieName + "=something",
-			a.refreshCookieName + "=something",
-		}},
-		"otherCookies": {"cookie": []string{a.accessCookieName + "=something", "blah=blah"}},
-	} {
-		t.Run(name, func(t *testing.T) {
-			ogCookies := md["cookie"]
-			ctx := grpc.NewContextWithServerTransportStream(
-				metadata.NewIncomingContext(context.TODO(),
-					md,
-				),
-				NewServerTransportStream(),
-			)
-			md, ok := metadata.FromIncomingContext(ctx)
-			if !ok {
-				t.Fatalf("expected ok, got: %v", ok)
-			}
-			ctx, err := a.SetGRPCMetadata(ctx, md, "accesstoken!", "refreshtoken!")
-			if err != nil {
-				t.Fatalf("expected no errors, got: %v", err)
-			}
-			if err := grpc.SendHeader(ctx, md); err != nil {
-				t.Fatalf("expected no errors, got: %v", err)
-			}
-			md, ok = metadata.FromIncomingContext(ctx)
-			if !ok {
-				t.Fatalf("expected ok, got: %v", ok)
-			}
-			c, ok := md["cookie"]
-			if !ok {
-				t.Fatalf("expected ok, got: %v", ok)
-			}
-			var accessCookie, refreshCookie string
-			for _, cookie := range c {
-				if strings.HasPrefix(cookie, a.accessCookieName) {
-					accessCookie = cookie
-				} else if strings.HasPrefix(cookie, a.refreshCookieName) {
-					refreshCookie = cookie
-				}
-				if refreshCookie != "" && accessCookie != "" {
-					break
-				}
-			}
-
-			exp := a.accessCookieName + "=accesstoken!"
-			if accessCookie != exp {
-				t.Fatalf("expected '%v', got '%v'", exp, accessCookie)
-			}
-			exp = a.refreshCookieName + "=refreshtoken!"
-			if refreshCookie != exp {
-				t.Fatalf("expected '%v', got '%v'", exp, refreshCookie)
-			}
-
-			for _, cookie := range c {
-				if strings.HasPrefix(cookie, a.accessCookieName) || strings.HasPrefix(cookie, a.refreshCookieName) {
-					continue
-				}
-				found := false
-
-				for _, ogCookie := range ogCookies {
-					if cookie == ogCookie {
-						found = true
-						break
-					}
-				}
-				if !found {
-					t.Fatal("SetGRPCMetadata did not maintain the previous cookie list")
-				}
-			}
-		})
-	}
-}
-
-func TestAuth(t *testing.T) {
-	a := NewTestAuth(t)
-	t.Run("SetCookie", func(t *testing.T) {
-		w := httptest.NewRecorder()
-		err := a.SetCookie(w, "access", "refresh", time.Now().Add(time.Hour))
-		if err != nil {
-			t.Fatalf("expected no errors, got: %v", err)
-		}
-
-		if w.Result().Cookies()[0].Value == "" {
-			t.Errorf("expected something, got empty string")
-		}
-
-		if got, want := w.Result().Cookies()[0].Path, "/"; got != want {
-			t.Fatalf("path=%s, want %s", got, want)
-		}
-	})
-
-	t.Run("KeyLength", func(t *testing.T) {
-		_, err := NewAuth(
-			logger.NewStandardLogger(os.Stdout),
-			"http://localhost:10101/",
-			[]string{"https://graph.microsoft.com/.default", "offline_access"},
-			"https://login.microsoftonline.com/4a137d66-d161-4ae4-b1e6-07e9920874b8/oauth2/v2.0/authorize",
-			"https://login.microsoftonline.com/4a137d66-d161-4ae4-b1e6-07e9920874b8/oauth2/v2.0/token",
-			"https://graph.microsoft.com/v1.0/me/transitiveMemberOf/microsoft.graph.group?$count=true",
-			"https://login.microsoftonline.com/common/oauth2/v2.0/logout",
-			"e9088663-eb08-41d7-8f65-efb5f54bbb71",
-			"DEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEF",
-			"DEADBEEFD",
-			[]string{},
-		)
-		if err == nil || !strings.Contains(err.Error(), "decoding secret key") {
-			t.Fatalf("expected error decoding secret key got: %v", err)
-		}
-	})
-	t.Run("GetSecretKey", func(t *testing.T) {
-		want, _ := hex.DecodeString("DEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEF")
-		if got := a.SecretKey(); !bytes.Equal(got, want) {
-			t.Fatalf("expected %v, got %v", got, want)
-		}
-	})
-}
-
-func TestAuthenticate(t *testing.T) {
-	cases := []struct {
-		name         string
-		uid          string
-		uname        string
-		exp          int64
-		refresh      bool
-		refreshToken string
-		malformed    bool
-		empty        bool
-		groups       []Group
-		err          error
-	}{
-		{
-			name:  "GoodToken",
-			uid:   "42",
-			uname: "A. Token",
-			groups: []Group{
-				{
-					GroupID:   "ac97c9e2-346b-42a2-b6da-18bcb61a32fe",
-					GroupName: "adminGroup",
-				},
-			},
-		},
-		{
-			name:      "Malformed",
-			malformed: true,
-			err:       fmt.Errorf("parsing auth token: token contains an invalid number of segments"),
-		},
-		{
-			name:  "Empty",
-			empty: true,
-			err:   fmt.Errorf("auth token is empty"),
-		},
-		{
-			name:  "ExpiredTokenNoRefresh",
-			uid:   "42",
-			uname: "A. Token",
-			groups: []Group{
-				{
-					GroupID:   "ac97c9e2-346b-42a2-b6da-18bcb61a32fe",
-					GroupName: "adminGroup",
-				},
-			},
-			exp: -17764800,
-			err: fmt.Errorf("token is expired: refreshing token: 400 Bad Request"),
-		},
-		{
-			name:  "ExpiredTokenYesRefresh",
-			uid:   "42",
-			uname: "A. Token",
-			groups: []Group{
-				{
-					GroupID:   "ac97c9e2-346b-42a2-b6da-18bcb61a32fe",
-					GroupName: "adminGroup",
-				},
-			},
-			refresh:      true,
-			refreshToken: "refreshToken",
-			exp:          -17764800,
-		},
-		{
-			name:  "ExpiredTokenYesRefreshButError",
-			uid:   "42",
-			uname: "A. Token",
-			groups: []Group{
-				{
-					GroupID:   "ac97c9e2-346b-42a2-b6da-18bcb61a32fe",
-					GroupName: "adminGroup",
-				},
-			},
-			refresh:      true,
-			refreshToken: "blah!!",
-			exp:          -17764800,
-			err:          fmt.Errorf("token is expired: refreshing token: 403 Forbidden"),
-		},
-	}
-	for _, test := range cases {
-		t.Run(test.name, func(t *testing.T) {
-			// setup the test
-			a := NewTestAuth(t)
-			token := ""
-			var err error
-			if !test.malformed && !test.empty {
-				tkn := jwt.New(jwt.SigningMethodHS256)
-				claims := tkn.Claims.(jwt.MapClaims)
-				claims["oid"] = test.uid
-				claims["name"] = test.uname
-				if test.exp != 0 {
-					claims["exp"] = float64(test.exp)
-				}
-				token, err = tkn.SignedString(a.SecretKey())
-				if err != nil {
-					t.Fatalf("unexpected error when signing token %v", err)
-				}
-			} else if !test.empty {
-				token = "asdfasdfasdfasdF"
-			}
-			if len(test.groups) > 0 {
-				a.groupsCache[token] = cachedGroups{time.Now(), test.groups}
-			}
-			if test.refresh {
-				srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-					if err := r.ParseForm(); err != nil {
-						t.Fatalf("unexpected error: %v", err)
-					}
-					refresh := r.Form.Get("refresh_token")
-					if refresh != test.refreshToken {
-						t.Fatalf("refresh token not passed properly, expected %v, got %v", test.refreshToken, refresh)
-						return
-					}
-					if refresh != "refreshToken" {
-						http.Error(w, "bad token", http.StatusForbidden)
-					}
-
-					tkn := jwt.New(jwt.SigningMethodHS256)
-					claims := tkn.Claims.(jwt.MapClaims)
-					claims["oid"] = test.uid
-					claims["name"] = test.uname
-					expiry := float64(time.Now().Add(2 * time.Hour).Unix())
-					claims["exp"] = expiry
-					fresh, err := tkn.SignedString(a.SecretKey())
-					if err != nil {
-						t.Fatalf("unexpected error when signing token %v", err)
-					}
-
-					a.groupsCache[fresh] = cachedGroups{time.Now(), test.groups}
-					fmt.Fprintf(w, `{"access_token": "`+fresh+`",  "refresh_token": "blah",  "token_type": "bearer",  "expires": `+strconv.FormatFloat(expiry, 'f', 0, 64)+` }`)
-				}))
-				defer srv.Close()
-				a.oAuthConfig.Endpoint.TokenURL = srv.URL
-			}
-
-			// do the actual testing
-			uinfo, err := a.Authenticate(token, test.refreshToken)
-			// okay this part kind of sucks bc we need to check errors and i
-			// dont want to write a whole new test for things that should have
-			// errors just to avoid this mess. errors.Is doesn't work either
-			if (test.err == nil && err != nil) || (test.err != nil && err == nil) {
-				t.Fatalf("expected %v, but got %v", test.err, err)
-			} else if test.err != nil && err != nil {
-				if test.err.Error() != err.Error() {
-					t.Fatalf("expected %v, but got %v", test.err, err)
-				} else {
-					return
-				}
-			}
-
-			if !reflect.DeepEqual(uinfo.Groups, test.groups) {
-				t.Fatalf("expected %v, got %v", test.groups, uinfo.Groups)
-			}
-			if !reflect.DeepEqual(uinfo.UserID, test.uid) {
-				t.Fatalf("expected %v, got %v", test.uid, uinfo.UserID)
-			}
-			if !reflect.DeepEqual(uinfo.UserName, test.uname) {
-				t.Fatalf("expected %v, got %v", test.uname, uinfo.UserName)
-			}
-		})
-	}
-}
-
-func TestAuthenticate_CleanCache(t *testing.T) {
-	// this deserves its own test bc it has gross setup required
-	t.Run("should clean", func(t *testing.T) {
-		a := NewTestAuth(t)
-		now := time.Now()
-		a.groupsCache["oldy"] = cachedGroups{now.Add(-24 * time.Hour), []Group{}}
-		a.groupsCache["goldy"] = cachedGroups{now.Add(-4 * time.Hour), []Group{}}
-		a.lastCacheClean = now.Add(-45 * time.Minute)
-
-		_, _ = a.Authenticate("this doesn't matter", "this doesn't matter?")
-		if a.lastCacheClean.Sub(now) <= time.Nanosecond {
-			t.Fatalf("cache should have been cleaned")
-		}
-		if _, ok := a.groupsCache["oldy"]; ok {
-			t.Errorf("oldy should have been deleted")
-		}
-		if _, ok := a.groupsCache["goldy"]; !ok {
-			t.Errorf("goldy should not have been deleted")
-		}
-	})
-	t.Run("shouldn't clean", func(t *testing.T) {
-		a := NewTestAuth(t)
-		now := time.Now()
-		a.groupsCache["oldy"] = cachedGroups{now.Add(-24 * time.Hour), []Group{}}
-		a.groupsCache["goldy"] = cachedGroups{now.Add(-4 * time.Hour), []Group{}}
-		a.lastCacheClean = now
-
-		_, _ = a.Authenticate("this doesn't matter", "this doesn't matter?")
-		if a.lastCacheClean.Sub(now) >= time.Nanosecond {
-			t.Fatalf("cache should not have been cleaned")
-		}
-		if _, ok := a.groupsCache["oldy"]; !ok {
-			t.Errorf("oldy should not have been deleted")
-		}
-		if _, ok := a.groupsCache["goldy"]; !ok {
-			t.Errorf("goldy should not have been deleted")
-		}
-	})
-}
-
-func TestGetGroups(t *testing.T) {
-	a := NewTestAuth(t)
-	a.groupsCache = map[string]cachedGroups{
-		"the world is changed": {
-			cacheTime: time.Now(),
-			groups: []Group{
-				{
-					GroupID:   "a han noston ned wilith",
-					GroupName: "I smell it in the air",
-				},
-			},
-		},
-	}
-	srvNext := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		body, err := json.Marshal(
-			Groups{
-				Groups: []Group{
-					{
-						GroupID:   "han mathon ne chae",
-						GroupName: "I feel it in the earth",
-					},
-				},
-			},
-		)
-		if err != nil {
-			t.Fatalf("unexpected error marshalling groups response: %v", err)
-		}
-		fmt.Fprintf(w, "%s", body)
-	}))
-	defer srvNext.Close()
-	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		body, err := json.Marshal(
-			Groups{
-				NextLink: srvNext.URL,
-				Groups: []Group{
-					{
-						GroupID:   "han mathon ne nen",
-						GroupName: "i feel it in the water",
-					},
-				},
-			},
-		)
-		if err != nil {
-			t.Fatalf("unexpected error marshalling groups response: %v", err)
-		}
-		fmt.Fprintf(w, "%s", body)
-	}))
-	defer srv.Close()
-	a.groupEndpoint = srv.URL
-
-	for name, test := range map[string]struct {
-		token  string
-		groups []Group
-	}{
-		"InCache": {
-			token: "the world is changed",
-			groups: []Group{
-				{
-					GroupID:   "a han noston ned wilith",
-					GroupName: "I smell it in the air",
-				},
-			},
-		},
-		"NotInCache": {
-			token: "i smell it in the air",
-			groups: []Group{
-				{
-					GroupID:   "han mathon ne nen",
-					GroupName: "i feel it in the water",
-				},
-				{
-					GroupID:   "han mathon ne chae",
-					GroupName: "I feel it in the earth",
-				},
-			},
-		},
-	} {
-		t.Run(name, func(t *testing.T) {
-			if got, err := a.getGroups(test.token); err != nil || !reflect.DeepEqual(got, test.groups) {
-				t.Errorf("expected %v, nil, got %v, %v", test.groups, got, err)
-			}
-		})
-	}
-}
-
-func TestDecodeHex(t *testing.T) {
-	t.Run("cantDecode", func(t *testing.T) {
-		_, err := decodeHex("gggg")
-		if err == nil {
-			t.Fatalf("expected err cannot decode slice, got nil")
-		}
-	})
-	t.Run("tooSmall", func(t *testing.T) {
-		_, err := decodeHex("DEADBEEF")
-		if err == nil {
-			t.Fatalf("expected err wrong length, got nil")
-		}
-	})
-	t.Run("tooBig", func(t *testing.T) {
-		_, err := decodeHex("DEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEF")
-		if err == nil {
-			t.Fatalf("expected err wrong length, got nil")
-		}
-	})
-	t.Run("justRight", func(t *testing.T) {
-		_, err := decodeHex("DEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEF")
-		if err != nil {
-			t.Fatalf("expected nil, got %v", err)
-		}
-	})
-}
-
-func TestHandlers(t *testing.T) {
-	a := NewTestAuth(t)
-	t.Run("login", func(t *testing.T) {
-		req := httptest.NewRequest("GET", "/login", nil)
-		w := httptest.NewRecorder()
-		a.Login(w, req)
-		resp := w.Result()
-		if resp.StatusCode != http.StatusTemporaryRedirect {
-			t.Fatalf("expected redirect, got %v", resp.StatusCode)
-		}
-		redirect := a.oAuthConfig.AuthCodeURL(a.oAuthConfig.Endpoint.AuthURL)
-		if got, err := resp.Location(); err != nil || got.String() != redirect {
-			t.Fatalf("expected %v, got %v", redirect, got.Path)
-		}
-	})
-	t.Run("logout", func(t *testing.T) {
-		req := httptest.NewRequest("GET", "/logout", nil)
-		w := httptest.NewRecorder()
-		req.AddCookie(
-			&http.Cookie{
-				Name:     a.accessCookieName,
-				Value:    "test",
-				Path:     "/",
-				Secure:   true,
-				HttpOnly: true,
-				SameSite: http.SameSiteStrictMode,
-				Expires:  time.Unix(3000000, 0),
-			},
-		)
-
-		req.AddCookie(
-			&http.Cookie{
-				Name:     a.refreshCookieName,
-				Value:    "test",
-				Path:     "/",
-				Secure:   true,
-				HttpOnly: true,
-				SameSite: http.SameSiteStrictMode,
-				Expires:  time.Unix(3000000, 0),
-			},
-		)
-
-		a.groupsCache["test"] = cachedGroups{}
-		a.Logout(w, req)
-		resp := w.Result()
-		if resp.StatusCode != http.StatusTemporaryRedirect {
-			t.Fatalf("expected redirect, got %v", resp.StatusCode)
-		}
-		redirect := fmt.Sprintf("%s?post_logout_redirect_uri=%s/", a.logoutEndpoint, a.fbURL)
-		if got, err := resp.Location(); err != nil || got.String() != redirect {
-			t.Fatalf("expected %v, got %v", redirect, got.Path)
-		}
-		for _, c := range resp.Cookies() {
-			if c.Name == a.accessCookieName || c.Name == a.refreshCookieName {
-				if c.Value != "" {
-					t.Fatalf("cookie not set to empty value!")
-				}
-				want := time.Unix(0, 0).Unix()
-				got := c.Expires.Unix()
-				if want != got {
-					t.Fatalf("expected %v, got %v", want, got)
-				}
-			}
-		}
-		if _, ok := a.groupsCache["test"]; ok {
-			t.Fatalf("groups not deleted!")
-		}
-	})
-	t.Run("redirectGood", func(t *testing.T) {
-		req := httptest.NewRequest("GET", "/redirect", nil)
-		w := httptest.NewRecorder()
-		tkn := jwt.New(jwt.SigningMethodHS256)
-		claims := tkn.Claims.(jwt.MapClaims)
-		claims["oid"] = "user id"
-		claims["name"] = "user name"
-		expiresIn := 2 * time.Hour
-		exp := time.Now().Add(expiresIn)
-		expiry := float64(exp.Unix())
-		claims["exp"] = expiry
-		fresh, err := tkn.SignedString(a.SecretKey())
-		if err != nil {
-			t.Fatalf("unexpected error when signing token %v", err)
-		}
-
-		srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-			body := `{"access_token": "` + fresh + `", "refresh_token": "blah", "expires_in": "` + strconv.Itoa(int(expiresIn.Seconds())) + `"}`
-			w.Header().Set("Content-Type", "application/json; charset=utf-8")
-			w.WriteHeader(http.StatusOK)
-			w.Write([]byte(body))
-		}))
-		a.oAuthConfig.Endpoint.TokenURL = srv.URL
-		a.Redirect(w, req)
-		resp := w.Result()
-		if resp.StatusCode != http.StatusTemporaryRedirect {
-			t.Fatalf("expected redirect, got %v", resp.StatusCode)
-		}
-		if got, err := resp.Location(); err != nil || got.String() != "/" {
-			t.Fatalf("expected %v, got %v", "/", got.Path)
-		}
-		cookies := resp.Cookies()
-		for _, c := range cookies {
-			if c.Name == a.accessCookieName && c.Value != fresh {
-				t.Fatalf("expected %v, got %v", exp, c.Value)
-			} else if c.Name == a.refreshCookieName && c.Value != "blah" {
-				t.Fatalf("expected %v, got %v", "blah", c.Value)
-			}
-		}
-	})
-
-	t.Run("redirectBad", func(t *testing.T) {
-		req := httptest.NewRequest("GET", "/redirect", nil)
-		w := httptest.NewRecorder()
-		srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-			http.Error(w, "Server Error", http.StatusInternalServerError)
-		}))
-		a.oAuthConfig.Endpoint.TokenURL = srv.URL
-		a.Redirect(w, req)
-		resp := w.Result()
-		if resp.StatusCode != http.StatusBadRequest {
-			t.Fatalf("expected BadRequest, got %v", resp.StatusCode)
-		}
-	})
-}
-
-// This type is used for mocking ServerTransportStreams in tests
-type ServerTransportStream struct {
-	md     metadata.MD
-	method string
-}
-
-func NewServerTransportStream() *ServerTransportStream {
-	return &ServerTransportStream{
-		md:     metadata.MD{},
-		method: "test",
-	}
-}
-
-func (s *ServerTransportStream) Method() string {
-	return s.method
-}
-
-func (s *ServerTransportStream) SetHeader(md metadata.MD) error {
-	s.md = md
-	return nil
-}
-
-func (s *ServerTransportStream) SendHeader(md metadata.MD) error {
-	_ = md
-	return nil
-}
-
-func (s *ServerTransportStream) SetTrailer(md metadata.MD) error {
-	_ = md
-	return nil
-}
-
-func TestCleanOAuthConfig(t *testing.T) {
-	a := NewTestAuth(t)
-	res := a.CleanOAuthConfig()
-	assertEqual("", res.ClientSecret, t)
-	assertEqual(a.oAuthConfig.ClientID, res.ClientID, t)
-	assertEqual(a.oAuthConfig.RedirectURL, res.RedirectURL, t)
-	assertEqual(a.oAuthConfig.Scopes, res.Scopes, t)
-	assertEqual(a.oAuthConfig.Endpoint, res.Endpoint, t)
-}
-
-func assertEqual(exp, got interface{}, t *testing.T) {
-	if !reflect.DeepEqual(exp, got) {
-		t.Fatalf("expected %v, got %v", exp, got)
-	}
-}
-
-func TestCheckAllowedNetworks(t *testing.T) {
-	tests := []struct {
-		requestIP     string
-		configuredIPs []string
-		isAdmin       bool
-	}{
-		{
-			requestIP:     "10.0.0.1",
-			configuredIPs: []string{"10.0.0.1"},
-			isAdmin:       true,
-		},
-		{
-			requestIP:     "10.0.0.3",
-			configuredIPs: []string{"10.0.0.1", "10.0.0.2"},
-			isAdmin:       false,
-		},
-		{
-			requestIP:     "10.0.0.2",
-			configuredIPs: []string{"10.0.0.1/30"},
-			isAdmin:       true,
-		},
-		// it is possible for the client IP to have a port
-		{
-			requestIP:     "10.0.0.2:22",
-			configuredIPs: []string{"10.0.0.1/30"},
-			isAdmin:       true,
-		},
-		{
-			requestIP:     "10.1.0.3",
-			configuredIPs: []string{"10.0.0.1/32"},
-			isAdmin:       false,
-		},
-		{
-			requestIP:     "10.0.0.254",
-			configuredIPs: []string{"10.0.0.1/24"},
-			isAdmin:       true,
-		},
-	}
-
-	for i, test := range tests {
-		t.Run(fmt.Sprintf("network-%d", i), func(t *testing.T) {
-			a := NewTestAuth(t)
-			if err := a.convertIP(test.configuredIPs); err != nil {
-				t.Fatalf("failed to convert IPs from strings to net.IP: %v", err)
-			}
-			got := a.CheckAllowedNetworks(test.requestIP)
-			if got != test.isAdmin {
-				t.Fatalf("expected %v, got %v", test.isAdmin, got)
-			}
-		})
-	}
-}
-
-func TestConvertIP(t *testing.T) {
-	tests := []struct {
-		configuredIPs []string
-		convertedIPs  []net.IPNet
-	}{
-		{
-			configuredIPs: []string{"10.0.0.1"},
-			convertedIPs: []net.IPNet{
-				{IP: net.ParseIP("10.0.0.1"), Mask: net.CIDRMask(32, 32)},
-			},
-		},
-		{
-			configuredIPs: []string{"10.0.0.1/30"},
-			convertedIPs: []net.IPNet{
-				{IP: net.ParseIP("10.0.0.0"), Mask: net.CIDRMask(30, 32)},
-			},
-		},
-	}
-
-	for i, test := range tests {
-		t.Run(fmt.Sprintf("network-%d", i), func(t *testing.T) {
-			a := NewTestAuth(t)
-			if err := a.convertIP(test.configuredIPs); err != nil {
-				t.Fatalf("failed to convert IPs from strings to net.IP: %v", err)
-			}
-
-			if len(a.allowedNetworks) != len(test.convertedIPs) {
-				t.Fatalf("expected len of %v networks, got %v", len(test.convertedIPs), len(a.allowedNetworks))
-			}
-
-			for i := range a.allowedNetworks {
-				expected, got := test.convertedIPs[i], a.allowedNetworks[i]
-				if got.IP.String() != expected.IP.String() {
-					t.Fatalf("for IP, expected %v, got %v", expected.IP, got.IP)
-				}
-				if got.Mask.String() != expected.Mask.String() {
-					t.Fatalf("for mask, expected %v, got %v", expected.Mask.String(), got.Mask.String())
-				}
-			}
-		})
-	}
-}

authn/context.go

@@ -1,54 +0,0 @@
-// Copyright 2022 Molecula Corp (DBA FeatureBase). All rights reserved.
-package authn
-
-import "context"
-
-// Empty struct to avoid allocations
-type contextKeyAccessToken struct{}
-type contextKeyRefreshToken struct{}
-type contextKeyUserInfo struct{}
-type contextKeyIndexes struct{}
-
-// GetAccessToken gets the access token from a context.
-func GetAccessToken(ctx context.Context) (token string, ok bool) {
-	token, ok = ctx.Value(contextKeyAccessToken{}).(string)
-	return
-}
-
-// WithAccessToken makes a new Context with an access token.
-func WithAccessToken(ctx context.Context, token string) context.Context {
-	return context.WithValue(ctx, contextKeyAccessToken{}, token)
-}
-
-// GetRefreshToken gets the refresh token from a context.
-func GetRefreshToken(ctx context.Context) (token string, ok bool) {
-	token, ok = ctx.Value(contextKeyRefreshToken{}).(string)
-	return
-}
-
-// WithRefreshToken makes a new Context with a refresh token.
-func WithRefreshToken(ctx context.Context, token string) context.Context {
-	return context.WithValue(ctx, contextKeyRefreshToken{}, token)
-}
-
-// GetUserInfo gets the UserInfo from a context.
-func GetUserInfo(ctx context.Context) (userInfo *UserInfo, ok bool) {
-	userInfo, ok = ctx.Value(contextKeyUserInfo{}).(*UserInfo)
-	return
-}
-
-// WithUserInfo makes a new Context with UserInfo.
-func WithUserInfo(ctx context.Context, userInfo *UserInfo) context.Context {
-	return context.WithValue(ctx, contextKeyUserInfo{}, userInfo)
-}
-
-// GetIndexes get the indices from a context.
-func GetIndexes(ctx context.Context) (indexes []string, ok bool) {
-	indexes, ok = ctx.Value(contextKeyIndexes{}).([]string)
-	return
-}
-
-// WithIndexes makes a new Context with a []string containing the indicies.
-func WithIndexes(ctx context.Context, indexes []string) context.Context {
-	return context.WithValue(ctx, contextKeyUserInfo{}, indexes)
-}

authz/authorization.go

@@ -1,130 +0,0 @@
-// Copyright 2022 Molecula Corp. (DBA FeatureBase).
-// SPDX-License-Identifier: Apache-2.0
-
-package authz
-
-import (
-	"fmt"
-	"io"
-
-	"github.com/featurebasedb/featurebase/v3/authn"
-
-	"gopkg.in/yaml.v2"
-)
-
-type GroupPermissions struct {
-	Permissions map[string]map[string]Permission `yaml:"user-groups"`
-	Admin       string                           `yaml:"admin"`
-}
-
-type Permission string
-
-const (
-	None  Permission = ""
-	Read  Permission = "read"
-	Write Permission = "write"
-	Admin Permission = "admin"
-)
-
-// Satisfies returns whether `p` satisfies the permissions required by `b`
-func (p Permission) Satisfies(b Permission) bool {
-	switch p {
-	case "":
-		return b == ""
-	case "read":
-		return b == "" || b == "read"
-	case "write":
-		return b == "" || b == "read" || b == "write"
-	case "admin":
-		return b == "" || b == "read" || b == "write" || b == "admin"
-	}
-	return false
-}
-
-func (p *GroupPermissions) ReadPermissionsFile(permsFile io.Reader) (err error) {
-	permsData, err := io.ReadAll(permsFile)
-
-	if err != nil {
-		return fmt.Errorf("reading permissions failed with error: %s", err)
-	}
-
-	err = yaml.UnmarshalStrict(permsData, &p)
-	if err != nil {
-		return fmt.Errorf("unmarshalling permissions failed with error: %s", err)
-	}
-
-	return
-}
-
-func (p *GroupPermissions) GetPermissions(user *authn.UserInfo, index string) (permission Permission, errors error) {
-	groups := user.Groups
-	if admin := p.IsAdmin(groups); admin {
-		return Admin, nil
-	}
-
-	allPermissions := map[Permission]bool{
-		Write: false,
-		Read:  false,
-	}
-
-	if len(groups) == 0 {
-		return None, fmt.Errorf("user is not part of any groups in identity provider")
-	}
-
-	var groupsDenied []string
-	for _, group := range groups {
-		if _, ok := p.Permissions[group.GroupID]; ok {
-			if perm, ok := p.Permissions[group.GroupID][index]; ok {
-				allPermissions[perm] = true
-			} else {
-				return None, fmt.Errorf("user %s does not have permission to index %s", user.UserID, index)
-			}
-		} else {
-			groupsDenied = append(groupsDenied, group.GroupID)
-		}
-	}
-
-	if len(groupsDenied) == len(groups) {
-		return None, fmt.Errorf("group(s) %s does not have permission to FeatureBase", groupsDenied)
-	}
-
-	if allPermissions[Write] {
-		return Write, nil
-	} else if allPermissions[Read] {
-		return Read, nil
-	} else {
-		return None, fmt.Errorf("no permissions found")
-	}
-}
-
-func (p *GroupPermissions) IsAdmin(groups []authn.Group) bool {
-	for _, group := range groups {
-		if p.Admin == group.GroupID {
-			return true
-		}
-	}
-	return false
-}
-
-func (p *GroupPermissions) GetAuthorizedIndexList(groups []authn.Group, desiredPermission Permission) (indexList []string) {
-	// if user is admin, find all indexes in permissions file and return them
-	if p.IsAdmin(groups) {
-		for groupId := range p.Permissions {
-			for index := range p.Permissions[groupId] {
-				indexList = append(indexList, index)
-			}
-		}
-		return indexList
-	}
-
-	for _, group := range groups {
-		if _, ok := p.Permissions[group.GroupID]; ok {
-			for index, permission := range p.Permissions[group.GroupID] {
-				if permission.Satisfies(desiredPermission) {
-					indexList = append(indexList, index)
-				}
-			}
-		}
-	}
-	return indexList
-}

authz/authorization_test.go

@@ -1,305 +0,0 @@
-// Copyright 2022 Molecula Corp. (DBA FeatureBase).
-// SPDX-License-Identifier: Apache-2.0
-package authz_test
-
-import (
-	"fmt"
-	"reflect"
-	"sort"
-	"strings"
-	"testing"
-
-	"github.com/featurebasedb/featurebase/v3/authn"
-	"github.com/featurebasedb/featurebase/v3/authz"
-)
-
-func TestAuth_ReadPermissionsFile(t *testing.T) {
-
-	singleInput := `user-groups:
-  "dca35310-ecda-4f23-86cd-876aee55906b":
-    "test": "read"
-admin: "ac97c9e2-346b-42a2-b6da-18bcb61a32fe"`
-
-	multiInput := `user-groups:
-  "dca35310-ecda-4f23-86cd-876aee55906b":
-    "test": "read"
-    "test2": "write"
-  "dca35310-ecda-4f23-86cd-876aee559900":
-    "test": "write"
-admin: "ac97c9e2-346b-42a2-b6da-18bcb61a32fe"`
-
-	singlePermission := authz.GroupPermissions{
-		Permissions: map[string]map[string]authz.Permission{
-			"dca35310-ecda-4f23-86cd-876aee55906b": {"test": authz.Read},
-		},
-		Admin: "ac97c9e2-346b-42a2-b6da-18bcb61a32fe",
-	}
-
-	multiPermission := authz.GroupPermissions{
-		Permissions: map[string]map[string]authz.Permission{
-			"dca35310-ecda-4f23-86cd-876aee55906b": {"test": authz.Read, "test2": authz.Write},
-			"dca35310-ecda-4f23-86cd-876aee559900": {"test": authz.Write}},
-		Admin: "ac97c9e2-346b-42a2-b6da-18bcb61a32fe",
-	}
-
-	tests := []struct {
-		input  string
-		output authz.GroupPermissions
-	}{
-		{singleInput, singlePermission},
-		{multiInput, multiPermission},
-	}
-
-	for i, test := range tests {
-		t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
-			permFile := strings.NewReader(test.input)
-
-			var p authz.GroupPermissions
-			err := p.ReadPermissionsFile(permFile)
-			if err != nil {
-				t.Fatalf("readPermissionsFile error: %s", err)
-			}
-
-			if !reflect.DeepEqual(p, test.output) {
-				t.Fatalf("expected output %s, but got %s", test.output, p)
-			}
-		},
-		)
-	}
-}
-
-func TestAuth_GetPermissions(t *testing.T) {
-
-	// initializes different example of permissions file in yaml
-	permissions1 := `"user-groups":
-  "dca35310-ecda-4f23-86cd-876aee55906b":
-    "test": "read"
-admin: "ac97c9e2-346b-42a2-b6da-18bcb61a32fe"`
-
-	permissions2 := `"user-groups":
-  "dca35310-ecda-4f23-86cd-876aee559900":
-    "test": "write"
-admin: "ac97c9e2-346b-42a2-b6da-18bcb61a32fe"`
-
-	permissions3 := `"user-groups":
-  "dca35310-ecda-4f23-86cd-876aee55906b":
-    "test": "write"
-    "test2": "read"
-  "dca35310-ecda-4f23-86cd-876aee559900":
-    "test": "read"
-admin: "ac97c9e2-346b-42a2-b6da-18bcb61a32fe"`
-
-	permissions4 := `"user-groups":
-  "dca35310-ecda-4f23-86cd-876aee559900":
-    "test": ""
-admin: "ac97c9e2-346b-42a2-b6da-18bcb61a32fe"`
-
-	// initializes groups that are returned from identity provider
-	groupName := "name"
-	groupsList1 := []authn.Group{}
-	groupsList2 := []authn.Group{{
-		GroupID:   "fake-group",
-		GroupName: groupName}}
-	groupsList3 := []authn.Group{
-		{GroupID: "dca35310-ecda-4f23-86cd-876aee55906b", GroupName: groupName},
-		{GroupID: "dca35310-ecda-4f23-86cd-876aee559900", GroupName: groupName},
-	}
-	groupsList4 := []authn.Group{{GroupID: "ac97c9e2-346b-42a2-b6da-18bcb61a32fe", GroupName: groupName}}
-
-	tests := []struct {
-		yamlData   string
-		groups     []authn.Group
-		index      string
-		userAccess authz.Permission
-		err        string
-	}{
-		{
-			permissions1,
-			groupsList1,
-			"test",
-			authz.None,
-			"user is not part of any groups in identity provider",
-		},
-		{
-			permissions1,
-			groupsList3,
-			"test1",
-			authz.None,
-			"does not have permission to index",
-		},
-		{
-			permissions2,
-			groupsList2,
-			"test",
-			authz.None,
-			"does not have permission to FeatureBase",
-		},
-		{
-			permissions1,
-			groupsList3,
-			"test",
-			authz.Read,
-			"",
-		},
-		{
-			permissions2,
-			groupsList3,
-			"test",
-			authz.Write,
-			"",
-		},
-		{
-			permissions3,
-			groupsList4,
-			"test",
-			authz.Admin,
-			"",
-		},
-		{
-			permissions4,
-			groupsList3,
-			"test",
-			authz.None,
-			"no permissions found",
-		},
-	}
-
-	for i, test := range tests {
-		t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
-
-			permFile := strings.NewReader(test.yamlData)
-
-			var p authz.GroupPermissions
-			if err := p.ReadPermissionsFile(permFile); err != nil {
-				t.Errorf("Error: %s", err)
-			}
-
-			p1, err := p.GetPermissions(&authn.UserInfo{Groups: test.groups}, test.index)
-
-			if p1 != test.userAccess {
-				t.Errorf("expected permission to be %s, but got %s", test.userAccess, p1)
-			}
-
-			if err != nil {
-				if !strings.Contains(err.Error(), test.err) {
-					t.Errorf("expected error to contain %s, but got %s", test.err, err.Error())
-				}
-			}
-
-		})
-	}
-}
-
-func TestAuth_IsAdmin(t *testing.T) {
-
-	group1 := []authn.Group{
-		{GroupID: "ac97c9e2-346b-42a2-b6da-18bcb61a32fe", GroupName: "admin-group"},
-	}
-
-	group2 := []authn.Group{
-		{GroupID: "dca35310-ecda-4f23-86cd-876aee55906b", GroupName: "group-name"},
-	}
-
-	groupPermissions := authz.GroupPermissions{
-		Permissions: map[string]map[string]authz.Permission{
-			"dca35310-ecda-4f23-86cd-876aee55906b": {"test": authz.Write},
-		},
-		Admin: "ac97c9e2-346b-42a2-b6da-18bcb61a32fe",
-	}
-
-	tests := []struct {
-		groups           []authn.Group
-		groupPermissions authz.GroupPermissions
-		output           bool
-	}{
-		{
-			group1, groupPermissions, true,
-		},
-		{
-			group2, groupPermissions, false,
-		},
-	}
-
-	for i, test := range tests {
-		t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
-			p := test.groupPermissions
-			resp := p.IsAdmin(test.groups)
-			if resp != test.output {
-				t.Errorf("expected %t, but got %t", test.output, resp)
-			}
-		})
-	}
-}
-
-func TestAuth_GetAuthorizedIndexList(t *testing.T) {
-
-	group1 := []authn.Group{
-		{GroupID: "dca35310-ecda-4f23-86cd-876aee55906b", GroupName: "group-name"},
-	}
-
-	group2 := []authn.Group{
-		{GroupID: "ac97c9e2-346b-42a2-b6da-18bcb61a32fe", GroupName: "admin-group"},
-	}
-
-	group3 := []authn.Group{
-		{GroupID: "dca35310-ecda-4f23-86cd-876aee559900", GroupName: "group-name"},
-	}
-
-	p := authz.GroupPermissions{
-		Permissions: map[string]map[string]authz.Permission{
-			"dca35310-ecda-4f23-86cd-876aee55906b": {
-				"test1": authz.Read,
-				"test2": authz.Write,
-			},
-			"dca35310-ecda-4f23-86cd-876aee559900": {
-				"test3": authz.Read,
-			},
-		},
-		Admin: "ac97c9e2-346b-42a2-b6da-18bcb61a32fe",
-	}
-
-	tests := []struct {
-		groups     []authn.Group
-		permission authz.Permission
-		output     []string
-	}{
-		{
-			group1,
-			authz.Read,
-			[]string{"test1", "test2"},
-		},
-		{
-			group1,
-			authz.Write,
-			[]string{"test2"},
-		},
-		{
-			group3,
-			authz.Write,
-			nil,
-		},
-		{
-			group2,
-			authz.Read,
-			[]string{"test1", "test2", "test3"},
-		},
-		{
-			group2,
-			authz.Write,
-			[]string{"test1", "test2", "test3"},
-		},
-	}
-
-	for i, test := range tests {
-		t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
-
-			indexList := p.GetAuthorizedIndexList(test.groups, test.permission)
-			sort.Strings(indexList)
-
-			if !reflect.DeepEqual(indexList, test.output) {
-				t.Errorf("expected %s, but got %s", test.output, indexList)
-			}
-		})
-	}
-
-}

batch/Dockerfile-test

@@ -1,11 +0,0 @@
-ARG GO_VERSION=1.19
-
-FROM golang:${GO_VERSION}
-
-WORKDIR /go/src/github.com/featurebasedb/featurebase/
-
-COPY . .
-
-WORKDIR /go/src/github.com/featurebasedb/featurebase/batch/
-
-CMD ["go","test","-v","-mod=vendor","-tags=odbc,dynamic","./..."]

batch/Dockerfile-wait

@@ -1,8 +0,0 @@
-FROM ubuntu:18.04
-
-RUN ["apt-get", "update", "-y"]
-RUN ["apt-get", "install", "-y", "curl", "netcat"]
-
-ADD wait.sh /wait
-
-ENTRYPOINT ["/wait"]

batch/Makefile

@@ -1,44 +0,0 @@
-GO ?= go
-
-# We allow setting a custom docker-compose "project". Multiple of the
-# same docker-compose environment can exist simultaneously as long as
-# they use different projects (the project name is prepended to
-# container names and such). This is useful in a CI environment where
-# we might be running multiple instances of the tests concurrently.
-PROJECT ?= batch
-DOCKER_COMPOSE = docker-compose -p $(PROJECT)
-
-vendor: ../go.mod
-	$(GO) mod vendor
-
-build-%:
-	$(DOCKER_COMPOSE) build $*
-
-test-all:
-	$(MAKE) startup
-	$(MAKE) test-run
-	$(MAKE) shutdown
-
-start-all: build-wait
-	$(DOCKER_COMPOSE) up -d featurebase
-	$(DOCKER_COMPOSE) run -T wait featurebase curl --silent --fail http://featurebase:10101/status
-
-startup: start-all
-
-shutdown:
-	$(DOCKER_COMPOSE) down -v --remove-orphans
-
-save-%-logs:
-	$(DOCKER_COMPOSE) logs $* > ./testdata/$(PROJECT)_$*_logs.txt
-
-TCMD ?= ./...
-# do "make startup", then e.g. "make test-run-local TCMD='-run=MyFavTest ./kafka'"
-test-run-local: vendor
-	pwd
-	$(DOCKER_COMPOSE) build batch-test
-	$(DOCKER_COMPOSE) run -T batch-test go test -mod=vendor -tags=odbc,dynamic $(TCMD)
-
-TPKG ?= ../...
-test-run: vendor
-	$(DOCKER_COMPOSE) build batch-test
-	$(DOCKER_COMPOSE) run -T batch-test bash -c "set -o pipefail; go test -v -mod=vendor -tags=odbc,dynamic ./... -covermode=atomic -coverpkg=$(TPKG) -coverprofile=/testdata/$(PROJECT)_base_coverage.out"

batch/README.md

@@ -1,46 +0,0 @@
-# batch
-
-The `batch` package provides a standard tool set for batching records in a way
-that is most performant for ingesting those records into FeatureBase. The main
-implementation is `Batch` (which can be initated with the `NewBatch()`
-function). The `NewBatch()` function takes an `Importer` which contains all of
- the methods required to interact with FeatureBase; these include methods for
- doing string/id translation as well as for importing shards of data.
-
- IDK uses the `batch` package internally. Another example where the `batch`
- package is used in the `sql3` package. When an "INSERT INTO" statement is
- executed, the SQL engine uses a `Batch` to do key translation and build import
- batches prior to doing the final import.
-## Integration tests
-
-To run the tests, you will need to install the following dependencies:
-
-1. [Docker](https://docs.docker.com/install/)
-2. [Docker Compose](https://docs.docker.com/compose/install/)
-
-In addition to these dependancies, you will need to be added to the molecula [Gitlab](https://registry.gitlab.com/molecula) account.
-
-First start the test environment. This is a docker-compose environment that includes featurebase.
-
-    make startup
-
-To build and run the integration tests, run:
-
-    make test-run-local
-
-Then to shut down the test environment, run:
-
-    make shutdown
-
-The previous command is equivalent to running the following:
-
-    make startup
-    sleep 30 # wait for services to come up
-    make test-run
-    make shutdown
-
-To run an individual test, you can run the command directly using docker-compose. Note that you must run `docker-compose build batch-test` for docker to run the latest code. Modify the following as needed:
-
-    make startup
-    docker-compose build batch-test
-    docker-compose run batch-test /usr/local/go/bin/go test -count=1 -mod=vendor -run=TestCmdMainOne .

batch/batcher.go

@@ -1,20 +0,0 @@
-package batch
-
-import (
-	"time"
-
-	"github.com/featurebasedb/featurebase/v3/dax"
-)
-
-// Batcher is an interface implemented by anything which can allocate new
-// batches.
-type Batcher interface {
-	NewBatch(cfg Config, tbl *dax.Table, fields []*dax.Field) (RecordBatch, error)
-}
-
-// Config is the configuration options passed to NewBatch for any implementation
-// of the Batcher interface.
-type Config struct {
-	Size         int
-	MaxStaleness time.Duration
-}

batch/convert.go

@@ -1,145 +0,0 @@
-package batch
-
-import (
-	"time"
-
-	featurebase "github.com/featurebasedb/featurebase/v3"
-	"github.com/featurebasedb/featurebase/v3/errors"
-)
-
-var (
-	MinTimestampNano = time.Unix(-1<<32, 0).UTC()       // 1833-11-24T17:31:44Z
-	MaxTimestampNano = time.Unix(1<<32, 0).UTC()        // 2106-02-07T06:28:16Z
-	MinTimestamp     = time.Unix(-62135596799, 0).UTC() // 0001-01-01T00:00:01Z
-	MaxTimestamp     = time.Unix(253402300799, 0).UTC() // 9999-12-31T23:59:59Z
-
-	ErrTimestampOutOfRange = errors.New("", "value provided for timestamp field is out of range")
-)
-
-type TimeUnit string
-
-const (
-	TimeUnitSeconds      = TimeUnit(featurebase.TimeUnitSeconds)
-	TimeUnitMilliseconds = TimeUnit(featurebase.TimeUnitMilliseconds)
-	TimeUnitMicroseconds = TimeUnit(featurebase.TimeUnitMicroseconds)
-	TimeUnitUSeconds     = TimeUnit(featurebase.TimeUnitUSeconds)
-	TimeUnitNanoseconds  = TimeUnit(featurebase.TimeUnitNanoseconds)
-)
-
-// TimestampToInt64 converts the provided timestamp to an int64 as the number of
-// units past the epoch.
-func TimestampToInt64(unit TimeUnit, epoch time.Time, ts time.Time) (int64, error) {
-	var err error
-
-	unit, err = validateTimeUnit(unit)
-	if err != nil {
-		return 0, errors.Wrap(err, "validating time unit")
-	}
-
-	epoch, err = validateEpoch(epoch)
-	if err != nil {
-		return 0, errors.Wrap(err, "validating epoch")
-	}
-
-	// Check if the epoch alone is out-of-range. If so, ingest should halt,
-	// regardless of state of the timestamp out-of-range CLI option.
-	if err := validateTimestamp(unit, epoch); err != nil {
-		return 0, errors.Wrap(err, "validating epoch")
-	}
-
-	epochAsInt64 := timestampToInt(unit, epoch)
-
-	// Check if the timestamp is out-of-range.
-	if err := validateTimestamp(unit, ts); err != nil {
-		return 0, errors.Wrapf(ErrTimestampOutOfRange, "validating timestamp: %s", ts)
-	}
-
-	tsAsInt64 := timestampToInt(unit, ts)
-
-	return tsAsInt64 - epochAsInt64, nil
-}
-
-// validateTimeUnit checks if the time unit is supported. If the provided unit
-// is blank, validateTimeUnit returns the default TimeUnit.
-func validateTimeUnit(unit TimeUnit) (TimeUnit, error) {
-	switch unit {
-	case "":
-		return TimeUnitSeconds, nil
-
-	case TimeUnitSeconds,
-		TimeUnitMilliseconds,
-		TimeUnitMicroseconds,
-		TimeUnitUSeconds,
-		TimeUnitNanoseconds:
-		return unit, nil
-	}
-
-	return "", errors.Errorf("unsupported time unit: %s", unit)
-}
-
-// validateEpoch checks if the epoch is supported. If the provided epoch
-// is "zero", validateEpoch returns the default epoch value.
-func validateEpoch(epoch time.Time) (time.Time, error) {
-	if epoch.IsZero() {
-		return time.Unix(0, 0), nil
-	}
-	return epoch, nil
-}
-
-// validateTimestamp checks if the timestamp is within the range of what FB accepts.
-func validateTimestamp(unit TimeUnit, ts time.Time) error {
-	// Min and Max timestamps that Featurebase accepts
-	var minStamp, maxStamp time.Time
-	switch unit {
-	case TimeUnitNanoseconds:
-		minStamp = MinTimestampNano
-		maxStamp = MaxTimestampNano
-	default:
-		minStamp = MinTimestamp
-		maxStamp = MaxTimestamp
-	}
-
-	if ts.Before(minStamp) || ts.After(maxStamp) {
-		return errors.Errorf("timestamp value (%v) must be within min: %v and max: %v", ts, minStamp, maxStamp)
-	}
-	return nil
-}
-
-// timestampToInt takes a time unit and a time.Time and converts it to an
-// integer value.
-func timestampToInt(unit TimeUnit, ts time.Time) int64 {
-	switch unit {
-	case TimeUnitSeconds:
-		return ts.Unix()
-	case TimeUnitMilliseconds:
-		return ts.UnixMilli()
-	case TimeUnitMicroseconds, TimeUnitUSeconds:
-		return ts.UnixMicro()
-	case TimeUnitNanoseconds:
-		return ts.UnixNano()
-	}
-	return 0
-}
-
-// intToTimestamp takes a timeunit and an integer value and converts it to
-// time.Time.
-func intToTimestamp(unit TimeUnit, val int64) (time.Time, error) {
-	switch unit {
-	case TimeUnitSeconds:
-		return time.Unix(val, 0).UTC(), nil
-	case TimeUnitMilliseconds:
-		return time.UnixMilli(val).UTC(), nil
-	case TimeUnitMicroseconds, TimeUnitUSeconds:
-		return time.UnixMicro(val).UTC(), nil
-	case TimeUnitNanoseconds:
-		return time.Unix(0, val).UTC(), nil
-	default:
-		return time.Time{}, errors.Errorf("Unknown time unit: '%v'", unit)
-	}
-}
-
-// Int64ToTimestamp converts the provided int64 to a timestamp based on the time unit
-// and epoch.
-func Int64ToTimestamp(unit TimeUnit, epoch time.Time, val int64) (time.Time, error) {
-	return intToTimestamp(unit, timestampToInt(unit, epoch)+val)
-}

batch/docker-compose.yml

@@ -1,29 +0,0 @@
-version: '3'
-
-services:
-  featurebase:
-    build:
-      context: ../.
-      dockerfile: ./Dockerfile-clustertests
-    environment:
-      PILOSA_DATA_DIR: /data
-      PILOSA_BIND: 0.0.0.0:10101
-      PILOSA_BIND_GRPC: 0.0.0.0:20101
-      PILOSA_ADVERTISE: featurebase:10101
-    command: /featurebase -test.run=TestRunMain -test.coverprofile=/testdata/batch_coverage.out server
-    volumes:
-      - ./testdata:/testdata
-
-  batch-test:
-    build:
-      context: ../.
-      dockerfile: ./batch/Dockerfile-test
-    volumes:
-      - ./testdata:/testdata
-
-  wait:
-    depends_on:
-      - "featurebase"
-    build:
-      context: .
-      dockerfile: Dockerfile-wait

batch/egpool/egpool.go

@@ -1,111 +0,0 @@
-// Copyright 2022 Molecula Corp. (DBA FeatureBase).
-// SPDX-License-Identifier: Apache-2.0
-package egpool
-
-import (
-	"errors"
-	"fmt"
-	"sync"
-)
-
-type Group struct {
-	PoolSize int
-
-	jobs chan func() error
-
-	sema     chan struct{}
-	errMu    sync.Mutex
-	firstErr error
-	errs     []error
-}
-
-func (eg *Group) Go(f func() error) {
-	if eg.PoolSize <= 0 {
-		eg.PoolSize = 1
-	}
-
-	if eg.jobs == nil {
-		eg.jobs = make(chan func() error)
-		eg.sema = make(chan struct{}, eg.PoolSize)
-	}
-
-	// Start the job in an idle worker if possible.
-	select {
-	case eg.jobs <- f:
-		return
-	default:
-	}
-
-	// Start a new worker if necessary.
-	select {
-	case eg.jobs <- f:
-		// A worker finished its previous job and took this one over.
-		return
-	case eg.sema <- struct{}{}:
-		// Start a new worker.
-		go eg.processJobs()
-		eg.jobs <- f
-	}
-}
-
-func (eg *Group) err(err error) {
-	eg.errMu.Lock()
-	defer eg.errMu.Unlock()
-
-	if eg.firstErr == nil {
-		eg.firstErr = err
-	}
-	eg.errs = append(eg.errs, err)
-}
-
-type PanicError struct {
-	Value interface{}
-}
-
-func (p PanicError) Error() string {
-	return fmt.Sprintf("panic: %v", p.Value)
-}
-
-var ErrGoexit = errors.New("runtime.Goexit used in job function")
-
-func (eg *Group) processJobs() {
-	// Notify pool of shutdown.
-	defer func() { <-eg.sema }()
-
-	// Handle panic and Goexit.
-	var finished bool
-	defer func() {
-		if !finished {
-			if p := recover(); p != nil {
-				eg.err(PanicError{p})
-			} else {
-				eg.err(ErrGoexit)
-			}
-		}
-	}()
-
-	// Run jobs from queue.
-	for jobFn := range eg.jobs {
-		err := jobFn()
-		if err != nil {
-			eg.err(err)
-		}
-	}
-
-	finished = true
-}
-
-func (eg *Group) Wait() error {
-	if eg.jobs == nil {
-		return nil
-	}
-	close(eg.jobs)
-	for i := 0; i < eg.PoolSize; i++ {
-		eg.sema <- struct{}{}
-	}
-	return eg.firstErr
-}
-
-func (eg *Group) Errors() []error {
-	return eg.errs
-}

batch/egpool/egpool_test.go

@@ -1,38 +0,0 @@
-// Copyright 2022 Molecula Corp. (DBA FeatureBase).
-// SPDX-License-Identifier: Apache-2.0
-package egpool_test
-
-import (
-	"errors"
-	"testing"
-
-	"github.com/featurebasedb/featurebase/v3/batch/egpool"
-)
-
-func TestEGPool(t *testing.T) {
-	eg := egpool.Group{}
-
-	a := make([]int, 10)
-
-	for i := 0; i < 10; i++ {
-		i := i
-		eg.Go(func() error {
-			a[i] = i
-			if i == 7 {
-				return errors.New("blah")
-			}
-			return nil
-		})
-	}
-
-	err := eg.Wait()
-	if err == nil || err.Error() != "blah" {
-		t.Errorf("expected err blah, got: %v", err)
-	}
-
-	for i := 0; i < 10; i++ {
-		if a[i] != i {
-			t.Errorf("expected a[%d] to be %d, but is %d", i, i, a[i])
-		}
-	}
-}

batch/error.go

@@ -1,8 +0,0 @@
-package batch
-
-import "github.com/pkg/errors"
-
-// Predefined batch-related errors.
-var (
-	ErrPreconditionFailed = errors.New("Precondition failed")
-)

batch/metrics.go

@@ -1,3 +0,0 @@
-// Copyright 2022 Molecula Corp. (DBA FeatureBase).
-// SPDX-License-Identifier: Apache-2.0
-package batch

batch/testdata/README.md

@@ -1,3 +0,0 @@
-# testdata
-
-This directory is used in CI tests. I think.

batch/wait.sh

@@ -1,26 +0,0 @@
-#!/bin/sh
-
-name=$1
-shift
-
-_start_ts=$(date +%s)
-elapsed=0
-timeout=120
-while :
-do
-    $@ > /dev/null
-    _ret=$?
-    _end_ts=$(date +%s)
-    if [ $_ret -eq 0 ]; then
-        echo "$name is available after $((_end_ts - _start_ts)) seconds."
-        break
-    else
-        echo "Waiting for $name after $((_end_ts - _start_ts)) seconds."
-    fi
-    sleep 1s
-    elapsed=$((elapsed+1))
-    if [ $elapsed -ge $timeout ]; then
-	exit 110
-    fi
-done
-set -ex

bench/LICENSE

@@ -0,0 +1,30 @@
+The following applies to the files errgroup.go and errgroup_test.go
+in the same directory this file resides:
+
+Copyright (c) 2009 The Go Authors. All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+   * Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+   * Redistributions in binary form must reproduce the above
+copyright notice, this list of conditions and the following disclaimer
+in the documentation and/or other materials provided with the
+distribution.
+   * Neither the name of Google Inc. nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

bench/bench.go

@@ -0,0 +1,41 @@
+package bench
+
+import "context"
+
+// Benchmark is an interface to guide the creation of new pilosa benchmarks or
+// benchmark components. It defines 2 methods, Init, and Run. These are separate
+// methods so that benchmark running code can time only the running of the
+// benchmark, and not any setup.
+type Benchmark interface {
+	// Init takes a list of hosts and an agent number. It is generally expected
+	// to set up a connection to pilosa using whatever client it chooses. The
+	// agentNum should be used to parameterize the benchmark's configuration if
+	// it is being run simultaneously on multiple "agents". E.G. the agentNum
+	// might be used to make a random seed different for each agent, or have
+	// each agent set a different set of bits. Init's doc string should document
+	// how the agentNum affects it. Every benchmark should have a 'Name' field
+	// set by init, which appears when the benchmark is marshalled to json as
+	// "name".
+	Init(hosts []string, agentNum int) error
+
+	// Run runs the benchmark. The return value of Run is kept generic so that
+	// any relevant statistics or metrics that may be specific to the benchmark
+	// in question can be reported. TODO guidelines for what gets included in
+	// results and what will get added by other stuff. Run does not need to
+	// report total run time in `results`, as that will be added by calling
+	// code.
+	Run(ctx context.Context) map[string]interface{}
+}
+
+// Command extends Benchmark by adding methods for configuring via command line flags and returning usage information.
+type Command interface {
+	Benchmark
+
+	// ConsumeFlags sets and parses flags, and then returns flagSet.Args(). This
+	// is so that multiple benchmarks can be specified at the command line.
+	ConsumeFlags(args []string) ([]string, error)
+
+	// Usage returns information on how to use this benchmark. The usage string
+	// should explain how the agent num affects the benchmark's operation.
+	Usage() string
+}

bench/client.go

@@ -0,0 +1,67 @@
+package bench
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"github.com/pilosa/pilosa"
+)
+
+func firstHostClient(hosts []string) (*pilosa.Client, error) {
+	client, err := pilosa.NewClient(hosts[0])
+	if err != nil {
+		return nil, err
+	}
+	return client, nil
+}
+
+func roundRobinClient(hosts []string, agentNum int) (*pilosa.Client, error) {
+	clientNum := agentNum % len(hosts)
+	return firstHostClient(hosts[clientNum:])
+}
+
+// HasClient provides a reusable component for Benchmark implementations which
+// provides the Init method, a ClientType argument and a cli internal variable.
+type HasClient struct {
+	client      *pilosa.Client
+	ClientType  string `json:"client-type"`
+	ContentType string `json:"content-type"`
+}
+
+// Init for HasClient looks at the ClientType field and creates a pilosa client
+// either using the first host in the list of hosts or based on the agent
+// number mod len(hosts)
+func (h *HasClient) Init(hosts []string, agentNum int) error {
+	var err error
+	switch h.ClientType {
+	case "single":
+		h.client, err = firstHostClient(hosts)
+	case "round_robin":
+		h.client, err = roundRobinClient(hosts, agentNum)
+	default:
+		err = fmt.Errorf("Unsupported ClientType: %v", h.ClientType)
+	}
+	if err != nil {
+		return err
+	}
+
+	switch h.ContentType {
+	case "protobuf":
+		return nil
+	case "pql":
+		return nil
+	default:
+		return fmt.Errorf("Unsupported ContentType: %v", h.ContentType)
+
+	}
+}
+
+func (h *HasClient) ExecuteQuery(contentType, db, query string, ctx context.Context) (interface{}, error) {
+	if contentType == "protobuf" {
+		return h.client.ExecuteQuery(ctx, db, query, true)
+	} else if contentType == "pql" {
+		return h.client.ExecutePQL(ctx, db, query)
+	} else {
+		return nil, errors.New("unsupport content type")
+	}
+}

bench/diagonal.go

@@ -0,0 +1,105 @@
+package bench
+
+import (
+	"fmt"
+
+	"flag"
+	"io/ioutil"
+
+	"context"
+	"time"
+)
+
+// DiagonalSetBits sets bits with increasing profile id and bitmap id.
+type DiagonalSetBits struct {
+	HasClient
+	Name          string `json:"name"`
+	BaseBitmapID  int    `json:"base-bitmap-id"`
+	BaseProfileID int    `json:"base-profile-id"`
+	Iterations    int    `json:"iterations"`
+	DB            string `json:"db"`
+}
+
+// Init sets up the pilosa client and modifies the configured values based on
+// the agent num.
+func (b *DiagonalSetBits) Init(hosts []string, agentNum int) error {
+	b.Name = "diagonal-set-bits"
+	b.BaseBitmapID = b.BaseBitmapID + (agentNum * b.Iterations)
+	b.BaseProfileID = b.BaseProfileID + (agentNum * b.Iterations)
+	return b.HasClient.Init(hosts, agentNum)
+}
+
+// Usage returns the usage message to be printed.
+func (b *DiagonalSetBits) Usage() string {
+	return `
+diagonal-set-bits sets bits with increasing profile id and bitmap id.
+
+Agent num offsets both the base profile id and base bitmap id by the number of
+iterations, so that only bits on the main diagonal are set, and agents don't
+overlap at all.
+
+Usage: diagonal-set-bits [arguments]
+
+The following arguments are available:
+
+	-base-bitmap-id int
+		bits being set will all be greater than BaseBitmapID
+
+	-base-profile-id int
+		profile id num to start from
+
+	-iterations int
+		number of bits to set
+
+	-db string
+		pilosa db to use
+
+	-client-type string
+		Can be 'single' (all agents hitting one host) or 'round_robin'
+
+	-content-type string
+		protobuf or pql
+`[1:]
+}
+
+// ConsumeFlags parses all flags up to the next non flag argument (argument does
+// not start with "-" and isn't the value of a flag). It returns the remaining
+// args.
+func (b *DiagonalSetBits) ConsumeFlags(args []string) ([]string, error) {
+	fs := flag.NewFlagSet("DiagonalSetBits", flag.ContinueOnError)
+	fs.SetOutput(ioutil.Discard)
+	fs.IntVar(&b.BaseBitmapID, "base-bitmap-id", 0, "")
+	fs.IntVar(&b.BaseProfileID, "base-profile-id", 0, "")
+	fs.IntVar(&b.Iterations, "iterations", 100, "")
+	fs.StringVar(&b.DB, "db", "benchdb", "")
+	fs.StringVar(&b.ClientType, "client-type", "single", "")
+	fs.StringVar(&b.ContentType, "content-type", "protobuf", "")
+
+	if err := fs.Parse(args); err != nil {
+		return nil, err
+	}
+	return fs.Args(), nil
+}
+
+// Run runs the DiagonalSetBits benchmark
+func (b *DiagonalSetBits) Run(ctx context.Context) map[string]interface{} {
+	results := make(map[string]interface{})
+	if b.client == nil {
+		results["error"] = fmt.Errorf("No client set for DiagonalSetBits")
+		return results
+	}
+	s := NewStats()
+	var start time.Time
+	for n := 0; n < b.Iterations; n++ {
+		query := fmt.Sprintf("SetBit(%d, 'frame.n', %d)", b.BaseBitmapID+n, b.BaseProfileID+n)
+		start = time.Now()
+		_, err := b.client.ExecuteQuery(ctx, b.DB, query, true)
+		if err != nil {
+			results["error"] = err
+			return results
+		}
+		s.Add(time.Now().Sub(start))
+	}
+	AddToResults(s, results)
+	return results
+}

bench/doc.go

@@ -0,0 +1,39 @@
+// bench contains benchmarks and common utilities useful to benchmarks
+//
+// In order to write new benchmarks, one must satisfy the Benchmark and Command
+// interfaces in bench.go. In order to use the benchmark from pilosactl, it
+// needs to be wired in in two places. The first is BagentCommand.ParseFlags,
+// where a case statement needs to be added, and the second is just adding the
+// benchmark to the BagentCommand.Usage usage string.
+//
+// When writing a new benchmark, there are a few things to keep in mind other
+// than just implementing the interface:
+//
+// 1. The benchmark should modify its own configuration in its Init method based
+// on the agentNum it is given. How it modifies is specific to the benchmark,
+// but the idea is that it should make sense to call the benchmark with the same
+// configuration, but multiple different agent numbers, and it should do useful
+// work each time (i.e. not just setting the same bits, or running the same
+// queries).
+//
+// 2. The Init method should do everything that needs to be done to get the
+// benchmark to a runnable state - all code in run should be the stuff that we
+// actually want to time.
+//
+// 3. The Run method does not need to report the total runtime - that is collected
+// by calling code.
+//
+// 4. Usage should follow the format in other benchmarks, and explain how the
+// benchmark uses agentNum to modify its behavior
+//
+//
+// Files:
+//
+// 1. client.go contains pilosa client code which is shared by many benchmarks
+//
+// 2. errgroup.go contains the ErrGroup implementation copied from golang.org/x/
+// so as not to pull in a bunch of useless deps.
+//
+// 3. stats.go contains useful code for gathering stats about a series of timed
+// operations.
+package bench

bench/errgroup.go

@@ -0,0 +1,67 @@
+// Copyright 2016 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package errgroup provides synchronization, error propagation, and Context
+// cancelation for groups of goroutines working on subtasks of a common task.
+package bench
+
+import (
+	"sync"
+
+	"context"
+)
+
+// An ErrGroup is a collection of goroutines working on subtasks that are part of
+// the same overall task.
+//
+// A zero ErrGroup is valid and does not cancel on error.
+type ErrGroup struct {
+	cancel func()
+
+	wg sync.WaitGroup
+
+	errOnce sync.Once
+	err     error
+}
+
+// WithContext returns a new ErrGroup and an associated Context derived from ctx.
+//
+// The derived Context is canceled the first time a function passed to Go
+// returns a non-nil error or the first time Wait returns, whichever occurs
+// first.
+func WithContext(ctx context.Context) (*ErrGroup, context.Context) {
+	ctx, cancel := context.WithCancel(ctx)
+	return &ErrGroup{cancel: cancel}, ctx
+}
+
+// Wait blocks until all function calls from the Go method have returned, then
+// returns the first non-nil error (if any) from them.
+func (g *ErrGroup) Wait() error {
+	g.wg.Wait()
+	if g.cancel != nil {
+		g.cancel()
+	}
+	return g.err
+}
+
+// Go calls the given function in a new goroutine.
+//
+// The first call to return a non-nil error cancels the group; its error will be
+// returned by Wait.
+func (g *ErrGroup) Go(f func() error) {
+	g.wg.Add(1)
+
+	go func() {
+		defer g.wg.Done()
+
+		if err := f(); err != nil {
+			g.errOnce.Do(func() {
+				g.err = err
+				if g.cancel != nil {
+					g.cancel()
+				}
+			})
+		}
+	}()
+}

bench/errgroup_test.go

@@ -0,0 +1,176 @@
+// Copyright 2016 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package bench_test
+
+import (
+	"errors"
+	"fmt"
+	"net/http"
+	"os"
+	"testing"
+
+	"context"
+	"github.com/pilosa/pilosa/bench"
+)
+
+var (
+	Web   = fakeSearch("web")
+	Image = fakeSearch("image")
+	Video = fakeSearch("video")
+)
+
+type Result string
+type Search func(ctx context.Context, query string) (Result, error)
+
+func fakeSearch(kind string) Search {
+	return func(_ context.Context, query string) (Result, error) {
+		return Result(fmt.Sprintf("%s result for %q", kind, query)), nil
+	}
+}
+
+// JustErrors illustrates the use of a ErrGroup in place of a sync.WaitGroup to
+// simplify goroutine counting and error handling. This example is derived from
+// the sync.WaitGroup example at https://golang.org/pkg/sync/#example_WaitGroup.
+func ExampleGroup_justErrors() {
+	var g bench.ErrGroup
+	var urls = []string{
+		"http://www.golang.org/",
+		"http://www.google.com/",
+		"http://www.somestupidname.com/",
+	}
+	for _, url := range urls {
+		// Launch a goroutine to fetch the URL.
+		url := url // https://golang.org/doc/faq#closures_and_goroutines
+		g.Go(func() error {
+			// Fetch the URL.
+			resp, err := http.Get(url)
+			if err == nil {
+				resp.Body.Close()
+			}
+			return err
+		})
+	}
+	// Wait for all HTTP fetches to complete.
+	if err := g.Wait(); err == nil {
+		fmt.Println("Successfully fetched all URLs.")
+	}
+}
+
+// Parallel illustrates the use of a ErrGroup for synchronizing a simple parallel
+// task: the "Google Search 2.0" function from
+// https://talks.golang.org/2012/concurrency.slide#46, augmented with a Context
+// and error-handling.
+func ExampleGroup_parallel() {
+	Google := func(ctx context.Context, query string) ([]Result, error) {
+		g, ctx := bench.WithContext(ctx)
+
+		searches := []Search{Web, Image, Video}
+		results := make([]Result, len(searches))
+		for i, search := range searches {
+			i, search := i, search // https://golang.org/doc/faq#closures_and_goroutines
+			g.Go(func() error {
+				result, err := search(ctx, query)
+				if err == nil {
+					results[i] = result
+				}
+				return err
+			})
+		}
+		if err := g.Wait(); err != nil {
+			return nil, err
+		}
+		return results, nil
+	}
+
+	results, err := Google(context.Background(), "golang")
+	if err != nil {
+		fmt.Fprintln(os.Stderr, err)
+		return
+	}
+	for _, result := range results {
+		fmt.Println(result)
+	}
+
+	// Output:
+	// web result for "golang"
+	// image result for "golang"
+	// video result for "golang"
+}
+
+func TestZeroGroup(t *testing.T) {
+	err1 := errors.New("errgroup_test: 1")
+	err2 := errors.New("errgroup_test: 2")
+
+	cases := []struct {
+		errs []error
+	}{
+		{errs: []error{}},
+		{errs: []error{nil}},
+		{errs: []error{err1}},
+		{errs: []error{err1, nil}},
+		{errs: []error{err1, nil, err2}},
+	}
+
+	for _, tc := range cases {
+		var g bench.ErrGroup
+
+		var firstErr error
+		for i, err := range tc.errs {
+			err := err
+			g.Go(func() error { return err })
+
+			if firstErr == nil && err != nil {
+				firstErr = err
+			}
+
+			if gErr := g.Wait(); gErr != firstErr {
+				t.Errorf("after %T.Go(func() error { return err }) for err in %v\n"+
+					"g.Wait() = %v; want %v",
+					g, tc.errs[:i+1], err, firstErr)
+			}
+		}
+	}
+}
+
+func TestWithContext(t *testing.T) {
+	errDoom := errors.New("group_test: doomed")
+
+	cases := []struct {
+		errs []error
+		want error
+	}{
+		{want: nil},
+		{errs: []error{nil}, want: nil},
+		{errs: []error{errDoom}, want: errDoom},
+		{errs: []error{errDoom, nil}, want: errDoom},
+	}
+
+	for _, tc := range cases {
+		g, ctx := bench.WithContext(context.Background())
+
+		for _, err := range tc.errs {
+			err := err
+			g.Go(func() error { return err })
+		}
+
+		if err := g.Wait(); err != tc.want {
+			t.Errorf("after %T.Go(func() error { return err }) for err in %v\n"+
+				"g.Wait() = %v; want %v",
+				g, tc.errs, err, tc.want)
+		}
+
+		canceled := false
+		select {
+		case <-ctx.Done():
+			canceled = true
+		default:
+		}
+		if !canceled {
+			t.Errorf("after %T.Go(func() error { return err }) for err in %v\n"+
+				"ctx.Done() was not closed",
+				g, tc.errs)
+		}
+	}
+}

bench/import.go

@@ -0,0 +1,203 @@
+package bench
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"math/rand"
+
+	"sort"
+
+	"github.com/pilosa/pilosa/pilosactl"
+)
+
+// NewImport returns an Import Benchmark which pilosactl importer configured.
+func NewImport(stdin io.Reader, stdout, stderr io.Writer) *Import {
+	return &Import{
+		ImportCommand: pilosactl.NewImportCommand(stdin, stdout, stderr),
+	}
+}
+
+// Import sets bits with increasing profile id and bitmap id.
+type Import struct {
+	Name              string `json:"name"`
+	BaseBitmapID      int64  `json:"base-bitmap-id"`
+	MaxBitmapID       int64  `json:"max-bitmap-id"`
+	BaseProfileID     int64  `json:"base-profile-id"`
+	MaxProfileID      int64  `json:"max-profile-id"`
+	RandomBitmapOrder bool   `json:"random-bitmap-order"`
+	MinBitsPerMap     int64  `json:"min-bits-per-map"`
+	MaxBitsPerMap     int64  `json:"max-bits-per-map"`
+	AgentControls     string `json:"agent-controls"`
+	Seed              int64  `json:"seed"`
+	numbits           int
+
+	*pilosactl.ImportCommand
+}
+
+// Usage returns the usage message to be printed.
+func (b *Import) Usage() string {
+	return `
+import generates an import file and imports using pilosa's bulk import interface
+
+Agent num can have various effects - see -agent-controls flag.
+
+Usage: import [arguments]
+
+The following arguments are available:
+
+	-base-bitmap-id int
+		bits being set will all be greater than this
+
+	-max-bitmap-id int
+		bits being set will all be less than this
+
+	-base-profile-id int
+		profile id num to start from
+
+	-max-profile-id int
+		maximum profile id to generate
+
+	-random-bitmap-order
+		if this option is set, the import file will not be sorted by bitmap id
+
+	-min-bits-per-map int
+		minimum number of bits set per bitmap
+
+	-max-bits-per-map int
+		maximum number of bits set per bitmap
+
+	-agent-controls string
+		can be 'height', 'width', or empty (TODO or square?)- increasing
+		number of agents modulates bitmap id range, profile id range,
+		or just sets more bits in the same range.
+
+	-seed int
+		seed for RNG
+
+	-db string
+		pilosa db to use
+
+	-frame string
+		frame to import into
+`[1:]
+}
+
+// ConsumeFlags parses all flags up to the next non flag argument (argument does
+// not start with "-" and isn't the value of a flag). It returns the remaining
+// args.
+func (b *Import) ConsumeFlags(args []string) ([]string, error) {
+	fs := flag.NewFlagSet("Import", flag.ContinueOnError)
+	fs.SetOutput(ioutil.Discard)
+	fs.Int64Var(&b.BaseBitmapID, "base-bitmap-id", 0, "")
+	fs.Int64Var(&b.MaxBitmapID, "max-bitmap-id", 1000, "")
+	fs.Int64Var(&b.BaseProfileID, "base-profile-id", 0, "")
+	fs.Int64Var(&b.MaxProfileID, "max-profile-id", 1000, "")
+	fs.BoolVar(&b.RandomBitmapOrder, "random-bitmap-order", false, "")
+	fs.Int64Var(&b.MinBitsPerMap, "min-bits-per-map", 0, "")
+	fs.Int64Var(&b.MaxBitsPerMap, "max-bits-per-map", 10, "")
+	fs.StringVar(&b.AgentControls, "agent-controls", "", "")
+	fs.Int64Var(&b.Seed, "seed", 0, "")
+	fs.StringVar(&b.Database, "db", "benchdb", "")
+	fs.StringVar(&b.Frame, "frame", "testframe", "")
+	fs.IntVar(&b.BufferSize, "buffer-size", 10000000, "")
+
+	if err := fs.Parse(args); err != nil {
+		return nil, err
+	}
+	return fs.Args(), nil
+}
+
+// Init generates import data based on the agent num and fields of 'b'.
+func (b *Import) Init(hosts []string, agentNum int) error {
+	if len(hosts) == 0 {
+		return fmt.Errorf("Need at least one host")
+	}
+	b.Name = "import"
+	b.Host = hosts[0]
+	// generate csv data
+	b.Seed = b.Seed + int64(agentNum)
+	switch b.AgentControls {
+	case "height":
+		numBitmapIDs := (b.MaxBitmapID - b.BaseBitmapID)
+		b.BaseBitmapID = b.BaseBitmapID + (numBitmapIDs * int64(agentNum))
+		b.MaxBitmapID = b.BaseBitmapID + numBitmapIDs
+	case "width":
+		numProfileIDs := (b.MaxProfileID - b.BaseProfileID)
+		b.BaseProfileID = b.BaseProfileID + (numProfileIDs * int64(agentNum))
+		b.MaxProfileID = b.BaseProfileID + numProfileIDs
+	case "":
+		break
+	default:
+		return fmt.Errorf("agent-controls: '%v' is not supported", b.AgentControls)
+	}
+	f, err := ioutil.TempFile("", "")
+	if err != nil {
+		return err
+	}
+	// set b.Paths)
+	num := GenerateImportCSV(f, b.BaseBitmapID, b.MaxBitmapID, b.BaseProfileID, b.MaxProfileID,
+		b.MinBitsPerMap, b.MaxBitsPerMap, b.Seed, b.RandomBitmapOrder)
+	b.numbits = num
+	// set b.Paths
+	b.Paths = []string{f.Name()}
+	return f.Close()
+}
+
+// Run runs the Import benchmark
+func (b *Import) Run(ctx context.Context) map[string]interface{} {
+	results := make(map[string]interface{})
+	results["numbits"] = b.numbits
+	results["db"] = b.Database
+	err := b.ImportCommand.Run(ctx)
+
+	if err != nil {
+		results["error"] = err.Error()
+	}
+	return results
+}
+
+// Int64Slice is a sortable slice of 64 bit signed ints
+type Int64Slice []int64
+
+func (s Int64Slice) Len() int           { return len(s) }
+func (s Int64Slice) Less(i, j int) bool { return s[i] < s[j] }
+func (s Int64Slice) Swap(i, j int)      { s[i], s[j] = s[j], s[i] }
+
+// GenerateImportCSV writes a generated csv to 'w' which is in the form pilosactl expects for imports.
+func GenerateImportCSV(w io.Writer, baseBitmapID, maxBitmapID, baseProfileID, maxProfileID, minBitsPerMap, maxBitsPerMap, seed int64, randomOrder bool) int {
+	src := rand.NewSource(seed)
+	rng := rand.New(src)
+
+	var bitmapIDs []int
+	if randomOrder {
+		bitmapIDs = rng.Perm(int(maxBitmapID - baseBitmapID))
+	}
+	numrows := 0
+	profileIDs := make(Int64Slice, maxBitsPerMap)
+	for i := baseBitmapID; i < maxBitmapID; i++ {
+		var bitmapID int64
+		if randomOrder {
+			bitmapID = int64(bitmapIDs[i-baseBitmapID])
+		} else {
+			bitmapID = int64(i)
+		}
+
+		numBitsToSet := rng.Int63n(maxBitsPerMap-minBitsPerMap) + minBitsPerMap
+		numrows += int(numBitsToSet)
+		for j := int64(0); j < numBitsToSet; j++ {
+			profileIDs[j] = rng.Int63n(maxProfileID-baseProfileID) + baseProfileID
+		}
+		profIDs := profileIDs[:numBitsToSet]
+		if !randomOrder {
+			sort.Sort(profIDs)
+		}
+		for j := int64(0); j < numBitsToSet; j++ {
+			fmt.Fprintf(w, "%d,%d\n", bitmapID, profIDs[j])
+		}
+
+	}
+	return numrows
+}

bench/import_test.go

@@ -0,0 +1,136 @@
+package bench_test
+
+import (
+	"bytes"
+	"testing"
+
+	"io/ioutil"
+
+	"os"
+
+	"github.com/pilosa/pilosa/bench"
+)
+
+func TestImportInit(t *testing.T) {
+	imp := bench.NewImport(os.Stdin, os.Stdout, os.Stderr)
+	imp.BaseBitmapID = 0
+	imp.MaxBitmapID = 10
+	imp.BaseProfileID = 0
+	imp.MaxProfileID = 10
+	imp.RandomBitmapOrder = false
+	imp.MinBitsPerMap = 2
+	imp.MaxBitsPerMap = 3
+	imp.AgentControls = "width"
+	imp.Seed = 0
+
+	imp.Init([]string{"blah"}, 2)
+	f, err := os.Open(imp.Paths[0])
+	if err != nil {
+		t.Fatalf("Couldn't open file: %v, err: %v", imp.Paths[0], err)
+	}
+	bytes, err := ioutil.ReadAll(f)
+	if err != nil {
+		t.Fatalf("error reading file: %v", err)
+	}
+
+	expected := `
+0,21
+0,22
+1,20
+1,22
+2,22
+2,26
+3,21
+3,23
+4,21
+4,22
+5,20
+5,28
+6,23
+6,27
+7,20
+7,20
+8,23
+8,29
+9,23
+9,29
+`[1:]
+
+	if string(bytes) != expected {
+		t.Fatalf("unexpected result: %v", string(bytes))
+	}
+}
+
+func TestGenerateImportCSVNonRand(t *testing.T) {
+	b := bytes.NewBuffer(make([]byte, 0))
+
+	bench.GenerateImportCSV(b, 0, 10, 20, 30, 2, 3, 2, false)
+
+	bytes, err := ioutil.ReadAll(b)
+	if err != nil {
+		t.Fatalf("Error reading buffer: %v", err)
+	}
+
+	expected := `
+0,21
+0,22
+1,20
+1,22
+2,22
+2,26
+3,21
+3,23
+4,21
+4,22
+5,20
+5,28
+6,23
+6,27
+7,20
+7,20
+8,23
+8,29
+9,23
+9,29
+`[1:]
+
+	if string(bytes) != expected {
+		t.Fatalf("unexpected value for generated csv: \n%v", string(bytes))
+	}
+}
+
+func TestGenerateImportCSVRand(t *testing.T) {
+	b := bytes.NewBuffer(make([]byte, 0))
+
+	bench.GenerateImportCSV(b, 0, 10, 21, 29, 1, 4, 0, true)
+
+	bytes, err := ioutil.ReadAll(b)
+	if err != nil {
+		t.Fatalf("Error reading buffer: %v", err)
+	}
+
+	expected := `
+8,25
+2,23
+3,22
+3,28
+3,28
+0,25
+0,23
+5,26
+5,23
+7,21
+7,23
+1,25
+6,23
+6,27
+6,25
+9,26
+4,22
+4,24
+`[1:]
+
+	if string(bytes) != expected {
+		t.Fatalf("unexpected value for generated csv: \n%v", string(bytes))
+	}
+}

bench/multidb.go

@@ -0,0 +1,97 @@
+package bench
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"io/ioutil"
+	"strconv"
+	"time"
+)
+
+// MultiDBSetBits sets bits with increasing profile id and bitmap id.
+type MultiDBSetBits struct {
+	HasClient
+	Name          string `json:"name"`
+	BaseBitmapID  int    `json:"base-bitmap-id"`
+	BaseProfileID int    `json:"base-profile-id"`
+	Iterations    int    `json:"iterations"`
+	Database      string `json:"database"`
+}
+
+// Init sets up the db name based on the agentNum and sets up the pilosa client.
+func (b *MultiDBSetBits) Init(hosts []string, agentNum int) error {
+	b.Name = "multi-db-set-bits"
+	b.Database = b.Database + strconv.Itoa(agentNum)
+	return b.HasClient.Init(hosts, agentNum)
+}
+
+// Usage returns the usage message to be printed.
+func (b *MultiDBSetBits) Usage() string {
+	return `
+multi-db-set-bits sets bits with increasing profile id and bitmap id using a different DB for each agent.
+
+Agent num changes the database being written to.
+
+Usage: multi-db-set-bits [arguments]
+
+The following arguments are available:
+
+	-base-bitmap-id int
+		bits being set will all be greater than base-bitmap-id
+
+	-base-profile-id int
+		profile id num to start from
+
+	-iterations int
+		number of bits to set
+
+	-client-type string
+		Can be 'single' (all agents hitting one host) or 'round_robin'
+
+	-content-type string
+		protobuf or pql
+
+`[1:]
+}
+
+// ConsumeFlags parses all flags up to the next non flag argument (argument does
+// not start with "-" and isn't the value of a flag). It returns the remaining
+// args.
+func (b *MultiDBSetBits) ConsumeFlags(args []string) ([]string, error) {
+	fs := flag.NewFlagSet("MultiDBSetBits", flag.ContinueOnError)
+	fs.SetOutput(ioutil.Discard)
+	fs.IntVar(&b.BaseBitmapID, "base-bitmap-id", 0, "")
+	fs.IntVar(&b.BaseProfileID, "base-profile-id", 0, "")
+	fs.IntVar(&b.Iterations, "iterations", 100, "")
+	fs.StringVar(&b.ClientType, "client-type", "single", "")
+	fs.StringVar(&b.ContentType, "content-type", "protobuf", "")
+
+	if err := fs.Parse(args); err != nil {
+		return nil, err
+	}
+	return fs.Args(), nil
+}
+
+// Run runs the MultiDBSetBits benchmark
+func (b *MultiDBSetBits) Run(ctx context.Context) map[string]interface{} {
+	results := make(map[string]interface{})
+	if b.client == nil {
+		results["error"] = fmt.Errorf("No client set for MultiDBSetBits")
+		return results
+	}
+	s := NewStats()
+	var start time.Time
+	for n := 0; n < b.Iterations; n++ {
+		query := fmt.Sprintf("SetBit(%d, 'frame.n', %d)", b.BaseBitmapID+n, b.BaseProfileID+n)
+		start = time.Now()
+		_, err := b.client.ExecuteQuery(ctx, b.Database, query, true)
+		if err != nil {
+			results["error"] = err
+			return results
+		}
+		s.Add(time.Now().Sub(start))
+	}
+	AddToResults(s, results)
+	return results
+}

bench/permutations.go

@@ -0,0 +1,64 @@
+package bench
+
+// PermutationGenerator provides a way to pass integer IDs through a permutation
+// map that is pseudorandom but repeatable. This could be done with rand.Perm,
+// but that would require storing a [Iterations]int64 array, which we want to avoid
+// for large values of Iterations.
+// It works by using a Linear Congruence Generator (https://en.wikipedia.org/wiki/Linear_congruential_generator)
+// with modulus m = Iterations,
+// c = an arbitrary prime,
+// a = computed to ensure the full period.
+// relevant stackoverflow: http://cs.stackexchange.com/questions/29822/lazily-computing-a-random-permutation-of-the-positive-integers
+type PermutationGenerator struct {
+	a int64
+	c int64
+	m int64
+}
+
+func NewPermutationGenerator(m int64, seed int64) *PermutationGenerator {
+	// figure out 'a' and 'c', return PermutationGenerator
+	a := LCGmultiplierFromModulus(m, seed)
+	c := int64(22695479)
+	return &PermutationGenerator{a, c, m}
+}
+
+func (p *PermutationGenerator) Next(n int64) int64 {
+	// run one step of the LCG
+	return (n*p.a + p.c) % p.m
+}
+
+// LCG parameters must satisfy three conditions:
+// 1. m and c are relatively prime (satisfied for prime c != m)
+// 2. a-1 is divisible by all prime factors of m
+// 3. a-1 is divisible by 4 if m is divisible by 4
+// Additionally, a seed can be used to select between different permutations
+func LCGmultiplierFromModulus(m int64, seed int64) int64 {
+	factors := primeFactors(m)
+	product := int64(1)
+	for p := range factors {
+		// satisfy condition 2
+		product *= p
+	}
+
+	if m%4 == 0 {
+		// satisfy condition 3
+		product *= 2
+	}
+
+	return product*seed + 1
+}
+
+// Returns map of {integerFactor: count, ...}
+// This is a naive algorithm that will not work well for large prime n.
+func primeFactors(n int64) map[int64]int {
+	factors := make(map[int64]int)
+	for i := int64(2); i <= n; i++ {
+		div, mod := n/i, n%i
+		for mod == 0 {
+			factors[i] += 1
+			n = div
+			div, mod = n/i, n%i
+		}
+	}
+	return factors
+}

bench/prettify.go

@@ -0,0 +1,41 @@
+package bench
+
+import "time"
+
+// wrapper type to force human-readable JSON output
+type PrettyDuration time.Duration
+
+// MarshalJSON returns a nicely formatted duration, instead of it just being
+// treated like an int.
+func (d PrettyDuration) MarshalJSON() ([]byte, error) {
+	s := time.Duration(d).String()
+	return []byte("\"" + s + "\""), nil
+}
+
+// Recursively replaces elements of ugly types with their pretty wrappers
+func Prettify(m map[string]interface{}) map[string]interface{} {
+	newmap := make(map[string]interface{})
+	for k, v := range m {
+		switch v.(type) {
+		case map[string]interface{}:
+			newmap[k] = Prettify(v.(map[string]interface{}))
+		case []time.Duration:
+			newslice := make([]PrettyDuration, len(v.([]time.Duration)))
+			slice := v.([]time.Duration)
+			for n, e := range slice {
+				newslice[n] = PrettyDuration(e)
+			}
+			newmap[k] = newslice
+		case time.Duration:
+			newmap[k] = PrettyDuration(v.(time.Duration))
+		default:
+			if interv, ok := v.([]map[string]interface{}); ok {
+				for i, iv := range interv {
+					interv[i] = Prettify(iv)
+				}
+			}
+			newmap[k] = v
+		}
+	}
+	return newmap
+}

bench/prettify_test.go

@@ -0,0 +1,93 @@
+package bench_test
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"os"
+	"testing"
+	"time"
+
+	"github.com/pilosa/pilosa/bench"
+)
+
+func prettyEncode(data map[string]interface{}) string {
+	pretty := bench.Prettify(data)
+	jsonString := new(bytes.Buffer)
+	enc := json.NewEncoder(jsonString)
+	enc.SetIndent("", "  ")
+	err := enc.Encode(pretty)
+	if err != nil {
+		fmt.Fprintln(os.Stderr, err)
+	}
+
+	return jsonString.String()
+}
+
+func TestPrettifyString(t *testing.T) {
+	res := make(map[string]interface{}, 1)
+	res["0"] = "foobar"
+	pretty := prettyEncode(res)
+
+	expected := `
+{
+  "0": "foobar"
+}
+`[1:]
+
+	if pretty != expected {
+		t.Fatalf("Pretty string doesn't match")
+	}
+}
+
+func TestPrettifyInt(t *testing.T) {
+	res := make(map[string]interface{}, 1)
+	res["0"] = 234567
+	pretty := prettyEncode(res)
+
+	expected := `
+{
+  "0": 234567
+}
+`[1:]
+
+	if pretty != expected {
+		t.Fatalf("Pretty int doesn't match")
+	}
+}
+
+func TestPrettifyDuration(t *testing.T) {
+	res := make(map[string]interface{}, 1)
+	res["0"] = time.Duration(234567)
+	pretty := prettyEncode(res)
+
+	expected := `
+{
+  "0": "234.567µs"
+}
+`[1:]
+
+	if pretty != expected {
+		t.Fatalf("Pretty duration doesn't match")
+	}
+}
+
+func TestPrettifyDurationSlice(t *testing.T) {
+	res := make(map[string]interface{}, 1)
+	res["0"] = []time.Duration{123, 234567, 34567890}
+	pretty := prettyEncode(res)
+
+	expected := `
+{
+  "0": [
+    "123ns",
+    "234.567µs",
+    "34.56789ms"
+  ]
+}
+`[1:]
+
+	if pretty != expected {
+		t.Fatalf("Pretty duration slice doesn't match")
+	}
+}

bench/query.go

@@ -0,0 +1,163 @@
+package bench
+
+import (
+	"math/rand"
+
+	"github.com/pilosa/pilosa/pql"
+)
+
+// NewQueryGenerator initializes a new QueryGenerator
+func NewQueryGenerator(seed int64) *QueryGenerator {
+	return &QueryGenerator{
+		IDToFrameFn: func(id uint64) string { return "frame.n" },
+		R:           rand.New(rand.NewSource(seed)),
+		Frames:      []string{"frame.n"},
+	}
+}
+
+// QueryGenerator holds the configuration and state for randomly generating
+// queries.
+type QueryGenerator struct {
+	IDToFrameFn func(id uint64) string
+	R           *rand.Rand
+	Frames      []string
+}
+
+// Random returns a randomly generated query.
+func (q *QueryGenerator) Random(maxN, depth, maxargs int, idmin, idmax uint64) pql.Call {
+	// TODO: handle depth==1 or 0
+	val := q.R.Intn(5)
+	switch val {
+	case 0:
+		return q.RandomTopN(maxN, depth, maxargs, idmin, idmax)
+	default:
+		return q.RandomBitmapCall(depth, maxargs, idmin, idmax)
+	}
+}
+
+// RandomTopN returns a randomly generated TopN query.
+func (q *QueryGenerator) RandomTopN(maxN, depth, maxargs int, idmin, idmax uint64) *pql.TopN {
+	frameIdx := q.R.Intn(len(q.Frames))
+	return &pql.TopN{
+		Frame: q.Frames[frameIdx],
+		N:     q.R.Intn(maxN-1) + 1,
+		Src:   q.RandomBitmapCall(depth, maxargs, idmin, idmax),
+	}
+}
+
+// RandomBitmapCall returns a randomly generate query which is a pql.BitmapCall.
+func (q *QueryGenerator) RandomBitmapCall(depth, maxargs int, idmin, idmax uint64) pql.BitmapCall {
+	if depth <= 1 {
+		bitmapID := q.R.Int63n(int64(idmax)-int64(idmin)) + int64(idmin)
+		return Bitmap(uint64(bitmapID), q.IDToFrameFn(uint64(bitmapID)))
+	}
+	call := q.R.Intn(4)
+	if call == 0 {
+		return q.RandomBitmapCall(1, 0, idmin, idmax)
+	}
+
+	var numargs int
+	if maxargs <= 2 {
+		numargs = 2
+	} else {
+		numargs = q.R.Intn(maxargs-2) + 2
+	}
+	calls := make([]pql.BitmapCall, numargs)
+	for i := 0; i < numargs; i++ {
+		calls[i] = q.RandomBitmapCall(depth-1, maxargs, idmin, idmax)
+	}
+
+	switch call {
+	case 1:
+		return Difference(calls...)
+	case 2:
+		return Intersect(calls...)
+	case 3:
+		return Union(calls...)
+	}
+	return nil
+}
+
+///////////////////////////////////////////////////
+// Helpers TODO: move elsewhere
+///////////////////////////////////////////////////
+
+func ClearBit(id uint64, frame string, profileID uint64) *pql.ClearBit {
+	return &pql.ClearBit{
+		ID:        id,
+		Frame:     frame,
+		ProfileID: profileID,
+	}
+}
+
+func Count(bm pql.BitmapCall) *pql.Count {
+	return &pql.Count{
+		Input: bm,
+	}
+}
+
+func Profile(id uint64) *pql.Profile {
+	return &pql.Profile{
+		ID: id,
+	}
+}
+
+func SetBit(id uint64, frame string, profileID uint64) *pql.SetBit {
+	return &pql.SetBit{
+		ID:        id,
+		Frame:     frame,
+		ProfileID: profileID,
+	}
+}
+
+func SetBitmapAttrs(id uint64, frame string, attrs map[string]interface{}) *pql.SetBitmapAttrs {
+	return &pql.SetBitmapAttrs{
+		ID:    id,
+		Frame: frame,
+		Attrs: attrs,
+	}
+}
+
+func SetProfileAttrs(id uint64, attrs map[string]interface{}) *pql.SetProfileAttrs {
+	return &pql.SetProfileAttrs{
+		ID:    id,
+		Attrs: attrs,
+	}
+}
+
+func TopN(frame string, n int, src pql.BitmapCall, bmids []uint64, field string, filters []interface{}) *pql.TopN {
+	return &pql.TopN{
+		Frame:     frame,
+		N:         n,
+		Src:       src,
+		BitmapIDs: bmids,
+		Field:     field,
+		Filters:   filters,
+	}
+}
+
+func Difference(bms ...pql.BitmapCall) *pql.Difference {
+	// TODO does this need to be limited to two inputs?
+	return &pql.Difference{
+		Inputs: bms,
+	}
+}
+
+func Intersect(bms ...pql.BitmapCall) *pql.Intersect {
+	return &pql.Intersect{
+		Inputs: bms,
+	}
+}
+
+func Union(bms ...pql.BitmapCall) *pql.Union {
+	return &pql.Union{
+		Inputs: bms,
+	}
+}
+
+func Bitmap(id uint64, frame string) *pql.Bitmap {
+	return &pql.Bitmap{
+		ID:    id,
+		Frame: frame,
+	}
+}

bench/query_test.go

@@ -0,0 +1,20 @@
+package bench_test
+
+import (
+	"github.com/pilosa/pilosa/bench"
+	"testing"
+)
+
+func TestRandomBitmapCall(t *testing.T) {
+	qm := bench.NewQueryGenerator(5)
+	bmc := qm.RandomBitmapCall(4, 3, 0, 1000)
+	t.Log(bmc.String())
+}
+
+func TestRandom(t *testing.T) {
+	for i := 99; i < 120; i++ {
+		qm := bench.NewQueryGenerator(int64(i))
+		call := qm.Random(10, 4, 3, 0, 1000)
+		t.Log(call)
+	}
+}

bench/random.go

@@ -0,0 +1,117 @@
+package bench
+
+import (
+	"fmt"
+
+	"flag"
+	"io/ioutil"
+
+	"context"
+	"math/rand"
+	"time"
+)
+
+// RandomSetBits sets bits randomly and deterministically based on a seed.
+type RandomSetBits struct {
+	HasClient
+	Name           string `json:"name"`
+	BaseBitmapID   int64  `json:"base-bitmap-id"`
+	BaseProfileID  int64  `json:"base-profile-id"`
+	BitmapIDRange  int64  `json:"bitmap-id-range"`
+	ProfileIDRange int64  `json:"profile-id-range"`
+	Iterations     int    `json:"iterations"`
+	Seed           int64  `json:"seed"`
+	DB             string `json:"db"`
+}
+
+// Init adds the agent num to the random seed and initializes the client.
+func (b *RandomSetBits) Init(hosts []string, agentNum int) error {
+	b.Name = "random-set-bits"
+	b.Seed = b.Seed + int64(agentNum)
+	return b.HasClient.Init(hosts, agentNum)
+}
+
+// Usage returns the usage message to be printed.
+func (b *RandomSetBits) Usage() string {
+	return `
+random-set-bits sets random bits
+
+Agent number modifies the random seed.
+
+Usage: random-set-bits [arguments]
+
+The following arguments are available:
+
+	-base-bitmap-id int
+		bits being set will all be greater than BaseBitmapID
+
+	-bitmap-id-range int
+		number of possible bitmap ids that can be set
+
+	-base-profile-id int
+		profile id num to start from
+
+	-profile-id-range int
+		number of possible profile ids that can be set
+
+	-iterations int
+		number of bits to set
+
+	-seed int
+		Seed for RNG
+
+	-db string
+		pilosa db to use
+
+	-client-type string
+		Can be 'single' (all agents hitting one host) or 'round_robin'
+
+	-content-type string
+		protobuf or pql
+`[1:]
+}
+
+// ConsumeFlags parses all flags up to the next non flag argument (argument does
+// not start with "-" and isn't the value of a flag). It returns the remaining
+// args.
+func (b *RandomSetBits) ConsumeFlags(args []string) ([]string, error) {
+	fs := flag.NewFlagSet("RandomSetBits", flag.ContinueOnError)
+	fs.SetOutput(ioutil.Discard)
+	fs.Int64Var(&b.BaseBitmapID, "base-bitmap-id", 0, "")
+	fs.Int64Var(&b.BitmapIDRange, "bitmap-id-range", 100000, "")
+	fs.Int64Var(&b.BaseProfileID, "base-profile-id", 0, "")
+	fs.Int64Var(&b.ProfileIDRange, "profile-id-range", 100000, "")
+	fs.Int64Var(&b.Seed, "seed", 1, "")
+	fs.IntVar(&b.Iterations, "iterations", 100, "")
+	fs.StringVar(&b.DB, "db", "benchdb", "")
+	fs.StringVar(&b.ClientType, "client-type", "single", "")
+	fs.StringVar(&b.ContentType, "content-type", "protobuf", "")
+
+	if err := fs.Parse(args); err != nil {
+		return nil, err
+	}
+	return fs.Args(), nil
+}
+
+// Run runs the RandomSetBits benchmark
+func (b *RandomSetBits) Run(ctx context.Context) map[string]interface{} {
+	src := rand.NewSource(b.Seed)
+	rng := rand.New(src)
+	results := make(map[string]interface{})
+	if b.client == nil {
+		results["error"] = fmt.Errorf("No client set for RandomSetBits")
+		return results
+	}
+	s := NewStats()
+	var start time.Time
+	for n := 0; n < b.Iterations; n++ {
+		bitmapID := rng.Int63n(b.BitmapIDRange)
+		profID := rng.Int63n(b.ProfileIDRange)
+		query := fmt.Sprintf("SetBit(%d, 'frame.n', %d)", b.BaseBitmapID+bitmapID, b.BaseProfileID+profID)
+		start = time.Now()
+		b.client.ExecuteQuery(ctx, b.DB, query, true)
+		s.Add(time.Now().Sub(start))
+	}
+	AddToResults(s, results)
+	return results
+}

bench/randquery.go

@@ -0,0 +1,119 @@
+package bench
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"io/ioutil"
+	"strings"
+	"time"
+)
+
+// RandomQuery queries randomly and deterministically based on a seed.
+type RandomQuery struct {
+	HasClient
+	Name          string   `json:"name"`
+	MaxDepth      int      `json:"max-depth"`
+	MaxArgs       int      `json:"max-args"`
+	MaxN          int      `json:"max-n"`
+	BaseBitmapID  int64    `json:"base-bitmap-id"`
+	BitmapIDRange int64    `json:"bitmap-id-range"`
+	Iterations    int      `json:"iterations"`
+	Seed          int64    `json:"seed"`
+	DBs           []string `json:"dbs"`
+}
+
+// Init adds the agent num to the random seed and initializes the client.
+func (b *RandomQuery) Init(hosts []string, agentNum int) error {
+	b.Name = "random-query"
+	b.Seed = b.Seed + int64(agentNum)
+	return b.HasClient.Init(hosts, agentNum)
+}
+
+// Usage returns the usage message to be printed.
+func (b *RandomQuery) Usage() string {
+	return `
+random-query constructs random queries
+
+Agent number modifies the random seed.
+
+Usage: random-query [arguments]
+
+The following arguments are available:
+
+	-max-depth int
+		Maximum nesting depth of queries
+
+	-max-args int
+		Maximum number of args for Union/Intersect/Difference Queries
+
+	-max-n int
+		Maximum N value for TopN queries.
+
+	-base-bitmap-id int
+		bitmap id to start from
+
+	-bitmap-id-range int
+		number of possible bitmap ids that can be set
+
+	-iterations int
+		number of bits to set
+
+	-seed int
+		Seed for RNG
+
+	-dbs string
+		Comma separated list of DBs to query against
+
+	-client-type string
+		Can be 'single' (all agents hitting one host) or 'round_robin'
+
+	-content-type string
+		protobuf or pql
+`[1:]
+}
+
+// ConsumeFlags parses all flags up to the next non flag argument (argument does
+// not start with "-" and isn't the value of a flag). It returns the remaining
+// args.
+func (b *RandomQuery) ConsumeFlags(args []string) ([]string, error) {
+	fs := flag.NewFlagSet("RandomQuery", flag.ContinueOnError)
+	fs.SetOutput(ioutil.Discard)
+	fs.IntVar(&b.MaxDepth, "max-depth", 4, "")
+	fs.IntVar(&b.MaxArgs, "max-args", 4, "")
+	fs.IntVar(&b.MaxN, "max-n", 4, "")
+	fs.Int64Var(&b.BaseBitmapID, "base-bitmap-id", 0, "")
+	fs.Int64Var(&b.BitmapIDRange, "bitmap-id-range", 100000, "")
+	fs.Int64Var(&b.Seed, "seed", 1, "")
+	fs.IntVar(&b.Iterations, "iterations", 100, "")
+	var dbs string
+	fs.StringVar(&dbs, "dbs", "benchdb", "")
+	fs.StringVar(&b.ClientType, "client-type", "single", "")
+	fs.StringVar(&b.ContentType, "content-type", "protobuf", "")
+
+	if err := fs.Parse(args); err != nil {
+		return nil, err
+	}
+	b.DBs = strings.Split(dbs, ",")
+	return fs.Args(), nil
+}
+
+// Run runs the RandomQuery benchmark
+func (b *RandomQuery) Run(ctx context.Context) map[string]interface{} {
+	results := make(map[string]interface{})
+	if b.client == nil {
+		results["error"] = fmt.Errorf("No client set for RandomQuery")
+		return results
+	}
+	qm := NewQueryGenerator(b.Seed)
+	s := NewStats()
+	var start time.Time
+	for n := 0; n < b.Iterations; n++ {
+		call := qm.Random(b.MaxN, b.MaxDepth, b.MaxArgs, uint64(b.BaseBitmapID), uint64(b.BitmapIDRange))
+		start = time.Now()
+		b.ExecuteQuery(b.ContentType, b.DBs[n%len(b.DBs)], call.String(), ctx)
+		s.Add(time.Now().Sub(start))
+	}
+	AddToResults(s, results)
+	return results
+}

bench/sliceheight.go

@@ -0,0 +1,145 @@
+package bench
+
+import (
+	"context"
+	"flag"
+	"io"
+	"io/ioutil"
+	"strconv"
+	"time"
+
+	"github.com/pilosa/pilosa"
+	"github.com/pilosa/pilosa/pql"
+)
+
+// NewSliceHeight creates a new slice height benchmark with stdin/out/err
+// initialized.
+func NewSliceHeight(stdin io.Reader, stdout, stderr io.Writer) *SliceHeight {
+	return &SliceHeight{
+		Stdin:  stdin,
+		Stdout: stdout,
+		Stderr: stderr,
+	}
+}
+
+// SliceHeight benchmark tests the effect of an increasing number of bitmaps in
+// a single slice on query time.
+type SliceHeight struct {
+	MaxTime time.Duration `json:"max-time"`
+	hosts   []string
+
+	Name          string `json:"name"`
+	MinBitsPerMap int64  `json:"min-bits-per-map"`
+	MaxBitsPerMap int64  `json:"max-bits-per-map"`
+	Seed          int64  `json:"seed"`
+	Database      string `json:"database"`
+	Frame         string `json:"frame"`
+
+	Stdin  io.Reader `json:"-"`
+	Stdout io.Writer `json:"-"`
+	Stderr io.Writer `json:"-"`
+}
+
+// Usage returns the usage message to be printed.
+func (b *SliceHeight) Usage() string {
+	return `
+slice-height repeatedly imports more bitmaps into a single slice and tests query times in between.
+
+Agent number has no effect on this benchmark.
+
+Usage: slice-height [arguments]
+
+The following arguments are available:
+
+	-max-time int
+		stop benchmark after this many seconds
+
+	-min-bits-per-map int
+		minimum number of bits set per bitmap
+
+	-max-bits-per-map int
+		maximum number of bits set per bitmap
+
+	-seed int
+		seed for RNG
+
+	-db string
+		pilosa db to use
+
+	-frame string
+		frame to import into
+`[1:]
+}
+
+// ConsumeFlags parses all flags up to the next non flag argument (argument does
+// not start with "-" and isn't the value of a flag). It returns the remaining
+// args.
+func (b *SliceHeight) ConsumeFlags(args []string) ([]string, error) {
+	fs := flag.NewFlagSet("SliceHeight", flag.ContinueOnError)
+	fs.SetOutput(ioutil.Discard)
+
+	maxTime := fs.Int("max-time", 30, "")
+	fs.Int64Var(&b.MinBitsPerMap, "min-bits-per-map", 0, "")
+	fs.Int64Var(&b.MaxBitsPerMap, "max-bits-per-map", 10, "")
+	fs.Int64Var(&b.Seed, "seed", 0, "")
+	fs.StringVar(&b.Database, "db", "benchdb", "")
+	fs.StringVar(&b.Frame, "frame", "testframe", "")
+
+	if err := fs.Parse(args); err != nil {
+		return nil, err
+	}
+	b.MaxTime = time.Duration(*maxTime) * time.Second
+	return fs.Args(), nil
+}
+
+// Init sets up the slice height benchmark.
+func (b *SliceHeight) Init(hosts []string, agentNum int) error {
+	b.Name = "slice-height"
+	b.hosts = hosts
+	return nil
+}
+
+// Run runs the SliceHeight benchmark
+func (b *SliceHeight) Run(ctx context.Context) map[string]interface{} {
+	results := make(map[string]interface{})
+
+	imp := NewImport(b.Stdin, b.Stdout, b.Stderr)
+	imp.MaxBitmapID = 100
+	imp.MaxProfileID = pilosa.SliceWidth
+	imp.MinBitsPerMap = b.MinBitsPerMap
+	imp.MaxBitsPerMap = b.MaxBitsPerMap
+	imp.Database = b.Database
+	imp.Frame = b.Frame
+
+	start := time.Now()
+
+	for i := 0; i > -1; i++ {
+		iresults := make(map[string]interface{})
+		results["iteration"+strconv.Itoa(i)] = iresults
+
+		genstart := time.Now()
+		imp.Init(b.hosts, 0)
+		gendur := time.Now().Sub(genstart)
+		iresults["csvgen"] = gendur
+
+		iresults["import"] = imp.Run(ctx)
+
+		qstart := time.Now()
+		q := &pql.TopN{Frame: b.Frame, N: 50}
+		_, err := imp.Client.ExecuteQuery(ctx, b.Database, q.String(), true)
+		if err != nil {
+			iresults["query_error"] = err.Error()
+		} else {
+			qdur := time.Now().Sub(qstart)
+			iresults["query"] = qdur
+		}
+		imp.BaseBitmapID = imp.MaxBitmapID
+		imp.MaxBitmapID = imp.MaxBitmapID * 10
+
+		if time.Now().Sub(start) > b.MaxTime {
+			break
+		}
+	}
+
+	return results
+}

bench/stats.go

@@ -0,0 +1,65 @@
+package bench
+
+import (
+	"math"
+	"time"
+)
+
+// Stats object helps track timing stats.
+type Stats struct {
+	Min            time.Duration
+	Max            time.Duration
+	Mean           time.Duration
+	sumSquareDelta float64
+	Total          time.Duration
+	Num            int64
+	All            []time.Duration
+	SaveAll        bool
+}
+
+// NewStats gets a Stats object.
+func NewStats() *Stats {
+	return &Stats{
+		Min: 1<<63 - 1,
+		All: make([]time.Duration, 0),
+	}
+}
+
+// Add adds a new time to the stats object.
+func (s *Stats) Add(td time.Duration) {
+	if s.SaveAll {
+		s.All = append(s.All, td)
+	}
+	s.Num += 1
+	s.Total += td
+	if td < s.Min {
+		s.Min = td
+	}
+	if td > s.Max {
+		s.Max = td
+	}
+
+	// online variance calculation
+	// https://en.wikipedia.org/wiki/Algorithms_for_calculating_variance#Online_algorithm
+	delta := td - s.Mean
+	s.Mean += delta / time.Duration(s.Num)
+	s.sumSquareDelta += float64(delta * (td - s.Mean))
+}
+
+// Avg returns the average of all durations Added to the Stats object.
+func (s *Stats) Avg() time.Duration {
+	return s.Total / time.Duration(s.Num)
+}
+
+// AddToResults serializes the summary of Stats and adds them to the results
+// map.
+func AddToResults(s *Stats, results map[string]interface{}) {
+	results["min"] = s.Min
+	results["max"] = s.Max
+	results["avg"] = s.Mean
+	variance := s.sumSquareDelta / float64(s.Num)
+	results["std"] = time.Duration(math.Sqrt(variance))
+	if s.SaveAll {
+		results["all"] = s.All
+	}
+}

bench/store.go

@@ -0,0 +1,39 @@
+package bench
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/s3"
+)
+
+// NewS3Uploader creates an S3Uploader with specified bucket and key
+func NewS3Uploader(bucket string, key string) *S3Uploader {
+	return &S3Uploader{
+		bucket,
+		s3.New(session.New(&aws.Config{})),
+		key,
+	}
+}
+
+// S3Uploader is an io.Writer for sending output to AWS S3 storage.
+type S3Uploader struct {
+	bucket  string
+	service *s3.S3
+	key     string
+}
+
+// Write writes data to the uploader's bucket/key
+func (u *S3Uploader) Write(data []byte) (int, error) {
+	// first return value of PutObject contains an ETag (hash) of the uploaded object, not needed here
+	fmt.Println(string(data))
+	_, err := u.service.PutObject(&s3.PutObjectInput{
+		Body:   strings.NewReader(string(data)),
+		Bucket: &u.bucket,
+		Key:    &u.key,
+	})
+
+	return 0, err
+}

bench/zipf.go

@@ -0,0 +1,191 @@
+package bench
+
+import (
+	"fmt"
+
+	"flag"
+	"io/ioutil"
+
+	"context"
+	"math"
+	"math/rand"
+	"time"
+)
+
+// Zipf sets random bits according to the Zipf-Mandelbrot distribution.
+// This distribution accepts two parameters, Exponent and Ratio, for both bitmaps and profiles.
+// It also uses PermutationGenerator to permute IDs randomly.
+type Zipf struct {
+	HasClient
+	Name            string  `json:"name"`
+	BaseBitmapID    int64   `json:"base-bitmap-id"`
+	BaseProfileID   int64   `json:"base-profile-id"`
+	BitmapIDRange   int64   `json:"bitmap-id-range"`
+	ProfileIDRange  int64   `json:"profile-id-range"`
+	Iterations      int     `json:"iterations"`
+	Seed            int64   `json:"seed"`
+	DB              string  `json:"db"`
+	BitmapExponent  float64 `json:"bitmap-exponent"`
+	BitmapRatio     float64 `json:"bitmap-ratio"`
+	ProfileExponent float64 `json:"profile-exponent"`
+	ProfileRatio    float64 `json:"profile-ratio"`
+	Operation       string  `json:"operation"`
+	bitmapRng       *rand.Zipf
+	profileRng      *rand.Zipf
+	bitmapPerm      *PermutationGenerator
+	profilePerm     *PermutationGenerator
+}
+
+// Usage returns the usage message to be printed.
+func (b *Zipf) Usage() string {
+	return `
+zipf sets random bits according to the Zipf distribution.
+This is a power-law distribution controlled by two parameters.
+Exponent, in the range (1, inf), with a default value of 1.001, controls
+the "sharpness" of the distribution, with higher exponent being sharper.
+Ratio, in the range (0, 1), with a default value of 0.25, controls the
+maximum variation of the distribution, with higher ratio being more uniform.
+
+Agent number modifies random seed.
+
+Usage: zipf [arguments]
+
+The following arguments are available:
+
+	-base-bitmap-id int
+		bits being set will all be greater than BaseBitmapID
+
+	-bitmap-id-range int
+		number of possible bitmap ids that can be set
+
+	-base-profile-id int
+		profile id num to start from
+
+	-profile-id-range int
+		number of possible profile ids that can be set
+
+	-iterations int
+		number of bits to set
+
+	-seed int
+		Seed for RNG
+
+	-db string
+		pilosa db to use
+
+	-bitmap-exponent float64
+		zipf exponent parameter for bitmap IDs
+
+	-bitmap-ratio float64
+		zipf probability ratio parameter for bitmap IDs
+
+	-profile-exponent float64
+		zipf exponent parameter for profile IDs
+
+	-profile-ratio float64
+		zipf probability ratio parameter for profile IDs
+
+	-client-type string
+		Can be 'single' (all agents hitting one host) or 'round_robin'
+
+	-operation string
+		Can be 'set' or 'clear'
+
+	-content-type string
+		protobuf or pql
+`[1:]
+}
+
+// ConsumeFlags parses all flags up to the next non flag argument (argument does
+// not start with "-" and isn't the value of a flag). It returns the remaining
+// args.
+func (b *Zipf) ConsumeFlags(args []string) ([]string, error) {
+	fs := flag.NewFlagSet("Zipf", flag.ContinueOnError)
+	fs.SetOutput(ioutil.Discard)
+	fs.Int64Var(&b.BaseBitmapID, "base-bitmap-id", 0, "")
+	fs.Int64Var(&b.BitmapIDRange, "bitmap-id-range", 100000, "")
+	fs.Int64Var(&b.BaseProfileID, "base-profile-id", 0, "")
+	fs.Int64Var(&b.ProfileIDRange, "profile-id-range", 100000, "")
+	fs.Int64Var(&b.Seed, "seed", 1, "")
+	fs.IntVar(&b.Iterations, "iterations", 100, "")
+	fs.StringVar(&b.DB, "db", "benchdb", "")
+	fs.Float64Var(&b.BitmapExponent, "bitmap-exponent", 1.01, "")
+	fs.Float64Var(&b.BitmapRatio, "bitmap-ratio", 0.25, "")
+	fs.Float64Var(&b.ProfileExponent, "profile-exponent", 1.01, "")
+	fs.Float64Var(&b.ProfileRatio, "profile-ratio", 0.25, "")
+	fs.StringVar(&b.ClientType, "client-type", "single", "")
+	fs.StringVar(&b.Operation, "operation", "set", "")
+	fs.StringVar(&b.ContentType, "content-type", "protobuf", "")
+
+	if err := fs.Parse(args); err != nil {
+		return nil, err
+	}
+	return fs.Args(), nil
+}
+
+// Offset is the true parameter used by the Zipf distribution, but the ratio,
+// as defined here, is a simpler, readable way to define the distribution.
+// Offset is in [1, inf), and its meaning depends on N (a pain for updating benchmark configs)
+// ratio is in (0, 1), and its meaning does not depend on N.
+// it is the ratio of the lowest probability in the distribution to the highest.
+// ratio=0.01 corresponds to a very small offset - the most skewed distribution for a given pair (N, exp)
+// ratio=0.99 corresponds to a very large offset - the most nearly uniform distribution for a given (N, exp)
+func getZipfOffset(N int64, exp, ratio float64) float64 {
+	z := math.Pow(ratio, 1/exp)
+	return z * float64(N-1) / (1 - z)
+}
+
+// Init sets up the benchmark based on the agent number and initializes the
+// client.
+func (b *Zipf) Init(hosts []string, agentNum int) error {
+	b.Name = "zipf"
+	b.Seed = b.Seed + int64(agentNum)
+	rnd := rand.New(rand.NewSource(b.Seed))
+	bitmapOffset := getZipfOffset(b.BitmapIDRange, b.BitmapExponent, b.BitmapRatio)
+	b.bitmapRng = rand.NewZipf(rnd, b.BitmapExponent, bitmapOffset, uint64(b.BitmapIDRange-1))
+	profileOffset := getZipfOffset(b.ProfileIDRange, b.ProfileExponent, b.ProfileRatio)
+	b.profileRng = rand.NewZipf(rnd, b.ProfileExponent, profileOffset, uint64(b.ProfileIDRange-1))
+
+	b.bitmapPerm = NewPermutationGenerator(b.BitmapIDRange, b.Seed)
+	b.profilePerm = NewPermutationGenerator(b.ProfileIDRange, b.Seed+1)
+
+	if b.Operation != "set" && b.Operation != "clear" {
+		return fmt.Errorf("Unsupported operation: \"%s\" (must be \"set\" or \"clear\")", b.Operation)
+	}
+
+	return b.HasClient.Init(hosts, agentNum)
+}
+
+// Run runs the Zipf benchmark
+func (b *Zipf) Run(ctx context.Context) map[string]interface{} {
+	results := make(map[string]interface{})
+	if b.client == nil {
+		results["error"] = fmt.Errorf("No client set for Zipf")
+		return results
+	}
+	operation := "SetBit"
+	if b.Operation == "clear" {
+		operation = "ClearBit"
+	}
+	s := NewStats()
+	var start time.Time
+	for n := 0; n < b.Iterations; n++ {
+		// generate IDs from Zipf distribution
+		bitmapIDOriginal := b.bitmapRng.Uint64()
+		profIDOriginal := b.profileRng.Uint64()
+		// permute IDs randomly, but repeatably
+		bitmapID := b.bitmapPerm.Next(int64(bitmapIDOriginal))
+		profID := b.profilePerm.Next(int64(profIDOriginal))
+
+		query := fmt.Sprintf("%s(%d, 'frame.n', %d)", operation, b.BaseBitmapID+int64(bitmapID), b.BaseProfileID+int64(profID))
+		start = time.Now()
+		_, err := b.client.ExecuteQuery(ctx, b.DB, query, true)
+		if err != nil {
+			results["error"] = fmt.Sprintf("Error executing query in zipf: %v", err)
+			return results
+		}
+		s.Add(time.Now().Sub(start))
+	}
+	AddToResults(s, results)
+	return results
+}

bitmap.go

@@ -0,0 +1,405 @@
+package pilosa
+
+// #cgo  CFLAGS:-mpopcnt
+
+import (
+	"encoding/json"
+	"sort"
+
+	"github.com/pilosa/pilosa/internal"
+	"github.com/pilosa/pilosa/roaring"
+)
+
+// Bitmap represents a set of bits.
+type Bitmap struct {
+	segments []BitmapSegment
+
+	// Attributes associated with the bitmap.
+	Attrs map[string]interface{}
+}
+
+// NewBitmap returns a new instance of Bitmap.
+func NewBitmap(bits ...uint64) *Bitmap {
+	bm := &Bitmap{}
+	for _, i := range bits {
+		bm.SetBit(i)
+	}
+	return bm
+}
+
+// Merge merges data from other into b.
+func (b *Bitmap) Merge(other *Bitmap) {
+	var segments []BitmapSegment
+
+	itr := newMergeSegmentIterator(b.segments, other.segments)
+	for s0, s1 := itr.next(); s0 != nil || s1 != nil; s0, s1 = itr.next() {
+		// Use the other bitmap's data if segment is missing.
+		if s0 == nil {
+			segments = append(segments, *s1)
+			continue
+		} else if s1 == nil {
+			segments = append(segments, *s0)
+			continue
+		}
+
+		// Otherwise merge.
+		s0.Merge(s1)
+		segments = append(segments, *s0)
+	}
+
+	b.segments = segments
+	b.InvalidateCount()
+}
+
+// IntersectionCount returns the number of intersections between b and other.
+func (b *Bitmap) IntersectionCount(other *Bitmap) uint64 {
+	var n uint64
+
+	itr := newMergeSegmentIterator(b.segments, other.segments)
+	for s0, s1 := itr.next(); s0 != nil || s1 != nil; s0, s1 = itr.next() {
+		// Ignore non-overlapping segments.
+		if s0 == nil || s1 == nil {
+			continue
+		}
+
+		n += s0.IntersectionCount(s1)
+	}
+	return n
+}
+
+// Intersect returns the itersection of b and other.
+func (b *Bitmap) Intersect(other *Bitmap) *Bitmap {
+	var segments []BitmapSegment
+
+	itr := newMergeSegmentIterator(b.segments, other.segments)
+	for s0, s1 := itr.next(); s0 != nil || s1 != nil; s0, s1 = itr.next() {
+		// Ignore non-overlapping segments.
+		if s0 == nil || s1 == nil {
+			continue
+		}
+		segments = append(segments, *s0.Intersect(s1))
+	}
+
+	return &Bitmap{segments: segments}
+}
+
+// Union returns the bitwise union of b and other.
+func (b *Bitmap) Union(other *Bitmap) *Bitmap {
+	var segments []BitmapSegment
+	itr := newMergeSegmentIterator(b.segments, other.segments)
+	for s0, s1 := itr.next(); s0 != nil || s1 != nil; s0, s1 = itr.next() {
+		if s1 == nil {
+			segments = append(segments, *s0)
+			continue
+		} else if s0 == nil {
+			segments = append(segments, *s1)
+			continue
+		}
+		segments = append(segments, *s0.Union(s1))
+	}
+
+	return &Bitmap{segments: segments}
+}
+
+// Difference returns the diff of b and other.
+func (b *Bitmap) Difference(other *Bitmap) *Bitmap {
+	var segments []BitmapSegment
+
+	itr := newMergeSegmentIterator(b.segments, other.segments)
+	for s0, s1 := itr.next(); s0 != nil || s1 != nil; s0, s1 = itr.next() {
+		if s0 == nil {
+			continue
+		} else if s1 == nil {
+			segments = append(segments, *s0)
+			continue
+		}
+		segments = append(segments, *s0.Difference(s1))
+	}
+
+	return &Bitmap{segments: segments}
+}
+
+// SetBit sets the i-th bit of the bitmap.
+func (b *Bitmap) SetBit(i uint64) (changed bool) {
+	return b.createSegmentIfNotExists(i / SliceWidth).SetBit(i)
+}
+
+// ClearBit clears the i-th bit of the bitmap.
+func (b *Bitmap) ClearBit(i uint64) (changed bool) {
+	s := b.segment(i / SliceWidth)
+	if s == nil {
+		return false
+	}
+	return s.ClearBit(i)
+}
+
+// segment returns a segment for a given slice.
+// Returns nil if segment does not exist.
+func (b *Bitmap) segment(slice uint64) *BitmapSegment {
+	if i := sort.Search(len(b.segments), func(i int) bool {
+		return b.segments[i].slice >= slice
+	}); i < len(b.segments) && b.segments[i].slice == slice {
+		return &b.segments[i]
+	}
+	return nil
+}
+
+func (b *Bitmap) createSegmentIfNotExists(slice uint64) *BitmapSegment {
+	i := sort.Search(len(b.segments), func(i int) bool {
+		return b.segments[i].slice >= slice
+	})
+
+	// Return exact match.
+	if i < len(b.segments) && b.segments[i].slice == slice {
+		return &b.segments[i]
+	}
+
+	// Insert new segment.
+	b.segments = append(b.segments, BitmapSegment{})
+	if i < len(b.segments) {
+		copy(b.segments[i+1:], b.segments[i:])
+	}
+	b.segments[i] = BitmapSegment{
+		slice:    slice,
+		writable: true,
+	}
+
+	return &b.segments[i]
+}
+
+// InvalidateCount updates the cached count in the bitmap.
+func (b *Bitmap) InvalidateCount() {
+	for i := range b.segments {
+		b.segments[i].InvalidateCount()
+	}
+}
+
+// Count returns the number of set bits in the bitmap.
+func (b *Bitmap) Count() uint64 {
+	var n uint64
+	for i := range b.segments {
+		n += b.segments[i].Count()
+	}
+	return n
+}
+
+// MarshalJSON returns a JSON-encoded byte slice of b.
+func (b *Bitmap) MarshalJSON() ([]byte, error) {
+	var o struct {
+		Attrs map[string]interface{} `json:"attrs"`
+		Bits  []uint64               `json:"bits"`
+	}
+	o.Bits = b.Bits()
+
+	o.Attrs = b.Attrs
+	if o.Attrs == nil {
+		o.Attrs = make(map[string]interface{})
+	}
+
+	return json.Marshal(&o)
+}
+
+// Bits returns the bits in b as a slice of ints.
+func (b *Bitmap) Bits() []uint64 {
+	a := make([]uint64, 0, b.Count())
+	for i := range b.segments {
+		a = append(a, b.segments[i].Bits()...)
+	}
+	return a
+}
+
+// encodeBitmap converts b into its internal representation.
+func encodeBitmap(b *Bitmap) *internal.Bitmap {
+	if b == nil {
+		return nil
+	}
+
+	return &internal.Bitmap{
+		Bits:  b.Bits(),
+		Attrs: encodeAttrs(b.Attrs),
+	}
+}
+
+// decodeBitmap converts b from its internal representation.
+func decodeBitmap(pb *internal.Bitmap) *Bitmap {
+	if pb == nil {
+		return nil
+	}
+
+	b := NewBitmap()
+	b.Attrs = decodeAttrs(pb.Attrs)
+	for _, v := range pb.Bits {
+		b.SetBit(v)
+	}
+	return b
+}
+
+// Union performs a union on a slice of bitmaps.
+func Union(bitmaps []*Bitmap) *Bitmap {
+	other := bitmaps[0]
+	for _, bm := range bitmaps[1:] {
+		other = other.Union(bm)
+	}
+	return other
+}
+
+// BitmapSegment holds a subset of a bitmap.
+// This could point to a mmapped roaring bitmap or an in-memory bitmap. The
+// width of the segment will always match the slice width.
+type BitmapSegment struct {
+	// Slice this segment belongs to
+	slice uint64
+
+	// Underlying raw bitmap implementation.
+	// This is an mmapped bitmap if writable is false. Otherwise
+	// it is a heap allocated bitmap which can be manipulated.
+	data     roaring.Bitmap
+	writable bool
+
+	// Bit count
+	n uint64
+}
+
+// Merge adds chunks from other to s.
+// Chunks in s are overwritten if they exist in other.
+func (s *BitmapSegment) Merge(other *BitmapSegment) {
+	s.ensureWritable()
+
+	itr := other.data.Iterator()
+	for v, eof := itr.Next(); !eof; v, eof = itr.Next() {
+		s.SetBit(v)
+	}
+}
+
+// IntersectionCount returns the number of intersections between s and other.
+func (s *BitmapSegment) IntersectionCount(other *BitmapSegment) uint64 {
+	return s.data.IntersectionCount(&other.data)
+}
+
+// Intersect returns the itersection of s and other.
+func (s *BitmapSegment) Intersect(other *BitmapSegment) *BitmapSegment {
+	data := s.data.Intersect(&other.data)
+
+	return &BitmapSegment{
+		data:  *data,
+		slice: s.slice,
+		n:     data.Count(),
+	}
+}
+
+// Union returns the bitwise union of s and other.
+func (s *BitmapSegment) Union(other *BitmapSegment) *BitmapSegment {
+	data := s.data.Union(&other.data)
+
+	return &BitmapSegment{
+		data:  *data,
+		slice: s.slice,
+		n:     data.Count(),
+	}
+}
+
+// Difference returns the diff of s and other.
+func (s *BitmapSegment) Difference(other *BitmapSegment) *BitmapSegment {
+	data := s.data.Difference(&other.data)
+
+	return &BitmapSegment{
+		data:  *data,
+		slice: s.slice,
+		n:     data.Count(),
+	}
+}
+
+// SetBit sets the i-th bit of the bitmap.
+func (s *BitmapSegment) SetBit(i uint64) (changed bool) {
+	s.ensureWritable()
+
+	changed, _ = s.data.Add(i)
+	if changed {
+		s.n++
+	}
+	return changed
+}
+
+// ClearBit clears the i-th bit of the bitmap.
+func (s *BitmapSegment) ClearBit(i uint64) (changed bool) {
+	s.ensureWritable()
+
+	changed, _ = s.data.Remove(i)
+	if changed {
+		s.n--
+	}
+	return changed
+}
+
+// InvalidateCount updates the cached count in the bitmap.
+func (s *BitmapSegment) InvalidateCount() {
+	s.n = s.data.Count()
+}
+
+// Bits returns a list of all bits set in the segment.
+func (s *BitmapSegment) Bits() []uint64 {
+	a := make([]uint64, 0, s.Count())
+	itr := s.data.Iterator()
+	for v, eof := itr.Next(); !eof; v, eof = itr.Next() {
+		a = append(a, v)
+	}
+	return a
+}
+
+// Count returns the number of set bits in the bitmap.
+func (s *BitmapSegment) Count() uint64 { return s.n }
+
+// ensureWritable clones the segment if it is pointing to non-writable data.
+func (s *BitmapSegment) ensureWritable() {
+	if s.writable {
+		return
+	}
+
+	s.data = *s.data.Clone()
+	s.writable = true
+}
+
+// mergeSegmentIterator produces an iterator that loops through two sets of segments.
+type mergeSegmentIterator struct {
+	a0, a1 []BitmapSegment
+}
+
+// newMergeSegmentIterator returns a new instance of mergeSegmentIterator.
+func newMergeSegmentIterator(a0, a1 []BitmapSegment) mergeSegmentIterator {
+	return mergeSegmentIterator{a0: a0, a1: a1}
+}
+
+// next returns the next set of segments.
+func (itr *mergeSegmentIterator) next() (s0, s1 *BitmapSegment) {
+	// Find current segments.
+	if len(itr.a0) > 0 {
+		s0 = &itr.a0[0]
+	}
+	if len(itr.a1) > 0 {
+		s1 = &itr.a1[0]
+	}
+
+	// Return if either or both are nil.
+	if s0 == nil && s1 == nil {
+		return
+	} else if s0 == nil {
+		itr.a1 = itr.a1[1:]
+		return
+	} else if s1 == nil {
+		itr.a0 = itr.a0[1:]
+		return
+	}
+
+	// Otherwise determine which is first.
+	if s0.slice < s1.slice {
+		itr.a0 = itr.a0[1:]
+		return s0, nil
+	} else if s0.slice > s1.slice {
+		itr.a1 = itr.a1[1:]
+		return s1, nil
+	}
+
+	// Return both if slices are equal.
+	itr.a0, itr.a1 = itr.a0[1:], itr.a1[1:]
+	return s0, s1
+}

broadcast.go

@@ -1,165 +0,0 @@
-// Copyright 2022 Molecula Corp. (DBA FeatureBase).
-// SPDX-License-Identifier: Apache-2.0
-package pilosa
-
-import (
-	"fmt"
-
-	"github.com/featurebasedb/featurebase/v3/disco"
-	"github.com/pkg/errors"
-)
-
-// Serializer is an interface for serializing pilosa types to bytes and back.
-type Serializer interface {
-	Marshal(Message) ([]byte, error)
-	Unmarshal([]byte, Message) error
-}
-
-// NopSerializer represents a Serializer that doesn't do anything.
-var NopSerializer Serializer = &nopSerializer{}
-
-type nopSerializer struct{}
-
-// Marshal is a no-op implementation of Serializer Marshal method.
-func (*nopSerializer) Marshal(Message) ([]byte, error) { return nil, nil }
-
-// Unmarshal is a no-op implementation of Serializer Unmarshal method.
-func (*nopSerializer) Unmarshal([]byte, Message) error { return nil }
-
-// broadcaster is an interface for broadcasting messages.
-type broadcaster interface {
-	SendSync(Message) error
-	SendAsync(Message) error
-	SendTo(*disco.Node, Message) error
-}
-
-// Message is the interface implemented by all core pilosa types which can be serialized to messages.
-// TODO add at least a single "isMessage()" method.
-type Message interface{}
-
-// NopBroadcaster represents a Broadcaster that doesn't do anything.
-var NopBroadcaster broadcaster = &nopBroadcaster{}
-
-type nopBroadcaster struct{}
-
-// SendSync A no-op implementation of Broadcaster SendSync method.
-func (nopBroadcaster) SendSync(Message) error { return nil }
-
-// SendAsync A no-op implementation of Broadcaster SendAsync method.
-func (nopBroadcaster) SendAsync(Message) error { return nil }
-
-// SendTo is a no-op implementation of Broadcaster SendTo method.
-func (nopBroadcaster) SendTo(*disco.Node, Message) error { return nil }
-
-// Broadcast message types.
-const (
-	messageTypeCreateShard = iota
-	messageTypeCreateIndex
-	messageTypeDeleteIndex
-	messageTypeCreateField
-	messageTypeDeleteField
-	messageTypeCreateView
-	messageTypeDeleteView
-	messageTypeClusterStatus
-	messageTypeUNUSED0 // used to be ResizeInstruction
-	messageTypeUNUSED1 // used to be ResizeInstructionComplete
-	messageTypeNodeState
-	messageTypeRecalculateCaches
-	messageTypeLoadSchemaMessage
-	messageTypeNodeEvent
-	messageTypeNodeStatus
-	messageTypeTransaction
-	messageTypeUNUSED2 // used to be ResizeNodeMessage
-	messageTypeUNUSED3 // used to be ResizeAbortMessage
-	messageTypeUpdateField
-	messageTypeDeleteDataframe
-)
-
-// MarshalInternalMessage serializes the pilosa message and adds pilosa internal
-// type info which is used by the internal messaging stuff.
-func MarshalInternalMessage(m Message, s Serializer) ([]byte, error) {
-	typ := getMessageType(m)
-	buf, err := s.Marshal(m)
-	if err != nil {
-		return nil, errors.Wrap(err, "marshaling")
-	}
-	return append([]byte{typ}, buf...), nil
-}
-
-func getMessage(typ byte) Message {
-	switch typ {
-	case messageTypeCreateShard:
-		return &CreateShardMessage{}
-	case messageTypeCreateIndex:
-		return &CreateIndexMessage{}
-	case messageTypeDeleteIndex:
-		return &DeleteIndexMessage{}
-	case messageTypeCreateField:
-		return &CreateFieldMessage{}
-	case messageTypeDeleteField:
-		return &DeleteFieldMessage{}
-	case messageTypeCreateView:
-		return &CreateViewMessage{}
-	case messageTypeDeleteView:
-		return &DeleteViewMessage{}
-	case messageTypeClusterStatus:
-		return &ClusterStatus{}
-	case messageTypeNodeState:
-		return &NodeStateMessage{}
-	case messageTypeRecalculateCaches:
-		return &RecalculateCaches{}
-	case messageTypeLoadSchemaMessage:
-		return &LoadSchemaMessage{}
-	case messageTypeNodeEvent:
-		return &NodeEvent{}
-	case messageTypeNodeStatus:
-		return &NodeStatus{}
-	case messageTypeTransaction:
-		return &TransactionMessage{}
-	case messageTypeUpdateField:
-		return &UpdateFieldMessage{}
-	case messageTypeDeleteDataframe:
-		return &DeleteDataframeMessage{}
-	default:
-		panic(fmt.Sprintf("unknown message type %d", typ))
-	}
-}
-
-func getMessageType(m Message) byte {
-	switch m.(type) {
-	case *CreateShardMessage:
-		return messageTypeCreateShard
-	case *CreateIndexMessage:
-		return messageTypeCreateIndex
-	case *DeleteIndexMessage:
-		return messageTypeDeleteIndex
-	case *CreateFieldMessage:
-		return messageTypeCreateField
-	case *DeleteFieldMessage:
-		return messageTypeDeleteField
-	case *CreateViewMessage:
-		return messageTypeCreateView
-	case *DeleteViewMessage:
-		return messageTypeDeleteView
-	case *ClusterStatus:
-		return messageTypeClusterStatus
-	case *NodeStateMessage:
-		return messageTypeNodeState
-	case *RecalculateCaches:
-		return messageTypeRecalculateCaches
-	case *LoadSchemaMessage:
-		return messageTypeLoadSchemaMessage
-	case *NodeEvent:
-		return messageTypeNodeEvent
-	case *NodeStatus:
-		return messageTypeNodeStatus
-	case *TransactionMessage:
-		return messageTypeTransaction
-	case *UpdateFieldMessage:
-		return messageTypeUpdateField
-	case *DeleteDataframeMessage:
-		return messageTypeDeleteDataframe
-	default:
-		panic(fmt.Sprintf("don't have type for message %#v", m))
-	}
-}

bsi.go

@@ -1,284 +0,0 @@
-// Copyright 2022 Molecula Corp. (DBA FeatureBase).
-// SPDX-License-Identifier: Apache-2.0
-package pilosa
-
-import (
-	"math/bits"
-
-	"github.com/featurebasedb/featurebase/v3/roaring"
-)
-
-// BSIData contains BSI-structured data.
-type BSIData []*Row
-
-// PivotDescending loops over nonzero BSI values in descending order.
-// For each value, the provided function is called with the value and a slice of the associated columns.
-// If limit or offset are not-nil, they will be applied.
-// Applying a limit or offset may modify the pointed-to value.
-func (bsi BSIData) PivotDescending(filter *Row, branch uint64, limit, offset *uint64, fn func(uint64, ...uint64)) {
-	// This "pivot" algorithm works by treating the BSI data as a tree.
-	// Each branch of this tree corresponds to a power-of-2-sized range of BSI values.
-	// Each range is subdivided into 2 ranges of half size, which form lower branches.
-	// Eventually, a range of width 1 cannot be subdivided and forms a leaf.
-	// At each branch and leaf, there is a bitmap of all columns within the corresponding range.
-	// The lower branches are formed as a difference or intersect of the upper branch's bitmap with the BSI bit that subdivides the range.
-	// This function uses a depth-first search over this virtual tree.
-
-	switch {
-	case !filter.Any():
-		// There are no remaining data.
-
-	case offset != nil && *offset >= filter.Count():
-		// Skip this entire branch.
-		*offset -= filter.Count()
-
-	case limit != nil && *limit == 0:
-		// The limit has been reached.
-		// No more data is necessary.
-
-	case len(bsi) == 0:
-		// This is a leaf node.
-		cols := filter.Columns()
-		if offset != nil {
-			cols = cols[*offset:]
-			*offset = 0
-		}
-		if limit != nil {
-			if *limit < uint64(len(cols)) {
-				cols = cols[:*limit]
-			}
-			*limit -= uint64(len(cols))
-		}
-		fn(branch, cols...)
-
-	default:
-		// Pivot over the highest bit.
-		upperBranch, lowerBranch := branch|(1<<uint(len(bsi)-1)), branch
-		splitBit := bsi[len(bsi)-1]
-		lowerBits := bsi[:len(bsi)-1]
-		lowerBits.PivotDescending(filter.Intersect(splitBit), upperBranch, limit, offset, fn)
-		lowerBits.PivotDescending(filter.Difference(splitBit), lowerBranch, limit, offset, fn)
-	}
-}
-
-/*
-// distribution generates a BSI histogram for the input.
-// TODO: I forgot what I was going to use this for.
-// Could probbably use this for:
-// - quartile queries
-// - TopN on int
-func (bsi bsiData) distribution(filter *Row) bsiData {
-	var dist bsiData
-	bsi.PivotDescending(filter, 0, nil, nil, func(count uint64, values ...uint64) {
-		dist.insert(count, uint64(len(values)))
-	})
-	return dist
-}
-*/
-
-var placeholderBitmap = roaring.NewBitmap()
-
-// AddBSI adds two BSI bitmaps together.
-// It does not handle sign and has no concept of overflow.
-func AddBSI(x, y BSIData) BSIData {
-	// Accumulate row segments.
-	segments := make([][]RowSegment, len(x)+len(y))
-	xsegs, ysegs := segments[:len(x)], segments[len(x):]
-	for i, r := range x {
-		xsegs[i] = r.Segments
-	}
-	for i, r := range y {
-		ysegs[i] = r.Segments
-	}
-
-	var dst BSIData
-	var xbitmaps, ybitmaps []*roaring.Bitmap
-	for {
-		// Find the next shard.
-		next := ^uint64(0)
-		for _, s := range segments {
-			if len(s) == 0 {
-				continue
-			}
-			shard := s[0].shard
-			if shard < next {
-				next = shard
-			}
-		}
-		if next == ^uint64(0) {
-			// There are no remaining shards.
-			break
-		}
-
-		// Accumulate bitmaps for this shard.
-		xbitmaps, ybitmaps = xbitmaps[:0], ybitmaps[:0]
-		for i, segs := range xsegs {
-			if len(segs) == 0 || segs[0].shard != next {
-				continue
-			}
-			xsegs[i] = segs[1:]
-			bm := segs[0].data
-			if !bm.Any() {
-				continue
-			}
-			for len(xbitmaps) < i {
-				xbitmaps = append(xbitmaps, placeholderBitmap)
-			}
-			xbitmaps = append(xbitmaps, bm)
-		}
-		for i, segs := range ysegs {
-			if len(segs) == 0 || segs[0].shard != next {
-				continue
-			}
-			ysegs[i] = segs[1:]
-			bm := segs[0].data
-			if !bm.Any() {
-				continue
-			}
-			for len(ybitmaps) < i {
-				ybitmaps = append(ybitmaps, placeholderBitmap)
-			}
-			ybitmaps = append(ybitmaps, bm)
-		}
-
-		// Add the shard values together.
-		var out []*roaring.Bitmap
-		switch {
-		case len(xbitmaps) == 0:
-			// There are no values in x.
-			out = ybitmaps
-		case len(ybitmaps) == 0:
-			// There are no values in y.
-			out = xbitmaps
-		default:
-			out = roaring.Add(xbitmaps, ybitmaps)
-		}
-
-		// Convert the bitmaps to output segments.
-		for i, b := range out {
-			if !b.Any() {
-				continue
-			}
-			for len(dst) <= i {
-				dst = append(dst, NewRow())
-			}
-			dst[i].Segments = append(dst[i].Segments, RowSegment{
-				shard:    next,
-				writable: true,
-				data:     b,
-				n:        b.Count(),
-			})
-		}
-	}
-
-	return dst
-}
-
-// rowBuilder builds a row quickly from individual values.
-// It is optimized for the case in which values are generated sequentially.
-type rowBuilder struct {
-	bm    *roaring.Bitmap
-	mask  *[1024]uint64
-	array []uint16
-	key   uint64
-	n     int32
-}
-
-// flushKey flushes the data at the current key to the bitmap.
-func (b *rowBuilder) flushKey() {
-	var c *roaring.Container
-	switch {
-	case b.mask != nil:
-		c = roaring.NewContainerBitmapN(b.mask[:], b.n)
-		b.mask = nil
-	case len(b.array) > 0:
-		c = roaring.NewContainerArrayCopy(b.array)
-		b.array = b.array[:0]
-	default:
-		return
-	}
-
-	if b.bm == nil {
-		b.bm = roaring.NewBitmap()
-	}
-	if old := b.bm.Containers.Get(b.key); old != nil {
-		c = roaring.Union(c, old)
-	}
-	b.bm.Containers.Put(b.key, c)
-}
-
-// Add a value to the bitmap.
-// Values must be added sequentially.
-func (b *rowBuilder) Add(v uint64) {
-	vkey := v / (1 << 16)
-	if b.key != vkey {
-		// This is a new key, so flush the old one.
-		b.flushKey()
-		b.key = vkey
-	}
-
-	if b.mask != nil {
-		// Add to the mask.
-		b.n += int32(1 &^ (b.mask[uint16(v)/64] >> (v % 64)))
-		b.mask[uint16(v)/64] |= 1 << (v % 64)
-		return
-	}
-
-	// Add to an array.
-	b.array = append(b.array, uint16(v))
-	if len(b.array) >= roaring.ArrayMaxSize {
-		// The array is too big.
-		// Convert it to a bitmask.
-		m := [1024]uint64{}
-		for _, v := range b.array {
-			m[v/64] |= 1 << (v % 64)
-		}
-		b.n = int32(len(b.array))
-		b.array = b.array[:0]
-		b.mask = &m
-	}
-}
-
-// Build a Row from stored data.
-// This resets the builder.
-func (b *rowBuilder) Build() *Row {
-	// Flush the active key to the bitmap.
-	b.flushKey()
-
-	// Remove the bitmap and convert it to a Row.
-	bm := b.bm
-	b.bm = nil
-	if bm == nil {
-		return NewRow()
-	}
-	return NewRowFromBitmap(bm)
-}
-
-// bsiBuilder assembles BSI data.
-// It is optimized for the case in which values are generated sequentially.
-type bsiBuilder []rowBuilder
-
-// Insert a value into the BSI data.
-// Columns must be inserted sequentially, and duplicates are not allowed.
-func (b *bsiBuilder) Insert(col, val uint64) {
-	for val != 0 {
-		i := bits.TrailingZeros64(val)
-		val &^= 1 << i
-		for len(*b) <= i {
-			*b = append(*b, rowBuilder{})
-		}
-		(*b)[i].Add(col)
-	}
-}
-
-// Build BSI data.
-// This resets the builder.
-func (b *bsiBuilder) Build() BSIData {
-	builders := *b
-	*b = builders[:0]
-	rows := make(BSIData, len(builders))
-	for i := range builders {
-		rows[i] = builders[i].Build()
-	}
-	return rows
-}

bsi_test.go

@@ -1,160 +0,0 @@
-// Copyright 2022 Molecula Corp. (DBA FeatureBase).
-// SPDX-License-Identifier: Apache-2.0
-package pilosa
-
-import (
-	"fmt"
-	"math/rand"
-	"sort"
-	"testing"
-)
-
-// TestBSIAdd does a number of iterations. For each iteration, it
-// generates a random number of ids, and two random values for each id
-// to add together.
-func TestBSIAdd(t *testing.T) {
-	// TODO wouldn't it be cool if our test suite had a randomized
-	// burn-in mode where you could run any test which supported it
-	// with a random seed and way more iterations?
-	rnd := rand.New(rand.NewSource(99))
-	//numZipf := rand.NewZipf(rnd, 1.5, 2, ShardWidth-1)
-	idZipf := rand.NewZipf(rnd, 1.8, 4, ShardWidth)
-	var builderA, builderB bsiBuilder
-	// a and b are generated slices of numbers to add together
-	var a, b []uint64
-	// idToIndex maps record ids to indexes in a and b
-	idToIndex := make(map[int]int)
-	// indexToID has the record id for each value in a and b
-	indexToID := []uint64{}
-
-	min := 999999999
-	max := 0
-
-	for iteration := 0; iteration < 1; iteration++ {
-		t.Run(fmt.Sprintf("%d", iteration), func(t *testing.T) {
-			// reset generated data
-			a, b = a[:0], b[:0]
-			indexToID = indexToID[:0]
-			for k := range idToIndex {
-				delete(idToIndex, k)
-			}
-
-			// z generates the values, they can be fairly large, but are usually small
-			z := rand.NewZipf(rnd, 1.3, 7, 1<<44)
-			id := -1
-			for i := 0; true; i++ {
-				// get the next id, skipping a random amount
-				id = id + int(idZipf.Uint64()+1)
-				if id >= ShardWidth {
-					if i < min {
-						min = i
-					}
-					if max < i {
-						max = i
-					}
-					break
-				}
-				idToIndex[id] = int(i)
-				indexToID = append(indexToID, uint64(id))
-
-				// append a random value to each data slice
-				a = append(a, z.Uint64())
-				b = append(b, z.Uint64())
-			}
-
-			// build the BSIs based on the data slices and generated IDs
-			for index, id := range indexToID {
-				va, vb := a[index], b[index]
-				builderA.Insert(uint64(id), va)
-				builderB.Insert(uint64(id), vb)
-			}
-			dataA, dataB := builderA.Build(), builderB.Build()
-			dataC := AddBSI(dataA, dataB)
-
-			// build results from added bsiData; results[i] should hold a[i]+b[i]
-			results := make([]uint64, len(a))
-			dataC.PivotDescending(NewRow().Union(dataC...), 0, nil, nil, func(count uint64, ids ...uint64) {
-				for _, id := range ids {
-					results[idToIndex[int(id)]] = count
-				}
-			})
-
-			for i, res := range results {
-				if res != a[i]+b[i] {
-					t.Errorf("Mismatch at %d\na: %v\nb: %v\nr: %v", i, a, b, results)
-				}
-			}
-		})
-	}
-}
-
-type bsiAddCase struct {
-	positions []uint64
-	a         []uint64
-	b         []uint64
-}
-
-func (b bsiAddCase) Len() int {
-	return len(b.positions)
-}
-
-// Less reports whether the element with
-// index i should sort before the element with index j.
-func (b bsiAddCase) Less(i, j int) bool {
-	return b.positions[i] < b.positions[j]
-}
-
-// Swap swaps the elements with indexes i and j.
-func (b bsiAddCase) Swap(i, j int) {
-	b.positions[i], b.positions[j] = b.positions[j], b.positions[i]
-	b.a[i], b.a[j] = b.a[j], b.a[i]
-	b.b[i], b.b[j] = b.b[j], b.b[i]
-}
-
-// TestBSIAddCases tests specific cases of bsiAdd (would generally be
-// pulled from randomly generated ones from TestBSIAdd upon failure).
-func TestBSIAddCases(t *testing.T) {
-	tests := []bsiAddCase{
-		{
-			positions: []uint64{161311, 611110, 82544, 996022, 836077, 64964, 480737, 156534, 240525, 580896, 239236, 54607, 1019438, 894260, 17570, 884645, 936658, 682651, 987695, 390274},
-			a:         []uint64{17, 1, 2846, 45437619, 23781, 36, 88, 168691, 13417, 1301, 10, 71, 0, 176, 1010, 21, 1, 509, 17, 4},
-			b:         []uint64{24, 288, 12737, 14, 150, 21, 24, 354, 0, 19, 5, 150, 3940, 121, 25, 621, 7, 9023592401, 6033, 7},
-		},
-		{
-			positions: []uint64{17570, 54607},
-			a:         []uint64{1010, 71},
-			b:         []uint64{25, 150},
-		},
-	}
-
-	var builderA, builderB bsiBuilder
-	for i, tst := range tests {
-		t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
-			if len(tst.a) != len(tst.b) || len(tst.a) != len(tst.positions) {
-				t.Fatalf("Malformed test, a is %d, but b is %d", len(tst.a), len(tst.b))
-			}
-			sort.Sort(tst)
-
-			for i := 0; i < len(tst.a); i++ {
-				builderA.Insert(tst.positions[i], tst.a[i])
-				builderB.Insert(tst.positions[i], tst.b[i])
-			}
-
-			dataA, dataB := builderA.Build(), builderB.Build()
-			dataC := AddBSI(dataA, dataB)
-			// maps id to count
-			results := make(map[uint64]uint64)
-			dataC.PivotDescending(NewRow().Union(dataC...), 0, nil, nil, func(count uint64, ids ...uint64) {
-				for _, id := range ids {
-					results[id] = count
-				}
-			})
-
-			for i, id := range tst.positions {
-				if results[id] != tst.a[i]+tst.b[i] {
-					t.Fatalf("value %d mismatch, id: %d. got %d, want %d", i, id, results[id], tst.a[i]+tst.b[i])
-				}
-			}
-		})
-	}
-}

buffer/filebuffer.go

@@ -1,110 +0,0 @@
-package buffer
-
-import (
-	"bytes"
-	"io"
-	"io/ioutil"
-	"os"
-	"sync"
-)
-
-// NewFileBuffer returns a file buffer which will use an in-memory buffer, until `max` bytes have been written, at which point it will write the contents of memory to a file, and continue writing future data to the file.
-// The file will be written to `temp` directory. The buffer fulfills the io.Reader and io.Writer interface
-func NewFileBuffer(max int, temp string) *FileBuffer {
-	return &FileBuffer{max: max, tempDir: temp}
-}
-
-type FileBuffer struct {
-	max     int
-	buf     bytes.Buffer
-	file    *os.File
-	tempDir string
-	reading bool
-	files   []*os.File
-	mu      sync.Mutex
-}
-
-func (fb *FileBuffer) Write(p []byte) (n int, err error) {
-	if fb.reading {
-		panic("cannot write after read")
-	}
-	if fb.file != nil {
-		return fb.file.Write(p)
-	}
-	n, err = fb.buf.Write(p)
-	if err != nil {
-		return
-	}
-	if fb.buf.Len() > fb.max {
-		fb.file, err = ioutil.TempFile(fb.tempDir, "filebuffer-")
-		if err != nil {
-			return
-		}
-		_, err = io.Copy(fb.file, &fb.buf)
-		fb.buf.Reset()
-	}
-	return
-}
-
-func (fb *FileBuffer) Len() (int64, error) {
-	if fb.file == nil {
-		return int64(fb.buf.Len()), nil
-	}
-	fi, err := fb.file.Stat()
-	if err != nil {
-		return 0, err
-	}
-
-	return fi.Size(), nil
-}
-
-func (fb *FileBuffer) Read(p []byte) (n int, err error) {
-	if fb.file != nil {
-		if !fb.reading {
-			fb.reading = true
-			_, err = fb.file.Seek(0, 0)
-			if err != nil {
-				return
-			}
-		}
-		return fb.file.Read(p)
-	}
-	fb.reading = true
-	return fb.buf.Read(p)
-}
-
-func (fb *FileBuffer) Close() error {
-	if fb.file != nil {
-		name := fb.file.Name()
-		if err := fb.file.Close(); err != nil {
-			return err
-		}
-		for _, f := range fb.files {
-			f.Close()
-		}
-		fb.files = fb.files[:0]
-		fb.file = nil
-		return os.Remove(name)
-	}
-	return nil
-}
-
-func (fb *FileBuffer) Reset() error {
-	fb.mu.Lock()
-	defer fb.mu.Unlock()
-	fb.reading = false
-	fb.buf.Reset()
-	return fb.Close()
-}
-
-func (fb *FileBuffer) NewReader() (io.Reader, error) {
-	fb.mu.Lock()
-	defer fb.mu.Unlock()
-	fb.reading = true
-	if fb.file == nil {
-		return bytes.NewReader(fb.buf.Bytes()), nil
-	}
-	f, err := os.OpenFile(fb.file.Name(), os.O_RDONLY, 0)
-	fb.files = append(fb.files, f)
-	return f, err
-}

bufferpool/bufferpool.go

@@ -1,262 +0,0 @@
-package bufferpool
-
-import (
-	"errors"
-	"fmt"
-	"sync"
-)
-
-// FrameID is the type for frame id
-type FrameID int
-
-// PageID is the type for page id
-type PageID int
-
-var pageSyncPool = sync.Pool{
-	New: func() any {
-		pg := new(Page)
-		pg.id = PageID(INVALID_PAGE)
-		pg.isDirty = false
-		pg.pinCount = 0
-		return pg
-	},
-}
-
-// BufferPool represents a buffer pool of pages
-type BufferPool struct {
-	// the underlying storage
-	diskManager DiskManager
-	// the actual pages in the buffer pool
-	pages []*Page
-	// the replacer that will elect replacements when buffer pool is full
-	replacer *ClockReplacer
-	// the list of free frames
-	freeList []FrameID
-	// the map of frames to page ids to frame ids
-	// frame ids are the offset into pages
-	// if you ask the pool for page 673, this will know at
-	// what offset in pages page 673 will exist
-	pageTable map[PageID]FrameID
-}
-
-// TODO(pok) implement a lazy writer
-//  * if free list is 'low' then
-//		* increase size of cache if there is physical memory available
-//		* write out old pages and boot them from the cache to increase free list
-
-// TODO(pok) implement a checkpoint that scans the pool and writes out dirty pages every
-// minute or so
-
-// NewBufferPool returns a buffer pool
-func NewBufferPool(maxSize int, diskManager DiskManager) *BufferPool {
-	freeList := make([]FrameID, 0)
-	pages := make([]*Page, maxSize)
-	for i := 0; i < maxSize; i++ {
-		frameNumber := FrameID(i)
-		freeList = append(freeList, frameNumber)
-	}
-	clockReplacer := NewClockReplacer(maxSize)
-	return &BufferPool{
-		diskManager: diskManager,
-		pages:       pages,
-		replacer:    clockReplacer,
-		freeList:    freeList,
-		pageTable:   make(map[PageID]FrameID),
-	}
-}
-
-// Dumps all the pages in the buffer pool
-func (b *BufferPool) Dump() {
-	fmt.Println()
-	fmt.Printf("------------------------------------------------------------------------------------------\n")
-	fmt.Printf("BUFFER POOL\n")
-	for _, p := range b.pages {
-		if p != nil {
-			p.Dump("")
-		}
-	}
-	fmt.Printf("------------------------------------------------------------------------------------------\n")
-	fmt.Println()
-}
-
-// FetchPage fetches the requested page from the buffer pool.
-func (b *BufferPool) FetchPage(pageID PageID) (*Page, error) {
-	// if it is in buffer pool already then just return it
-	if frameID, ok := b.pageTable[pageID]; ok {
-		page := b.pages[frameID]
-		page.pinCount++
-		b.replacer.Pin(frameID)
-		return page, nil
-	}
-
-	// not in the buffer pool so try the free list or
-	// the replacer will vote a page off the island
-	frameID, isFromFreeList, err := b.getFrameID()
-	if err != nil {
-		return nil, err
-	}
-
-	if !isFromFreeList {
-		// if it didn't come from the freelist then
-		// remove page from current frame, writing it out if dirty
-		currentPage := b.pages[frameID]
-		if currentPage != nil {
-			if currentPage.isDirty {
-				b.diskManager.WritePage(currentPage)
-			}
-
-			delete(b.pageTable, currentPage.id)
-		}
-	}
-
-	// if we got to here, sorry, have to do an I/O
-	page, err := b.diskManager.ReadPage(pageID)
-	if err != nil {
-		return nil, err
-	}
-	page.pinCount = 1
-	b.pageTable[pageID] = frameID
-	pageSyncPool.Put(b.pages[frameID])
-	b.pages[frameID] = page
-	b.replacer.Pin(frameID)
-
-	return page, nil
-}
-
-// UnpinPage unpins the target page from the buffer pool
-func (b *BufferPool) UnpinPage(pageID PageID) error {
-	if frameID, ok := b.pageTable[pageID]; ok {
-		page := b.pages[frameID]
-		page.DecPinCount()
-
-		if page.pinCount <= 0 {
-			b.replacer.Unpin(frameID)
-		}
-		return nil
-	}
-
-	return errors.New("could not find page")
-}
-
-// FlushPage Flushes the target page to disk
-func (b *BufferPool) FlushPage(pageID PageID) bool {
-	if frameID, ok := b.pageTable[pageID]; ok {
-		page := b.pages[frameID]
-		page.DecPinCount()
-
-		b.diskManager.WritePage(page)
-		page.isDirty = false
-
-		return true
-	}
-	return false
-}
-
-// NewPage allocates a new page in the buffer pool with the disk manager help
-func (b *BufferPool) NewPage() (*Page, error) {
-	// get a free frame
-	frameID, isFromFreeList, err := b.getFrameID()
-	if err != nil {
-		return nil, err
-	}
-
-	if !isFromFreeList {
-		// remove page from current frame
-		currentPage := b.pages[frameID]
-		if currentPage != nil {
-			if currentPage.isDirty {
-				b.diskManager.WritePage(currentPage)
-			}
-
-			delete(b.pageTable, currentPage.id)
-		}
-	}
-
-	// allocates new page
-	pageID, err := b.diskManager.AllocatePage()
-	if err != nil {
-		return nil, err
-	}
-	page := &Page{pageID, 1, false, [PAGE_SIZE]byte{}}
-	page.WritePageNumber(int32(pageID))
-	page.WriteFreeSpaceOffset(int16(PAGE_SIZE))
-	page.WriteNextPointer(int32(INVALID_PAGE))
-	page.WritePrevPointer(int32(INVALID_PAGE))
-
-	// update the frame table
-	b.pageTable[pageID] = frameID
-	pageSyncPool.Put(b.pages[frameID])
-	b.pages[frameID] = page
-
-	return page, nil
-}
-
-// ScratchPage returns a page outside the buffer pool - do not use if you intend the page
-// to be in the buffer pool (use NewPage() for that)
-// ScratchPage is intended to be used in cases where you need the Page primitives
-// and will copy the scratch page back over a real page later
-func (b *BufferPool) ScratchPage() *Page {
-	page := &Page{
-		id:       PageID(INVALID_PAGE),
-		pinCount: 0,
-		isDirty:  false,
-		data:     [PAGE_SIZE]byte{},
-	}
-	page.WritePageNumber(int32(INVALID_PAGE))
-	page.WriteFreeSpaceOffset(int16(PAGE_SIZE))
-	page.WriteNextPointer(int32(INVALID_PAGE))
-	page.WritePrevPointer(int32(INVALID_PAGE))
-	return page
-}
-
-// DeletePage deletes a page from the buffer pool
-func (b *BufferPool) DeletePage(pageID PageID) error {
-	var frameID FrameID
-	var ok bool
-	if frameID, ok = b.pageTable[pageID]; !ok {
-		return nil
-	}
-
-	page := b.pages[frameID]
-
-	if page.pinCount > 0 {
-		return errors.New("pin count greater than 0")
-	}
-	delete(b.pageTable, page.id)
-	b.replacer.Pin(frameID)
-	b.diskManager.DeallocatePage(pageID)
-
-	b.freeList = append(b.freeList, frameID)
-
-	return nil
-}
-
-// FlushAllpages flushes all the pages in the buffer pool to disk
-// Yeah, never call this unless you know what you are doing
-func (b *BufferPool) FlushAllpages() {
-	for pageID := range b.pageTable {
-		b.FlushPage(pageID)
-	}
-}
-
-func (b *BufferPool) getFrameID() (FrameID, bool, error) {
-	if len(b.freeList) > 0 {
-		frameID, newFreeList := b.freeList[0], b.freeList[1:]
-		b.freeList = newFreeList
-		return frameID, true, nil
-	}
-
-	victim, err := b.replacer.Victim()
-	return victim, false, err
-}
-
-// OnDiskSize exposes the on disk size of the backing store
-// behind this buffer pool
-func (b *BufferPool) OnDiskSize() int64 {
-	return b.diskManager.FileSize()
-}
-
-// Close closes the buffer pool
-func (b *BufferPool) Close() {
-	b.diskManager.Close()
-}

bufferpool/circularlist.go

@@ -1,93 +0,0 @@
-package bufferpool
-
-import (
-	"errors"
-)
-
-type circularListNode struct {
-	key   interface{}
-	value interface{}
-	next  *circularListNode
-	prev  *circularListNode
-}
-
-type circularList struct {
-	head     *circularListNode
-	tail     *circularListNode
-	size     int
-	capacity int
-}
-
-func newCircularList(maxSize int) *circularList {
-	return &circularList{nil, nil, 0, maxSize}
-}
-
-func (c *circularList) find(key interface{}) *circularListNode {
-	ptr := c.head
-	for i := 0; i < c.size; i++ {
-		if ptr.key == key {
-			return ptr
-		}
-		ptr = ptr.next
-	}
-	return nil
-}
-
-func (c *circularList) hasKey(key interface{}) bool {
-	return c.find(key) != nil
-}
-
-func (c *circularList) insert(key interface{}, value interface{}) error {
-	if c.size == c.capacity {
-		return errors.New("list is full")
-	}
-	newNode := &circularListNode{key, value, nil, nil}
-	if c.size == 0 {
-		newNode.next = newNode
-		newNode.prev = newNode
-		c.head = newNode
-		c.tail = newNode
-		c.size++
-		return nil
-	}
-
-	node := c.find(key)
-	if node != nil {
-		node.value = value
-		return nil
-	}
-
-	newNode.next = c.head
-	newNode.prev = c.tail
-	c.tail.next = newNode
-	if c.head == c.tail {
-		c.head.next = newNode
-	}
-	c.tail = newNode
-	c.head.prev = c.tail
-
-	c.size++
-	return nil
-}
-
-func (c *circularList) remove(key interface{}) {
-	node := c.find(key)
-	if node == nil {
-		return
-	}
-	if c.size == 1 {
-		c.head = nil
-		c.tail = nil
-		c.size--
-		return
-	}
-	if node == c.head {
-		c.head = c.head.next
-	}
-	if node == c.tail {
-		c.tail = c.tail.prev
-	}
-	node.next.prev = node.prev
-	node.prev.next = node.next
-	c.size--
-}

bufferpool/clockreplacer.go

@@ -1,64 +0,0 @@
-package bufferpool
-
-import "errors"
-
-// ClockReplacer implements a clock replacer algorithm
-type ClockReplacer struct {
-	cList     *circularList
-	clockHand **circularListNode
-}
-
-// NewClockReplacer instantiates a new clock replacer
-func NewClockReplacer(poolSize int) *ClockReplacer {
-	cList := newCircularList(poolSize)
-	return &ClockReplacer{cList, &cList.head}
-}
-
-// Victim removes the victim frame as defined by the replacement policy
-func (c *ClockReplacer) Victim() (FrameID, error) {
-	if c.cList.size == 0 {
-		return FrameID(INVALID_PAGE), errors.New("no victims available")
-	}
-	var victimFrameID FrameID
-	currentNode := (*c.clockHand)
-	for {
-
-		if currentNode.value.(bool) {
-			currentNode.value = false
-			c.clockHand = &currentNode.next
-		} else {
-			frameID := currentNode.key.(FrameID)
-			victimFrameID = frameID
-			c.clockHand = &currentNode.next
-			c.cList.remove(currentNode.key)
-			return victimFrameID, nil
-		}
-	}
-}
-
-// Unpin unpins a frame, indicating that it can now be victimized
-func (c *ClockReplacer) Unpin(id FrameID) {
-	if !c.cList.hasKey(id) {
-		c.cList.insert(id, true)
-		if c.cList.size == 1 {
-			c.clockHand = &c.cList.head
-		}
-	}
-}
-
-// Pin pins a frame, indicating that it should not be victimized until it is unpinned
-func (c *ClockReplacer) Pin(id FrameID) {
-	node := c.cList.find(id)
-	if node == nil {
-		return
-	}
-	if (*c.clockHand) == node {
-		c.clockHand = &(*c.clockHand).next
-	}
-	c.cList.remove(id)
-}
-
-// Size returns the size of the clock
-func (c *ClockReplacer) Size() int {
-	return c.cList.size
-}

bufferpool/diskmanager.go

@@ -1,21 +0,0 @@
-package bufferpool
-
-// DiskManager is responsible for interacting with disk
-type DiskManager interface {
-	// reads a page from the disk
-	ReadPage(PageID) (*Page, error)
-	// writes a page to the disk
-	WritePage(*Page) error
-
-	// allocates a page
-	AllocatePage() (PageID, error)
-
-	// deallocates a page
-	DeallocatePage(PageID) error
-
-	// returns on disk file size
-	FileSize() int64
-
-	// closes and does any clean up
-	Close()
-}

bufferpool/inmemdiskmanager.go

@@ -1,162 +0,0 @@
-package bufferpool
-
-import (
-	"errors"
-	"fmt"
-	"os"
-
-	uuid "github.com/satori/go.uuid"
-)
-
-// InMemDiskSpillingDiskManager is a memory implementation for a DiskManager interface
-// that can spill to disk when a threshold is reached
-type InMemDiskSpillingDiskManager struct {
-	// tracks the number of pages
-	numPages int
-
-	onDiskPages int
-
-	// tracks the number of pages we can consume before spilling
-	thresholdPages int
-	hasSpilled     *struct{}
-	fd             *os.File
-
-	// the data buffer
-	data []byte
-}
-
-// NewInMemDiskSpillingDiskManager returns a in-memory version of disk manager
-func NewInMemDiskSpillingDiskManager(thresholdPages int) *InMemDiskSpillingDiskManager {
-	dm := &InMemDiskSpillingDiskManager{
-		numPages:       0,
-		thresholdPages: thresholdPages,
-		data:           make([]byte, 0),
-	}
-	return dm
-}
-
-// ReadPage reads a page from pages
-func (d *InMemDiskSpillingDiskManager) ReadPage(pageID PageID) (*Page, error) {
-	// check we're not asking for page out of range
-	if pageID < 0 || int(pageID) >= d.numPages {
-		return nil, errors.New("page not found")
-	}
-	// check that the offset is within range
-	offset := int(pageID) * PAGE_SIZE
-
-	var page = pageSyncPool.Get().(*Page)
-	// we have to do this stupid check because if -cpuprofile is set for go test, this
-	// the previous line return a weird nil-ish thing...
-	if page == (*Page)(nil) {
-		page = pageSyncPool.New().(*Page)
-	}
-	page.id = pageID
-
-	// do the read
-	if d.hasSpilled == nil {
-		if offset+PAGE_SIZE > len(d.data) {
-			return nil, errors.New("offset out of range")
-		}
-		b := copy(page.data[:], d.data[offset:offset+PAGE_SIZE])
-		fmt.Printf("bytes read: %d", b)
-	} else {
-		var err error
-		if offset+PAGE_SIZE > d.numPages*PAGE_SIZE {
-			return nil, errors.New("offset out of range")
-		}
-		_, err = d.fd.ReadAt(page.data[:], int64(offset))
-		if err != nil {
-			return nil, err
-		}
-	}
-	return page, nil
-}
-
-// WritePage writes a page in memory to pages
-func (d *InMemDiskSpillingDiskManager) WritePage(page *Page) error {
-	// make sure the offset is sensible
-	offset := int(page.ID()) * PAGE_SIZE
-	// do the write
-	if d.hasSpilled == nil {
-		if offset+PAGE_SIZE > len(d.data) {
-			return errors.New("offset out of range")
-		}
-		copy(d.data[offset:], page.data[:])
-	} else {
-		var err error
-		if offset+PAGE_SIZE > d.numPages*PAGE_SIZE {
-			return errors.New("offset out of range")
-		}
-		_, err = d.fd.WriteAt(page.data[:], int64(offset))
-		if err != nil {
-			return err
-		}
-		// err = d.fd.Sync()
-		// if err != nil {
-		// 	return err
-		// }
-	}
-	return nil
-}
-
-// AllocatePage allocates a page and returns the page number
-func (d *InMemDiskSpillingDiskManager) AllocatePage() (PageID, error) {
-	d.numPages = d.numPages + 1
-	pageID := PageID(d.numPages - 1)
-
-	if d.hasSpilled == nil {
-		// we have not spilled (yet), so make storage bigger
-		newData := make([]byte, PAGE_SIZE)
-		d.data = append(d.data, newData...)
-
-		// check to see if we need to spill
-		if d.numPages > d.thresholdPages {
-			fileUUID, err := uuid.NewV4()
-			if err != nil {
-				return PageID(INVALID_PAGE), err
-			}
-			// TODO(pok) we should try to tell the OS not to cache this file
-			d.fd, err = os.CreateTemp("", fmt.Sprintf("fb-ehash-%s", fileUUID.String()))
-			if err != nil {
-				return PageID(INVALID_PAGE), err
-			}
-			_, err = d.fd.WriteAt(d.data, 0)
-			if err != nil {
-				return PageID(INVALID_PAGE), err
-			}
-			d.data = []byte{}
-			d.hasSpilled = &struct{}{}
-		}
-	} else {
-		if d.numPages >= d.onDiskPages {
-			// grow the file by a chunk - 512 pages
-			d.onDiskPages += 512
-			var err error
-			size := int64(d.onDiskPages * PAGE_SIZE)
-			_, err = d.fd.WriteAt([]byte{0}, size-1)
-			if err != nil {
-				return PageID(INVALID_PAGE), err
-			}
-		}
-	}
-
-	return pageID, nil
-}
-
-// DeallocatePage removes page from disk
-func (d *InMemDiskSpillingDiskManager) DeallocatePage(pageID PageID) error {
-	// nothing to do right now
-	return nil
-}
-
-func (d *InMemDiskSpillingDiskManager) FileSize() int64 {
-	return int64(len(d.data))
-}
-
-func (d *InMemDiskSpillingDiskManager) Close() {
-	// close and delete the file if we spilled
-	if d.fd != nil {
-		_ = d.fd.Close()
-		os.Remove(d.fd.Name())
-	}
-}

bufferpool/page.go

@@ -1,371 +0,0 @@
-package bufferpool
-
-import (
-	"encoding/binary"
-	"errors"
-	"fmt"
-)
-
-const PAGE_SIZE int = 8192
-
-const INVALID_PAGE int = -1
-
-const PAGE_TYPE_BTREE_INTERNAL = 10
-const PAGE_TYPE_BTREE_LEAF = 11
-const PAGE_TYPE_HASH_TABLE = 12
-
-// PAGE
-// page size 8192 bytes
-// byte aligned, big endian
-
-// |====================================================|
-// | offset | length        |                           |
-// |----------------------------------------------------|
-// | header                                             |
-// |====================================================|
-// | 0      | 4             |  pageNumber (int32)       |
-// | 4      | 2             |  pageType (int16)         |
-// | 6      | 2             |  slotCount (int16)        |
-// | 8      | 2             |  localDepth (int16)       |
-// | 10     | 2             |  freeSpaceOffset (int16)  |
-// | 12     | 4             |  prevPointer (int32)      |
-// | 16     | 4             |  nextPointer (int32)      |
-// |====================================================|
-// | <start of slot array 1..slotCount>                 |
-// |----------------------------------------------------|
-// | 20     | slotcount     |  slot entry is 2 int16    |
-// |        |  * slotwidth  |   values (payloadOffset,  |
-// |        |  * #slots     |   payloadLength)          |
-// |----------------------------------------------------|
-// | <free space>                                       |
-// |----------------------------------------------------|
-// | <payload starting at freeSpaceOffset>              |
-// |  payload entries are keylength (int16), key bytes, |
-// |  payload length (int32), payload bytes             |
-// |====================================================|
-
-const PAGE_NUMBER_OFFSET = 0        // offset 0, length 4, end 4
-const PAGE_TYPE_OFFSET = 4          // offset 4, length 2, end 6
-const PAGE_SLOT_COUNT_OFFSET = 6    // offset 6, length 2, end 8
-const PAGE_LOCAL_DEPTH_OFFSET = 8   // offset 8, length 2, end 10
-const PAGE_FREE_SPACE_OFFSET = 10   // offset 10, length 2, end 12
-const PAGE_PREV_POINTER_OFFSET = 12 // offset 12, length 4, end 16
-const PAGE_NEXT_POINTER_OFFSET = 16 // offset 16, length 4, end 20
-const PAGE_SLOTS_START_OFFSET = 20  // offset 20
-
-// PAGE_SLOT_LENGTH is the size of the page slot key/value.
-//
-// key offset int16    	//offset 0, length 2, end 2
-// value offset int16	    //offset 2, length 2, end 4
-const PAGE_SLOT_LENGTH = 4
-
-// Page represents a page on disk
-type Page struct {
-	id       PageID
-	pinCount int
-	isDirty  bool
-	data     [PAGE_SIZE]byte
-}
-
-type PageSlot struct {
-	KeyOffset   int16
-	ValueOffset int16
-}
-
-func (s *PageSlot) KeyBytes(page *Page) []byte {
-	offset := s.KeyOffset
-	keyLen := int16(binary.BigEndian.Uint16(page.data[offset:]))
-	offset += 2
-	result := make([]byte, keyLen)
-	copy(result, page.data[offset:offset+keyLen])
-	return result
-}
-
-func (s *PageSlot) KeyAsInt(page *Page) int32 {
-	return int32(binary.BigEndian.Uint32(page.data[s.KeyOffset+2:]))
-}
-
-func (s *PageSlot) ValueBytes(page *Page) []byte {
-	offset := s.ValueOffset
-	valueLen := int32(binary.BigEndian.Uint32(page.data[offset:]))
-	offset += 4
-	result := make([]byte, valueLen)
-	copy(result, page.data[offset:int32(offset)+valueLen])
-	return result
-}
-
-func (s *PageSlot) ValueAsPagePointer(page *Page) int32 {
-	return int32(binary.BigEndian.Uint32(page.data[s.ValueOffset+4:]))
-}
-
-type PageChunk struct {
-	KeyLength int16
-	KeyBytes  []byte
-	// TODO(pok) ValueBytes can be up to int32 long
-	// this requires an overflow page mechanism, that is not implemented
-	// yet, so be aware of this when storing stuff...
-	ValueLength int32
-	ValueBytes  []byte
-}
-
-func (pc *PageChunk) Length() int {
-	return 2 + len(pc.KeyBytes) + 4 + len(pc.ValueBytes)
-}
-
-func (pc *PageChunk) ComputeKeyOffset(pageOffset int) int {
-	return pageOffset
-}
-
-func (pc *PageChunk) ComputeValueOffset(pageOffset int) int {
-	return pageOffset + 2 + len(pc.KeyBytes)
-}
-
-func (p *Page) WritePageNumber(pageNumber int32) {
-	p.id = PageID(pageNumber)
-	binary.BigEndian.PutUint32(p.data[PAGE_NUMBER_OFFSET:], uint32(pageNumber))
-	p.isDirty = true
-}
-
-func (p *Page) ReadPageNumber() int {
-	return int(binary.BigEndian.Uint32(p.data[PAGE_NUMBER_OFFSET:]))
-}
-
-func (p *Page) WritePageType(pageType int16) {
-	binary.BigEndian.PutUint16(p.data[PAGE_TYPE_OFFSET:], uint16(pageType))
-	p.isDirty = true
-}
-
-func (p *Page) ReadPageType() int16 {
-	return int16(binary.BigEndian.Uint16(p.data[PAGE_TYPE_OFFSET:]))
-}
-
-func (p *Page) WriteSlotCount(slotCount int16) {
-	binary.BigEndian.PutUint16(p.data[PAGE_SLOT_COUNT_OFFSET:], uint16(slotCount))
-	p.isDirty = true
-}
-
-func (p *Page) ReadSlotCount() int16 {
-	return int16(binary.BigEndian.Uint16(p.data[PAGE_SLOT_COUNT_OFFSET:]))
-}
-
-func (p *Page) WriteLocalDepth(localDepth int16) {
-	binary.BigEndian.PutUint16(p.data[PAGE_LOCAL_DEPTH_OFFSET:], uint16(localDepth))
-	p.isDirty = true
-}
-
-func (p *Page) ReadLocalDepth() int16 {
-	return int16(binary.BigEndian.Uint16(p.data[PAGE_LOCAL_DEPTH_OFFSET:]))
-}
-
-func (p *Page) ReadSlot(slot int16) PageSlot {
-	offset := PAGE_SLOTS_START_OFFSET + PAGE_SLOT_LENGTH*slot
-	keyOffset := int16(binary.BigEndian.Uint16(p.data[offset:]))
-	offset += 2
-	valueOffset := int16(binary.BigEndian.Uint16(p.data[offset:]))
-	return PageSlot{
-		KeyOffset:   keyOffset,
-		ValueOffset: valueOffset,
-	}
-}
-
-func (p *Page) WriteSlot(slot int16, value PageSlot) {
-	offset := PAGE_SLOTS_START_OFFSET + PAGE_SLOT_LENGTH*slot
-	binary.BigEndian.PutUint16(p.data[offset:], uint16(value.KeyOffset))
-	offset += 2
-	binary.BigEndian.PutUint16(p.data[offset:], uint16(value.ValueOffset))
-}
-
-func (p *Page) WriteFreeSpaceOffset(offset int16) {
-	binary.BigEndian.PutUint16(p.data[PAGE_FREE_SPACE_OFFSET:], uint16(offset))
-	p.isDirty = true
-}
-
-func (p *Page) ReadFreeSpaceOffset() int16 {
-	return int16(binary.BigEndian.Uint16(p.data[PAGE_FREE_SPACE_OFFSET:]))
-}
-
-func (p *Page) WritePrevPointer(prevPointer int32) {
-	binary.BigEndian.PutUint32(p.data[PAGE_PREV_POINTER_OFFSET:], uint32(prevPointer))
-	p.isDirty = true
-}
-
-func (p *Page) ReadPrevPointer() int {
-	return int(binary.BigEndian.Uint32(p.data[PAGE_PREV_POINTER_OFFSET:]))
-}
-
-func (p *Page) WriteNextPointer(nextPointer int32) {
-	binary.BigEndian.PutUint32(p.data[PAGE_NEXT_POINTER_OFFSET:], uint32(nextPointer))
-	p.isDirty = true
-}
-
-func (p *Page) ReadNextPointer() int {
-	return int(binary.BigEndian.Uint32(p.data[PAGE_NEXT_POINTER_OFFSET:]))
-}
-
-func (p *Page) WriteChunk(offset int16, chunk PageChunk) {
-	binary.BigEndian.PutUint16(p.data[offset:], uint16(chunk.KeyLength))
-	offset += 2
-	copy(p.data[offset:], chunk.KeyBytes)
-	offset += int16(len(chunk.KeyBytes))
-	binary.BigEndian.PutUint32(p.data[offset:], uint32(chunk.ValueLength))
-	offset += 4
-	copy(p.data[offset:], chunk.ValueBytes)
-	p.isDirty = true
-}
-
-func (p *Page) ReadChunk(offset int16) PageChunk {
-	keyLen := int16(binary.BigEndian.Uint16(p.data[offset:]))
-	offset += 2
-	keyBytes := make([]byte, keyLen)
-	copy(keyBytes, p.data[offset:offset+keyLen])
-	offset += keyLen
-	valueLen := int32(binary.BigEndian.Uint32(p.data[offset:]))
-	offset += 4
-	valueBytes := make([]byte, valueLen)
-	copy(valueBytes, p.data[offset:int32(offset)+valueLen])
-	return PageChunk{
-		KeyLength:   keyLen,
-		KeyBytes:    keyBytes,
-		ValueLength: valueLen,
-		ValueBytes:  valueBytes,
-	}
-}
-
-func (p *Page) FreeSpace() int16 {
-	freeSpaceOffset := p.ReadFreeSpaceOffset()
-	freespace := freeSpaceOffset - (p.ReadSlotCount()*PAGE_SLOT_LENGTH + PAGE_SLOT_LENGTH + PAGE_SLOTS_START_OFFSET)
-	return freespace
-}
-
-func (p *Page) WriteKeyValueInSlot(slotNumber int16, key []byte, value []byte) error {
-	freeSpaceOffset := p.ReadFreeSpaceOffset()
-
-	// build a chunk
-	chunk := PageChunk{
-		KeyLength:   int16(len(key)),
-		KeyBytes:    key,
-		ValueLength: int32(len(value)),
-		ValueBytes:  value,
-	}
-
-	// compute the new free space offset
-	freeSpaceOffset -= int16(chunk.Length())
-
-	// check we won't blow free space on page
-	slotCount := p.ReadSlotCount()
-	slotEndOffset := slotCount*PAGE_SLOT_LENGTH + PAGE_SLOT_LENGTH + PAGE_SLOTS_START_OFFSET
-
-	// DEBUG!!
-	//fmt.Printf("freeSpaceOffset: %d, slotCount: %d, slotCount*4 + 4 + 20: %d, freeSpace: %d\n", freeSpaceOffset, slotCount, slotEndOffset, freeSpaceOffset-slotEndOffset)
-
-	if freeSpaceOffset-slotEndOffset <= 0 {
-		return errors.New("page is full")
-	}
-
-	keyOffset := chunk.ComputeKeyOffset(int(freeSpaceOffset))
-	valueOffset := chunk.ComputeValueOffset(int(freeSpaceOffset))
-
-	p.WriteChunk(freeSpaceOffset, chunk)
-
-	// update the free space offset
-	p.WriteFreeSpaceOffset(int16(freeSpaceOffset))
-
-	// make a slot
-	slot := PageSlot{
-		KeyOffset:   int16(keyOffset),
-		ValueOffset: int16(valueOffset),
-	}
-	// write the slot
-	p.WriteSlot(slotNumber, slot)
-
-	return nil
-}
-
-func (p *Page) WritePage(page *Page) {
-	// copy everything but pageNumber & pageType
-	offset := PAGE_SLOT_COUNT_OFFSET
-	copy(page.data[offset:], p.data[offset:offset+PAGE_SIZE-offset])
-}
-
-func (p *Page) PinCount() int {
-	return p.pinCount
-}
-
-func (p *Page) ID() PageID {
-	return p.id
-}
-
-func (p *Page) DecPinCount() {
-	if p.pinCount > 0 {
-		p.pinCount--
-	}
-}
-
-type PageSlotIterator struct {
-	page      *Page
-	slotCount int16
-	cursor    int16
-}
-
-func NewPageSlotIterator(page *Page, fromSlot int16) *PageSlotIterator {
-	i := &PageSlotIterator{
-		page:      page,
-		slotCount: page.ReadSlotCount(),
-		cursor:    fromSlot,
-	}
-	return i
-}
-
-func (i *PageSlotIterator) Next() *PageSlot {
-	if i.cursor < i.slotCount {
-		s := i.page.ReadSlot(i.cursor)
-		i.cursor++
-		return &s
-	}
-	return nil
-}
-
-func (i *PageSlotIterator) Cursor() int16 {
-	return i.cursor
-}
-
-func (pg *Page) Dump(label string) {
-	indent := 0
-	if len(label) > 0 {
-		fmt.Printf("%s%s:\n", fmt.Sprintf("%*s", indent, ""), label)
-		indent += 4
-	}
-	pageType := pg.ReadPageType()
-	fmt.Printf("%sPAGE(%d) pageType: %d slotCount: %d, prevPtr: %d, nextPtr: %d\n", fmt.Sprintf("%*s", indent, ""), pg.ID(), pageType, pg.ReadSlotCount(), pg.ReadPrevPointer(), pg.ReadNextPointer())
-	fmt.Printf("%sKEYS: -->\n", fmt.Sprintf("%*s", indent, ""))
-	indent += 4
-
-	// get the keys off the page
-	keys := make([]int, 0)
-	pointers := make([]int, 0)
-	iter := NewPageSlotIterator(pg, 0)
-	for {
-		ps := iter.Next()
-		if ps == nil {
-			break
-		}
-		keys = append(keys, int(ps.KeyAsInt(pg)))
-		if pageType == /*nodeTypeInternal*/ 10 {
-			pointers = append(pointers, int(ps.ValueAsPagePointer(pg)))
-		}
-	}
-
-	if pageType == /*nodeTypeLeaf*/ 11 {
-		for _, key := range keys {
-			fmt.Printf("%s(%d)\n", fmt.Sprintf("%*s", indent, ""), key)
-		}
-	} else {
-		for idx, key := range keys {
-			ptr := pointers[idx]
-			fmt.Printf("%s(%d, %d)\n", fmt.Sprintf("%*s", indent, ""), key, ptr)
-		}
-		ptr := pg.ReadNextPointer()
-		fmt.Printf("%s(-->, %d)\n", fmt.Sprintf("%*s", indent, ""), ptr)
-	}
-
-}

build/build.go

@@ -0,0 +1,29 @@
+package build
+
+import (
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"os/exec"
+)
+
+func Binary(pkg, goos, goarch string) (io.Reader, error) {
+	binFile, err := ioutil.TempFile("", "pilosactl")
+	if err != nil {
+		return nil, fmt.Errorf("build binary: %v", err)
+	}
+	com := exec.Command("go", "build", "-o", binFile.Name(), pkg)
+	com.Env = append([]string{"GOOS=" + goos, "GOARCH=" + goarch}, os.Environ()...)
+
+	err = com.Run()
+	if err != nil {
+		return nil, err
+	}
+
+	f, err := os.Open(binFile.Name())
+	if err != nil {
+		return nil, err
+	}
+	return f, nil
+}

cache.go

@@ -1,97 +1,69 @@
-// Copyright 2022 Molecula Corp. (DBA FeatureBase).
-// SPDX-License-Identifier: Apache-2.0
 package pilosa
 
 import (
 	"bytes"
-	"encoding/json"
 	"fmt"
 	"io"
 	"sort"
-	"sync"
 	"time"
 
-	"github.com/featurebasedb/featurebase/v3/lru"
-	pb "github.com/featurebasedb/featurebase/v3/proto"
-	"github.com/pkg/errors"
+	"github.com/golang/groupcache/lru"
+	"github.com/pilosa/pilosa/internal"
 )
 
-const (
-	// thresholdFactor is used to calculate the threshold for new items entering the cache
-	thresholdFactor = 1.1
-)
-
-// cache represents a cache of counts.
-type cache interface {
-	Add(id uint64, n uint64)
-	BulkAdd(id uint64, n uint64)
-	Get(id uint64) uint64
+// Cache represents a cache for bitmap counts.
+type Cache interface {
+	Add(bitmapID uint64, n uint64)
+	Get(bitmapID uint64) uint64
 	Len() int
 
-	// Returns a list of all IDs.
-	IDs() []uint64
+	// Returns a list of all bitmap IDs.
+	BitmapIDs() []uint64
 
-	// Soft ask for the cache to be rebuilt - may not if it has been done recently.
+	// Updates the cache, if necessary.
 	Invalidate()
 
-	// Rebuilds the cache.
-	Recalculate()
-
 	// Returns an ordered list of the top ranked bitmaps.
-	Top() []bitmapPair
-
-	// Clear removes everything from the cache. If possible it should leave allocated structures in place to be reused.
-	Clear()
+	Top() []BitmapPair
 }
 
-// lruCache represents a least recently used Cache implementation.
-type lruCache struct {
+// LRUCache represents a least recently used Cache implemenation.
+type LRUCache struct {
 	cache  *lru.Cache
 	counts map[uint64]uint64
-	// maxEntries is saved to support Clear which recreates the cache.
-	maxEntries uint32
 }
 
-// newLRUCache returns a new instance of LRUCache.
-func newLRUCache(maxEntries uint32) *lruCache {
-	c := &lruCache{
-		cache:      lru.New(int(maxEntries)),
-		counts:     make(map[uint64]uint64),
-		maxEntries: maxEntries,
+// NewLRUCache returns a new instance of LRUCache.
+func NewLRUCache(maxEntries int) *LRUCache {
+	c := &LRUCache{
+		cache:  lru.New(maxEntries),
+		counts: make(map[uint64]uint64),
 	}
 	c.cache.OnEvicted = c.onEvicted
 	return c
 }
 
-// BulkAdd adds a count to the cache unsorted. You should Invalidate after completion.
-func (c *lruCache) BulkAdd(id, n uint64) {
-	c.Add(id, n)
+// Add adds a bitmap to the cache.
+func (c *LRUCache) Add(bitmapID, n uint64) {
+	c.cache.Add(bitmapID, n)
+	c.counts[bitmapID] = n
 }
 
-// Add adds a count to the cache.
-func (c *lruCache) Add(id, n uint64) {
-	c.cache.Add(id, n)
-	c.counts[id] = n
-}
-
-// Get returns a count for a given id.
-func (c *lruCache) Get(id uint64) uint64 {
-	n, _ := c.cache.Get(id)
+// Get returns a bitmap with a given id.
+func (c *LRUCache) Get(bitmapID uint64) uint64 {
+	n, _ := c.cache.Get(bitmapID)
 	nn, _ := n.(uint64)
 	return nn
 }
 
 // Len returns the number of items in the cache.
-func (c *lruCache) Len() int { return c.cache.Len() }
+func (c *LRUCache) Len() int { return c.cache.Len() }
 
 // Invalidate is a no-op.
-func (c *lruCache) Invalidate() {}
+func (c *LRUCache) Invalidate() {}
 
-// Recalculate is a no-op.
-func (c *lruCache) Recalculate() {}
-
-// IDs returns a list of all IDs in the cache.
-func (c *lruCache) IDs() []uint64 {
+// BitmapIDs returns a list of all bitmap IDs in the cache.
+func (c *LRUCache) BitmapIDs() []uint64 {
 	a := make([]uint64, 0, len(c.counts))
 	for id := range c.counts {
 		a = append(a, id)
@@ -101,382 +73,183 @@ func (c *lruCache) IDs() []uint64 {
 }
 
 // Top returns all counts in the cache.
-func (c *lruCache) Top() []bitmapPair {
-	a := make([]bitmapPair, 0, len(c.counts))
+func (c *LRUCache) Top() []BitmapPair {
+	a := make([]BitmapPair, 0, len(c.counts))
 	for id, n := range c.counts {
-		a = append(a, bitmapPair{
+		a = append(a, BitmapPair{
 			ID:    id,
-			Count: n,
+			Count: uint64(n),
 		})
 	}
-	pairs := bitmapPairs(a)
-	sort.Sort(&pairs)
+	sort.Sort(BitmapPairs(a))
 	return a
 }
 
-func (c *lruCache) Clear() {
-	for k := range c.counts {
-		delete(c.counts, k)
-	}
-	c.cache = lru.New(int(c.maxEntries))
-}
-
-func (c *lruCache) onEvicted(key lru.Key, _ interface{}) { delete(c.counts, key.(uint64)) }
+func (c *LRUCache) onEvicted(key lru.Key, _ interface{}) { delete(c.counts, key.(uint64)) }
 
 // Ensure LRUCache implements Cache.
-var _ cache = &lruCache{}
+var _ Cache = &LRUCache{}
 
-// rankCache represents a cache with sorted entries.
-type rankCache struct {
-	// TODO why does this have a lock and lruCache doesn't?
-	mu           sync.Mutex
-	entries      map[uint64]uint64
-	rankings     bitmapPairs // cached, ordered list
-	rankingsRead bool
-	dirty        bool
+// RankCache represents a cache with sorted entries.
+type RankCache struct {
+	entries  map[uint64]uint64
+	rankings []BitmapPair // cached, ordered list
 
 	updateN    int
 	updateTime time.Time
 
-	// maxEntries is the user defined size of the cache
-	maxEntries uint32
-
-	// thresholdBuffer is used the calculate the lowest cached threshold value
-	// This threshold determines what new items are added to the cache
-	thresholdBuffer int
-
-	// thresholdValue is the value of the last item in the cache
-	thresholdValue uint64
+	ThresholdLength int
+	ThresholdIndex  int
+	ThresholdValue  uint64
 }
 
 // NewRankCache returns a new instance of RankCache.
-func NewRankCache(maxEntries uint32) *rankCache {
-	return &rankCache{
-		maxEntries:      maxEntries,
-		thresholdBuffer: int(thresholdFactor * float64(maxEntries)),
-		entries:         make(map[uint64]uint64),
+func NewRankCache() *RankCache {
+	return &RankCache{
+		entries: make(map[uint64]uint64),
 	}
 }
 
-func (c *rankCache) Clear() {
-	c.mu.Lock()
-	defer c.mu.Unlock()
-	for k := range c.entries {
-		delete(c.entries, k)
-	}
-	c.rankings = c.rankings[:0]
-	c.rankingsRead = false
-	c.dirty = false
-
-	c.updateN = 0
-	c.updateTime = time.Time{}
-	c.thresholdValue = 0
-}
-
-// Add adds a count to the cache.
-func (c *rankCache) Add(id uint64, n uint64) {
-	c.mu.Lock()
-	defer c.mu.Unlock()
-
-	// Flag the cache as dirty.
-	// This forces recalculation if top is called before the cache is recalculated.
-	c.dirty = true
-
-	// Ignore if the column count is below the threshold,
-	// unless the count is 0, which is effectively used
-	// to clear the cache value.
-	if n < c.thresholdValue && n > 0 {
-		delete(c.entries, id)
+// Add adds a bitmap to the cache.
+func (c *RankCache) Add(bitmapID uint64, n uint64) {
+	// Ignore if the bit count on the bitmap is below the threshold.
+	if n < c.ThresholdValue {
 		return
 	}
 
-	c.entries[id] = n
+	// Add to cache.
+	c.entries[bitmapID] = n
 
-	c.invalidate()
-}
-
-// BulkAdd adds a count to the cache unsorted. You should Invalidate after completion.
-func (c *rankCache) BulkAdd(id uint64, n uint64) {
-	c.mu.Lock()
-	defer c.mu.Unlock()
-
-	// Flag the cache as dirty.
-	// This forces recalculation if top is called before the cache is recalculated.
-	c.dirty = true
-
-	if n < c.thresholdValue {
-		delete(c.entries, id)
-		return
-	}
-
-	c.entries[id] = n
-
-	// FB-1206: Periodically invalidate the cache when we are bulk loading
-	// as this can take up an upbounded amount of memory. This is especially
-	// true when restoring shards as all rows will be added.
-	if len(c.entries) > int(2*c.maxEntries) {
-		CounterRecalculateCache.Inc()
-		c.recalculate()
+	// If size is larger than the threshold then trim it.
+	if len(c.entries) > c.ThresholdLength {
+		c.update()
+		for id, n := range c.entries {
+			if n <= c.ThresholdValue {
+				delete(c.entries, id)
+			}
+		}
 	}
 }
 
-// Get returns a count for a given id.
-func (c *rankCache) Get(id uint64) uint64 {
-	c.mu.Lock()
-	defer c.mu.Unlock()
-	return c.entries[id]
-}
+// Get returns a bitmap with a given id.
+func (c *RankCache) Get(bitmapID uint64) uint64 { return c.entries[bitmapID] }
 
 // Len returns the number of items in the cache.
-func (c *rankCache) Len() int {
-	c.mu.Lock()
-	defer c.mu.Unlock()
-	return len(c.entries)
-}
+func (c *RankCache) Len() int { return len(c.entries) }
 
-// IDs returns a list of all IDs in the cache.
-func (c *rankCache) IDs() []uint64 {
-	c.mu.Lock()
-	defer c.mu.Unlock()
-	if len(c.entries) == 0 {
-		return nil
-	}
-	ids := make([]uint64, 0, len(c.entries))
+// BitmapIDs returns a list of all bitmap IDs in the cache.
+func (c *RankCache) BitmapIDs() []uint64 {
+	a := make([]uint64, 0, len(c.entries))
 	for id := range c.entries {
-		ids = append(ids, id)
+		a = append(a, id)
 	}
-	sort.Sort(uint64Slice(ids))
-	return ids
+	sort.Sort(uint64Slice(a))
+	return a
 }
 
-// Invalidate recalculates the entries by rank.
-func (c *rankCache) Invalidate() {
-	c.mu.Lock()
-	defer c.mu.Unlock()
-	c.invalidate()
-}
-
-// Recalculate rebuilds the cache.
-func (c *rankCache) Recalculate() {
-	c.mu.Lock()
-	defer c.mu.Unlock()
-	CounterRecalculateCache.Inc()
-	c.recalculate()
-}
-
-func (c *rankCache) invalidate() {
-	// Don't invalidate more than once every X seconds.
-	// TODO: consider making this configurable.
-	if time.Since(c.updateTime).Seconds() < 10 {
-		// Skipping recalculation means that the ranked cache's growth is unbounded.
-		// This is somewhat necessary for now since recalculation is not cheap.
-		// The cache will remain flagged as dirty and will be recalculated if Top is called.
-		// This may cause unexpected memory growth, so record it in metrics for debugging purposes.
-		CounterInvalidateCacheSkipped.Inc()
-		// Ensure that we're marked as dirty even if we weren't otherwise.
-		c.dirty = true
-		return
+// Invalidate reorders the entries, if necessary.
+func (c *RankCache) Invalidate() {
+	// Update if there aren't many items or it hasn't been updated recently.
+	if len(c.rankings) < 50 || (c.updateN > 0 && time.Since(c.updateTime) > 5*time.Minute) {
+		c.update()
 	}
-	CounterInvalidateCache.Inc()
-	c.recalculate()
 }
 
-func (c *rankCache) recalculate() {
-	if c.rankingsRead {
-		c.rankings = nil
-		c.rankingsRead = false
-	}
-
+// update reorders the entries by rank.
+func (c *RankCache) update() {
 	// Convert cache to a sorted list.
-	rankings := c.rankings[:0]
-	if cap(rankings) < len(c.entries) {
-		rankings = make([]bitmapPair, 0, len(c.entries))
-	}
-	for id, cnt := range c.entries {
-		rankings = append(rankings, bitmapPair{
+	rankings := make([]BitmapPair, 0, len(c.entries))
+	for id, n := range c.entries {
+		rankings = append(rankings, BitmapPair{
 			ID:    id,
-			Count: cnt,
+			Count: n,
 		})
 	}
-	c.rankings = rankings
-	sort.Sort(&c.rankings)
+	sort.Sort(BitmapPairs(rankings))
 
 	// Store the count of the item at the threshold index.
-	length := len(c.rankings)
-	GaugeRankCacheLength.Set(float64(length))
-
-	var removeItems []bitmapPair // cached, ordered list
-	if length > int(c.maxEntries) {
-		c.thresholdValue = rankings[c.maxEntries].Count
-		removeItems = c.rankings[c.maxEntries:]
-		c.rankings = c.rankings[0:c.maxEntries]
+	c.rankings = rankings
+	if len(c.rankings) > c.ThresholdIndex {
+		c.ThresholdValue = rankings[c.ThresholdIndex].Count
 	} else {
-		c.thresholdValue = 1
+		c.ThresholdValue = 1
 	}
 
 	// Reset counters.
 	c.updateTime, c.updateN = time.Now(), 0
-
-	// If size is larger than the threshold then trim it.
-	if len(c.entries) > c.thresholdBuffer {
-		CounterCacheThresholdReached.Inc()
-		for _, pair := range removeItems {
-			delete(c.entries, pair.ID)
-		}
-	}
-
-	// The cache is no longer dirty.
-	c.dirty = false
 }
 
-// Top returns an ordered list of pairs.
-func (c *rankCache) Top() []bitmapPair {
-	c.mu.Lock()
-	defer c.mu.Unlock()
-
-	if c.dirty {
-		// The cache is dirty, so we need to recalculate it to get a consistent view.
-		CounterReadDirtyCache.Inc()
-		c.recalculate()
-	}
-
-	c.rankingsRead = true
-	return c.rankings
-}
+// Top returns an ordered list of bitmaps.
+func (c *RankCache) Top() []BitmapPair { return c.rankings }
 
 // WriteTo writes the cache to w.
-func (c *rankCache) WriteTo(w io.Writer) (n int64, err error) {
+func (c *RankCache) WriteTo(w io.Writer) (n int64, err error) {
 	panic("FIXME: TODO")
 }
 
 // ReadFrom read from r into the cache.
-func (c *rankCache) ReadFrom(r io.Reader) (n int64, err error) {
+func (c *RankCache) ReadFrom(r io.Reader) (n int64, err error) {
 	panic("FIXME: TODO")
 }
 
 // Ensure RankCache implements Cache.
-var _ cache = &rankCache{}
+var _ Cache = &RankCache{}
 
-// bitmapPair represents a id/count pair with an associated identifier.
-type bitmapPair struct {
+// BitmapPair represents a bitmap with an associated identifier.
+type BitmapPair struct {
 	ID    uint64
 	Count uint64
 }
 
-// bitmapPairs is a sortable list of BitmapPair objects.
-type bitmapPairs []bitmapPair
+// BitmapPairs is a sortable list of BitmapPair objects.
+type BitmapPairs []BitmapPair
 
-func (p *bitmapPairs) Swap(i, j int)      { (*p)[i], (*p)[j] = (*p)[j], (*p)[i] }
-func (p *bitmapPairs) Len() int           { return len(*p) }
-func (p *bitmapPairs) Less(i, j int) bool { return (*p)[i].Count > (*p)[j].Count }
+func (p BitmapPairs) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }
+func (p BitmapPairs) Len() int           { return len(p) }
+func (p BitmapPairs) Less(i, j int) bool { return p[i].Count > p[j].Count }
 
-// Pair holds an id/count pair.
 type Pair struct {
-	ID    uint64 `json:"id"`
-	Key   string `json:"key"`
+	Key   uint64 `json:"key"`
 	Count uint64 `json:"count"`
 }
 
-// PairField is a Pair with its associated field.
-type PairField struct {
-	Pair  Pair
-	Field string
-}
-
-func (p PairField) Clone() (r PairField) {
-	return PairField{
-		Pair:  p.Pair,
-		Field: p.Field,
+func encodePair(p Pair) *internal.Pair {
+	return &internal.Pair{
+		Key:   p.Key,
+		Count: p.Count,
 	}
 }
 
-// ToTable implements the ToTabler interface.
-func (p PairField) ToTable() (*pb.TableResponse, error) {
-	return pb.RowsToTable(p, 1)
-}
-
-// ToRows implements the ToRowser interface.
-func (p PairField) ToRows(callback func(*pb.RowResponse) error) error {
-	if p.Pair.Key != "" {
-		return callback(&pb.RowResponse{
-			Headers: []*pb.ColumnInfo{
-				{Name: p.Field, Datatype: "string"},
-				{Name: "count", Datatype: "uint64"},
-			},
-			Columns: []*pb.ColumnResponse{
-				{ColumnVal: &pb.ColumnResponse_StringVal{StringVal: p.Pair.Key}},
-				{ColumnVal: &pb.ColumnResponse_Uint64Val{Uint64Val: p.Pair.Count}},
-			},
-		})
-	} else {
-		return callback(&pb.RowResponse{
-			Headers: []*pb.ColumnInfo{
-				{Name: p.Field, Datatype: "uint64"},
-				{Name: "count", Datatype: "uint64"},
-			},
-			Columns: []*pb.ColumnResponse{
-				{ColumnVal: &pb.ColumnResponse_Uint64Val{Uint64Val: p.Pair.ID}},
-				{ColumnVal: &pb.ColumnResponse_Uint64Val{Uint64Val: p.Pair.Count}},
-			},
-		})
+func decodePair(pb *internal.Pair) Pair {
+	return Pair{
+		Key:   pb.Key,
+		Count: pb.Count,
 	}
 }
 
-// MarshalJSON marshals PairField into a JSON-encoded byte slice,
-// excluding `Field`.
-func (p PairField) MarshalJSON() ([]byte, error) {
-	return json.Marshal(p.Pair)
-}
-
-// Pairs is a sortable slice of Pair objects.
 type Pairs []Pair
 
 func (p Pairs) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }
 func (p Pairs) Len() int           { return len(p) }
 func (p Pairs) Less(i, j int) bool { return p[i].Count > p[j].Count }
 
-// pairHeap is a heap implementation over a group of Pairs.
-type pairHeap struct {
-	Pairs
-}
-
-// Less implemets the Sort interface.
-// reports whether the element with index i should sort before the element with index j.
-func (p pairHeap) Less(i, j int) bool { return p.Pairs[i].Count < p.Pairs[j].Count }
-
-// Push appends the element onto the Pair slice.
-func (p *Pairs) Push(x interface{}) {
-	// Push and Pop use pointer receivers because they modify the slice's length,
-	// not just its contents.
-	*p = append(*p, x.(Pair))
-}
-
-// Pop removes the minimum element from the Pair slice.
-func (p *Pairs) Pop() interface{} {
-	old := *p
-	n := len(old)
-	x := old[n-1]
-	*p = old[0 : n-1]
-	return x
-}
-
 // Add merges other into p and returns a new slice.
 func (p Pairs) Add(other []Pair) []Pair {
 	// Create lookup of key/counts.
 	m := make(map[uint64]uint64, len(p))
 	for _, pair := range p {
-		m[pair.ID] = pair.Count
+		m[pair.Key] = pair.Count
 	}
 
 	// Add/merge from other.
 	for _, pair := range other {
-		m[pair.ID] += pair.Count
+		m[pair.Key] += pair.Count
 	}
 
 	// Convert back to slice.
 	a := make([]Pair, 0, len(m))
 	for k, v := range m {
-		a = append(a, Pair{ID: k, Count: v})
+		a = append(a, Pair{Key: k, Count: v})
 	}
 	return a
 }
@@ -485,7 +258,7 @@ func (p Pairs) Add(other []Pair) []Pair {
 func (p Pairs) Keys() []uint64 {
 	a := make([]uint64, len(p))
 	for i := range p {
-		a[i] = p[i].ID
+		a[i] = p[i].Key
 	}
 	return a
 }
@@ -494,7 +267,7 @@ func (p Pairs) String() string {
 	var buf bytes.Buffer
 	buf.WriteString("Pairs(")
 	for i := range p {
-		fmt.Fprintf(&buf, "%d/%d", p[i].ID, p[i].Count)
+		fmt.Fprintf(&buf, "%d/%d", p[i].Key, p[i].Count)
 		if i < len(p)-1 {
 			buf.WriteString(", ")
 		}
@@ -503,82 +276,22 @@ func (p Pairs) String() string {
 	return buf.String()
 }
 
-// PairsField is a Pairs object with its associated field.
-type PairsField struct {
-	Pairs []Pair
-	Field string
+func encodePairs(a Pairs) []*internal.Pair {
+	other := make([]*internal.Pair, len(a))
+	for i := range a {
+		other[i] = encodePair(a[i])
+	}
+	return other
 }
 
-func (p *PairsField) Clone() (r *PairsField) {
-	r = &PairsField{
-		Pairs: make([]Pair, len(p.Pairs)),
-		Field: p.Field,
+func decodePairs(a []*internal.Pair) []Pair {
+	other := make([]Pair, len(a))
+	for i := range a {
+		other[i] = decodePair(a[i])
 	}
-	copy(r.Pairs, p.Pairs)
-	return
+	return other
 }
 
-// ToTable implements the ToTabler interface.
-func (p *PairsField) ToTable() (*pb.TableResponse, error) {
-	return pb.RowsToTable(p, len(p.Pairs))
-}
-
-// ToRows implements the ToRowser interface.
-func (p *PairsField) ToRows(callback func(*pb.RowResponse) error) error {
-	// Determine if the ID has string keys.
-	var stringKeys bool
-	if len(p.Pairs) > 0 {
-		if p.Pairs[0].Key != "" {
-			stringKeys = true
-		}
-	}
-
-	dtype := "uint64"
-	if stringKeys {
-		dtype = "string"
-	}
-	ci := []*pb.ColumnInfo{
-		{Name: p.Field, Datatype: dtype},
-		{Name: "count", Datatype: "uint64"},
-	}
-	for _, pair := range p.Pairs {
-		if stringKeys {
-			if err := callback(&pb.RowResponse{
-				Headers: ci,
-				Columns: []*pb.ColumnResponse{
-					{ColumnVal: &pb.ColumnResponse_StringVal{StringVal: pair.Key}},
-					{ColumnVal: &pb.ColumnResponse_Uint64Val{Uint64Val: uint64(pair.Count)}},
-				}}); err != nil {
-				return errors.Wrap(err, "calling callback")
-			}
-		} else {
-			if err := callback(&pb.RowResponse{
-				Headers: ci,
-				Columns: []*pb.ColumnResponse{
-					{ColumnVal: &pb.ColumnResponse_Uint64Val{Uint64Val: uint64(pair.ID)}},
-					{ColumnVal: &pb.ColumnResponse_Uint64Val{Uint64Val: uint64(pair.Count)}},
-				}}); err != nil {
-				return errors.Wrap(err, "calling callback")
-			}
-		}
-		ci = nil //only send on the first
-	}
-	return nil
-}
-
-// MarshalJSON marshals PairsField into a JSON-encoded byte slice,
-// excluding `Field`.
-func (p PairsField) MarshalJSON() ([]byte, error) {
-	return json.Marshal(p.Pairs)
-}
-
-// int64Slice represents a sortable slice of int64 numbers.
-type int64Slice []int64
-
-func (p int64Slice) Len() int           { return len(p) }
-func (p int64Slice) Less(i, j int) bool { return p[i] < p[j] }
-func (p int64Slice) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }
-
 // uint64Slice represents a sortable slice of uint64 numbers.
 type uint64Slice []uint64
 
@@ -586,23 +299,31 @@ func (p uint64Slice) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }
 func (p uint64Slice) Len() int           { return len(p) }
 func (p uint64Slice) Less(i, j int) bool { return p[i] < p[j] }
 
-// nopCache represents a no-op Cache implementation.
-type nopCache struct{}
+// merge combines p and other to a unique sorted set of values.
+// p and other must both have unique sets and be sorted.
+func (p uint64Slice) merge(other []uint64) []uint64 {
+	ret := make([]uint64, 0, len(p))
 
-// Ensure NopCache implements Cache.
-var globalNopCache cache = nopCache{}
+	i, j := 0, 0
+	for i < len(p) && j < len(other) {
+		a, b := p[i], other[j]
+		if a == b {
+			ret = append(ret, a)
+			i, j = i+1, j+1
+		} else if a < b {
+			ret = append(ret, a)
+			i++
+		} else {
+			ret = append(ret, b)
+			j++
+		}
+	}
 
-func (c nopCache) Add(uint64, uint64)     {}
-func (c nopCache) BulkAdd(uint64, uint64) {}
-func (c nopCache) Get(uint64) uint64      { return 0 }
-func (c nopCache) IDs() []uint64          { return []uint64{} }
+	if i < len(p) {
+		ret = append(ret, p[i:]...)
+	} else if j < len(other) {
+		ret = append(ret, other[j:]...)
+	}
 
-func (c nopCache) Invalidate()  {}
-func (c nopCache) Len() int     { return 0 }
-func (c nopCache) Recalculate() {}
-
-func (c nopCache) Clear() {}
-
-func (c nopCache) Top() []bitmapPair {
-	return []bitmapPair{}
+	return ret
 }

cache_test.go

@@ -1,85 +0,0 @@
-// Copyright 2022 Molecula Corp. (DBA FeatureBase).
-// SPDX-License-Identifier: Apache-2.0
-package pilosa_test
-
-import (
-	"reflect"
-	"testing"
-
-	pilosa "github.com/featurebasedb/featurebase/v3"
-)
-
-// Ensure cache stays constrained to its configured size.
-func TestCache_Rank_Size(t *testing.T) {
-	cacheSize := uint32(3)
-	cache := pilosa.NewRankCache(cacheSize)
-	for i := 1; i < int(2*cacheSize); i++ {
-		cache.Add(uint64(i), 3)
-	}
-	cache.Recalculate()
-	if cache.Len() != int(cacheSize) {
-		t.Fatalf("unexpected cache Size: %d!=%d expected\n", cache.Len(), cacheSize)
-	}
-}
-
-// Ensure cache entries set below threshold are handled appropriately.
-func TestCache_Rank_Threshold(t *testing.T) {
-	cacheSize := uint32(5)
-	cache := pilosa.NewRankCache(cacheSize)
-	for i := 1; i < int(2*cacheSize); i++ {
-		cache.Add(uint64(i), 3)
-	}
-
-	// Set the cache value for rows 4 and 5 to a number below the threshold
-	// value (which is 3), and ensure that they gets zeroed out.
-	cache.Add(4, 1)
-	cache.BulkAdd(5, 1)
-	cache.Recalculate()
-
-	if cache.Get(4) != 0 {
-		t.Fatalf("unexpected cache value after Add: %d!=%d expected\n", cache.Get(4), 0)
-	}
-	if cache.Get(5) != 0 {
-		t.Fatalf("unexpected cache value after BulkAdd: %d!=%d expected\n", cache.Get(5), 0)
-	}
-}
-
-// Test that consecutive writes show up in Top.
-// On later writes, the cache skips recalculation to save CPU time.
-// This used to mean that the later writes would not show up in Top.
-// Now, the cache is flagged as dirty and recalculated during the call to Top.
-func TestCache_Rank_Dirty(t *testing.T) {
-	cacheSize := uint32(5)
-	cache := pilosa.NewRankCache(cacheSize)
-
-	type pair struct{ ID, Count uint64 }
-	expect := []pair{
-		{5, 2},
-		{4, 1},
-	}
-
-	for _, v := range expect {
-		cache.Add(v.ID, v.Count)
-	}
-
-	var got []pair //nolint:prealloc
-	for _, p := range cache.Top() {
-		got = append(got, pair(p))
-	}
-
-	if !reflect.DeepEqual(expect, got) {
-		t.Fatalf("wrote %v but got %v", expect, got)
-	}
-}
-
-func TestCache_Rank_BulkAdd(t *testing.T) {
-	const cacheSize = 10
-	cache := pilosa.NewRankCache(uint32(cacheSize))
-
-	for i := uint64(0); i < 1000; i++ {
-		cache.BulkAdd(i, i)
-		if n := cache.Len(); n > cacheSize*2 {
-			t.Fatalf("entry count exceed 2x cache size: %d", n)
-		}
-	}
-}

catcher.go

@@ -1,233 +0,0 @@
-// Copyright 2022 Molecula Corp. (DBA FeatureBase).
-// SPDX-License-Identifier: Apache-2.0
-package pilosa
-
-import (
-	"github.com/featurebasedb/featurebase/v3/roaring"
-	txkey "github.com/featurebasedb/featurebase/v3/short_txkey"
-	"github.com/featurebasedb/featurebase/v3/vprint"
-)
-
-// catcher is useful to report error locations with a
-// Stack dump before the complexity
-// of the executor_test swallows up
-// the location of a PanicOn.
-type catcherTx struct {
-	b Tx
-}
-
-func newCatcherTx(b Tx) *catcherTx {
-	return &catcherTx{b: b}
-}
-
-func init() {
-	// keep golangci-lint happy
-	_ = newCatcherTx
-}
-
-var _ Tx = (*catcherTx)(nil)
-
-func (c *catcherTx) ImportRoaringBits(index, field, view string, shard uint64, rit roaring.RoaringIterator, clear bool, log bool, rowSize uint64) (changed int, rowSet map[uint64]int, err error) {
-	defer func() {
-		if r := recover(); r != nil {
-			vprint.AlwaysPrintf("see ImportRoaringBits() PanicOn '%v' at '%v'", r, vprint.Stack())
-			vprint.PanicOn(r)
-		}
-	}()
-	return c.b.ImportRoaringBits(index, field, view, shard, rit, clear, log, rowSize)
-}
-
-func (c *catcherTx) Rollback() {
-	defer func() {
-		if r := recover(); r != nil {
-			vprint.AlwaysPrintf("see Rollback() PanicOn '%v' at '%v'", r, vprint.Stack())
-			vprint.PanicOn(r)
-		}
-	}()
-	c.b.Rollback()
-}
-
-func (c *catcherTx) Commit() error {
-
-	defer func() {
-		if r := recover(); r != nil {
-			vprint.AlwaysPrintf("see Commit() PanicOn '%v' at '%v'", r, vprint.Stack())
-			vprint.PanicOn(r)
-		}
-	}()
-	return c.b.Commit()
-}
-
-func (c *catcherTx) RoaringBitmap(index, field, view string, shard uint64) (*roaring.Bitmap, error) {
-
-	defer func() {
-		if r := recover(); r != nil {
-			vprint.AlwaysPrintf("see RoaringBitmap() PanicOn '%v' at '%v'", r, vprint.Stack())
-			vprint.PanicOn(r)
-		}
-	}()
-	return c.b.RoaringBitmap(index, field, view, shard)
-}
-
-func (c *catcherTx) Container(index, field, view string, shard uint64, key uint64) (ct *roaring.Container, err error) {
-
-	defer func() {
-		if r := recover(); r != nil {
-			vprint.AlwaysPrintf("see Container() PanicOn '%v' at '%v'", r, vprint.Stack())
-			vprint.PanicOn(r)
-		}
-	}()
-	return c.b.Container(index, field, view, shard, key)
-}
-
-func (c *catcherTx) PutContainer(index, field, view string, shard uint64, key uint64, rc *roaring.Container) error {
-
-	defer func() {
-		if r := recover(); r != nil {
-			vprint.AlwaysPrintf("see PutContainer() PanicOn '%v' at '%v'", r, vprint.Stack())
-			vprint.PanicOn(r)
-		}
-	}()
-	return c.b.PutContainer(index, field, view, shard, key, rc)
-}
-
-func (c *catcherTx) RemoveContainer(index, field, view string, shard uint64, key uint64) error {
-
-	defer func() {
-		if r := recover(); r != nil {
-			vprint.AlwaysPrintf("see RemoveContainer() PanicOn '%v' at '%v'", r, vprint.Stack())
-			vprint.PanicOn(r)
-		}
-	}()
-	return c.b.RemoveContainer(index, field, view, shard, key)
-}
-
-func (c *catcherTx) Add(index, field, view string, shard uint64, a ...uint64) (changeCount int, err error) {
-
-	defer func() {
-		if r := recover(); r != nil {
-			vprint.AlwaysPrintf("see Add() PanicOn '%v' at '%v'", r, vprint.Stack())
-			vprint.PanicOn(r)
-		}
-	}()
-	return c.b.Add(index, field, view, shard, a...)
-}
-
-func (c *catcherTx) Remove(index, field, view string, shard uint64, a ...uint64) (changeCount int, err error) {
-
-	defer func() {
-		if r := recover(); r != nil {
-			vprint.AlwaysPrintf("see Remove() PanicOn '%v' at '%v'", r, vprint.Stack())
-			vprint.PanicOn(r)
-		}
-	}()
-	return c.b.Remove(index, field, view, shard, a...)
-}
-
-func (c *catcherTx) Removed(index, field, view string, shard uint64, a ...uint64) (changed []uint64, err error) {
-
-	defer func() {
-		if r := recover(); r != nil {
-			vprint.AlwaysPrintf("see Removed() PanicOn '%v' at '%v'", r, vprint.Stack())
-			vprint.PanicOn(r)
-		}
-	}()
-	return c.b.Removed(index, field, view, shard, a...)
-}
-
-func (c *catcherTx) Contains(index, field, view string, shard uint64, key uint64) (exists bool, err error) {
-
-	defer func() {
-		if r := recover(); r != nil {
-			vprint.AlwaysPrintf("see Contains() PanicOn '%v' at '%v'", r, vprint.Stack())
-			vprint.PanicOn(r)
-		}
-	}()
-	return c.b.Contains(index, field, view, shard, key)
-}
-
-func (c *catcherTx) ContainerIterator(index, field, view string, shard uint64, firstRoaringContainerKey uint64) (citer roaring.ContainerIterator, found bool, err error) {
-
-	defer func() {
-		if r := recover(); r != nil {
-			vprint.AlwaysPrintf("see ContainerIterator() PanicOn '%v' at '%v'", r, vprint.Stack())
-			vprint.PanicOn(r)
-		}
-	}()
-	return c.b.ContainerIterator(index, field, view, shard, firstRoaringContainerKey)
-}
-
-func (c *catcherTx) Count(index, field, view string, shard uint64) (uint64, error) {
-
-	defer func() {
-		if r := recover(); r != nil {
-			vprint.AlwaysPrintf("see Count() PanicOn '%v' at '%v'", r, vprint.Stack())
-			vprint.PanicOn(r)
-		}
-	}()
-	return c.b.Count(index, field, view, shard)
-}
-
-func (c *catcherTx) Max(index, field, view string, shard uint64) (uint64, error) {
-
-	defer func() {
-		if r := recover(); r != nil {
-			vprint.AlwaysPrintf("see Max() PanicOn '%v' at '%v'", r, vprint.Stack())
-			vprint.PanicOn(r)
-		}
-	}()
-	return c.b.Max(index, field, view, shard)
-}
-
-func (c *catcherTx) Min(index, field, view string, shard uint64) (uint64, bool, error) {
-
-	defer func() {
-		if r := recover(); r != nil {
-			vprint.AlwaysPrintf("see Min() PanicOn '%v' at '%v'", r, vprint.Stack())
-			vprint.PanicOn(r)
-		}
-	}()
-	return c.b.Min(index, field, view, shard)
-}
-
-func (c *catcherTx) CountRange(index, field, view string, shard uint64, start, end uint64) (n uint64, err error) {
-
-	defer func() {
-		if r := recover(); r != nil {
-			vprint.AlwaysPrintf("see CountRange() PanicOn '%v' at '%v'", r, vprint.Stack())
-			vprint.PanicOn(r)
-		}
-	}()
-	return c.b.CountRange(index, field, view, shard, start, end)
-}
-
-func (c *catcherTx) OffsetRange(index, field, view string, shard, offset, start, end uint64) (other *roaring.Bitmap, err error) {
-
-	defer func() {
-		if r := recover(); r != nil {
-			vprint.AlwaysPrintf("see OffsetRange() PanicOn '%v' at '%v'", r, vprint.Stack())
-			vprint.PanicOn(r)
-		}
-	}()
-	return c.b.OffsetRange(index, field, view, shard, offset, start, end)
-}
-
-func (c *catcherTx) Type() string {
-	return c.b.Type()
-}
-
-func (c *catcherTx) ApplyFilter(index, field, view string, shard uint64, ckey uint64, filter roaring.BitmapFilter) (err error) {
-	return GenericApplyFilter(c, index, field, view, shard, ckey, filter)
-}
-
-func (c *catcherTx) ApplyRewriter(index, field, view string, shard uint64, ckey uint64, filter roaring.BitmapRewriter) (err error) {
-	return c.b.ApplyRewriter(index, field, view, shard, ckey, filter)
-}
-
-func (c *catcherTx) GetSortedFieldViewList(idx *Index, shard uint64) (fvs []txkey.FieldView, err error) {
-	return c.b.GetSortedFieldViewList(idx, shard)
-}
-
-func (c *catcherTx) GetFieldSizeBytes(index, field string) (uint64, error) {
-	return 0, nil
-}

cli/Makefile

@@ -1,15 +0,0 @@
-.PHONY: test testv test-integration testv-integration
-
-GO=go
-
-test:
-	$(GO) test ./... -short
-
-testv:
-	$(GO) test -v ./... -short
-
-test-integration:
-	$(GO) test . -count 1 -timeout 20m -run TestCLIIntegration/$(RUN)
-
-testv-integration:
-	$(GO) test -v . -count 1 -timeout 20m -run TestCLIIntegration/$(RUN)

cli/batch/inserter.go

@@ -1,8 +0,0 @@
-package batch
-
-// Inserter can be implemented by anything which can handle a SQL statement
-// representing a write operation. An example is `BULK INSERT`. The Insert()
-// method on this interface does not return any results other than an error.
-type Inserter interface {
-	Insert(sql string) error
-}

cli/batch/sql.go

@@ -1,215 +0,0 @@
-package batch
-
-import (
-	"encoding/json"
-	"fmt"
-	"strings"
-	"time"
-
-	fbbatch "github.com/featurebasedb/featurebase/v3/batch"
-	"github.com/featurebasedb/featurebase/v3/dax"
-	"github.com/featurebasedb/featurebase/v3/errors"
-	"github.com/featurebasedb/featurebase/v3/pql"
-)
-
-// Ensure type implements interface.
-var _ fbbatch.Batcher = (*sqlBatcher)(nil)
-
-type sqlBatcher struct {
-	inserter Inserter
-	fields   []*dax.Field
-}
-
-func NewSQLBatcher(i Inserter, flds []*dax.Field) *sqlBatcher {
-	return &sqlBatcher{
-		inserter: i,
-		fields:   flds,
-	}
-}
-
-func (b *sqlBatcher) NewBatch(cfg fbbatch.Config, tbl *dax.Table, flds []*dax.Field) (fbbatch.RecordBatch, error) {
-	fields := flds
-	if b.fields != nil {
-		fields = b.fields
-	}
-	return &sqlBatch{
-		table:        tbl,
-		fields:       fields,
-		size:         cfg.Size,
-		maxStaleness: cfg.MaxStaleness,
-		ids:          make([]interface{}, 0, cfg.Size),
-		rows:         make([][]interface{}, 0, cfg.Size),
-		inserter:     b.inserter,
-	}, nil
-}
-
-// Ensure type implements interface.
-var _ fbbatch.RecordBatch = (*sqlBatch)(nil)
-
-type sqlBatch struct {
-	table  *dax.Table
-	fields []*dax.Field
-	size   int
-
-	ids  []interface{}
-	rows [][]interface{}
-
-	// staleTime tracks the time the first record of the batch was inserted
-	// plus the maxStaleness, in order to raise ErrBatchNowStale if the
-	// maxStaleness has elapsed
-	staleTime    time.Time
-	maxStaleness time.Duration
-
-	// inserter handles SQL INSERT statements generated for each batch.
-	inserter Inserter
-}
-
-func (b *sqlBatch) Add(rec fbbatch.Row) error {
-	// Clear rec.Values and rec.Clears upon return.
-	defer func() {
-		for i := range rec.Values {
-			rec.Values[i] = nil
-		}
-		for k := range rec.Clears {
-			delete(rec.Clears, k)
-		}
-	}()
-
-	if len(b.ids) == cap(b.ids) {
-		return fbbatch.ErrBatchAlreadyFull
-	}
-	if len(rec.Values) != len(b.fields) {
-		return errors.Errorf("record needs to match up with batch fields, got %d fields and %d record", len(b.fields), len(rec.Values))
-	}
-
-	// Append the ID to b.ids.
-	b.ids = append(b.ids, rec.ID)
-
-	// Convert decimal fields (which come in as int64, along with the scale in
-	// field) to pql.Decimal.
-	for i, fld := range b.fields {
-		switch b.fields[i].Type {
-		case dax.BaseTypeDecimal:
-			if val, ok := rec.Values[i].(int64); ok {
-				rec.Values[i] = pql.NewDecimal(val, fld.Options.Scale)
-			}
-		case dax.BaseTypeTimestamp:
-			if val, ok := rec.Values[i].(int64); ok {
-				ts := time.Unix(val, 0)
-				rec.Values[i] = ts.Format(time.RFC3339)
-			}
-		}
-	}
-
-	// Append the record to b.rows.
-	vals := make([]interface{}, 0, len(rec.Values))
-	vals = append(vals, rec.Values...)
-	b.rows = append(b.rows, vals)
-
-	// Check for batch full or stale.
-	if len(b.ids) == cap(b.ids) {
-		return fbbatch.ErrBatchNowFull
-	}
-	if b.maxStaleness != time.Duration(0) { // set maxStaleness to 0 to disable staleness checking
-		if len(b.ids) == 1 {
-			b.staleTime = time.Now().Add(b.maxStaleness)
-		} else if time.Now().After(b.staleTime) {
-			return fbbatch.ErrBatchNowStale
-		}
-	}
-	return nil
-}
-
-func (b *sqlBatch) Import() error {
-	if len(b.rows) == 0 {
-		return nil
-	}
-
-	// Construct the BULK INSERT statement based on the table and fields.
-	sql, err := buildBulkInsert(b.table, b.fields, b.ids, b.rows)
-	if err != nil {
-		return errors.Wrap(err, "building bulk insert statement")
-	}
-
-	// Reset batch data.
-	b.reset()
-
-	// Submit the SQL statement.
-	return b.inserter.Insert(sql)
-}
-
-func (b *sqlBatch) reset() {
-	b.ids = b.ids[:0]
-	b.rows = b.rows[:0]
-}
-
-func (b *sqlBatch) Len() int {
-	return len(b.rows)
-}
-
-func (b *sqlBatch) Flush() error {
-	return nil
-}
-
-func buildBulkInsert(tbl *dax.Table, fields []*dax.Field, ids []interface{}, rows [][]interface{}) (string, error) {
-	// Validation.
-	if tbl.Name == "" {
-		return "", errors.New(errors.ErrUncoded, "table name is required")
-	} else if len(fields) == 0 {
-		return "", errors.New(errors.ErrUncoded, "at least one field is required")
-	}
-
-	var sb strings.Builder
-
-	sb.WriteString(`BULK INSERT INTO `)
-	sb.WriteString(string(tbl.Name))
-	sb.WriteString(` (_id,`)
-
-	flds := make([]string, 0, len(fields))
-	maps := make([]string, 0, len(fields))
-	for i := range fields {
-		flds = append(flds, string(fields[i].Name))
-		maps = append(maps, fmt.Sprintf("'$.col_%d' %s", i, fields[i].FullType()))
-	}
-	// Fields
-	sb.WriteString(strings.Join(flds, ","))
-
-	// MAP
-	keyType := dax.BaseTypeID
-	if tbl.StringKeys() {
-		keyType = dax.BaseTypeString
-	}
-	sb.WriteString(`) MAP ('$._id' `)
-	sb.WriteString(keyType)
-	sb.WriteString(`,`)
-	sb.WriteString(strings.Join(maps, ","))
-	sb.WriteString(`) FROM x'`)
-
-	// Row values.
-
-	// m is a map representing a single row to be marshalled and added to the
-	// bulk insert as one line in the NDJSON payload. We re-use the map for each
-	// row.
-	m := make(map[string]interface{})
-	for i := range rows {
-		// Write the ID value.
-		m[string(dax.PrimaryKeyFieldName)] = ids[i]
-		// Write the rest of the data values.
-		for col := range rows[i] {
-			m[fmt.Sprintf("col_%d", col)] = rows[i][col]
-		}
-
-		// Marshal the map to json and add to the sql statement.
-		if j, err := json.Marshal(m); err != nil {
-			return "", errors.Wrap(err, "marshalling row to json")
-		} else {
-			sb.Write(j)
-			sb.WriteString("\n")
-		}
-	}
-
-	// WITH
-	sb.WriteString(fmt.Sprintf(`' WITH BATCHSIZE %d FORMAT 'NDJSON' INPUT 'STREAM'`, len(rows)))
-
-	return sb.String(), nil
-}

cli/batch/sql_test.go

@@ -1,47 +0,0 @@
-package batch
-
-import (
-	"testing"
-
-	"github.com/featurebasedb/featurebase/v3/dax"
-	"github.com/stretchr/testify/assert"
-)
-
-func TestBatchSQL(t *testing.T) {
-	tbl := &dax.Table{
-		Name: "foo",
-	}
-	fields := []*dax.Field{
-		{
-			Name: "name",
-			Type: dax.BaseTypeString,
-		},
-		{
-			Name: "age",
-			Type: dax.BaseTypeInt,
-		},
-	}
-	ids := []interface{}{
-		0, 1, 2,
-	}
-	rows := [][]interface{}{
-		{
-			[]interface{}{"Alice", int64(11)},
-		},
-		{
-			[]interface{}{"Bob", int64(22)},
-		},
-		{
-			[]interface{}{"Carl,Comma", int64(33)},
-		},
-	}
-
-	s, err := buildBulkInsert(tbl, fields, ids, rows)
-	assert.NoError(t, err)
-
-	exp := `BULK INSERT INTO foo (_id,name,age) MAP ('$._id' id,'$.col_0' string,'$.col_1' int) FROM x'{"_id":0,"col_0":["Alice",11]}
-{"_id":1,"col_0":["Bob",22]}
-{"_id":2,"col_0":["Carl,Comma",33]}
-' WITH BATCHSIZE 3 FORMAT 'NDJSON' INPUT 'STREAM'`
-	assert.Equal(t, exp, s)
-}

cli/buffer.go

@@ -1,91 +0,0 @@
-package cli
-
-import (
-	"io"
-	"strings"
-
-	"github.com/featurebasedb/featurebase/v3/errors"
-)
-
-// buffer is a query buffer for SQL statements. Note that this is not a query
-// buffer as you would find on a database server (buffering query results).
-// Rather, this buffers the working SQL statement. The buffer has two
-// components: the buffer of query parts making up the working, incomplete SQL
-// statement, and the last completed SQL statement submitted to the Queryer.
-type buffer struct {
-	parts     []queryPart
-	lastQuery query
-
-	hasBatchFile bool
-}
-
-func newBuffer() *buffer {
-	return &buffer{}
-}
-
-// addPart adds the given queryPart to the buffer. If the part is of type
-// `partTerminator` (which is generally singified in the CLI by a ";"), the
-// buffer will finalize the query and return it. In all other cases, the
-// returned query is nil.
-func (b *buffer) addPart(part queryPart) (query, error) {
-	// Check for part type compatibility. For example, multiple batchFile parts
-	// are not allowed in the same query.
-	switch part.(type) {
-	case *partBatchFile:
-		if b.hasBatchFile {
-			return nil, errors.Errorf("multiple batch files in one query is not supported")
-		}
-		b.hasBatchFile = true
-	case *partTerminator:
-		return b.finalize(), nil
-	}
-
-	b.parts = append(b.parts, part)
-	return nil, nil
-}
-
-// finalize copies the contents (queryParts) of buffer to lastQuery and then
-// resets the buffer. It returns the query that was finalized.
-func (b *buffer) finalize() query {
-	q := make(query, len(b.parts))
-	copy(q, b.parts)
-	b.lastQuery = q
-	b.reset()
-	return q
-
-}
-
-// print returns the contents of the buffer as a string. This is generally used
-// to visually inspect the state of the buffer (for example, when a user issues
-// a `\p` meta-command in the CLI).
-func (b *buffer) print() string {
-	if len(b.parts) > 0 {
-		return query(b.parts).String()
-	} else if b.lastQuery != nil {
-		return b.lastQuery.String() + ";"
-	}
-	return "Query buffer is empty."
-}
-
-// reset clears the buffer. It returns a message which may optionally be used to
-// display to a user.
-func (b *buffer) reset() string {
-	b.parts = b.parts[:0]
-	b.hasBatchFile = false
-	return "Query buffer reset (cleared)."
-}
-
-func (b *buffer) Reader() io.Reader {
-	if len(b.parts) > 0 {
-		return query(b.parts).Reader()
-	} else if b.lastQuery != nil {
-		r := b.lastQuery.Reader()
-		// TODO(tlt): terminating the query here results in a line feed just
-		// before the semi-colon (for example, when you print out the query
-		// buffer using `\w [FILE]`). The removal and re-introduction of line
-		// feeds is kind of a mess.
-		term := strings.NewReader(";")
-		return io.MultiReader(r, term)
-	}
-	return strings.NewReader("")
-}