diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml
index 6b9de74c0..d2ba46246 100644
--- a/.gitlab/.gitlab-ci.yml
+++ b/.gitlab/.gitlab-ci.yml
@@ -1,7 +1,7 @@
-include:
-  - template: Security/SAST.gitlab-ci.yml
-  - template: Security/License-Scanning.gitlab-ci.yml
-  - template: Security/Dependency-Scanning.gitlab-ci.yml
+# include:
+#   - template: Security/SAST.gitlab-ci.yml
+#   - template: Security/License-Scanning.gitlab-ci.yml
+#   - template: Security/Dependency-Scanning.gitlab-ci.yml
 
 variables:
   GOVERSION: "1.17.8"
@@ -16,35 +16,35 @@ stages:
   - post build
   - nonblocking
 
-smoke build:
-  image: golang:$GOVERSION
-  stage: lint
-  allow_failure: false
-  rules:
-    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
-  script:
-    - echo "Let's just see if it compiles... (sometimes the linter gives unclear errors if it doesn't)"
-    - go build ./...
-
-golangci-lint:
-  image: golangci/golangci-lint:v1.39.0
-  stage: lint
-  allow_failure: false
-  rules:
-    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
-  script:
-    - echo "Checking for issues in new code"
-    - golangci-lint run
-
-go mod tidy:
-  stage: lint
-  image: golang:$GOVERSION
-  rules:
-    - if: '$CI_COMMIT_TAG == null && ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web")'
-  script:
-    - go mod tidy
-    - git diff --exit-code -- go.mod go.sum
-
+#smoke build:
+#  image: golang:$GOVERSION
+#  stage: lint
+#  allow_failure: false
+#  rules:
+#    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
+#  script:
+#    - echo "Let's just see if it compiles... (sometimes the linter gives unclear errors if it doesn't)"
+#    - go build ./...
+#
+#golangci-lint:
+#  image: golangci/golangci-lint:v1.39.0
+#  stage: lint
+#  allow_failure: false
+#  rules:
+#    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
+#  script:
+#    - echo "Checking for issues in new code"
+#    - golangci-lint run
+#
+#go mod tidy:
+#  stage: lint
+#  image: golang:$GOVERSION
+#  rules:
+#    - if: '$CI_COMMIT_TAG == null && ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web")'
+#  script:
+#    - go mod tidy
+#    - git diff --exit-code -- go.mod go.sum
+#
 build lattice:
   stage: test
   image: node:14
@@ -64,110 +64,110 @@ build lattice:
   artifacts:
     paths:
       - lattice.tar.gz
-
-run jest tests:
-  stage: test
-  image: node:14
-  variables:
-    CI: "true"
-  rules:
-    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
-  script:
-    - echo "Testing lattice..."
-    - cd lattice
-    - npm install --force
-    - npm test -- --coverage --testResultsProcessor=jest-sonar-reporter
-  artifacts:
-    paths:
-      - lattice/coverage/lcov.info
-
-run go tests:
-  stage: test
-  image: golang:$GOVERSION
-  rules:
-    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
-  retry: 1
-  script:
-    - echo "Running featurebase unit tests..."
-    - go test -timeout=30m ./...
-  tags:
-    - aws
-
-run go tests race:
-  stage: nonblocking # don't let this job block any other jobs because it takes much longer than the other tests.
-  image: golang:$GOVERSION
-  rules:
-    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
-  retry: 1
-  needs: [] # don't wait to start running this.
-  script:
-    - echo "Running featurebase race tests..."
-    - go test -race -v -timeout=90m ./...
-  tags:
-    - aws
-
-run go tests shardwidth22:
-  stage: test
-  image: golang:$GOVERSION
-  rules:
-    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
-  script:
-    - echo "Running featurebase shardwidth22 tests..."
-    - go test -timeout=30m -tags=shardwidth22 ./...
-  tags:
-    - aws
-    
-# we do coverage reporting from the future tests because the json
-# output is very difficult to human-read. The alternative would be to
-# run the regular tests twice and also run the future tests.
-run go tests future:
-  stage: test
-  image: golang:1.18
-  rules:
-    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
-  retry: 1
-  script:
-    - echo "Running featurebase unit tests..."
-    - PKG_LIST=$(go list ./... | grep -v internal/clustertests | paste -s -d, -)
-    - go test -timeout=30m -json -coverprofile=coverage.out -covermode=atomic -coverpkg=${PKG_LIST} ./... | tee test-report.out
-  artifacts:
-    paths:
-      - coverage.out
-      - test-report.out
-  tags:
-    - aws
-
-upload to sonarcloud:
-  stage: integration
-  image: sonarsource/sonar-scanner-cli:4.6
-  variables:
-    SONAR_TOKEN: $SONAR_TOKEN
-  rules:
-    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
-  script:
-    - sonar-scanner -Dsonar.projectKey=molecula_featurebase -Dsonar.organization=molecula -Dsonar.sources=. -Dsonar.host.url=https://sonarcloud.io -Dsonar.go.coverage.reportPaths=coverage.out,results/coverage*.out -Dsonar.go.tests.reportPaths=test-report.out,results/report*  -Dsonar.javascript.lcov.reportPaths=lattice/coverage/lcov.info
-  needs:
-    - job: run go tests future
-    - job: run jest tests 
-    - job: clustertests
-    
-build for linux amd64:
-  stage: build
-  image: golang:$GOVERSION
-  rules:
-    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
-  script:
-    - rm -r lattice
-    - tar -xvf lattice.tar.gz 
-    - go get -v -u github.com/rakyll/statik 
-    - /go/bin/statik -src=lattice
-    - GOOS="linux" GOARCH="amd64" make build FLAGS="-o featurebase_linux_amd64"
-    - GOOS="linux" GOARCH="amd64" go build -o roaring-migrate_linux_amd64 ./cmd/roaring-migrate
-  artifacts:
-    paths:
-      - featurebase_linux_amd64
-      - roaring-migrate_linux_amd64
-
+#
+#run jest tests:
+#  stage: test
+#  image: node:14
+#  variables:
+#    CI: "true"
+#  rules:
+#    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
+#  script:
+#    - echo "Testing lattice..."
+#    - cd lattice
+#    - npm install --force
+#    - npm test -- --coverage --testResultsProcessor=jest-sonar-reporter
+#  artifacts:
+#    paths:
+#      - lattice/coverage/lcov.info
+#
+#run go tests:
+#  stage: test
+#  image: golang:$GOVERSION
+#  rules:
+#    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
+#  retry: 1
+#  script:
+#    - echo "Running featurebase unit tests..."
+#    - go test -timeout=30m ./...
+#  tags:
+#    - aws
+#
+#run go tests race:
+#  stage: nonblocking # don't let this job block any other jobs because it takes much longer than the other tests.
+#  image: golang:$GOVERSION
+#  rules:
+#    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
+#  retry: 1
+#  needs: [] # don't wait to start running this.
+#  script:
+#    - echo "Running featurebase race tests..."
+#    - go test -race -v -timeout=90m ./...
+#  tags:
+#    - aws
+#
+#run go tests shardwidth22:
+#  stage: test
+#  image: golang:$GOVERSION
+#  rules:
+#    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
+#  script:
+#    - echo "Running featurebase shardwidth22 tests..."
+#    - go test -timeout=30m -tags=shardwidth22 ./...
+#  tags:
+#    - aws
+#    
+## we do coverage reporting from the future tests because the json
+## output is very difficult to human-read. The alternative would be to
+## run the regular tests twice and also run the future tests.
+#run go tests future:
+#  stage: test
+#  image: golang:1.18
+#  rules:
+#    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
+#  retry: 1
+#  script:
+#    - echo "Running featurebase unit tests..."
+#    - PKG_LIST=$(go list ./... | grep -v internal/clustertests | paste -s -d, -)
+#    - go test -timeout=30m -json -coverprofile=coverage.out -covermode=atomic -coverpkg=${PKG_LIST} ./... | tee test-report.out
+#  artifacts:
+#    paths:
+#      - coverage.out
+#      - test-report.out
+#  tags:
+#    - aws
+#
+#upload to sonarcloud:
+#  stage: integration
+#  image: sonarsource/sonar-scanner-cli:4.6
+#  variables:
+#    SONAR_TOKEN: $SONAR_TOKEN
+#  rules:
+#    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
+#  script:
+#    - sonar-scanner -Dsonar.projectKey=molecula_featurebase -Dsonar.organization=molecula -Dsonar.sources=. -Dsonar.host.url=https://sonarcloud.io -Dsonar.go.coverage.reportPaths=coverage.out,results/coverage*.out -Dsonar.go.tests.reportPaths=test-report.out,results/report*  -Dsonar.javascript.lcov.reportPaths=lattice/coverage/lcov.info
+#  needs:
+#    - job: run go tests future
+#    - job: run jest tests 
+#    - job: clustertests
+#    
+#build for linux amd64:
+#  stage: build
+#  image: golang:$GOVERSION
+#  rules:
+#    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
+#  script:
+#    - rm -r lattice
+#    - tar -xvf lattice.tar.gz 
+#    - go get -v -u github.com/rakyll/statik 
+#    - /go/bin/statik -src=lattice
+#    - GOOS="linux" GOARCH="amd64" make build FLAGS="-o featurebase_linux_amd64"
+#    - GOOS="linux" GOARCH="amd64" go build -o roaring-migrate_linux_amd64 ./cmd/roaring-migrate
+#  artifacts:
+#    paths:
+#      - featurebase_linux_amd64
+#      - roaring-migrate_linux_amd64
+#
 build for linux arm64:
   stage: build
   image: golang:$GOVERSION
@@ -184,223 +184,279 @@ build for linux arm64:
     paths:
       - featurebase_linux_arm64
       - roaring-migrate_linux_arm64
-
-build for darwin amd64:
-  stage: build
-  image: golang:$GOVERSION
-  rules:
-    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
-  script:
-    - rm -r lattice
-    - tar -xvf lattice.tar.gz 
-    - go get -v -u github.com/rakyll/statik 
-    - /go/bin/statik -src=lattice
-    - GOOS="darwin" GOARCH="amd64" make build FLAGS="-o featurebase_darwin_amd64"
-    - GOOS="darwin" GOARCH="amd64" go build -o roaring-migrate_darwin_amd64 ./cmd/roaring-migrate
-  artifacts:
-    paths:
-      - featurebase_darwin_amd64
-      - roaring-migrate_darwin_amd64
-
-build for darwin arm64:
-  stage: build
-  image: golang:$GOVERSION
-  rules:
-    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
-  script:
-    - rm -r lattice
-    - tar -xvf lattice.tar.gz 
-    - go get -v -u github.com/rakyll/statik 
-    - /go/bin/statik -src=lattice
-    - GOOS="darwin" GOARCH="arm64" make build FLAGS="-o featurebase_darwin_arm64"
-    - GOOS="darwin" GOARCH="arm64" go build -o roaring-migrate_darwin_arm64 ./cmd/roaring-migrate
-  artifacts:
-    paths:
-      - featurebase_darwin_arm64
-      - roaring-migrate_darwin_arm64
-
-package for linux amd64:
-  stage: build
-  image: golang:$GOVERSION
-  rules:
-    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
-  variables:
-    GOOS: "linux"
-    GOARCH: "amd64"
-  script:
-    - echo 'deb [trusted=yes] https://repo.goreleaser.com/apt/ /' | tee /etc/apt/sources.list.d/goreleaser.list
-    - apt update && apt install nfpm=2.11.3
-    - make package
-  artifacts:
-    paths:
-      - "*.deb"
-      - "*.rpm"
-
-package for linux arm64:
-  stage: build
-  image: golang:$GOVERSION
-  rules:
-    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
-  variables:
-    GOOS: "linux"
-    GOARCH: "arm64"
-  script:
-    - echo 'deb [trusted=yes] https://repo.goreleaser.com/apt/ /' | tee /etc/apt/sources.list.d/goreleaser.list
-    - apt update && apt install nfpm=2.11.3
-    - make package
-  artifacts:
-    paths:
-      - "*.deb"
-      - "*.rpm"
-
-build amd container fb:
-  stage: build
-  needs:
-    - "build for linux amd64"
-  tags:
-    - shell
-  rules:
-    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
-  before_script:    
-    - echo "${DOCKER_DEPLOY_TOKEN}" | docker login -u ${DOCKER_DEPLOY_USER} --password-stdin ${CI_REGISTRY}
-  script:    
-    - tag=${CI_REGISTRY_IMAGE}/featurebase:linux-amd64-${CI_COMMIT_REF_SLUG}
-    - docker build --build-arg GO_VERSION=$GOVERSION --build-arg ARCH=amd64 -t $tag -f .gitlab/Dockerfile .
-    - docker push $tag
-    - echo Created docker featurebase image with tag "$tag"
-
-build arm container fb:
-  stage: build
-  needs:
-    - "build for linux arm64"
-  tags:
-    - shell
-  rules:
-    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
-  before_script:    
-    - echo "${DOCKER_DEPLOY_TOKEN}" | docker login -u ${DOCKER_DEPLOY_USER} --password-stdin ${CI_REGISTRY}
-  script:    
-    - tag=${CI_REGISTRY_IMAGE}/featurebase:linux-arm64-${CI_COMMIT_REF_SLUG}
-    - docker build --build-arg GO_VERSION=$GOVERSION --build-arg ARCH=arm64 -t $tag -f .gitlab/Dockerfile .
-    - docker push $tag
-    - echo Created docker featurebase image with tag "$tag"
-
-
-# clustertests doesn't run in docker, and requires several things to be set up on the runner to work:
-# 1. Install Go, make sure it's on the path
-# 2. Make sure "make" is installed
-# 3. make sure docker/docker-compose is installed
-# 4. make sure the git config is done `git config --global --add url."ssh://git@github.com/".insteadOf "https://github.com/"`
-# 5. Add deploy key github.com/molecula/featurebase/settings/keys and add public key in .ssh folder of gitlab-runner user
-clustertests:
-  variables:
-    PROJECT: clustertests_${CI_CONCURRENT_ID}
-  stage: integration
-  tags:
-    - shell
-  retry: 1
-  rules:
-    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
-  script:
-    - rm -rf  internal/clustertests/results && mkdir -p internal/clustertests/results && chown gitlab-runner:gitlab-runner  internal/clustertests/results
-    - make clustertests
-    - mv internal/clustertests/results/ results/
-  artifacts:
-    paths:
-      - results/coverage*.out
-
-authclustertests:
-  variables:
-    PROJECT: authclustertests_${CI_CONCURRENT_ID}
-  stage: integration
-  retry: 1
-  tags:
-    - shell
-  rules:
-    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
-  script:
-    - rm -rf  internal/clustertests/results && mkdir -p internal/clustertests/results && chown gitlab-runner:gitlab-runner  internal/clustertests/results
-    - make authclustertests
-    - rm -rf internal/clustertests/results
-
-
-external lookup tests:
-  stage: integration
-  image: golang:$GOVERSION
-  # TODO: no rules here, do we need to add the rules line?
-  variables:
-    POSTGRES_DB: $POSTGRES_DB
-    POSTGRES_USER: $POSTGRES_USER
-    POSTGRES_PASSWORD: $POSTGRES_PASSWORD
-    POSTGRES_HOST_AUTH_METHOD: trust
-  services:
-    - postgres:13.5
-  script:
-    - apt-get update --allow-releaseinfo-change -y
-    - apt-get install -y postgresql-client
-    - go test . -run "^TestExternalLookup" -externalLookupDSN postgresql://$POSTGRES_USER:$POSTGRES_PASSWORD@postgres/$POSTGRES_DB?sslmode=disable
-
-
-smoke test:
-  stage: integration
-  image: registry.gitlab.com/gitlab-org/cloud-deploy/aws-base:latest
-  variables:
-    PROFILE: "service-terraform"
-    AWS_SSH_PRIVATE_KEY: $AWS_FBCI_SSH_KEY
-    AWS_ACCESS_KEY_ID: $AWS_FBCI_ACCESS_KEY_ID
-    AWS_SECRET_ACCESS_KEY: $AWS_FBCI_SECRET_ACCESS_KEY
-    TF_VAR_cluster_prefix: ""
-  tags:
-    - aws
-    - docker
-    - fbsmoke
-  rules:
-    - if: '$CI_PIPELINE_SOURCE == "push"'
-  before_script:
-    - apt-get update && apt-get install -y gnupg software-properties-common curl git
-    - curl -fsSL https://apt.releases.hashicorp.com/gpg | apt-key add -
-    - apt-add-repository "deb [arch=amd64] https://apt.releases.hashicorp.com $(lsb_release -cs) main"
-    - apt-get update && apt-get install terraform
-    - aws configure set aws_access_key_id $AWS_FBCI_ACCESS_KEY_ID --profile $PROFILE
-    - aws configure set aws_secret_access_key $AWS_FBCI_SECRET_ACCESS_KEY --profile $PROFILE
-    - aws configure set region "us-east-2" --profile $PROFILE
-    - aws configure set aws_profile $PROFILE
-    - echo $AWS_FBCI_SSH_KEY > gitlab-featurebase-ci.pem
-    - chmod 400 gitlab-featurebase-ci.pem
-    - 'which ssh-agent || ( apt-get update -y && apt-get install openssh-client -y )'
-    - eval $(ssh-agent -s)
-    - mkdir -p ~/.ssh
-    - echo $AWS_FBCI_SSH_KEY > /root/.ssh/gitlab-featurebase-ci.pem
-    - chmod 400 /root/.ssh/gitlab-featurebase-ci.pem
-    - echo "$AWS_FBCI_SSH_KEY" | ssh-add -
-    - chmod 700 /root/.ssh
-    - '[[ -f /.dockerenv ]] && echo -e "Host *\n\tStrictHostKeyChecking no\n\n" > ~/.ssh/config'
-    - apt update && apt -y install jq wget
-    - wget -q https://go.dev/dl/go$GOVERSION.linux-amd64.tar.gz
-    - tar -C /usr/local -xzf go$GOVERSION.linux-amd64.tar.gz
-    - export PATH=$PATH:/usr/local/go/bin
-    - TF_VAR_cluster_prefix="smoke-$(openssl rand -base64 12 | tr -d /=+ | cut -c -16)"
-    - echo "Cluster Prefix --> $TF_VAR_cluster_prefix"
-    # download datagen for FB-1270 repro test. TODO replace w/ locally built datagen once we merge IDK into FB
-    - aws s3 cp s3://molecula-artifact-storage/idk/master/_latest/idk-linux-arm64/datagen ./datagen_linux_arm64
-    - chmod +x ./datagen_linux_arm64
-  script:
-    - ./qa/scripts/setupSmokeTest.sh
-    - ./qa/scripts/testSmokeTest.sh
-    - ./qa/scripts/bug_repro_tests.sh
-  after_script:
-    - ./qa/scripts/teardownSmokeTest.sh
-  needs:
-    - job: build for linux arm64
-  artifacts:
-    when: always
-    paths:
-      - report.xml
-    reports:
-      junit: report.xml
-
-
-gauntlet:
+#
+#build for darwin amd64:
+#  stage: build
+#  image: golang:$GOVERSION
+#  rules:
+#    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
+#  script:
+#    - rm -r lattice
+#    - tar -xvf lattice.tar.gz 
+#    - go get -v -u github.com/rakyll/statik 
+#    - /go/bin/statik -src=lattice
+#    - GOOS="darwin" GOARCH="amd64" make build FLAGS="-o featurebase_darwin_amd64"
+#    - GOOS="darwin" GOARCH="amd64" go build -o roaring-migrate_darwin_amd64 ./cmd/roaring-migrate
+#  artifacts:
+#    paths:
+#      - featurebase_darwin_amd64
+#      - roaring-migrate_darwin_amd64
+#
+#build for darwin arm64:
+#  stage: build
+#  image: golang:$GOVERSION
+#  rules:
+#    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
+#  script:
+#    - rm -r lattice
+#    - tar -xvf lattice.tar.gz 
+#    - go get -v -u github.com/rakyll/statik 
+#    - /go/bin/statik -src=lattice
+#    - GOOS="darwin" GOARCH="arm64" make build FLAGS="-o featurebase_darwin_arm64"
+#    - GOOS="darwin" GOARCH="arm64" go build -o roaring-migrate_darwin_arm64 ./cmd/roaring-migrate
+#  artifacts:
+#    paths:
+#      - featurebase_darwin_arm64
+#      - roaring-migrate_darwin_arm64
+#
+#package for linux amd64:
+#  stage: build
+#  image: golang:$GOVERSION
+#  rules:
+#    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
+#  variables:
+#    GOOS: "linux"
+#    GOARCH: "amd64"
+#  script:
+#    - echo 'deb [trusted=yes] https://repo.goreleaser.com/apt/ /' | tee /etc/apt/sources.list.d/goreleaser.list
+#    - apt update && apt install nfpm=2.11.3
+#    - make package
+#  artifacts:
+#    paths:
+#      - "*.deb"
+#      - "*.rpm"
+#
+#package for linux arm64:
+#  stage: build
+#  image: golang:$GOVERSION
+#  rules:
+#    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
+#  variables:
+#    GOOS: "linux"
+#    GOARCH: "arm64"
+#  script:
+#    - echo 'deb [trusted=yes] https://repo.goreleaser.com/apt/ /' | tee /etc/apt/sources.list.d/goreleaser.list
+#    - apt update && apt install nfpm=2.11.3
+#    - make package
+#  artifacts:
+#    paths:
+#      - "*.deb"
+#      - "*.rpm"
+#
+#build amd container fb:
+#  stage: build
+#  needs:
+#    - "build for linux amd64"
+#  tags:
+#    - shell
+#  rules:
+#    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
+#  before_script:    
+#    - echo "${DOCKER_DEPLOY_TOKEN}" | docker login -u ${DOCKER_DEPLOY_USER} --password-stdin ${CI_REGISTRY}
+#  script:    
+#    - tag=${CI_REGISTRY_IMAGE}/featurebase:linux-amd64-${CI_COMMIT_REF_SLUG}
+#    - docker build --build-arg GO_VERSION=$GOVERSION --build-arg ARCH=amd64 -t $tag -f .gitlab/Dockerfile .
+#    - docker push $tag
+#    - echo Created docker featurebase image with tag "$tag"
+#
+#build arm container fb:
+#  stage: build
+#  needs:
+#    - "build for linux arm64"
+#  tags:
+#    - shell
+#  rules:
+#    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
+#  before_script:    
+#    - echo "${DOCKER_DEPLOY_TOKEN}" | docker login -u ${DOCKER_DEPLOY_USER} --password-stdin ${CI_REGISTRY}
+#  script:    
+#    - tag=${CI_REGISTRY_IMAGE}/featurebase:linux-arm64-${CI_COMMIT_REF_SLUG}
+#    - docker build --build-arg GO_VERSION=$GOVERSION --build-arg ARCH=arm64 -t $tag -f .gitlab/Dockerfile .
+#    - docker push $tag
+#    - echo Created docker featurebase image with tag "$tag"
+#
+#
+## clustertests doesn't run in docker, and requires several things to be set up on the runner to work:
+## 1. Install Go, make sure it's on the path
+## 2. Make sure "make" is installed
+## 3. make sure docker/docker-compose is installed
+## 4. make sure the git config is done `git config --global --add url."ssh://git@github.com/".insteadOf "https://github.com/"`
+## 5. Add deploy key github.com/molecula/featurebase/settings/keys and add public key in .ssh folder of gitlab-runner user
+#clustertests:
+#  variables:
+#    PROJECT: clustertests_${CI_CONCURRENT_ID}
+#  stage: integration
+#  tags:
+#    - shell
+#  retry: 1
+#  rules:
+#    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
+#  script:
+#    - rm -rf  internal/clustertests/results && mkdir -p internal/clustertests/results && chown gitlab-runner:gitlab-runner  internal/clustertests/results
+#    - make clustertests
+#    - mv internal/clustertests/results/ results/
+#  artifacts:
+#    paths:
+#      - results/coverage*.out
+#
+#authclustertests:
+#  variables:
+#    PROJECT: authclustertests_${CI_CONCURRENT_ID}
+#  stage: integration
+#  retry: 1
+#  tags:
+#    - shell
+#  rules:
+#    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
+#  script:
+#    - rm -rf  internal/clustertests/results && mkdir -p internal/clustertests/results && chown gitlab-runner:gitlab-runner  internal/clustertests/results
+#    - make authclustertests
+#    - rm -rf internal/clustertests/results
+#
+#
+#external lookup tests:
+#  stage: integration
+#  image: golang:$GOVERSION
+#  # TODO: no rules here, do we need to add the rules line?
+#  variables:
+#    POSTGRES_DB: $POSTGRES_DB
+#    POSTGRES_USER: $POSTGRES_USER
+#    POSTGRES_PASSWORD: $POSTGRES_PASSWORD
+#    POSTGRES_HOST_AUTH_METHOD: trust
+#  services:
+#    - postgres:13.5
+#  script:
+#    - apt-get update --allow-releaseinfo-change -y
+#    - apt-get install -y postgresql-client
+#    - go test . -run "^TestExternalLookup" -externalLookupDSN postgresql://$POSTGRES_USER:$POSTGRES_PASSWORD@postgres/$POSTGRES_DB?sslmode=disable
+#
+#
+#smoke test:
+#  stage: integration
+#  image: registry.gitlab.com/gitlab-org/cloud-deploy/aws-base:latest
+#  variables:
+#    PROFILE: "service-terraform"
+#    AWS_SSH_PRIVATE_KEY: $AWS_FBCI_SSH_KEY
+#    AWS_ACCESS_KEY_ID: $AWS_FBCI_ACCESS_KEY_ID
+#    AWS_SECRET_ACCESS_KEY: $AWS_FBCI_SECRET_ACCESS_KEY
+#    TF_VAR_cluster_prefix: ""
+#  tags:
+#    - aws
+#    - docker
+#    - fbsmoke
+#  rules:
+#    - if: '$CI_PIPELINE_SOURCE == "push"'
+#  before_script:
+#    - apt-get update && apt-get install -y gnupg software-properties-common curl git
+#    - curl -fsSL https://apt.releases.hashicorp.com/gpg | apt-key add -
+#    - apt-add-repository "deb [arch=amd64] https://apt.releases.hashicorp.com $(lsb_release -cs) main"
+#    - apt-get update && apt-get install terraform
+#    - aws configure set aws_access_key_id $AWS_FBCI_ACCESS_KEY_ID --profile $PROFILE
+#    - aws configure set aws_secret_access_key $AWS_FBCI_SECRET_ACCESS_KEY --profile $PROFILE
+#    - aws configure set region "us-east-2" --profile $PROFILE
+#    - aws configure set aws_profile $PROFILE
+#    - echo $AWS_FBCI_SSH_KEY > gitlab-featurebase-ci.pem
+#    - chmod 400 gitlab-featurebase-ci.pem
+#    - 'which ssh-agent || ( apt-get update -y && apt-get install openssh-client -y )'
+#    - eval $(ssh-agent -s)
+#    - mkdir -p ~/.ssh
+#    - echo $AWS_FBCI_SSH_KEY > /root/.ssh/gitlab-featurebase-ci.pem
+#    - chmod 400 /root/.ssh/gitlab-featurebase-ci.pem
+#    - echo "$AWS_FBCI_SSH_KEY" | ssh-add -
+#    - chmod 700 /root/.ssh
+#    - '[[ -f /.dockerenv ]] && echo -e "Host *\n\tStrictHostKeyChecking no\n\n" > ~/.ssh/config'
+#    - apt update && apt -y install jq wget
+#    - wget -q https://go.dev/dl/go$GOVERSION.linux-amd64.tar.gz
+#    - tar -C /usr/local -xzf go$GOVERSION.linux-amd64.tar.gz
+#    - export PATH=$PATH:/usr/local/go/bin
+#    - TF_VAR_cluster_prefix="smoke-$(openssl rand -base64 12 | tr -d /=+ | cut -c -16)"
+#    - echo "Cluster Prefix --> $TF_VAR_cluster_prefix"
+#    # download datagen for FB-1270 repro test. TODO replace w/ locally built datagen once we merge IDK into FB
+#    - aws s3 cp s3://molecula-artifact-storage/idk/master/_latest/idk-linux-arm64/datagen ./datagen_linux_arm64
+#    - chmod +x ./datagen_linux_arm64
+#  script:
+#    - ./qa/scripts/setupSmokeTest.sh
+#    - ./qa/scripts/testSmokeTest.sh
+#    - ./qa/scripts/bug_repro_tests.sh
+#  after_script:
+#    - ./qa/scripts/teardownSmokeTest.sh
+#  needs:
+#    - job: build for linux arm64
+#  artifacts:
+#    when: always
+#    paths:
+#      - report.xml
+#    reports:
+#      junit: report.xml
+#
+#
+#samsung-gauntlet:
+#  stage: gauntlet
+#  timeout: 4h
+#  image: registry.gitlab.com/gitlab-org/cloud-deploy/aws-base:latest
+#  variables:
+#    FBCI_PROFILE: "service-terraform"
+#    INFRA_PROFILE: "service-gitlab"
+#    AWS_SSH_PRIVATE_KEY: $AWS_FBCI_SSH_KEY
+#    AWS_ACCESS_KEY_ID: $AWS_FBCI_ACCESS_KEY_ID
+#    AWS_SECRET_ACCESS_KEY: $AWS_FBCI_SECRET_ACCESS_KEY
+#    ASG_NAME: "gitlab-runners"
+#    TF_VAR_cluster_prefix: ""
+#  tags:
+#    - aws
+#    - docker
+#    - fbsmoke
+#  rules:
+#    - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && ($CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web")'
+#  before_script:
+#    - apt-get update && apt-get install -y gnupg software-properties-common curl git
+#    - curl -fsSL https://apt.releases.hashicorp.com/gpg | apt-key add -
+#    - apt-add-repository "deb [arch=amd64] https://apt.releases.hashicorp.com $(lsb_release -cs) main"
+#    - apt-get update && apt-get install terraform
+#    - aws configure set aws_access_key_id $AWS_FBCI_ACCESS_KEY_ID --profile $FBCI_PROFILE
+#    - aws configure set aws_secret_access_key $AWS_FBCI_SECRET_ACCESS_KEY --profile $FBCI_PROFILE
+#    - aws configure set region "us-east-2" --profile $FBCI_PROFILE
+#    - aws configure set aws_profile $FBCI_PROFILE
+#    - aws configure set aws_access_key_id $AWS_INFRA_ACCESS_KEY_ID --profile $INFRA_PROFILE
+#    - aws configure set aws_secret_access_key $AWS_INFRA_SECRET_ACCESS_KEY --profile $INFRA_PROFILE
+#    - aws configure set region "us-east-2" --profile $INFRA_PROFILE
+#    - echo $AWS_FBCI_SSH_KEY > gitlab-featurebase-ci.pem
+#    - chmod 400 gitlab-featurebase-ci.pem
+#    - 'which ssh-agent || ( apt-get update -y && apt-get install openssh-client -y )'
+#    - eval $(ssh-agent -s)
+#    - mkdir -p ~/.ssh
+#    - echo $AWS_FBCI_SSH_KEY > /root/.ssh/gitlab-featurebase-ci.pem
+#    - chmod 400 /root/.ssh/gitlab-featurebase-ci.pem
+#    - echo "$AWS_FBCI_SSH_KEY" | ssh-add -
+#    - chmod 700 /root/.ssh
+#    - '[[ -f /.dockerenv ]] && echo -e "Host *\n\tStrictHostKeyChecking no\n\n" > ~/.ssh/config'
+#    - apt update && apt -y install jq wget
+#    - wget -q https://go.dev/dl/go1.17.8.linux-amd64.tar.gz
+#    - tar -C /usr/local -xzf go1.17.8.linux-amd64.tar.gz
+#    - export PATH=$PATH:/usr/local/go/bin
+#    - TF_VAR_cluster_prefix="gauntlet-$(openssl rand -base64 12 | tr -d /=+ | cut -c -16)"
+#    - echo "Cluster Prefix --> $TF_VAR_cluster_prefix"
+#    - export INSTANCE_ID=$(curl --silent --fail "http://169.254.169.254/latest/meta-data/instance-id" | tee instance_id)
+#    - aws autoscaling set-instance-protection --instance-ids "$INSTANCE_ID" --auto-scaling-group-name $ASG_NAME --protected-from-scale-in --profile $INFRA_PROFILE
+#  script:
+#    - ./qa/scripts/setupSamsungGauntlet.sh
+#    - ./qa/scripts/testSamsungGauntlet.sh
+#  after_script:
+#    - ./qa/scripts/teardownSamsungGauntlet.sh || true # leaving dangling resources is better than dangling ASG instances that can't be terminated
+#    - export INSTANCE_ID=$(cat instance_id)
+#    - aws autoscaling set-instance-protection --instance-ids "$INSTANCE_ID" --auto-scaling-group-name $ASG_NAME --no-protected-from-scale-in --profile $INFRA_PROFILE
+#
+backup-restore-gauntlet:
   stage: gauntlet
   timeout: 4h
   image: registry.gitlab.com/gitlab-org/cloud-deploy/aws-base:latest
@@ -416,8 +472,6 @@ gauntlet:
     - aws
     - docker
     - fbsmoke
-  rules:
-    - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && ($CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web")'
   before_script:
     - apt-get update && apt-get install -y gnupg software-properties-common curl git
     - curl -fsSL https://apt.releases.hashicorp.com/gpg | apt-key add -
@@ -441,163 +495,165 @@ gauntlet:
     - chmod 700 /root/.ssh
     - '[[ -f /.dockerenv ]] && echo -e "Host *\n\tStrictHostKeyChecking no\n\n" > ~/.ssh/config'
     - apt update && apt -y install jq wget
-    - wget -q https://go.dev/dl/go1.17.5.linux-amd64.tar.gz
-    - tar -C /usr/local -xzf go1.17.5.linux-amd64.tar.gz
+    - wget -q https://go.dev/dl/go1.17.8.linux-amd64.tar.gz
+    - tar -C /usr/local -xzf go1.17.8.linux-amd64.tar.gz
     - export PATH=$PATH:/usr/local/go/bin
     - TF_VAR_cluster_prefix="gauntlet-$(openssl rand -base64 12 | tr -d /=+ | cut -c -16)"
     - echo "Cluster Prefix --> $TF_VAR_cluster_prefix"
     - export INSTANCE_ID=$(curl --silent --fail "http://169.254.169.254/latest/meta-data/instance-id" | tee instance_id)
     - aws autoscaling set-instance-protection --instance-ids "$INSTANCE_ID" --auto-scaling-group-name $ASG_NAME --protected-from-scale-in --profile $INFRA_PROFILE
   script:
-    - ./qa/scripts/setupSamsungGauntlet.sh
-    - ./qa/scripts/testSamsungGauntlet.sh
+    - ./qa/scripts/setupBackupRestoreGauntlet.sh
+    - ./qa/scripts/testBackupRestoreGauntlet.sh
   after_script:
-    - ./qa/scripts/teardownSamsungGauntlet.sh || true # leaving dangling resources is better than dangling ASG instances that can't be terminated
+    - ./qa/scripts/teardownBackupRestoreGauntlet.sh || true # leaving dangling resources is better than dangling ASG instances that can't be terminated
     - export INSTANCE_ID=$(cat instance_id)
     - aws autoscaling set-instance-protection --instance-ids "$INSTANCE_ID" --auto-scaling-group-name $ASG_NAME --no-protected-from-scale-in --profile $INFRA_PROFILE
+  needs:
+    - build for linux arm64
   
-s3 dump:
-  stage: post build
-  variables:
-    PROFILE: "service-fb-ci"
-    AWS_SSH_PRIVATE_KEY: $AWS_FBCI_SSH_KEY
-    AWS_ACCESS_KEY_ID: $AWS_FBCI_ACCESS_KEY_ID
-    AWS_SECRET_ACCESS_KEY: $AWS_FBCI_SECRET_ACCESS_KEY
-  tags:
-    - shell
-  rules:
-    - if: '$CI_COMMIT_TAG == null && ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "web")'
-  script:
-    - aws configure set aws_access_key_id $AWS_FBCI_ACCESS_KEY_ID
-    - aws configure set aws_secret_access_key $AWS_FBCI_SECRET_ACCESS_KEY
-    - aws configure set region "us-east-2"
-    - aws configure set aws_profile $PROFILE
-    - aws s3 cp featurebase_linux_amd64 s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/${CI_COMMIT_SHORT_SHA}/featurebase_linux_amd64
-    - aws s3 cp featurebase_linux_amd64 s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/_latest/featurebase_linux_amd64
-    - aws s3 cp roaring-migrate_linux_amd64 s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/${CI_COMMIT_SHORT_SHA}/roaring-migrate_linux_amd64
-    - aws s3 cp roaring-migrate_linux_amd64 s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/_latest/roaring-migrate_linux_amd64
-    - aws s3 cp featurebase_linux_arm64 s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/${CI_COMMIT_SHORT_SHA}/featurebase_linux_arm64
-    - aws s3 cp featurebase_linux_arm64 s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/_latest/featurebase_linux_arm64
-    - aws s3 cp roaring-migrate_linux_arm64 s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/${CI_COMMIT_SHORT_SHA}/roaring-migrate_linux_arm64
-    - aws s3 cp roaring-migrate_linux_arm64 s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/_latest/roaring-migrate_linux_arm64
-    - aws s3 cp featurebase_darwin_amd64 s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/${CI_COMMIT_SHORT_SHA}/featurebase_darwin_amd64
-    - aws s3 cp featurebase_darwin_amd64 s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/_latest/featurebase_darwin_amd64
-    - aws s3 cp roaring-migrate_darwin_amd64 s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/${CI_COMMIT_SHORT_SHA}/roaring-migrate_darwin_amd64
-    - aws s3 cp roaring-migrate_darwin_amd64 s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/_latest/roaring-migrate_darwin_amd64
-    - aws s3 cp featurebase_darwin_arm64 s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/${CI_COMMIT_SHORT_SHA}/featurebase_darwin_arm64
-    - aws s3 cp featurebase_darwin_arm64 s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/_latest/featurebase_darwin_arm64
-    - aws s3 cp roaring-migrate_darwin_arm64 s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/${CI_COMMIT_SHORT_SHA}/roaring-migrate_darwin_arm64
-    - aws s3 cp roaring-migrate_darwin_arm64 s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/_latest/roaring-migrate_darwin_arm64
-  needs:
-    - job: build for darwin amd64
-    - job: build for darwin arm64
-    - job: build for linux amd64
-    - job: build for linux arm64
-
-perf_able:
-  stage: performance
-  rules:
-    - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $CI_PIPELINE_SOURCE == "push"'
-  trigger:
-    include: .gitlab/.perf-able-gitlab-ci.yml
-  variables:
-    PARENT_PIPELINE_ID: $CI_PIPELINE_ID
-
-# only run on merge to master, the whole process takes about 10 minutes in total
-# (which is a long time)
-perf_delete:
-  stage: performance
-  rules:
-    - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $CI_PIPELINE_SOURCE == "push"'
-  timeout: 2h
-  image: registry.gitlab.com/gitlab-org/cloud-deploy/aws-base:latest
-  variables:
-    PROFILE: "service-terraform"
-    INFRA_PROFILE: "service-gitlab"
-    AWS_SSH_PRIVATE_KEY: $AWS_FBCI_SSH_KEY
-    AWS_ACCESS_KEY_ID: $AWS_FBCI_ACCESS_KEY_ID
-    AWS_SECRET_ACCESS_KEY: $AWS_FBCI_SECRET_ACCESS_KEY
-    ASG_NAME: "gitlab-runners"
-    TF_VAR_cluster_prefix: ""
-  tags:
-    - aws
-    - docker
-    - fbsmoke
-  before_script:
-    - apt-get update && apt-get install -y gnupg software-properties-common curl git
-    - curl -fsSL https://apt.releases.hashicorp.com/gpg | apt-key add -
-    - apt-add-repository "deb [arch=amd64] https://apt.releases.hashicorp.com $(lsb_release -cs) main"
-    - apt-get update && apt-get install terraform 
-    - apt-get update && apt-get install python3.7
-    - aws configure set aws_access_key_id $AWS_FBCI_ACCESS_KEY_ID --profile $PROFILE
-    - aws configure set aws_secret_access_key $AWS_FBCI_SECRET_ACCESS_KEY --profile $PROFILE
-    - aws configure set region "us-east-2" --profile $PROFILE
-    - aws configure set aws_profile $PROFILE
-    - aws configure set aws_access_key_id $AWS_INFRA_ACCESS_KEY_ID --profile $INFRA_PROFILE
-    - aws configure set aws_secret_access_key $AWS_INFRA_SECRET_ACCESS_KEY --profile $INFRA_PROFILE
-    - aws configure set region "us-east-2" --profile $INFRA_PROFILE
-    - echo $AWS_FBCI_SSH_KEY > gitlab-featurebase-ci.pem
-    - chmod 400 gitlab-featurebase-ci.pem
-    - 'which ssh-agent || ( apt-get update -y && apt-get install openssh-client -y )'
-    - eval $(ssh-agent -s)
-    - mkdir -p ~/.ssh
-    - echo $AWS_FBCI_SSH_KEY > /root/.ssh/gitlab-featurebase-ci.pem
-    - chmod 400 /root/.ssh/gitlab-featurebase-ci.pem
-    - echo "$AWS_FBCI_SSH_KEY" | ssh-add -
-    - chmod 700 /root/.ssh
-    - '[[ -f /.dockerenv ]] && echo -e "Host *\n\tStrictHostKeyChecking no\n\n" > ~/.ssh/config'
-    - apt update && apt -y install jq wget
-    - wget -q https://go.dev/dl/go1.17.5.linux-amd64.tar.gz
-    - tar -C /usr/local -xzf go1.17.5.linux-amd64.tar.gz
-    - export PATH=$PATH:/usr/local/go/bin
-    - TF_VAR_cluster_prefix="delete-$(openssl rand -base64 12 | tr -d /=+ | cut -c -16)"
-    - echo "Cluster Prefix --> $TF_VAR_cluster_prefix"
-    - export INSTANCE_ID=$(curl --silent --fail "http://169.254.169.254/latest/meta-data/instance-id" | tee instance_id)
-    - aws autoscaling set-instance-protection --instance-ids "$INSTANCE_ID" --auto-scaling-group-name $ASG_NAME --protected-from-scale-in --profile $INFRA_PROFILE
-  script:
-    - ./qa/scripts/perf/delete/deleteSetup.sh
-    - ./qa/scripts/perf/delete/deleteTest.sh
-  after_script:
-    - ./qa/scripts/perf/delete/deleteTeardown.sh || true
-    - export INSTANCE_ID=$(cat instance_id)
-    - aws autoscaling set-instance-protection --instance-ids "$INSTANCE_ID" --auto-scaling-group-name $ASG_NAME --no-protected-from-scale-in --profile $INFRA_PROFILE
-  needs:
-    - job: build for linux arm64
-
-s3 dump tag:
-  stage: post build
-  variables:
-    PROFILE: "service-fb-ci"
-    AWS_SSH_PRIVATE_KEY: $AWS_FBCI_SSH_KEY
-    AWS_ACCESS_KEY_ID: $AWS_FBCI_ACCESS_KEY_ID
-    AWS_SECRET_ACCESS_KEY: $AWS_FBCI_SECRET_ACCESS_KEY
-    LOCATION: molecula-artifact-storage/featurebase/_tags
-  tags:
-    - shell
-  rules:
-    - if: '$CI_COMMIT_TAG != null && ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "web")'
-  script:
-    - aws configure set aws_access_key_id $AWS_FBCI_ACCESS_KEY_ID
-    - aws configure set aws_secret_access_key $AWS_FBCI_SECRET_ACCESS_KEY
-    - aws configure set region "us-east-2"
-    - aws configure set aws_profile $PROFILE
-    - |
-      for goos in "darwin" "linux"; do
-        for goarch in "amd64" "arm64"; do
-          dir=featurebase-${CI_COMMIT_TAG}-${goos}-${goarch}
-          echo "Directory ${dir}"
-          mkdir $dir
-          mv featurebase_${goos}_${goarch} ${dir}/featurebase
-          mv roaring-migrate_${goos}_${goarch} ${dir}/roaring-migrate
-          cp NOTICE install/featurebase.conf install/featurebase.*.service ${dir}/
-          tar cvzf ${dir}.tar.gz ${dir}
-          aws s3 cp ${dir} s3://${LOCATION}/${CI_COMMIT_TAG}/${dir}/ --recursive
-          aws s3 cp ${dir}.tar.gz s3://${LOCATION}/${CI_COMMIT_TAG}/
-        done
-      done
-
-  needs:
-    - job: build for darwin amd64
-    - job: build for darwin arm64
-    - job: build for linux amd64
-    - job: build for linux arm64
+#s3 dump:
+#  stage: post build
+#  variables:
+#    PROFILE: "service-fb-ci"
+#    AWS_SSH_PRIVATE_KEY: $AWS_FBCI_SSH_KEY
+#    AWS_ACCESS_KEY_ID: $AWS_FBCI_ACCESS_KEY_ID
+#    AWS_SECRET_ACCESS_KEY: $AWS_FBCI_SECRET_ACCESS_KEY
+#  tags:
+#    - shell
+#  rules:
+#    - if: '$CI_COMMIT_TAG == null && ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "web")'
+#  script:
+#    - aws configure set aws_access_key_id $AWS_FBCI_ACCESS_KEY_ID
+#    - aws configure set aws_secret_access_key $AWS_FBCI_SECRET_ACCESS_KEY
+#    - aws configure set region "us-east-2"
+#    - aws configure set aws_profile $PROFILE
+#    - aws s3 cp featurebase_linux_amd64 s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/${CI_COMMIT_SHORT_SHA}/featurebase_linux_amd64
+#    - aws s3 cp featurebase_linux_amd64 s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/_latest/featurebase_linux_amd64
+#    - aws s3 cp roaring-migrate_linux_amd64 s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/${CI_COMMIT_SHORT_SHA}/roaring-migrate_linux_amd64
+#    - aws s3 cp roaring-migrate_linux_amd64 s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/_latest/roaring-migrate_linux_amd64
+#    - aws s3 cp featurebase_linux_arm64 s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/${CI_COMMIT_SHORT_SHA}/featurebase_linux_arm64
+#    - aws s3 cp featurebase_linux_arm64 s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/_latest/featurebase_linux_arm64
+#    - aws s3 cp roaring-migrate_linux_arm64 s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/${CI_COMMIT_SHORT_SHA}/roaring-migrate_linux_arm64
+#    - aws s3 cp roaring-migrate_linux_arm64 s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/_latest/roaring-migrate_linux_arm64
+#    - aws s3 cp featurebase_darwin_amd64 s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/${CI_COMMIT_SHORT_SHA}/featurebase_darwin_amd64
+#    - aws s3 cp featurebase_darwin_amd64 s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/_latest/featurebase_darwin_amd64
+#    - aws s3 cp roaring-migrate_darwin_amd64 s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/${CI_COMMIT_SHORT_SHA}/roaring-migrate_darwin_amd64
+#    - aws s3 cp roaring-migrate_darwin_amd64 s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/_latest/roaring-migrate_darwin_amd64
+#    - aws s3 cp featurebase_darwin_arm64 s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/${CI_COMMIT_SHORT_SHA}/featurebase_darwin_arm64
+#    - aws s3 cp featurebase_darwin_arm64 s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/_latest/featurebase_darwin_arm64
+#    - aws s3 cp roaring-migrate_darwin_arm64 s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/${CI_COMMIT_SHORT_SHA}/roaring-migrate_darwin_arm64
+#    - aws s3 cp roaring-migrate_darwin_arm64 s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/_latest/roaring-migrate_darwin_arm64
+#  needs:
+#    - job: build for darwin amd64
+#    - job: build for darwin arm64
+#    - job: build for linux amd64
+#    - job: build for linux arm64
+#
+#perf_able:
+#  stage: performance
+#  rules:
+#    - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $CI_PIPELINE_SOURCE == "push"'
+#  trigger:
+#    include: .gitlab/.perf-able-gitlab-ci.yml
+#  variables:
+#    PARENT_PIPELINE_ID: $CI_PIPELINE_ID
+#
+## only run on merge to master, the whole process takes about 10 minutes in total
+## (which is a long time)
+#perf_delete:
+#  stage: performance
+#  rules:
+#    - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $CI_PIPELINE_SOURCE == "push"'
+#  timeout: 2h
+#  image: registry.gitlab.com/gitlab-org/cloud-deploy/aws-base:latest
+#  variables:
+#    PROFILE: "service-terraform"
+#    INFRA_PROFILE: "service-gitlab"
+#    AWS_SSH_PRIVATE_KEY: $AWS_FBCI_SSH_KEY
+#    AWS_ACCESS_KEY_ID: $AWS_FBCI_ACCESS_KEY_ID
+#    AWS_SECRET_ACCESS_KEY: $AWS_FBCI_SECRET_ACCESS_KEY
+#    ASG_NAME: "gitlab-runners"
+#    TF_VAR_cluster_prefix: ""
+#  tags:
+#    - aws
+#    - docker
+#    - fbsmoke
+#  before_script:
+#    - apt-get update && apt-get install -y gnupg software-properties-common curl git
+#    - curl -fsSL https://apt.releases.hashicorp.com/gpg | apt-key add -
+#    - apt-add-repository "deb [arch=amd64] https://apt.releases.hashicorp.com $(lsb_release -cs) main"
+#    - apt-get update && apt-get install terraform 
+#    - apt-get update && apt-get install python3.7
+#    - aws configure set aws_access_key_id $AWS_FBCI_ACCESS_KEY_ID --profile $PROFILE
+#    - aws configure set aws_secret_access_key $AWS_FBCI_SECRET_ACCESS_KEY --profile $PROFILE
+#    - aws configure set region "us-east-2" --profile $PROFILE
+#    - aws configure set aws_profile $PROFILE
+#    - aws configure set aws_access_key_id $AWS_INFRA_ACCESS_KEY_ID --profile $INFRA_PROFILE
+#    - aws configure set aws_secret_access_key $AWS_INFRA_SECRET_ACCESS_KEY --profile $INFRA_PROFILE
+#    - aws configure set region "us-east-2" --profile $INFRA_PROFILE
+#    - echo $AWS_FBCI_SSH_KEY > gitlab-featurebase-ci.pem
+#    - chmod 400 gitlab-featurebase-ci.pem
+#    - 'which ssh-agent || ( apt-get update -y && apt-get install openssh-client -y )'
+#    - eval $(ssh-agent -s)
+#    - mkdir -p ~/.ssh
+#    - echo $AWS_FBCI_SSH_KEY > /root/.ssh/gitlab-featurebase-ci.pem
+#    - chmod 400 /root/.ssh/gitlab-featurebase-ci.pem
+#    - echo "$AWS_FBCI_SSH_KEY" | ssh-add -
+#    - chmod 700 /root/.ssh
+#    - '[[ -f /.dockerenv ]] && echo -e "Host *\n\tStrictHostKeyChecking no\n\n" > ~/.ssh/config'
+#    - apt update && apt -y install jq wget
+#    - wget -q https://go.dev/dl/go1.17.8.linux-amd64.tar.gz
+#    - tar -C /usr/local -xzf go1.17.8.linux-amd64.tar.gz
+#    - export PATH=$PATH:/usr/local/go/bin
+#    - TF_VAR_cluster_prefix="delete-$(openssl rand -base64 12 | tr -d /=+ | cut -c -16)"
+#    - echo "Cluster Prefix --> $TF_VAR_cluster_prefix"
+#    - export INSTANCE_ID=$(curl --silent --fail "http://169.254.169.254/latest/meta-data/instance-id" | tee instance_id)
+#    - aws autoscaling set-instance-protection --instance-ids "$INSTANCE_ID" --auto-scaling-group-name $ASG_NAME --protected-from-scale-in --profile $INFRA_PROFILE
+#  script:
+#    - ./qa/scripts/perf/delete/deleteSetup.sh
+#    - ./qa/scripts/perf/delete/deleteTest.sh
+#  after_script:
+#    - ./qa/scripts/perf/delete/deleteTeardown.sh || true
+#    - export INSTANCE_ID=$(cat instance_id)
+#    - aws autoscaling set-instance-protection --instance-ids "$INSTANCE_ID" --auto-scaling-group-name $ASG_NAME --no-protected-from-scale-in --profile $INFRA_PROFILE
+#  needs:
+#    - job: build for linux arm64
+#
+#s3 dump tag:
+#  stage: post build
+#  variables:
+#    PROFILE: "service-fb-ci"
+#    AWS_SSH_PRIVATE_KEY: $AWS_FBCI_SSH_KEY
+#    AWS_ACCESS_KEY_ID: $AWS_FBCI_ACCESS_KEY_ID
+#    AWS_SECRET_ACCESS_KEY: $AWS_FBCI_SECRET_ACCESS_KEY
+#    LOCATION: molecula-artifact-storage/featurebase/_tags
+#  tags:
+#    - shell
+#  rules:
+#    - if: '$CI_COMMIT_TAG != null && ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "web")'
+#  script:
+#    - aws configure set aws_access_key_id $AWS_FBCI_ACCESS_KEY_ID
+#    - aws configure set aws_secret_access_key $AWS_FBCI_SECRET_ACCESS_KEY
+#    - aws configure set region "us-east-2"
+#    - aws configure set aws_profile $PROFILE
+#    - |
+#      for goos in "darwin" "linux"; do
+#        for goarch in "amd64" "arm64"; do
+#          dir=featurebase-${CI_COMMIT_TAG}-${goos}-${goarch}
+#          echo "Directory ${dir}"
+#          mkdir $dir
+#          mv featurebase_${goos}_${goarch} ${dir}/featurebase
+#          mv roaring-migrate_${goos}_${goarch} ${dir}/roaring-migrate
+#          cp NOTICE install/featurebase.conf install/featurebase.*.service ${dir}/
+#          tar cvzf ${dir}.tar.gz ${dir}
+#          aws s3 cp ${dir} s3://${LOCATION}/${CI_COMMIT_TAG}/${dir}/ --recursive
+#          aws s3 cp ${dir}.tar.gz s3://${LOCATION}/${CI_COMMIT_TAG}/
+#        done
+#      done
+#
+#  needs:
+#    - job: build for darwin amd64
+#    - job: build for darwin arm64
+#    - job: build for linux amd64
+#    - job: build for linux arm64
diff --git a/qa/scripts/backupRestoreTest.sh b/qa/scripts/backupRestoreTest.sh
new file mode 100755
index 000000000..794898e41
--- /dev/null
+++ b/qa/scripts/backupRestoreTest.sh
@@ -0,0 +1,112 @@
+#!/bin/bash
+
+HOSTS=($@)
+/data/datagen --source custom --custom-config /data/backup_test_datagen.yaml --pilosa.index=backup_test --pilosa.hosts=$HOSTS --pilosa.batch-size=1000
+# make sure backup doesn't fail 
+if (( $? != 0 )); then
+    echo "datagen failed"
+    exit 1
+fi
+
+# kill a node somehow 
+echo "hosts we've got"
+for host in ${HOSTS[@]}; do
+    echo $host;
+done
+
+KILLNODE=${HOSTS[1]}
+echo "getting checksum from $KILLNODE"
+firstCheckSum=$(featurebase chksum --host $KILLNODE)
+echo "first check sum: $firstCheckSum"
+if (( $? != 0 ))
+then 
+    echo "getting checksum from $KILLNODE failed"
+    exit 1
+fi
+
+echo "pausing node $KILLNODE"
+ssh -A -o "StrictHostKeyChecking no" ec2-user@${KILLNODE} "sudo systemctl stop featurebase" 
+if (( $? != 0 ))
+then 
+    echo "pausing node $KILLNODE failed"
+    exit 1
+fi
+
+# backup from ingest node
+featurebase backup --host=${HOSTS[2]} --retry-period=0s --output=backupWOOO
+# make sure backup doesn't fail 
+if (( $? != 0 )); then
+    echo "backup failed!!!!!!!!!!"
+    exit 1
+fi
+
+echo "stopping all featurebase nodes"
+# kill all featurebase nodes and wipe the data directory
+for host in ${HOSTS[@]}; do
+    echo "stopping featurebase on ${host}"
+    ssh -A -o "StrictHostKeyChecking no" ec2-user@${host} "sudo systemctl stop featurebase && sudo find /data/featurebase/ -mindepth 1 -delete" 
+    if (( $? != 0 )); then 
+        echo "emptying and stopping node failed"
+        exit 1
+    fi
+done
+
+echo "starting all featurebase nodes"
+# start all featurebase nodes - systemd
+for host in ${HOSTS[@]}; do
+    echo "starting featurebase on ${host}"
+    ssh -A -o "StrictHostKeyChecking no" ec2-user@${host} "sudo systemctl restart featurebase" 
+    if (( $? != 0 )); then 
+        echo "restarting featurebase failed"
+        exit 1
+    fi
+done
+
+echo "waiting for featurebase to start"
+# wait until we can connect to one of the hosts
+for i in {0..24}; do 
+    echo "checking ${HOSTS[1]}:10101/status"
+    curl -v ${HOSTS[1]}:10101/status
+    S=$(curl -s ${HOSTS[1]}:10101/status | jq -r ".state"); 
+    if [[ $S == "NORMAL" ]]; then 
+        echo "hosts up after $i tries"; 
+        break;
+    fi
+    echo "attempt $i resulted in $S"
+    sleep 5
+done
+
+
+if [[ $S != "NORMAL" ]]; then
+    echo "couldn't connect, featurebase never stable"
+    exit 1
+fi
+
+echo "restoring featurebase"
+# featurebase restore 
+featurebase restore -s=backupWOOO --host=${HOSTS[1]}
+
+# make sure it doesn't fail
+if (( $? != 0 )); then
+    echo "restore failed!!!!"
+    exit 1
+fi
+
+# make sure it's the same data
+echo "getting checksum from $KILLNODE"
+secondChkSum=$(featurebase chksum --host $KILLNODE)
+echo "second checkSum $secondChkSum"
+if (( $? != 0 )); then 
+    echo "getting checksum from $KILLNODE failed"
+    exit 1
+fi
+
+if [[ $firstCheckSum == $secondChkSum ]]; then 
+    echo "checksums match"
+    exit 0
+else
+    echo "first check sum $firstCheckSum"
+    echo "second check sum $secondCheckSum"
+    echo "check sums don't match"
+    exit 1
+fi 
diff --git a/qa/scripts/backup_test_datagen.yaml b/qa/scripts/backup_test_datagen.yaml
new file mode 100644
index 000000000..c9b8292c5
--- /dev/null
+++ b/qa/scripts/backup_test_datagen.yaml
@@ -0,0 +1,61 @@
+fields:
+  - name: "a_random_string"
+    type: "string" # (default StringField (non-mutex))
+    generator_type: "random-string" # used to generate random strings rather than pulling from known set
+    min_len: 8
+    max_len: 12
+    charset: "AB" # set of possible characters to pull from when generating random string
+  - name: "id"
+    type: "uint"
+    distribution: "sequential"
+    min: 0
+    max: 1000 # 2%24
+    step: 1
+    repeat: false
+  - name: "type"
+    type: "int" # (default IntField)
+    min: 0
+    max: 3
+    distribution: "zipfian"
+    s: 1.1
+    v: 5.1
+  - name: "ts"
+    type: "timestamp"
+    min_date: 2006-01-02T15:04:05.001Z # RFC3339Nano
+    max_date: 2007-01-02T15:04:05.001Z # RFC3339Nano
+    distribution: "increasing" # only "increasing" is supported right now
+    min_step_duration: "10ns"
+    max_step_duration: "200ms" 
+  - name: "slice"
+    type: "uint-set" # (default IDArrayField)
+    min: 0
+    max: 35000
+    distribution: "zipfian"
+    s: 1.1
+    v: 5.1
+    min_num: 1
+    max_num: 50
+
+# idk_params describe how data from "fields" should be ingested by IDK
+idk_params:
+  primary_key_config:
+    field: "a_random_string" # if this is a single field named "id" then we'll use uint IDs, if it's empty we'll autogen ids, and if it's anything else we'll do string keys... yes this is a bit hacky, needs to be cleaned up.
+  # fields is keyed by names of fields from top level "fields". It is
+  # not required that all fields appear here, those that don't will
+  # use the default ingestion.
+  fields:
+    id:
+      - type: "ID"
+    type:
+      - type: "ID"
+    a_decimal_field:
+      - type: "Decimal"
+        scale: 4
+    ts:
+      - type: "RecordTime"
+        layout: "2006-01-02T15:04:05Z"
+        epoch: 1970-01-01T00:00:00.0Z
+        name: "na"
+    slice:
+      - type: "IDArray"
+        time_quantum: "YMD"
diff --git a/qa/scripts/perf/delete/deleteSetup.sh b/qa/scripts/perf/delete/deleteSetup.sh
index 6cadbcd6f..f3194217b 100755
--- a/qa/scripts/perf/delete/deleteSetup.sh
+++ b/qa/scripts/perf/delete/deleteSetup.sh
@@ -50,7 +50,7 @@ echo "}"
 
 DEPLOYED_INGEST_IPS_LEN=`echo "$DEPLOYED_INGEST_IPS" | wc -l`
 
-#wait until we can connect to one of the hosts
+# wait until we can connect to one of the hosts
 for i in {0..24}
 do 
     ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no -o ConnectTimeout=10 ec2-user@${DATANODE0} "pwd"
diff --git a/qa/scripts/setupBackupRestoreGauntlet.sh b/qa/scripts/setupBackupRestoreGauntlet.sh
new file mode 100755
index 000000000..2fa611ff9
--- /dev/null
+++ b/qa/scripts/setupBackupRestoreGauntlet.sh
@@ -0,0 +1,79 @@
+#!/bin/bash
+
+# To run script: ./setupBackupRestoreGauntlet.sh
+export TF_IN_AUTOMATION=1
+
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+source $SCRIPT_DIR/utilCluster.sh
+
+pushd ./qa/tf/gauntlet/backuprestore
+echo "Running terraform init..."
+terraform init -input=false
+echo "Running terraform apply..."
+terraform apply -input=false -auto-approve
+terraform output -json > outputs.json
+popd
+
+# get the first ingest host
+INGESTNODE0=$(cat ./qa/tf/gauntlet/backuprestore/outputs.json | jq -r '[.ingest_ips][0]["value"][0]')
+echo "using INGESTNODE0 ${INGESTNODE0}"
+
+# get the first data host
+DATANODE0=$(cat ./qa/tf/gauntlet/backuprestore/outputs.json | jq -r '[.data_node_ips][0]["value"][0]')
+echo "using DATANODE0 ${DATANODE0}"
+
+
+DEPLOYED_CLUSTER_PREFIX=$(cat ./qa/tf/gauntlet/backuprestore/outputs.json | jq -r '[.cluster_prefix][0]["value"]')
+echo "Using DEPLOYED_CLUSTER_PREFIX: ${DEPLOYED_CLUSTER_PREFIX}"
+
+DEPLOYED_CLUSTER_REPLICA_COUNT=$(cat ./qa/tf/gauntlet/backuprestore/outputs.json | jq -r '[.fb_cluster_replica_count][0]["value"]')
+echo "Using DEPLOYED_CLUSTER_REPLICA_COUNT: ${DEPLOYED_CLUSTER_REPLICA_COUNT}"
+
+DEPLOYED_DATA_IPS=$(cat ./qa/tf/gauntlet/backuprestore/outputs.json | jq -r '[.data_node_ips][0]["value"][]')
+echo "DEPLOYED_DATA_IPS: {"
+echo "${DEPLOYED_DATA_IPS}"
+echo "}"
+
+DEPLOYED_DATA_IPS_LEN=`echo "$DEPLOYED_DATA_IPS" | wc -l`
+
+DEPLOYED_INGEST_IPS=$(cat ./qa/tf/gauntlet/backuprestore/outputs.json | jq -r '[.ingest_ips][0]["value"][]')
+echo "DEPLOYED_INGEST_IPS: {"
+echo "${DEPLOYED_INGEST_IPS}"
+echo "}"
+
+DEPLOYED_INGEST_IPS_LEN=`echo "$DEPLOYED_INGEST_IPS" | wc -l`
+
+#wait until we can connect to one of the hosts
+for i in {0..24}
+do 
+    ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no -o ConnectTimeout=10 ec2-user@${DATANODE0} "pwd"
+    if [ $? -eq 0 ]
+    then
+        echo "Cluster is up after ${i} tries." 
+        break
+    fi
+    sleep 10
+done
+
+ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no -o ConnectTimeout=10 ec2-user@${DATANODE0} "pwd"
+if [ $? -ne 0 ]
+then
+    echo "Unable to connect to cluster - giving up" 
+    exit 1
+fi
+
+setupClusterNodes
+
+# verify featurebase running
+echo "Verifying featurebase cluster running..."
+curl -s http://${DATANODE0}:10101/status
+if (( $? != 0 )) 
+then 
+    echo "Featurebase cluster not running"
+    exit 1
+fi
+
+echo "Cluster running."
+
+
+
diff --git a/qa/scripts/teardownBackupRestoreGauntlet.sh b/qa/scripts/teardownBackupRestoreGauntlet.sh
new file mode 100755
index 000000000..0baa5c74b
--- /dev/null
+++ b/qa/scripts/teardownBackupRestoreGauntlet.sh
@@ -0,0 +1,7 @@
+#!/bin/bash
+
+# To run script: ./teardownBackupRestoreGauntlet.sh
+
+cd qa/tf/gauntlet/backuprestore
+export TF_IN_AUTOMATION=1
+terraform destroy -auto-approve
diff --git a/qa/scripts/testBackupRestoreGauntlet.sh b/qa/scripts/testBackupRestoreGauntlet.sh
new file mode 100755
index 000000000..a814901c9
--- /dev/null
+++ b/qa/scripts/testBackupRestoreGauntlet.sh
@@ -0,0 +1,51 @@
+#!/bin/bash
+
+source ./qa/scripts/utilCluster.sh
+
+# get the first ingest host
+INGESTNODE0=$(cat ./qa/tf/gauntlet/backuprestore/outputs.json | jq -r '[.ingest_ips][0]["value"][0]')
+echo "using INGESTNODE0 ${INGESTNODE0}"
+
+# get the first data host
+DATANODE0=$(cat ./qa/tf/gauntlet/backuprestore/outputs.json | jq -r '[.data_node_ips][0]["value"][0]')
+echo "using DATANODE0 ${DATANODE0}"
+
+# get the data node host ips
+HOSTS=($( cat ./qa/tf/gauntlet/backuprestore/outputs.json | jq -r '.data_node_ips.value' | tr -d '[],"'))
+
+echo "using hosts:"
+for host in ${HOSTS[@]}; do
+    echo $host;
+done
+
+installDatagen $INGESTNODE0
+
+# copy the tests over to ingest node
+scp -r -i ~/.ssh/gitlab-featurebase-ci.pem ./qa/scripts/backupRestoreTest.sh ec2-user@${INGESTNODE0}:/data
+if (( $? != 0 )) 
+then 
+    echo "test copy failed"
+    exit 1
+fi
+
+# copy the datagen over to ingest node
+scp -r -i ~/.ssh/gitlab-featurebase-ci.pem ./qa/scripts/backup_test_datagen.yaml ec2-user@${INGESTNODE0}:/data
+if (( $? != 0 )) 
+then 
+    echo "test copy failed"
+    exit 1
+fi
+
+
+echo "running backup test with hosts: ${HOSTS[@]}"
+ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o "StrictHostKeyChecking no" ec2-user@${INGESTNODE0} "cd /data; ./backupRestoreTest.sh ${HOSTS[@]}"
+TESTRESULT=$?
+
+if (( $TESTRESULT != 0 )) 
+then 
+    echo "backup test failed"
+else
+    echo "backup test complete"
+fi
+
+exit $TESTRESULT 
diff --git a/qa/scripts/utilCluster.sh b/qa/scripts/utilCluster.sh
index e083e082e..625bace9a 100644
--- a/qa/scripts/utilCluster.sh
+++ b/qa/scripts/utilCluster.sh
@@ -230,3 +230,55 @@ setupClusterNodes() {
     setupIngestNodes
 
 }
+
+installDatagen() {
+    INGESTNODE0=$1
+    # download datagen
+    aws s3 cp s3://molecula-artifact-storage/idk/master/_latest/idk-linux-arm64/datagen datagen
+    if (( $? != 0 ))
+    then
+        echo "datagen binary copy failed"
+        exit 1
+    fi
+
+    # make it executable
+    chmod +x datagen 
+    if (( $? != 0 ))
+    then
+        echo "couldn't make datagen executable"
+        exit 1
+    fi
+
+    # copy it over to the ingest node
+    scp -r -i ~/.ssh/gitlab-featurebase-ci.pem ./datagen ec2-user@${INGESTNODE0}:/data
+    if (( $? != 0 )) 
+    then 
+        echo "datagen copy failed"
+        exit 1
+    fi
+
+    # setup the yum repo needed for librdkafka onto the ingest node
+    scp -r -i ~/.ssh/gitlab-featurebase-ci.pem ./qa/scripts/perf/delete/confluent ec2-user@${INGESTNODE0}:/data
+    if (( $? != 0 )) 
+    then 
+        echo "confluent repo setup copy failed"
+        exit 1
+    fi
+
+    echo "setting up confluent repo"
+    ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o "StrictHostKeyChecking no" ec2-user@${INGESTNODE0} "sudo mv /data/confluent /etc/yum.repos.d" 
+    if (( $? != 0 ))
+    then 
+        echo "setting up confluent repo failed"
+        exit 1
+    fi
+
+
+    echo "installing librdkafka on ingest node"
+    ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o "StrictHostKeyChecking no" ec2-user@${INGESTNODE0} "sudo rpm --import http://packages.confluent.io/rpm/3.1/archive.key && sudo yum clean all && sudo yum install librdkafka-devel -y"
+    if (( $? != 0 ))
+    then 
+        echo "librdkafka install failed"
+        exit 1
+    fi
+}
diff --git a/qa/tf/gauntlet/backuprestore/README.md b/qa/tf/gauntlet/backuprestore/README.md
new file mode 100644
index 000000000..e7b633516
--- /dev/null
+++ b/qa/tf/gauntlet/backuprestore/README.md
@@ -0,0 +1,35 @@
+With terraform installed (`brew install terraform` if not)...
+
+You can do `terraform plan` -> `terraform apply` to spin up a cluster, `terraform destroy` to tear one down.
+
+## Other prerequisites:
+Please read these carefully.
+
+Be in the `tf` directory (e.g., when you try to run a `terraform` command, the output of `pwd` should be `.../featurebase/qa/tf`)
+
+Currently, the path to the terraform module is using a local reference, i.e., in `main.tf`, the source line is assuming that you have `molecula-terraform` project installed locally, such that the `molecular-terraform` project and `featurebase` have the same parent directory (e.g., `...A/featurebase/qa/tf` and `...A/molecular-terraform/aws/.modules/featurebase-cluster` should both be valid paths).
+
+In addition, you must currently have a local copy of the `fb901` branch for the `molecular-terraform` project (located in the previously specified directory).
+
+Last thing, there is a key that is currently in 1Password (in the `Shared` vault, called `gitlab-featurebase-ci AWS key`) that must be in `~/.ssh/`, `chmod 400`, named `gitlab-featurebase-ci.pem`. You need this key to SSH to these instances. Assuming an `~/.ssh/config` like the following (append to the top of yours)
+```
+Host test_*
+  User ec2-user
+  IdentityFile ~/.ssh/gitlab-featurebase-ci.pem
+Host test_ingest
+  HostName 3.143.237.165
+Host test_node
+  HostName 10.0.1.142
+  ProxyJump test_ingest
+```
+except with the `test_ingest`'s `HostName` being the public, `ingest_ips` output from `terraform output` and `test_node`'s `HostName` being one of the private, `data_node_ips` output from `terraform output`. (Hopefully the rationale to use the ssh config to do the jumping like this makes sense; you can do `ssh test_ingest` or `ssh test_node` with minimal further fiddling.)
+
+OR specify cert to us directly thus:
+
+`ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem ec2-user@ip_address`
+
+-A is used to ensure key forwarding.
+
+### TODOs
+* We need a `user-data.sh` script which sets up/installs featurebase (possibly installs go, most likely pulls the artifacts from GitLab; sets up featurebase on both the node and data workers).
+* Logs get sent to DataDog?
diff --git a/qa/tf/gauntlet/backuprestore/main.tf b/qa/tf/gauntlet/backuprestore/main.tf
new file mode 100644
index 000000000..87a68d0f0
--- /dev/null
+++ b/qa/tf/gauntlet/backuprestore/main.tf
@@ -0,0 +1,17 @@
+module "samsung-cluster" {
+  source                   = "../../.modules/featurebase-cluster"
+  cluster_prefix           = var.cluster_prefix
+  region                   = var.region
+  profile                  = var.profile
+  fb_data_node_type        = "m6g.xlarge"
+  fb_data_disk_iops        = 10000
+  fb_data_node_count       = 3
+  fb_ingest_type           = "m6g.large"
+  fb_cluster_replica_count = 3
+  fb_ingest_disk_iops      = 10000
+  fb_ingest_node_count     = 1
+  vpc_id                   = "vpc-05a26a122f961dc2b"
+  vpc_cidr_block           = "10.0.0.0/16"
+  vpc_public_subnets       = ["subnet-066b4b922b54e51a2", "subnet-037b8884269a69025", "subnet-08482631514426210", ]
+  vpc_private_subnets      = ["subnet-0319dde319380326f", "subnet-0517ca9a646d80f88", "subnet-05a7b685ed27eb1cf", ]
+}
diff --git a/qa/tf/gauntlet/backuprestore/outputs.tf b/qa/tf/gauntlet/backuprestore/outputs.tf
new file mode 100644
index 000000000..c00860a34
--- /dev/null
+++ b/qa/tf/gauntlet/backuprestore/outputs.tf
@@ -0,0 +1,19 @@
+output "ingest_ips" {
+    description = "List of ingest IPs"
+    value = module.samsung-cluster.ingest_ips
+}
+
+output "data_node_ips" {
+  description = "List of data node IPs"
+  value = module.samsung-cluster.data_node_ips
+}
+
+output "cluster_prefix" {
+  description = "The cluster prefix used"
+  value = module.samsung-cluster.cluster_prefix
+}
+
+output "fb_cluster_replica_count" {
+  description = "The cluster replica count used"
+  value = module.samsung-cluster.fb_cluster_replica_count
+}
diff --git a/qa/tf/gauntlet/backuprestore/provider.tf b/qa/tf/gauntlet/backuprestore/provider.tf
new file mode 100644
index 000000000..c0fc95d9d
--- /dev/null
+++ b/qa/tf/gauntlet/backuprestore/provider.tf
@@ -0,0 +1,4 @@
+provider "aws" {
+  region  = var.region
+  profile = var.profile
+}
\ No newline at end of file
diff --git a/qa/tf/gauntlet/backuprestore/tf.auto.tfvars b/qa/tf/gauntlet/backuprestore/tf.auto.tfvars
new file mode 100644
index 000000000..ac6de62a6
--- /dev/null
+++ b/qa/tf/gauntlet/backuprestore/tf.auto.tfvars
@@ -0,0 +1,2 @@
+region  = "us-east-2"
+profile = "service-terraform"
\ No newline at end of file
diff --git a/qa/tf/gauntlet/backuprestore/variables.tf b/qa/tf/gauntlet/backuprestore/variables.tf
new file mode 100644
index 000000000..e55c7936d
--- /dev/null
+++ b/qa/tf/gauntlet/backuprestore/variables.tf
@@ -0,0 +1,14 @@
+variable "region" {
+  description = "The AWS region in which the VPC should be built"
+  type        = string
+}
+
+variable "profile" {
+  description = "The name of the AWS profile Terraform should use for auth."
+  type        = string
+}
+
+variable "cluster_prefix" {
+  type        = string
+  description = "This is a identifier that will be prefixed to created resources"
+}