Merge pull request #1932 from molecula/able-perf

Able perf
2026-09-07 09:05:55 +00:00 · 2022-02-24 12:56:19 -06:00 · 2022-02-24 12:56:19 -06:00 · c9368d5e7a
commit c9368d5e7a
parent 94cfecf0d3 4fef282416
23 changed files with 10960 additions and 742 deletions
--- a/.gitignore
+++ b/.gitignore
@ -20,4 +20,5 @@ launch.json
 __pycache__/
 report.xml
 outputs.json
-builds/
+builds/
+*.tfstate.backup
--- a/.gitlab/.gitlab-ci.yml
+++ b/.gitlab/.gitlab-ci.yml
@ -288,6 +288,7 @@ clustertests:
  stage: integration
  tags:
    - shell
+  retry: 1
  rules:
    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
  script:
@ -302,6 +303,7 @@ authclustertests:
  variables:
    PROJECT: authclustertests_${CI_CONCURRENT_ID}
  stage: integration
+  retry: 1
  tags:
    - shell
  rules:
@ -450,7 +452,7 @@ s3 dump:
  tags:
    - shell
  rules:
-    - if: '$CI_COMMIT_TAG == null && ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web")'
+    - if: '$CI_COMMIT_TAG == null && ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "web")'
  script:
    - aws configure set aws_access_key_id $AWS_FBCI_ACCESS_KEY_ID
    - aws configure set aws_secret_access_key $AWS_FBCI_SECRET_ACCESS_KEY
@ -480,11 +482,12 @@ s3 dump:

 perf_able:
  stage: performance
+  rules:
+    - if: '$CI_PIPELINE_SOURCE == "push"'
  trigger:
    include: .gitlab/.perf-able-gitlab-ci.yml
-  rules:
-    - changes:
-      - featurebase/*
+  variables:
+    PARENT_PIPELINE_ID: $CI_PIPELINE_ID

 s3 dump tag:
  stage: post build
@ -497,7 +500,7 @@ s3 dump tag:
  tags:
    - shell
  rules:
-    - if: '$CI_COMMIT_TAG != null && ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web")'
+    - if: '$CI_COMMIT_TAG != null && ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "web")'
  script:
    - aws configure set aws_access_key_id $AWS_FBCI_ACCESS_KEY_ID
    - aws configure set aws_secret_access_key $AWS_FBCI_SECRET_ACCESS_KEY
--- a/.gitlab/.perf-able-gitlab-ci.yml
+++ b/.gitlab/.perf-able-gitlab-ci.yml
@ -1,11 +1,60 @@
 stages:
-  - loadtest
+  - performance

-loadtest:
-  image:
-    name: loadimpact/k6:latest
-    entrypoint: ['']
-  stage: loadtest
+perf_able:
+  stage: performance
+  image: registry.gitlab.com/gitlab-org/cloud-deploy/aws-base:latest
+  variables:
+    PROFILE: "service-terraform"
+    INFRA_PROFILE: "service-gitlab"
+    AWS_SSH_PRIVATE_KEY: $AWS_FBCI_SSH_KEY
+    AWS_ACCESS_KEY_ID: $AWS_FBCI_ACCESS_KEY_ID
+    AWS_SECRET_ACCESS_KEY: $AWS_FBCI_SECRET_ACCESS_KEY
+    ASG_NAME: "gitlab-runners"
+    TF_VAR_cluster_prefix: ""
+  tags:
+    - aws
+    - docker
+    - fbsmoke
+  before_script:
+    - apt-get update && apt-get install -y gnupg software-properties-common curl git
+    - curl -fsSL https://apt.releases.hashicorp.com/gpg | apt-key add -
+    - apt-add-repository "deb [arch=amd64] https://apt.releases.hashicorp.com $(lsb_release -cs) main"
+    - apt-get update && apt-get install terraform
+    - aws configure set aws_access_key_id $AWS_FBCI_ACCESS_KEY_ID --profile $PROFILE
+    - aws configure set aws_secret_access_key $AWS_FBCI_SECRET_ACCESS_KEY --profile $PROFILE
+    - aws configure set region "us-east-2" --profile $PROFILE
+    - aws configure set aws_profile $PROFILE
+    - aws configure set aws_access_key_id $AWS_INFRA_ACCESS_KEY_ID --profile $INFRA_PROFILE
+    - aws configure set aws_secret_access_key $AWS_INFRA_SECRET_ACCESS_KEY --profile $INFRA_PROFILE
+    - aws configure set region "us-east-2" --profile $INFRA_PROFILE
+    - echo $AWS_FBCI_SSH_KEY > gitlab-featurebase-ci.pem
+    - chmod 400 gitlab-featurebase-ci.pem
+    - 'which ssh-agent || ( apt-get update -y && apt-get install openssh-client -y )'
+    - eval $(ssh-agent -s)
+    - mkdir -p ~/.ssh
+    - echo $AWS_FBCI_SSH_KEY > /root/.ssh/gitlab-featurebase-ci.pem
+    - chmod 400 /root/.ssh/gitlab-featurebase-ci.pem
+    - echo "$AWS_FBCI_SSH_KEY" | ssh-add -
+    - chmod 700 /root/.ssh
+    - '[[ -f /.dockerenv ]] && echo -e "Host *\n\tStrictHostKeyChecking no\n\n" > ~/.ssh/config'
+    - apt update && apt -y install jq wget
+    - wget -q https://go.dev/dl/go1.17.5.linux-amd64.tar.gz
+    - tar -C /usr/local -xzf go1.17.5.linux-amd64.tar.gz
+    - export PATH=$PATH:/usr/local/go/bin
+    - TF_VAR_cluster_prefix="able-$(openssl rand -base64 12 | tr -d /=+ | cut -c -16)"
+    - echo "Cluster Prefix --> $TF_VAR_cluster_prefix"
+    - export INSTANCE_ID=$(curl --silent --fail "http://169.254.169.254/latest/meta-data/instance-id" | tee instance_id)
+    - aws autoscaling set-instance-protection --instance-ids "$INSTANCE_ID" --auto-scaling-group-name $ASG_NAME --protected-from-scale-in --profile $INFRA_PROFILE
  script:
-    - echo "executing local k6 in k6 container..."
-    - k6 run ./qa/scripts/perf/able/script.js
+    - ./qa/scripts/perf/able/ableSetup.sh
+    - ./qa/scripts/perf/able/ableTest.sh
+  after_script:
+    - ./qa/scripts/perf/able/ableTeardown.sh || true
+    - export INSTANCE_ID=$(cat instance_id)
+    - aws autoscaling set-instance-protection --instance-ids "$INSTANCE_ID" --auto-scaling-group-name $ASG_NAME --no-protected-from-scale-in --profile $INFRA_PROFILE
+  needs:
+    - pipeline: $PARENT_PIPELINE_ID
+      job: build for linux arm64
+    - pipeline: $PARENT_PIPELINE_ID
+      job: build for linux amd64
--- a/qa/scripts/perf/able/able.yaml
+++ b/qa/scripts/perf/able/able.yaml
@ -0,0 +1,96 @@
+fields:
+  - name: "id"
+    type: uint # (default IDField (non-mutex))
+    distribution: "sequential"
+    min: 0
+    max: 1000000000 # 1B
+    repeat: false # if false, data generation stops when we hit >= max. only available with sequential
+    step: 1
+  - name: "age"
+    type: int
+    distribution: "uniform" # uniform or zipfian # TODO should totally add some kind of poission, normal, gaussian, bimodal
+    min: 15
+    max: 107
+    null_chance: 0.01
+  - name: "education_level"
+    type: string
+    source_file: "values/education.txt"
+    distribution: "zipfian"
+    s: 1.1
+    v: 5.1
+  - name: "gender"
+    type: string
+    source_file: "values/gender.txt"
+    distribution: "fixed"
+  - name: "income_bracket"
+    type: string
+    source_file: "values/income.txt"
+  - name: "domain"
+    type: "string-set"
+    min_num: 1
+    max_num: 6
+    source_file: "values/opendns-top-domains-10K.txt"
+    distribution: "zipfian"
+    s: 1.5
+    v: 4.3
+  - name: "timestamp"
+    type: "timestamp" # (default TimestampField)
+    min_date: 2006-01-02T15:04:05.001Z # RFC3339Nano
+    max_date: 2010-01-02T15:04:05.001Z # RFC3339Nano
+    distribution: "increasing" # only "increasing" is supported right now
+    min_step_duration: "10us"
+    max_step_duration: "100ms" # generated values will add randomly between 1s and 1h to previous value starting at min_date.
+    repeat: false # stop at > max_date unless repeat=true... then go back to min.
+  - name: "political_party"
+    type: "string"
+    source_file: "values/political_parties.txt"
+    distribution: "zipfian"
+    s: 1.0001
+    v: 1.0001
+  - name: "ltv"
+    type: "float" # use idk_params to choose a scale
+    min_float: 0.2
+    max_float: 1500
+    distribution: "uniform" # only supported value
+  - name: "hobby"
+    type: "string-set"
+    source_file: "values/hobbies.txt"
+    distribution: "zipfian"
+    min_num: 0
+    max_num: 4
+    s: 1.3
+    v: 2.5
+
+# idk_params describe how data from "fields" should be ingested by IDK
+idk_params:
+  primary_key_config:
+    field: "id" # if this is a single field named "id" then we'll use uint IDs, if it's empty we'll autogen ids, and if it's anything else we'll do string keys... yes this is a bit hacky, needs to be cleaned up.
+  # fields is keyed by names of fields from top level "fields". It is
+  # not required that all fields appear here, those that don't will
+  # use the default ingestion.
+  fields:
+    id:
+      - type: "ID"
+    timestamp:
+      - type: "RecordTime"
+        layout: "2006-01-02T15:04:05Z"
+        epoch: 1970-01-01T00:00:00.0Z
+        name: "na"
+    domain:
+      - type: "StringArray"
+        time_quantum: "YMD"
+    ltv:
+      - type: "Decimal"
+        scale: 2
+    income_bracket:
+      - type: "String"
+        mutex: true
+    education_level:
+      - type: "String"
+        mutex: true
+    gender:
+      - type: "String"
+        mutex: true
+    political_party:
+      - type: "String"
+        mutex: true
--- a/qa/scripts/perf/able/ableSetup.sh
+++ b/qa/scripts/perf/able/ableSetup.sh
@ -13,7 +13,7 @@ fi
 SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
 source $SCRIPT_DIR/../../utilCluster.sh

-pushd ./qa/tf/gauntlet/able
+pushd ./qa/tf/perf/able
 echo "Running terraform init..."
 terraform init -input=false
 echo "Running terraform apply..."
@ -22,28 +22,28 @@ terraform output -json > outputs.json
 popd

 # get the first ingest host
-INGESTNODE0=$(cat ./qa/tf/gauntlet/able/outputs.json | jq -r '[.ingest_ips][0]["value"][0]')
+INGESTNODE0=$(cat ./qa/tf/perf/able/outputs.json | jq -r '[.ingest_ips][0]["value"][0]')
 echo "using INGESTNODE0 ${INGESTNODE0}"

 # get the first data host
-DATANODE0=$(cat ./qa/tf/gauntlet/able/outputs.json | jq -r '[.data_node_ips][0]["value"][0]')
+DATANODE0=$(cat ./qa/tf/perf/able/outputs.json | jq -r '[.data_node_ips][0]["value"][0]')
 echo "using DATANODE0 ${DATANODE0}"


-DEPLOYED_CLUSTER_PREFIX=$(cat ./qa/tf/gauntlet/able/outputs.json | jq -r '[.cluster_prefix][0]["value"]')
+DEPLOYED_CLUSTER_PREFIX=$(cat ./qa/tf/perf/able/outputs.json | jq -r '[.cluster_prefix][0]["value"]')
 echo "Using DEPLOYED_CLUSTER_PREFIX: ${DEPLOYED_CLUSTER_PREFIX}"

-DEPLOYED_CLUSTER_REPLICA_COUNT=$(cat ./qa/tf/gauntlet/able/outputs.json | jq -r '[.fb_cluster_replica_count][0]["value"]')
+DEPLOYED_CLUSTER_REPLICA_COUNT=$(cat ./qa/tf/perf/able/outputs.json | jq -r '[.fb_cluster_replica_count][0]["value"]')
 echo "Using DEPLOYED_CLUSTER_REPLICA_COUNT: ${DEPLOYED_CLUSTDEPLOYED_CLUSTER_REPLICA_COUNTER_PREFIX}"

-DEPLOYED_DATA_IPS=$(cat ./qa/tf/gauntlet/able/outputs.json | jq -r '[.data_node_ips][0]["value"][]')
+DEPLOYED_DATA_IPS=$(cat ./qa/tf/perf/able/outputs.json | jq -r '[.data_node_ips][0]["value"][]')
 echo "DEPLOYED_DATA_IPS: {"
 echo "${DEPLOYED_DATA_IPS}"
 echo "}"

 DEPLOYED_DATA_IPS_LEN=`echo "$DEPLOYED_DATA_IPS" | wc -l`

-DEPLOYED_INGEST_IPS=$(cat ./qa/tf/gauntlet/able/outputs.json | jq -r '[.ingest_ips][0]["value"][]')
+DEPLOYED_INGEST_IPS=$(cat ./qa/tf/perf/able/outputs.json | jq -r '[.ingest_ips][0]["value"][]')
 echo "DEPLOYED_INGEST_IPS: {"
 echo "${DEPLOYED_INGEST_IPS}"
 echo "}"
--- a/qa/scripts/perf/able/ableTeardown.sh
+++ b/qa/scripts/perf/able/ableTeardown.sh
@ -2,6 +2,6 @@

 # To run script: ./ableTeardown.sh

-cd qa/tf/gauntlet/able
+cd qa/tf/perf/able
 export TF_IN_AUTOMATION=1
 terraform destroy -auto-approve
--- a/qa/scripts/perf/able/ableTest.sh
+++ b/qa/scripts/perf/able/ableTest.sh
@ -1,12 +1,66 @@
 #!/bin/bash

 # get the first ingest host
-INGESTNODE0=$(cat ./qa/tf/gauntlet/able/outputs.json | jq -r '[.ingest_ips][0]["value"][0]')
+INGESTNODE0=$(cat ./qa/tf/perf/able/outputs.json | jq -r '[.ingest_ips][0]["value"][0]')
 echo "using INGESTNODE0 ${INGESTNODE0}"

 # get the first data host
-DATANODE0=$(cat ./qa/tf/gauntlet/able/outputs.json | jq -r '[.data_node_ips][0]["value"][0]')
+DATANODE0=$(cat ./qa/tf/perf/able/outputs.json | jq -r '[.data_node_ips][0]["value"][0]')
 echo "using DATANODE0 ${DATANODE0}"

+# leaving this here because K6 is timing out and need to work out why
+# ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o "StrictHostKeyChecking no" ec2-user@${INGESTNODE0} "wget https://github.com/grafana/k6/releases/download/v0.36.0/k6-v0.36.0-linux-arm64.tar.gz"
+# ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o "StrictHostKeyChecking no" ec2-user@${INGESTNODE0} "tar -xvf k6-v0.36.0-linux-arm64.tar.gz"
+# ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o "StrictHostKeyChecking no" ec2-user@${INGESTNODE0} "mkdir bin"
+# ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o "StrictHostKeyChecking no" ec2-user@${INGESTNODE0} "mv ./k6-v0.36.0-linux-arm64/k6 ./bin"

-echo "Done."
+echo "Copying tests to remote"
+scp -r -i ~/.ssh/gitlab-featurebase-ci.pem ./qa/scripts/perf/able/*.js ec2-user@${INGESTNODE0}:/data
+if (( $? != 0 )) 
+then 
+    echo "Copy failed"
+    exit 1
+fi
+
+# copy restore data to ingest node
+echo "Copying restore data from S3"
+ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o "StrictHostKeyChecking no" ec2-user@${INGESTNODE0} "aws s3 cp s3://molecula-perf-storage/able/perf-able-seg.tar.xz /data/perf-able-seg.tar.xz --no-progress" 
+if (( $? != 0 )) 
+then 
+    echo "Copy failed"
+    exit 1
+fi
+
+# untar data
+echo "Untarring data"
+ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o "StrictHostKeyChecking no" ec2-user@${INGESTNODE0} "cd /data; tar -xf perf-able-seg.tar.xz" 
+if (( $? != 0 )) 
+then 
+    echo "Untarring failed"
+    exit 1
+fi
+
+# restore data
+echo "Restoring data"
+ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o "StrictHostKeyChecking no" ec2-user@${INGESTNODE0} "cd /data; featurebase restore --host http://${DATANODE0}:10101 -s /data/data/backup > restore.out" 
+if (( $? != 0 )) 
+then 
+    echo "Restoring failed"
+    exit 1
+fi
+
+# run test
+echo "Running perf test"
+# leaving this here because K6 is timing out and need to work out why
+#ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o "StrictHostKeyChecking no" ec2-user@${INGESTNODE0} "/home/ec2-user/bin/k6 run -e DATANODE0=test.k6.io /data/highcardinalitygroupby.js" 
+ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o "StrictHostKeyChecking no" ec2-user@${INGESTNODE0} "curl ${DATANODE0}:10101/index/seg/query -X POST -o /data/response.json -d 'GroupBy(Rows(education_level), Rows(gender), Rows(political_party), Rows(domain))'" 
+ABLETESTRESULT=$?
+
+if (( $ABLETESTRESULT != 0 )) 
+then 
+    echo "able perf test complete with failures"
+else
+    echo "able test complete"
+fi
+
+exit $ABLETESTRESULT 
--- a/qa/scripts/perf/able/generateTestData.sh
+++ b/qa/scripts/perf/able/generateTestData.sh
@ -0,0 +1,33 @@
+#!/bin/bash
+
+
+# for --pilosa.hosts
+PILOSA_HOSTS=""
+
+DEPLOYED_DATA_IPS=$(cat ./qa/tf/perf/able/outputs.json | jq -r '[.data_node_ips][0]["value"][]')
+echo "DEPLOYED_DATA_IPS: {"
+echo "${DEPLOYED_DATA_IPS}"
+echo "}"
+
+DEPLOYED_DATA_IPS_LEN=`echo "$DEPLOYED_DATA_IPS" | wc -l`
+
+generatePilosaHostsString() {
+    IFS=$'\n'
+    cnt=0
+    for ip in $DEPLOYED_DATA_IPS
+    do
+        if (($cnt + 1 != $DEPLOYED_DATA_IPS_LEN)) 
+        then
+            PILOSA_HOSTS="${PILOSA_HOSTS}p${cnt}=$ip:10101,"
+        else
+            PILOSA_HOSTS="${PILOSA_HOSTS}p${cnt}=$ip:10101"
+        fi
+        cnt=$((cnt+1))
+    done
+
+    echo "PILOSA_HOSTS: ${PILOSA_HOSTS}"
+}
+
+generatePilosaHostsString
+
+datagen -s custom --custom-config=./able.yaml --pilosa.index=seg --pilosa.batch-size=1048576 --pilosa.hosts ${PILOSA_HOSTS}
--- a/qa/scripts/perf/able/highcardinalitygroupby.js
+++ b/qa/scripts/perf/able/highcardinalitygroupby.js
@ -0,0 +1,11 @@
+import http from 'k6/http';
+import { sleep } from 'k6';
+	
+export default function () {
+    const params = {
+        timeout: '1800s',
+    };
+
+    let res = http.post(`http://${__ENV.DATANODE0}:10101/index/seg/query`, "GroupBy(Rows(education_level), Rows(gender), Rows(political_party), Rows(domain), aggregate=Sum(field=age))", params);
+    sleep(1);
+}
--- a/qa/scripts/perf/able/script.js
+++ b/qa/scripts/perf/able/script.js
@ -1,7 +0,0 @@
-import http from 'k6/http';
-import { sleep } from 'k6';
-	
-export default function () {
-    http.get('https://test.k6.io');
-    sleep(1);
-}
--- a/qa/scripts/perf/able/values/education.txt
+++ b/qa/scripts/perf/able/values/education.txt
@ -0,0 +1,6 @@
+Some High School
+High School
+Some College
+College
+Master's
+Doctorate
--- a/qa/scripts/perf/able/values/gender.txt
+++ b/qa/scripts/perf/able/values/gender.txt
@ -0,0 +1,5 @@
+Male,0.48
+Female,0.48
+Transgender,0.01
+Other,0.01
+Unspecified,0.02
--- a/qa/scripts/perf/able/values/hobbies.txt
+++ b/qa/scripts/perf/able/values/hobbies.txt
@ -0,0 +1,642 @@
+Lego building
+Watching movies
+Watch making
+Slacklining
+BMX
+Cricket
+Sketching
+Satellite watching
+Volunteering
+Radio-controlled model playing
+Stone collecting
+Picnicking
+Hydroponics
+Karate
+Roller skating
+Skateboarding
+Element collecting
+Weaving
+Beach volleyball
+Archery
+Livestreaming
+Stone skipping
+Trapshooting
+Filmmaking
+Diorama
+Makeup
+Rugby league football
+Community activism
+Field hockey
+Backpacking
+Slot car
+Insect collecting
+VR Gaming
+Video making
+Bowling
+Sled dog racing
+Skiing
+Web design
+Sand art
+Public speaking
+Movie memorabilia collecting
+Gardening
+Wikipedia editing
+Croquet
+Mathematics
+Rail transport modeling
+Darts
+Judo
+Equestrianism
+Figure Skating
+Scrapbooking
+Airbrushing
+Photography
+Climbing
+Tourism
+Journaling
+Flower growing
+Wood carving
+Fashion design
+Polo
+Slot car racing
+Reading
+Electronic games
+Martial arts
+Bell ringing
+ Air sports
+Skipping rope
+Bowling
+Caving
+Leather crafting
+Construction
+Bus riding
+Flag football
+Anime
+Whittling
+Aerospace
+Sun bathing
+Music
+Running
+Diving
+Plastic art
+Stamp collecting
+Gymnastics
+Kabaddi
+Coin collecting
+Video editing
+Stripping
+Cribbage
+Candy making
+Amateur geology
+Motor sports
+Sculpting
+Transit map collecting
+Refinishing
+Surfing
+Swimming
+Skateboarding
+Knowledge/word games
+Tether car
+Poi
+Manga
+ Action figure
+Teaching
+Blacksmithing
+Fingerpainting
+Audiophile
+Spreadsheets
+Scouting
+Frisbee
+Metal detecting
+Book collecting
+Radio-controlled model playing
+Films
+Karaoke
+Wargaming
+Biology
+DJing
+Axe throwing
+Volleyball
+Life Science
+Fossil hunting
+Beachcombing
+Sudoku
+Cross-stitch
+Ephemera collecting
+Puzzles
+Hiking/backpacking
+Digital hoarding
+Horseshoes
+Amateur astronomy
+Book discussion clubs
+Model building
+Ceramics
+Telling jokes
+Gardening
+Renaissance fair
+Record collecting
+Collecting
+Taxidermy
+Flying
+Zumba
+ Archaeology
+Quidditch
+Playing musical instruments
+Tapestry
+Perfume
+Philately
+Business
+Microbiology
+Rafting
+Postcrossing
+Whisky
+Botany
+Badminton
+Chatting
+Board sports
+Groundhopping
+Inventing
+Paragliding
+Shooting sport
+Esports
+Sport stacking
+Proverbs
+Marching band
+Feng shui decorating
+Car tuning
+Sociology
+Writing music
+Robot combat
+Parkour
+Shogi
+Weightlifting
+Fashion
+Safari
+Motorcycling
+Pool
+Meteorology
+Auto audiophilia
+Mushroom hunting/mycology
+Radio-controlled model playing
+Miniature art
+Video game developing
+Medical science
+Herp keeping
+Shoemaking
+Gongfu tea
+Dowsing
+Microscopy
+Welding
+Woodworking
+Clothesmaking
+Fingerprint collecting
+Crossword puzzles
+Breadmaking
+Ice hockey
+Dolls
+Curling
+Sailing
+Mazes (indoor/outdoor)
+Fishkeeping
+Ticket collecting
+Flower arranging
+Nail art
+Couponing
+Skimboarding
+Fishing
+Figure skating
+Herping
+Surfing
+Go
+Vintage clothing
+Shortwave listening
+Water sports
+Darts
+Bonsai
+Lomography
+Crocheting
+Meditation
+Cornhole
+Railway journeys
+Cardistry
+Book restoration
+Graffiti
+Decorating
+Yo-yoing
+Speedcubing
+Lotology (lottery ticket collecting)
+Houseplant care
+Cryptography
+Quilling
+Powerlifting
+Cheesemaking
+Table tennis
+Public transport riding
+Pet adoption & fostering
+Magnet fishing
+Hooping
+Bridge
+Rubik's Cube
+Beekeeping
+Digital arts
+Foreign language learning
+Race walking
+Fusilately (phonecard collecting)
+Fishfarming
+Jigsaw puzzles
+Reviewing Gadgets
+Entrepreneurship
+Pickleball
+Wine tasting
+Footbag
+Astronomy
+Stuffed toy collecting
+Roller derby
+Astrology
+Furniture building
+Lapidary
+Iceboat racing
+High-power rocketry
+Reiki
+Baking
+Automobilism
+Witchcraft
+Walking
+Aerial silk
+Gongoozling
+Learning
+Cartophily (card collecting)
+Paintball
+Genealogy
+Do it yourself
+Volleyball
+Science and technology studies
+Horsemanship
+Swimming
+Needlepoint
+Fishkeeping
+Vintage cars
+Basketball
+Qigong
+Video game collecting
+Writing
+Vacation
+Nordic skating
+Powerboat racing
+Baseball
+Candle making
+Whale watching
+Knot tying
+Ice skating
+Debate
+Checkers (draughts)
+Board/tabletop games
+Model engineering
+VR Gaming
+Palmistry
+Air hockey
+Pole dancing
+Modeling
+Puppetry
+Memory training
+Sculling or rowing
+Seashell collecting
+Poetry
+Role-playing games
+Flying model planes
+Tennis polo
+Gymnastics
+Metalworking
+Scutelliphily
+Eating
+Pet sitting
+Fruit picking
+Farming
+Survivalism
+Fly tying
+Wax sealing
+Sea glass collecting
+Antiquing
+Metal detecting
+Guerrilla gardening
+Dance
+Birdwatching
+Skiing
+Jujitsu
+Hiking
+Model aircraft
+Model United Nations
+Jukskei
+Leaves
+Drama
+Lacrosse
+LARPing
+Home improvement
+Skydiving
+Snowmobiling
+Meteorology
+Fantasy sports
+Blogging
+Hobby horsing
+Knife throwing
+English
+Soapmaking
+Talking
+Lace making
+Driving
+Engraving
+Kung fu
+Laser tag
+Composting
+Sledding
+Croquet
+Railway studies
+Magic
+Kite flying
+Acting
+Juggling
+Travel
+Glassblowing
+Baton twirling
+Boxing
+Kart racing
+Comic book collecting
+Meditation
+Mineral collecting
+Dancing
+Antiquities
+Ultimate frisbee
+Planning
+Pole dancing
+Snorkeling
+Zoo visiting
+Animation
+Rock painting
+Exhibition drill
+Stamp collecting
+People-watching
+Knife collecting
+Herbalism
+Knitting
+Karting
+Tennis
+Drink mixing
+Kombucha brewing
+Chemistry
+Badminton
+Lock picking
+Letterboxing
+Storm chasing
+Sports memorabilia
+Tai chi
+Calligraphy
+Weight training
+Pin (lapel)
+Coffee roasting
+Unicycling
+Ghost hunting
+Archery
+Museum visiting
+Card games
+Dog sport
+Herping
+Netball
+Video gaming
+Trade fair visiting
+Baseball
+plush collecting
+Car fixing & building
+Tatebanko
+BASE jumping
+Gold prospecting
+Animal fancy
+Jogging
+Gunsmithing
+Shooting
+Long-distance running
+Quizzes
+Canoeing
+Aquascaping
+Practical jokes
+Tattooing
+Social studies
+Vehicle restoration
+Cheerleading
+Proofreading and editing
+Fishing
+Squash
+Tarot
+Sewing
+Birdwatching
+Cycling
+Button collecting
+Animation
+Art
+Giving advice
+Handball
+Die-cast toy
+Jewelry making
+Deltiology (postcard collecting)
+Brazilian jiu-jitsu
+Coloring
+Podcast hosting
+Couch surfing
+Reading
+Compact discs
+Bullet journaling
+Hunting
+Australian rules football
+Origami
+Tea bag collecting
+Webtooning
+Longboarding
+Auto detailing
+Hacking
+Kendama
+Photography
+Pilates
+Snowboarding
+Pressed flower craft
+Conlanging
+Beatboxing
+Amateur radio
+Freestyle football
+Mountaineering
+Rock tumbling
+Yoga
+Bus spotting
+Tour skating
+Rock balancing
+Camping
+Sculling or rowing
+Performance
+Djembe
+Entertaining
+Chess
+Cleaning
+Electronics
+Vinyl Records
+Beauty pageants
+Auto racing
+Climbing
+Road biking
+Gingerbread house making
+Distro Hopping
+Geocaching
+Snowshoeing
+Creative writing
+Taekwondo
+Radio-controlled car racing
+Worldbuilding
+Car riding
+Stand-up comedy
+Flying disc
+Dog walking
+Phillumeny
+Foraging
+Singing
+Barbershop Music
+Confectionery
+Amusement park visiting
+Inline skating
+Knife making
+History
+Breakdancing
+Experimenting
+Color guard
+Painting
+Soccer
+Backgammon
+City trip
+Marbles
+Renovating
+Speed skating
+Handball
+Gaming
+Triathlon
+Mountain biking
+Machining
+Art collecting
+Baton twirling
+Horseback riding
+Benchmarking
+Philately
+Tourism
+Wrestling
+Disc golf
+Flower collecting and pressing
+Fitness
+Acroyoga
+Beer tasting
+Video gaming
+Lacrosse
+Bodybuilding
+Thrifting
+Topiary
+3D printing
+Crystals
+Orienteering
+Noodling
+Geocaching
+Orienteering
+Winemaking
+Watching documentaries
+Pet
+Drawing
+Photography
+Airsoft
+Homebrewing
+Aircraft spotting
+Mini Golf
+Storytelling
+Pickleball
+Shuffleboard
+Cooking
+Rock climbing
+Vegetable farming
+Radio-controlled model playing
+Billiards
+Association football
+Embroidery
+Waxing
+Physics
+Hobby tunneling
+Scuba diving
+Kayaking
+Videography
+Tennis
+Slot car
+Table tennis
+Golfing
+Dog training
+Craft
+Mahjong
+Cycling
+Thru-hiking
+Fencing
+Airsoft
+Humor
+Mycology
+Rail transport modelling
+Sports science
+Table football
+Trainspotting
+Minimalism
+Urban exploration
+Macrame
+Computer programming
+Horseback riding
+Cue sports
+Magic
+Pyrography
+Ice skating
+Upcycling
+Shoes
+Power Nap
+Pen Spinning
+Jumping rope
+Astronomy
+Pottery
+Martial arts
+Butterfly watching
+Hula hooping
+Water polo
+Geography
+Chess
+Rugby
+Cosplaying
+Racquetball
+Shopping
+Graphic design
+Binge-watching
+Kitesurfing
+Research
+Model racing
+Listening to podcasts
+Radio-controlled model collecting
+Research
+Rapping
+Poker
+Rappelling
+Watching television
+Listening to music
+Mechanics
+Philosophy
+Recipe creation
+Quilting
+Fossicking
+Social media
+Word searches
+Massaging
+Dominoes
+Longboarding
+Scuba Diving
+Dining
+Hardware
+Communication
+Ant-keeping
+Canyoning
+Dandyism
+Psychology
+Softball
+Table tennis playing
--- a/qa/scripts/perf/able/values/income.txt
+++ b/qa/scripts/perf/able/values/income.txt
@ -0,0 +1,7 @@
+$0-$14,200
+$14,201-$54,200
+$54,201-$86,350
+$86,351-$164,900
+$164,901-$209,400
+$209,401-$523,600
+$523,601 or more
--- a/qa/scripts/perf/able/values/opendns-top-domains-10K.txt
+++ b/qa/scripts/perf/able/values/opendns-top-domains-10K.txt
--- a/qa/scripts/perf/able/values/political_parties.txt
+++ b/qa/scripts/perf/able/values/political_parties.txt
@ -0,0 +1,7 @@
+Democrat
+Republican
+Independent
+Libertarian
+Green
+Federalist
+Whig
--- a/qa/tf/.modules/featurebase-cluster/main.tf
+++ b/qa/tf/.modules/featurebase-cluster/main.tf
@ -257,6 +257,27 @@ resource "aws_iam_role" "fb_cluster_node_role" {
    })
  }

+  inline_policy {
+    name = "s3_perms"
+    policy = jsonencode({
+      Version = "2012-10-17"
+      Statement = [
+        {
+            Sid = "VisualEditor0",
+            Effect = "Allow",
+            Action = ["s3:PutObject", "s3:GetObject"],
+            Resource = "arn:aws:s3:::molecula-perf-storage/*"
+        },
+        {
+            Sid = "VisualEditor1",
+            Effect = "Allow",
+            Action = "s3:PutObject",
+            Resource = "arn:aws:s3:::molecula-artifact-storage/*"
+        }
+      ]
+    })
+  }
+
  tags = {
    Prefix = "${var.cluster_prefix}"
    Name   = "${var.cluster_prefix}-fb_cluster_node_role"
--- a/qa/tf/ci/smoketest/terraform.tfstate.backup
+++ b/qa/tf/ci/smoketest/terraform.tfstate.backup
@ -1,710 +0,0 @@
-{
-  "version": 4,
-  "terraform_version": "1.1.2",
-  "serial": 220,
-  "lineage": "0f5e8a05-0e94-e86f-f384-26086bd40585",
-  "outputs": {
-    "cluster_prefix": {
-      "value": "gauntlet-wFQOOzXB51R3ebr",
-      "type": "string"
-    },
-    "data_node_ips": {
-      "value": [
-        "10.0.1.16"
-      ],
-      "type": [
-        "tuple",
-        [
-          "string"
-        ]
-      ]
-    },
-    "fb_cluster_replica_count": {
-      "value": 1,
-      "type": "number"
-    },
-    "ingest_ips": {
-      "value": [
-        "3.142.172.86"
-      ],
-      "type": [
-        "tuple",
-        [
-          "string"
-        ]
-      ]
-    }
-  },
-  "resources": [
-    {
-      "module": "module.ci-cluster",
-      "mode": "data",
-      "type": "aws_ami",
-      "name": "amazon_linux_2",
-      "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
-      "instances": [
-        {
-          "schema_version": 0,
-          "attributes": {
-            "architecture": "arm64",
-            "arn": "arn:aws:ec2:us-east-2::image/ami-088e1f338c3b87d1a",
-            "block_device_mappings": [
-              {
-                "device_name": "/dev/xvda",
-                "ebs": {
-                  "delete_on_termination": "true",
-                  "encrypted": "false",
-                  "iops": "0",
-                  "snapshot_id": "snap-0f9ae89577e61b172",
-                  "throughput": "0",
-                  "volume_size": "8",
-                  "volume_type": "gp2"
-                },
-                "no_device": "",
-                "virtual_name": ""
-              }
-            ],
-            "creation_date": "2022-01-05T21:55:03.000Z",
-            "description": "Amazon Linux 2 LTS Arm64 AMI 2.0.20211223.0 arm64 HVM gp2",
-            "ena_support": true,
-            "executable_users": null,
-            "filter": [
-              {
-                "name": "architecture",
-                "values": [
-                  "arm64"
-                ]
-              },
-              {
-                "name": "name",
-                "values": [
-                  "amzn2-ami-hvm-*"
-                ]
-              },
-              {
-                "name": "virtualization-type",
-                "values": [
-                  "hvm"
-                ]
-              }
-            ],
-            "hypervisor": "xen",
-            "id": "ami-088e1f338c3b87d1a",
-            "image_id": "ami-088e1f338c3b87d1a",
-            "image_location": "amazon/amzn2-ami-hvm-2.0.20211223.0-arm64-gp2",
-            "image_owner_alias": "amazon",
-            "image_type": "machine",
-            "kernel_id": null,
-            "most_recent": true,
-            "name": "amzn2-ami-hvm-2.0.20211223.0-arm64-gp2",
-            "name_regex": null,
-            "owner_id": "137112412989",
-            "owners": [
-              "amazon"
-            ],
-            "platform": null,
-            "platform_details": "Linux/UNIX",
-            "product_codes": [],
-            "public": true,
-            "ramdisk_id": null,
-            "root_device_name": "/dev/xvda",
-            "root_device_type": "ebs",
-            "root_snapshot_id": "snap-0f9ae89577e61b172",
-            "sriov_net_support": "simple",
-            "state": "available",
-            "state_reason": {
-              "code": "UNSET",
-              "message": "UNSET"
-            },
-            "tags": {},
-            "usage_operation": "RunInstances",
-            "virtualization_type": "hvm"
-          },
-          "sensitive_attributes": []
-        }
-      ]
-    },
-    {
-      "module": "module.ci-cluster",
-      "mode": "managed",
-      "type": "aws_iam_instance_profile",
-      "name": "fb_cluster_node_profile",
-      "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
-      "instances": [
-        {
-          "schema_version": 0,
-          "attributes": {
-            "arn": "arn:aws:iam::977373308795:instance-profile/gauntlet-wFQOOzXB51R3ebr-fb_cluster_node_profile",
-            "create_date": "2022-01-12T15:00:15Z",
-            "id": "gauntlet-wFQOOzXB51R3ebr-fb_cluster_node_profile",
-            "name": "gauntlet-wFQOOzXB51R3ebr-fb_cluster_node_profile",
-            "name_prefix": null,
-            "path": "/",
-            "role": "gauntlet-wFQOOzXB51R3ebr-fb_cluster_node",
-            "tags": null,
-            "tags_all": {},
-            "unique_id": "AIPA6HD75E55WG6AVJLA4"
-          },
-          "sensitive_attributes": [],
-          "private": "bnVsbA==",
-          "dependencies": [
-            "module.ci-cluster.aws_iam_role.fb_cluster_node_role"
-          ]
-        }
-      ]
-    },
-    {
-      "module": "module.ci-cluster",
-      "mode": "managed",
-      "type": "aws_iam_role",
-      "name": "fb_cluster_node_role",
-      "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
-      "instances": [
-        {
-          "schema_version": 0,
-          "attributes": {
-            "arn": "arn:aws:iam::977373308795:role/gauntlet-wFQOOzXB51R3ebr-fb_cluster_node",
-            "assume_role_policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Sid\":\"\",\"Effect\":\"Allow\",\"Principal\":{\"Service\":\"ec2.amazonaws.com\"},\"Action\":\"sts:AssumeRole\"}]}",
-            "create_date": "2022-01-12T15:00:12Z",
-            "description": "",
-            "force_detach_policies": false,
-            "id": "gauntlet-wFQOOzXB51R3ebr-fb_cluster_node",
-            "inline_policy": [
-              {
-                "name": "ec2_read_all",
-                "policy": "{\"Statement\":[{\"Action\":[\"ec2:Describe*\"],\"Effect\":\"Allow\",\"Resource\":\"*\"}],\"Version\":\"2012-10-17\"}"
-              }
-            ],
-            "managed_policy_arns": [],
-            "max_session_duration": 3600,
-            "name": "gauntlet-wFQOOzXB51R3ebr-fb_cluster_node",
-            "name_prefix": "",
-            "path": "/",
-            "permissions_boundary": null,
-            "tags": null,
-            "tags_all": {},
-            "unique_id": "AROA6HD75E55YDDESCKUN"
-          },
-          "sensitive_attributes": [],
-          "private": "bnVsbA=="
-        }
-      ]
-    },
-    {
-      "module": "module.ci-cluster",
-      "mode": "managed",
-      "type": "aws_instance",
-      "name": "fb_cluster_nodes",
-      "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
-      "instances": [
-        {
-          "index_key": 0,
-          "schema_version": 1,
-          "attributes": {
-            "ami": "ami-088e1f338c3b87d1a",
-            "arn": "arn:aws:ec2:us-east-2:977373308795:instance/i-03a8897456f88b2a4",
-            "associate_public_ip_address": false,
-            "availability_zone": "us-east-2a",
-            "capacity_reservation_specification": [
-              {
-                "capacity_reservation_preference": "open",
-                "capacity_reservation_target": []
-              }
-            ],
-            "cpu_core_count": 2,
-            "cpu_threads_per_core": 1,
-            "credit_specification": [],
-            "disable_api_termination": false,
-            "ebs_block_device": [
-              {
-                "delete_on_termination": true,
-                "device_name": "/dev/sdb",
-                "encrypted": false,
-                "iops": 3000,
-                "kms_key_id": "",
-                "snapshot_id": "",
-                "tags": {},
-                "throughput": 125,
-                "volume_id": "vol-0028ea6c90c8ed849",
-                "volume_size": 100,
-                "volume_type": "gp3"
-              }
-            ],
-            "ebs_optimized": false,
-            "enclave_options": [
-              {
-                "enabled": false
-              }
-            ],
-            "ephemeral_block_device": [],
-            "get_password_data": false,
-            "hibernation": false,
-            "host_id": null,
-            "iam_instance_profile": "gauntlet-wFQOOzXB51R3ebr-fb_cluster_node_profile",
-            "id": "i-03a8897456f88b2a4",
-            "instance_initiated_shutdown_behavior": "stop",
-            "instance_state": "running",
-            "instance_type": "m6g.large",
-            "ipv6_address_count": 0,
-            "ipv6_addresses": [],
-            "key_name": "gauntlet-wFQOOzXB51R3ebr-gitlab-ci",
-            "launch_template": [],
-            "metadata_options": [
-              {
-                "http_endpoint": "enabled",
-                "http_put_response_hop_limit": 1,
-                "http_tokens": "optional"
-              }
-            ],
-            "monitoring": true,
-            "network_interface": [],
-            "outpost_arn": "",
-            "password_data": "",
-            "placement_group": "",
-            "placement_partition_number": null,
-            "primary_network_interface_id": "eni-0deaddc1bee29d648",
-            "private_dns": "ip-10-0-1-16.us-east-2.compute.internal",
-            "private_ip": "10.0.1.16",
-            "public_dns": "",
-            "public_ip": "",
-            "root_block_device": [
-              {
-                "delete_on_termination": true,
-                "device_name": "/dev/xvda",
-                "encrypted": false,
-                "iops": 3000,
-                "kms_key_id": "",
-                "tags": null,
-                "throughput": 125,
-                "volume_id": "vol-00eb233b46ec8746f",
-                "volume_size": 20,
-                "volume_type": "gp3"
-              }
-            ],
-            "secondary_private_ips": [],
-            "security_groups": [],
-            "source_dest_check": true,
-            "subnet_id": "subnet-050b1219d78f2db1b",
-            "tags": {
-              "Name": "gauntlet-wFQOOzXB51R3ebr-featurebase-cluster-0",
-              "Prefix": "gauntlet-wFQOOzXB51R3ebr",
-              "Role": "cluster_node"
-            },
-            "tags_all": {
-              "Name": "gauntlet-wFQOOzXB51R3ebr-featurebase-cluster-0",
-              "Prefix": "gauntlet-wFQOOzXB51R3ebr",
-              "Role": "cluster_node"
-            },
-            "tenancy": "default",
-            "timeouts": null,
-            "user_data": null,
-            "user_data_base64": null,
-            "volume_tags": null,
-            "vpc_security_group_ids": [
-              "sg-060f8471c4271acb8"
-            ]
-          },
-          "sensitive_attributes": [],
-          "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjo2MDAwMDAwMDAwMDAsImRlbGV0ZSI6MTIwMDAwMDAwMDAwMCwidXBkYXRlIjo2MDAwMDAwMDAwMDB9LCJzY2hlbWFfdmVyc2lvbiI6IjEifQ==",
-          "dependencies": [
-            "module.ci-cluster.aws_iam_instance_profile.fb_cluster_node_profile",
-            "module.ci-cluster.aws_iam_role.fb_cluster_node_role",
-            "module.ci-cluster.aws_key_pair.gitlab-featurebase-ci",
-            "module.ci-cluster.aws_security_group.featurebase",
-            "module.ci-cluster.data.aws_ami.amazon_linux_2"
-          ]
-        }
-      ]
-    },
-    {
-      "module": "module.ci-cluster",
-      "mode": "managed",
-      "type": "aws_instance",
-      "name": "fb_ingest",
-      "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
-      "instances": [
-        {
-          "index_key": 0,
-          "schema_version": 1,
-          "attributes": {
-            "ami": "ami-088e1f338c3b87d1a",
-            "arn": "arn:aws:ec2:us-east-2:977373308795:instance/i-012b6f6ad4a0295a8",
-            "associate_public_ip_address": true,
-            "availability_zone": "us-east-2a",
-            "capacity_reservation_specification": [
-              {
-                "capacity_reservation_preference": "open",
-                "capacity_reservation_target": []
-              }
-            ],
-            "cpu_core_count": 2,
-            "cpu_threads_per_core": 1,
-            "credit_specification": [],
-            "disable_api_termination": false,
-            "ebs_block_device": [
-              {
-                "delete_on_termination": true,
-                "device_name": "/dev/sdb",
-                "encrypted": false,
-                "iops": 3000,
-                "kms_key_id": "",
-                "snapshot_id": "",
-                "tags": {},
-                "throughput": 125,
-                "volume_id": "vol-0278897f78cb4f99e",
-                "volume_size": 100,
-                "volume_type": "gp3"
-              }
-            ],
-            "ebs_optimized": false,
-            "enclave_options": [
-              {
-                "enabled": false
-              }
-            ],
-            "ephemeral_block_device": [],
-            "get_password_data": false,
-            "hibernation": false,
-            "host_id": null,
-            "iam_instance_profile": "gauntlet-wFQOOzXB51R3ebr-fb_cluster_node_profile",
-            "id": "i-012b6f6ad4a0295a8",
-            "instance_initiated_shutdown_behavior": "stop",
-            "instance_state": "running",
-            "instance_type": "m6g.large",
-            "ipv6_address_count": 0,
-            "ipv6_addresses": [],
-            "key_name": "gauntlet-wFQOOzXB51R3ebr-gitlab-ci",
-            "launch_template": [],
-            "metadata_options": [
-              {
-                "http_endpoint": "enabled",
-                "http_put_response_hop_limit": 1,
-                "http_tokens": "optional"
-              }
-            ],
-            "monitoring": true,
-            "network_interface": [],
-            "outpost_arn": "",
-            "password_data": "",
-            "placement_group": "",
-            "placement_partition_number": null,
-            "primary_network_interface_id": "eni-0bb5e02c328f3c371",
-            "private_dns": "ip-10-0-101-66.us-east-2.compute.internal",
-            "private_ip": "10.0.101.66",
-            "public_dns": "",
-            "public_ip": "3.142.172.86",
-            "root_block_device": [
-              {
-                "delete_on_termination": true,
-                "device_name": "/dev/xvda",
-                "encrypted": false,
-                "iops": 3000,
-                "kms_key_id": "",
-                "tags": null,
-                "throughput": 125,
-                "volume_id": "vol-073cdccfd75958e27",
-                "volume_size": 20,
-                "volume_type": "gp3"
-              }
-            ],
-            "secondary_private_ips": [],
-            "security_groups": [],
-            "source_dest_check": true,
-            "subnet_id": "subnet-066b4b922b54e51a2",
-            "tags": {
-              "Name": "gauntlet-wFQOOzXB51R3ebr-featurebase-ingest-0",
-              "Prefix": "gauntlet-wFQOOzXB51R3ebr",
-              "Role": "ingest_node"
-            },
-            "tags_all": {
-              "Name": "gauntlet-wFQOOzXB51R3ebr-featurebase-ingest-0",
-              "Prefix": "gauntlet-wFQOOzXB51R3ebr",
-              "Role": "ingest_node"
-            },
-            "tenancy": "default",
-            "timeouts": null,
-            "user_data": null,
-            "user_data_base64": null,
-            "volume_tags": null,
-            "vpc_security_group_ids": [
-              "sg-07e67c395f920042c"
-            ]
-          },
-          "sensitive_attributes": [],
-          "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjo2MDAwMDAwMDAwMDAsImRlbGV0ZSI6MTIwMDAwMDAwMDAwMCwidXBkYXRlIjo2MDAwMDAwMDAwMDB9LCJzY2hlbWFfdmVyc2lvbiI6IjEifQ==",
-          "dependencies": [
-            "module.ci-cluster.aws_iam_instance_profile.fb_cluster_node_profile",
-            "module.ci-cluster.aws_iam_role.fb_cluster_node_role",
-            "module.ci-cluster.aws_key_pair.gitlab-featurebase-ci",
-            "module.ci-cluster.aws_security_group.ingest",
-            "module.ci-cluster.data.aws_ami.amazon_linux_2"
-          ]
-        }
-      ]
-    },
-    {
-      "module": "module.ci-cluster",
-      "mode": "managed",
-      "type": "aws_key_pair",
-      "name": "gitlab-featurebase-ci",
-      "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
-      "instances": [
-        {
-          "schema_version": 1,
-          "attributes": {
-            "arn": "arn:aws:ec2:us-east-2:977373308795:key-pair/gauntlet-wFQOOzXB51R3ebr-gitlab-ci",
-            "fingerprint": "69:e5:4b:15:8d:1f:22:61:de:08:a9:ee:f3:29:5c:69",
-            "id": "gauntlet-wFQOOzXB51R3ebr-gitlab-ci",
-            "key_name": "gauntlet-wFQOOzXB51R3ebr-gitlab-ci",
-            "key_name_prefix": "",
-            "key_pair_id": "key-0a49a5ef950bc7f0c",
-            "public_key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC91hhpVHNonAG7ku2ugpxEskf9KHeyHJPQJT26OHrMUw7R+T5A8TjqSzTau07sXQ/E9SO3ebV8SJ5PqeaQOnQB8VEvVNK0DjQH7ppvNg1Rfs42FZT9ttzTMvOjsSbK3vZTHXdoKQEdC9NxBwSkFIRGQojK1HUOq9xGrw31fA1OjSwlpLcbx7yyg18lcqW6UOptnVR8U9Yy9qQ5jZF1HtkQ6L9J+gv4o1UyNAUK2bopeGiXpBc3PQ/CFaFT2h/aqLBP66qAHsHVyAFD3PIRtplC5EHa8jXDgLacEls0uF7Q3kRPxvzcuo4g4VkOn1rDy9qH3vd2hT3aKVnM73FIDUiL",
-            "tags": null,
-            "tags_all": {}
-          },
-          "sensitive_attributes": [],
-          "private": "eyJzY2hlbWFfdmVyc2lvbiI6IjEifQ=="
-        }
-      ]
-    },
-    {
-      "module": "module.ci-cluster",
-      "mode": "managed",
-      "type": "aws_security_group",
-      "name": "featurebase",
-      "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
-      "instances": [
-        {
-          "schema_version": 1,
-          "attributes": {
-            "arn": "arn:aws:ec2:us-east-2:977373308795:security-group/sg-060f8471c4271acb8",
-            "description": "Allow featurebase inbound traffic",
-            "egress": [
-              {
-                "cidr_blocks": [
-                  "0.0.0.0/0"
-                ],
-                "description": "",
-                "from_port": 0,
-                "ipv6_cidr_blocks": [
-                  "::/0"
-                ],
-                "prefix_list_ids": [],
-                "protocol": "-1",
-                "security_groups": [],
-                "self": false,
-                "to_port": 0
-              }
-            ],
-            "id": "sg-060f8471c4271acb8",
-            "ingress": [
-              {
-                "cidr_blocks": [
-                  "0.0.0.0/0"
-                ],
-                "description": "SSH",
-                "from_port": 22,
-                "ipv6_cidr_blocks": [
-                  "::/0"
-                ],
-                "prefix_list_ids": [],
-                "protocol": "tcp",
-                "security_groups": [],
-                "self": false,
-                "to_port": 22
-              },
-              {
-                "cidr_blocks": [
-                  "0.0.0.0/0"
-                ],
-                "description": "icmp from Anywhere",
-                "from_port": -1,
-                "ipv6_cidr_blocks": [],
-                "prefix_list_ids": [],
-                "protocol": "icmp",
-                "security_groups": [],
-                "self": false,
-                "to_port": -1
-              },
-              {
-                "cidr_blocks": [
-                  "10.0.0.0/16"
-                ],
-                "description": "etcd from internal 2",
-                "from_port": 10401,
-                "ipv6_cidr_blocks": [],
-                "prefix_list_ids": [],
-                "protocol": "tcp",
-                "security_groups": [],
-                "self": false,
-                "to_port": 10401
-              },
-              {
-                "cidr_blocks": [
-                  "10.0.0.0/16"
-                ],
-                "description": "etcd from internal",
-                "from_port": 10301,
-                "ipv6_cidr_blocks": [],
-                "prefix_list_ids": [],
-                "protocol": "tcp",
-                "security_groups": [],
-                "self": false,
-                "to_port": 10301
-              },
-              {
-                "cidr_blocks": [
-                  "10.0.0.0/8",
-                  "172.31.0.0/16"
-                ],
-                "description": "GRPC from Internal",
-                "from_port": 20101,
-                "ipv6_cidr_blocks": [],
-                "prefix_list_ids": [],
-                "protocol": "tcp",
-                "security_groups": [],
-                "self": false,
-                "to_port": 20101
-              },
-              {
-                "cidr_blocks": [
-                  "10.0.0.0/8",
-                  "172.31.0.0/16"
-                ],
-                "description": "HTTP from Internal",
-                "from_port": 10101,
-                "ipv6_cidr_blocks": [],
-                "prefix_list_ids": [],
-                "protocol": "tcp",
-                "security_groups": [],
-                "self": false,
-                "to_port": 10101
-              },
-              {
-                "cidr_blocks": [
-                  "10.0.0.0/8",
-                  "172.31.0.0/16"
-                ],
-                "description": "PostgreSQL from Internal",
-                "from_port": 55432,
-                "ipv6_cidr_blocks": [],
-                "prefix_list_ids": [],
-                "protocol": "tcp",
-                "security_groups": [],
-                "self": false,
-                "to_port": 55432
-              }
-            ],
-            "name": "gauntlet-wFQOOzXB51R3ebr-allow_featurebase",
-            "name_prefix": "",
-            "owner_id": "977373308795",
-            "revoke_rules_on_delete": false,
-            "tags": {
-              "Name": "allow_featurebase"
-            },
-            "tags_all": {
-              "Name": "allow_featurebase"
-            },
-            "timeouts": null,
-            "vpc_id": "vpc-05a26a122f961dc2b"
-          },
-          "sensitive_attributes": [],
-          "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjo2MDAwMDAwMDAwMDAsImRlbGV0ZSI6OTAwMDAwMDAwMDAwfSwic2NoZW1hX3ZlcnNpb24iOiIxIn0="
-        }
-      ]
-    },
-    {
-      "module": "module.ci-cluster",
-      "mode": "managed",
-      "type": "aws_security_group",
-      "name": "ingest",
-      "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
-      "instances": [
-        {
-          "schema_version": 1,
-          "attributes": {
-            "arn": "arn:aws:ec2:us-east-2:977373308795:security-group/sg-07e67c395f920042c",
-            "description": "Allow ingest inbound traffic",
-            "egress": [
-              {
-                "cidr_blocks": [
-                  "0.0.0.0/0"
-                ],
-                "description": "",
-                "from_port": 0,
-                "ipv6_cidr_blocks": [
-                  "::/0"
-                ],
-                "prefix_list_ids": [],
-                "protocol": "-1",
-                "security_groups": [],
-                "self": false,
-                "to_port": 0
-              }
-            ],
-            "id": "sg-07e67c395f920042c",
-            "ingress": [
-              {
-                "cidr_blocks": [
-                  "0.0.0.0/0"
-                ],
-                "description": "",
-                "from_port": 10101,
-                "ipv6_cidr_blocks": [
-                  "::/0"
-                ],
-                "prefix_list_ids": [],
-                "protocol": "tcp",
-                "security_groups": [],
-                "self": false,
-                "to_port": 10101
-              },
-              {
-                "cidr_blocks": [
-                  "0.0.0.0/0"
-                ],
-                "description": "SSH",
-                "from_port": 22,
-                "ipv6_cidr_blocks": [
-                  "::/0"
-                ],
-                "prefix_list_ids": [],
-                "protocol": "tcp",
-                "security_groups": [],
-                "self": false,
-                "to_port": 22
-              },
-              {
-                "cidr_blocks": [
-                  "0.0.0.0/0"
-                ],
-                "description": "icmp from Anywhere",
-                "from_port": -1,
-                "ipv6_cidr_blocks": [],
-                "prefix_list_ids": [],
-                "protocol": "icmp",
-                "security_groups": [],
-                "self": false,
-                "to_port": -1
-              }
-            ],
-            "name": "gauntlet-wFQOOzXB51R3ebr-allow_ingest",
-            "name_prefix": "",
-            "owner_id": "977373308795",
-            "revoke_rules_on_delete": false,
-            "tags": {
-              "Name": "allow_ingest"
-            },
-            "tags_all": {
-              "Name": "allow_ingest"
-            },
-            "timeouts": null,
-            "vpc_id": "vpc-05a26a122f961dc2b"
-          },
-          "sensitive_attributes": [],
-          "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjo2MDAwMDAwMDAwMDAsImRlbGV0ZSI6OTAwMDAwMDAwMDAwfSwic2NoZW1hX3ZlcnNpb24iOiIxIn0="
-        }
-      ]
-    }
-  ]
-}
--- a/qa/tf/gauntlet/able/main.tf
+++ b/qa/tf/gauntlet/able/main.tf
--- a/qa/tf/gauntlet/able/outputs.tf
+++ b/qa/tf/gauntlet/able/outputs.tf
--- a/qa/tf/gauntlet/able/provider.tf
+++ b/qa/tf/gauntlet/able/provider.tf
--- a/qa/tf/gauntlet/able/tf.auto.tfvars
+++ b/qa/tf/gauntlet/able/tf.auto.tfvars
--- a/qa/tf/gauntlet/able/variables.tf
+++ b/qa/tf/gauntlet/able/variables.tf