2026-09-07 09:05:55 +00:00 · 2021-12-21 15:44:20 -07:00 · 2021-12-21 15:44:20 -07:00 · fa9ae13363
commit fa9ae13363
parent 63d8686e22
35 changed files with 1251 additions and 340 deletions
--- a/.gitignore
+++ b/.gitignore
@ -13,3 +13,8 @@ pilosa
 *.dot
 .idea/
 .*.swp
+.terraform/
+*.tfstate
+launch.json
+.terraform.lock.hcl
+__pycache__/
--- a/.gitlab/.gitlab-ci.yml
+++ b/.gitlab/.gitlab-ci.yml
@ -1,7 +1,7 @@
-include:
-  - template: Security/SAST.gitlab-ci.yml
-  - template: Security/License-Scanning.gitlab-ci.yml
-  - template: Security/Dependency-Scanning.gitlab-ci.yml
+# include:
+#   - template: Security/SAST.gitlab-ci.yml
+#   - template: Security/License-Scanning.gitlab-ci.yml
+#   - template: Security/Dependency-Scanning.gitlab-ci.yml

 .go-cache:
    variables:
@ -14,29 +14,31 @@ variables:
  GOVERSION: "1.16.10"

 stages:
-  - lint
+  # - lint
  - test
  - build
-  - integration
+  # - integration
+  - gauntlet

-golangci-lint:
-  image: golangci/golangci-lint:v1.39.0
-  stage: lint
-  extends: .go-cache
-  allow_failure: false
-  rules:
-    - if: '$CI_PIPELINE_SOURCE == "push"'
-  script:
-    - echo "Checking for issues in new code"
-    - golangci-lint run -v 
+# golangci-lint:
+#   image: golangci/golangci-lint:v1.39.0
+#   stage: lint
+#   extends: .go-cache
+#   allow_failure: false
+#   rules:
+#     - if: '$CI_PIPELINE_SOURCE == "push"'
+#   script:
+#     - echo "Checking for issues in new code"
+#     - golangci-lint run -v 

 build lattice:
  stage: test
  image: node:14
  variables:
    CI: "false"
-  rules:
-    - if: '$CI_PIPELINE_SOURCE == "push"'
+  # TODO: For now, always do this
+  # rules:
+  #   - if: '$CI_PIPELINE_SOURCE == "push"'
  script:
    - cd lattice
    - yarn install
@ -50,82 +52,83 @@ build lattice:
    paths:
      - lattice.tar.gz

-run jest tests:
-  stage: test
-  image: node:14
-  variables:
-    CI: "true"
-  rules:
-    - if: '$CI_PIPELINE_SOURCE == "push"'
-  script:
-    - echo "Testing lattice..."
-    - cd lattice
-    - npm install --force
-    - npm test -- --coverage --testResultsProcessor=jest-sonar-reporter
-  artifacts:
-    paths:
-      - lattice/coverage/lcov.info
+# run jest tests:
+#   stage: test
+#   image: node:14
+#   variables:
+#     CI: "true"
+#   rules:
+#     - if: '$CI_PIPELINE_SOURCE == "push"'
+#   script:
+#     - echo "Testing lattice..."
+#     - cd lattice
+#     - npm install --force
+#     - npm test -- --coverage --testResultsProcessor=jest-sonar-reporter
+#   artifacts:
+#     paths:
+#       - lattice/coverage/lcov.info

-run go tests:
-  stage: test
-  image: golang:$GOVERSION
-  extends: .go-cache
-  rules:
-    - if: '$CI_PIPELINE_SOURCE == "push"'
-  script:
-    - echo "Running featurebase unit tests..."
-    - PKG_LIST=$(go list ./... | grep -v internal/clustertests | paste -s -d, -)
-    - go test -coverprofile=coverage.out -covermode=atomic -coverpkg=${PKG_LIST} ./...
-  artifacts:
-    paths:
-      - coverage.out
+# run go tests:
+#   stage: test
+#   image: golang:$GOVERSION
+#   extends: .go-cache
+#   rules:
+#     - if: '$CI_PIPELINE_SOURCE == "push"'
+#   script:
+#     - echo "Running featurebase unit tests..."
+#     - PKG_LIST=$(go list ./... | grep -v internal/clustertests | paste -s -d, -)
+#     - go test -coverprofile=coverage.out -covermode=atomic -coverpkg=${PKG_LIST} ./...
+#   artifacts:
+#     paths:
+#       - coverage.out

-run go tests future:
-  stage: test
-  image: golang:1.17.3
-  extends: .go-cache
-  rules:
-    - if: '$CI_PIPELINE_SOURCE == "push"'
-  script:
-    - echo "Running featurebase unit tests..."
-    - PKG_LIST=$(go list ./... | grep -v internal/clustertests | paste -s -d, -)
-    - go test -coverprofile=coverage.out -covermode=atomic -coverpkg=${PKG_LIST} ./...
-  artifacts:
-    paths:
-      - coverage.out
+# run go tests future:
+#   stage: test
+#   image: golang:1.17.3
+#   extends: .go-cache
+#   rules:
+#     - if: '$CI_PIPELINE_SOURCE == "push"'
+#   script:
+#     - echo "Running featurebase unit tests..."
+#     - PKG_LIST=$(go list ./... | grep -v internal/clustertests | paste -s -d, -)
+#     - go test -coverprofile=coverage.out -covermode=atomic -coverpkg=${PKG_LIST} ./...
+#   artifacts:
+#     paths:
+#       - coverage.out


-run go tests with output:
-  stage: test
-  image: golang:$GOVERSION
-  rules:
-    - if: '$CI_PIPELINE_SOURCE == "push"'
-  script:
-    - echo "Running featurebase unit tests to capture JSON output..."
-    - go test -json > test-report.out
-  artifacts:
-    paths:
-      - test-report.out
+# run go tests with output:
+#   stage: test
+#   image: golang:$GOVERSION
+#   rules:
+#     - if: '$CI_PIPELINE_SOURCE == "push"'
+#   script:
+#     - echo "Running featurebase unit tests to capture JSON output..."
+#     - go test -json > test-report.out
+#   artifacts:
+#     paths:
+#       - test-report.out

-upload to sonarcloud:
-  stage: test
-  image: sonarsource/sonar-scanner-cli:4.6
-  variables:
-    SONAR_TOKEN: $SONAR_TOKEN
-  rules:
-    - if: '$CI_PIPELINE_SOURCE == "push"'
-  script:
-    - sonar-scanner -Dsonar.projectKey=molecula_featurebase -Dsonar.organization=molecula -Dsonar.sources=. -Dsonar.host.url=https://sonarcloud.io -Dsonar.go.coverage.reportPaths=coverage.out -Dsonar.go.tests.reportPaths=test-report.out  -Dsonar.javascript.lcov.reportPaths=lattice/coverage/lcov.info
-  needs:
-    - job: run go tests
-    - job: run go tests with output
-    - job: run jest tests 
+# upload to sonarcloud:
+#   stage: test
+#   image: sonarsource/sonar-scanner-cli:4.6
+#   variables:
+#     SONAR_TOKEN: $SONAR_TOKEN
+#   rules:
+#     - if: '$CI_PIPELINE_SOURCE == "push"'
+#   script:
+#     - sonar-scanner -Dsonar.projectKey=molecula_featurebase -Dsonar.organization=molecula -Dsonar.sources=. -Dsonar.host.url=https://sonarcloud.io -Dsonar.go.coverage.reportPaths=coverage.out -Dsonar.go.tests.reportPaths=test-report.out  -Dsonar.javascript.lcov.reportPaths=lattice/coverage/lcov.info
+#   needs:
+#     - job: run go tests
+#     - job: run go tests with output
+#     - job: run jest tests 
    
 build for linux amd64:
  stage: build
  image: golang:$GOVERSION
-  rules:
-    - if: '$CI_PIPELINE_SOURCE == "push"'
+  # TODO: For now, run always
+  # rules:
+  #   - if: '$CI_PIPELINE_SOURCE == "push"'
  script:
    - rm -r lattice
    - tar -xvf lattice.tar.gz 
@ -151,95 +154,137 @@ build for linux arm64:
    paths:
      - featurebase_linux_arm64

-build for darwin amd64:
-  stage: build
-  image: golang:$GOVERSION
-  rules:
-    - if: '$CI_PIPELINE_SOURCE == "push"'
-  script:
-    - rm -r lattice
-    - tar -xvf lattice.tar.gz 
-    - go get -v -u github.com/rakyll/statik 
-    - /go/bin/statik -src=lattice
-    - GOOS="darwin" GOARCH="amd64" make build FLAGS="-o featurebase_darwin_amd64"
-  artifacts:
-    paths:
-      - featurebase_darwin_amd64
+# build for darwin amd64:
+#   stage: build
+#   image: golang:$GOVERSION
+#   rules:
+#     - if: '$CI_PIPELINE_SOURCE == "push"'
+#   script:
+#     - rm -r lattice
+#     - tar -xvf lattice.tar.gz 
+#     - go get -v -u github.com/rakyll/statik 
+#     - /go/bin/statik -src=lattice
+#     - GOOS="darwin" GOARCH="amd64" make build FLAGS="-o featurebase_darwin_amd64"
+#   artifacts:
+#     paths:
+#       - featurebase_darwin_amd64

-build for darwin arm64:
-  stage: build
-  image: golang:$GOVERSION
-  rules:
-    - if: '$CI_PIPELINE_SOURCE == "push"'
-  script:
-    - rm -r lattice
-    - tar -xvf lattice.tar.gz 
-    - go get -v -u github.com/rakyll/statik 
-    - /go/bin/statik -src=lattice
-    - GOOS="darwin" GOARCH="arm64" make build FLAGS="-o featurebase_darwin_arm64"
-  artifacts:
-    paths:
-      - featurebase_darwin_arm64
+# build for darwin arm64:
+#   stage: build
+#   image: golang:$GOVERSION
+#   rules:
+#     - if: '$CI_PIPELINE_SOURCE == "push"'
+#   script:
+#     - rm -r lattice
+#     - tar -xvf lattice.tar.gz 
+#     - go get -v -u github.com/rakyll/statik 
+#     - /go/bin/statik -src=lattice
+#     - GOOS="darwin" GOARCH="arm64" make build FLAGS="-o featurebase_darwin_arm64"
+#   artifacts:
+#     paths:
+#       - featurebase_darwin_arm64

-package for linux amd64:
-  stage: build
-  image: golang:$GOVERSION
-  rules:
-    - if: '$CI_PIPELINE_SOURCE == "push"'
-  variables:
-    GOOS: "linux"
-    GOARCH: "amd64"
-  script:
-    - echo 'deb [trusted=yes] https://repo.goreleaser.com/apt/ /' | tee /etc/apt/sources.list.d/goreleaser.list
-    - apt update && apt install nfpm
-    - make package
-  artifacts:
-    paths:
-      - "*.deb"
-      - "*.rpm"
+# package for linux amd64:
+#   stage: build
+#   image: golang:$GOVERSION
+#   rules:
+#     - if: '$CI_PIPELINE_SOURCE == "push"'
+#   variables:
+#     GOOS: "linux"
+#     GOARCH: "amd64"
+#   script:
+#     - echo 'deb [trusted=yes] https://repo.goreleaser.com/apt/ /' | tee /etc/apt/sources.list.d/goreleaser.list
+#     - apt update && apt install nfpm
+#     - make package
+#   artifacts:
+#     paths:
+#       - "*.deb"
+#       - "*.rpm"

 # Build a FB Docker image with CI/CD and push to the GitLab registry.
-build container fb:
-  image: docker:stable
-  stage: build
-  needs:
-    - "build for linux amd64"
-  tags:
-    - shell
-  rules:
-    - if: '$CI_PIPELINE_SOURCE == "push"'
-  before_script:    
-    - echo "${DOCKER_DEPLOY_TOKEN}" | docker login -u ${DOCKER_DEPLOY_USER} --password-stdin ${CI_REGISTRY}
-  script:    
-    - tag=${CI_REGISTRY_IMAGE}/server:${CI_COMMIT_REF_SLUG}
-    - docker build --build-arg GO_VERSION=$GOVERSION -t $tag -f .gitlab/Dockerfile .
-    - docker push $tag
-    - echo Created docker featurebase image with tag "$tag"
+# build container fb:
+#   image: docker:stable
+#   stage: build
+#   needs:
+#     - "build for linux amd64"
+#   tags:
+#     - shell
+#   rules:
+#     - if: '$CI_PIPELINE_SOURCE == "push"'
+#   before_script:    
+#     - echo "${DOCKER_DEPLOY_TOKEN}" | docker login -u ${DOCKER_DEPLOY_USER} --password-stdin ${CI_REGISTRY}
+#   script:    
+#     - tag=${CI_REGISTRY_IMAGE}/server:${CI_COMMIT_REF_SLUG}
+#     - docker build --build-arg GO_VERSION=$GOVERSION -t $tag -f .gitlab/Dockerfile .
+#     - docker push $tag
+#     - echo Created docker featurebase image with tag "$tag"

 # deploy EC2 instance, configure and run featurebase 
-deploy node for linux amd64:
-  stage: integration
+# deploy node for linux amd64:
+#   stage: integration
+#   image: registry.gitlab.com/gitlab-org/cloud-deploy/aws-base:latest
+#   variables:
+#     PROFILE: "default"
+#     AWS_SSH_PRIVATE_KEY: $AWS_SSH_PRIVATE_KEY
+#   rules:
+#     - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'
+#   before_script:
+#     - aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID
+#     - aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY
+#     - aws configure set region "us-east-2"
+#     - aws configure set aws_profile $PROFILE
+#     - echo $AWS_SSH_PRIVATE_KEY > gitlab-featurebase-dev.pem
+#     - chmod 400 gitlab-featurebase-dev.pem
+#     - 'which ssh-agent || ( apt-get update -y && apt-get install openssh-client -y )'
+#     - eval `ssh-agent -s`
+#     - mkdir -p ~/.ssh 
+#     - echo "$AWS_SSH_PRIVATE_KEY" | ssh-add -
+#     - chmod 700 /root/.ssh
+#     - '[[ -f /.dockerenv ]] && echo -e "Host *\n\tStrictHostKeyChecking no\n\n" > ~/.ssh/config'
+#     - apt update && apt -y install jq
+#   script:
+#     - ./qa/scripts/deployNode.sh $PROFILE
+#   needs:
+#     - job: build for linux amd64
+
+gauntlet:
+  stage: gauntlet
+  timeout: 4h
  image: registry.gitlab.com/gitlab-org/cloud-deploy/aws-base:latest
  variables:
    PROFILE: "default"
-    AWS_SSH_PRIVATE_KEY: $AWS_SSH_PRIVATE_KEY
-  rules:
-    - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'
+    AWS_SSH_PRIVATE_KEY: $AWS_FBCI_SSH_KEY
+    AWS_ACCESS_KEY_ID: $AWS_FBCI_ACCESS_KEY_ID
+    AWS_SECRET_ACCESS_KEY: $AWS_FBCI_SECRET_ACCESS_KEY
+  # TODO: For now, run always
+  # rules:
+  #   - if: '$CI_PIPELINE_SOURCE == "schedule" && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'
  before_script:
-    - aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID
-    - aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY
+    - apt-get update && apt-get install -y gnupg software-properties-common curl git
+    - curl -fsSL https://apt.releases.hashicorp.com/gpg | apt-key add -
+    - apt-add-repository "deb [arch=amd64] https://apt.releases.hashicorp.com $(lsb_release -cs) main"
+    - apt-get update && apt-get install terraform
+    - aws configure set aws_access_key_id $AWS_FBCI_ACCESS_KEY_ID
+    - aws configure set aws_secret_access_key $AWS_FBCI_SECRET_ACCESS_KEY
    - aws configure set region "us-east-2"
    - aws configure set aws_profile $PROFILE
-    - echo $AWS_SSH_PRIVATE_KEY > gitlab-featurebase-dev.pem
-    - chmod 400 gitlab-featurebase-dev.pem
+    - echo $AWS_FBCI_SSH_KEY > gitlab-featurebase-ci.pem
+    - chmod 400 gitlab-featurebase-ci.pem
    - 'which ssh-agent || ( apt-get update -y && apt-get install openssh-client -y )'
-    - eval `ssh-agent -s`
-    - mkdir -p ~/.ssh 
-    - echo "$AWS_SSH_PRIVATE_KEY" | ssh-add -
+    - eval $(ssh-agent -s)
+    - mkdir -p ~/.ssh
+    - echo $AWS_FBCI_SSH_KEY > /root/.ssh/gitlab-featurebase-ci.pem
+    - chmod 400 /root/.ssh/gitlab-featurebase-ci.pem
+    - echo "$AWS_FBCI_SSH_KEY" | ssh-add -
    - chmod 700 /root/.ssh
    - '[[ -f /.dockerenv ]] && echo -e "Host *\n\tStrictHostKeyChecking no\n\n" > ~/.ssh/config'
-    - apt update && apt -y install jq
+    - apt update && apt -y install jq wget
+    - wget https://go.dev/dl/go1.17.5.linux-amd64.tar.gz
+    - tar -C /usr/local -xzf go1.17.5.linux-amd64.tar.gz
+    - export PATH=$PATH:/usr/local/go/bin
  script:
-    - ./qa/scripts/deployNode.sh $PROFILE
-  needs:
-    - job: build for linux amd64
+    - ./qa/scripts/setupSamsungGauntlet.sh
+    - ./qa/scripts/testSamsungGauntlet.sh
+  after_script:
+    - ./qa/scripts/teardownSamsungGauntlet.sh
+  needs: ["build for linux arm64"]
--- a/qa/scripts/deployNode.sh
+++ b/qa/scripts/deployNode.sh
@ -2,6 +2,9 @@

 # To run script: ./deployNode.sh $PROFILE

+# default to the VPC initially created
+VPC=${VPC:-vpc-0582f594d7d2ca2d4}
+
 function deploy_node() {
    # get AMI, security group and subnet ID 
    AMI=$(aws ssm get-parameters --names /aws/service/ami-amazon-linux-latest/amzn2-ami-hvm-x86_64-ebs --query 'Parameters[0].[Value]' --output text --profile $PROFILE)
@ -10,13 +13,13 @@ function deploy_node() {
        exit 1
    fi

-    SECURITY_GROUP=$(aws ec2 describe-security-groups --filters Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978 Name=group-name,Values=default --query 'SecurityGroups[*].[GroupId]' --output text --profile $PROFILE)
+    SECURITY_GROUP=$(aws ec2 describe-security-groups --filters "Name=vpc-id,Values=$VPC" Name=group-name,Values=default --query 'SecurityGroups[*].[GroupId]' --output text --profile $PROFILE)
    if [[ $? > 0 ]]; then 
        echo "aws session manager failed to find security group"
        exit 1
    fi
    
-    SUBNET_ID=$(aws ec2 describe-subnets --filters 'Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978' 'Name=availability-zone,Values=us-east-2a' --query 'Subnets[0].SubnetId' --output text --profile $PROFILE)
+    SUBNET_ID=$(aws ec2 describe-subnets --filters "Name=vpc-id,Values=$VPC" 'Name=availability-zone,Values=us-east-2a' --query 'Subnets[0].SubnetId' --output text --profile $PROFILE)
    if [[ $? > 0 ]]; then 
        echo "aws session manager failed to find subnet ID"
        exit 1
--- a/qa/scripts/deployPerf.sh
+++ b/qa/scripts/deployPerf.sh
@ -1,152 +0,0 @@
-#!/bin/bash
-
-# To run script: ./deployNode.sh $PROFILE
-
-# default to the VPC initially created
-VPC=${VPC:-vpc-0582f594d7d2ca2d4}
-
-INSTANCE_ID=""
-
-function log() {
-    fmt=$1
-    shift
-    printf "$fmt\n" "$@" >&2
-}
-
-function terminate() {
-    if [ -n "$INSTANCE_ID" ]; then
-        log "shutting down instance ID %s" "$INSTANCE_ID"
-        doAws ec2 terminate-instances --instance-ids "$INSTANCE_ID"
-    fi
-}
-
-# shut down instance on exit if we have created one
-trap terminate 0
-
-function doAws() {
-        aws "$@" --profile "$PROFILE"
-}
-
-# SCP files to ec2-user@$IP
-function doScp() {
-    scp -o StrictHostKeyChecking=no -i ~/.ssh/gitlab-featurebase-ci.pem "$@" ec2-user@$IP:.
-}
-
-# Run command as ec2-user@$IP
-function doSsh() {
-    ssh -o StrictHostKeyChecking=no -i ~/.ssh/gitlab-featurebase-ci.pem ec2-user@$IP "$@"
-}
-
-# We need an amd64 Linux binary
-function prep_binary() {
-    GOOS=linux GOARCH=amd64 make build && mv featurebase featurebase_linux_amd64
-}
-
-function check_running() {
-    existing_states=$(doAws ec2 describe-instances --query 'Reservations[*].Instances[*].State.Name' --output text)
-    log "instance states: %s" "$existing_states"
-    case " $existing_states " in
-    *" running "*)
-        log "existing instance in running state, not restarting"
-        return 1
-        ;;
-    esac
-}
-
-function get_config() {
-    # get AMI, security group and subnet ID
-    AMI=$(doAws ssm get-parameters --names "/aws/service/ami-amazon-linux-latest/amzn2-ami-hvm-x86_64-ebs" --query 'Parameters[0].[Value]' --output text)
-    if [[ $? > 0 ]]; then
-        log "aws session manager failed to find AMI"
-        return 1
-    fi
-
-    SECURITY_GROUP=$(doAws ec2 describe-security-groups --filters "Name=vpc-id,Values=$VPC" 'Name=group-name,Values=default' --query 'SecurityGroups[*].[GroupId]' --output text)
-    if [[ $? > 0 || -z "$SECURITY_GROUP" ]]; then
-        log "aws session manager failed to find security group"
-        return 1
-    fi
-
-    SUBNET_ID=$(aws ec2 describe-subnets --filters 'Name=vpc-id,Values='"$VPC" 'Name=availability-zone,Values=us-east-2a' 'Name=tag:Name,Values=fbci-vpc-public-us-east-2a' --query 'Subnets[0].SubnetId' --output text --profile $PROFILE)
-    if [[ $? > 0 || -z "$SUBNET_ID" ]]; then
-        log "aws session manager failed to find subnet ID"
-        return 1
-    fi
-}
-
-function deploy_node() {
-    # launch EC2 instance and get instance ID
-    aws ec2 run-instances --image-id "$AMI" --instance-type "$INSTANCE" --security-group-ids "$SECURITY_GROUP" --subnet-id "$SUBNET_ID" --key-name gitlab-featurebase-ci --tag-specifications 'ResourceType=instance,Tags=[{Key=Name,Value=linux-amd64-node}]' --profile $PROFILE --user-data file://./qa/scripts/cloud-init.sh --iam-instance-profile Name=featurebase-ci-ssm > config.json
-    if [[ $? > 0 ]]; then
-        echo "aws run-instances failed to launch a new EC2 instance"
-        exit 1
-    fi
-
-    INSTANCE_ID=$(jq '.Instances | .[0] |.InstanceId' config.json | tr -d '"')
-    echo "aws run-instances succeeded in launching a new EC2 instance with instance ID: " $INSTANCE_ID
-}
-
-function initialize_featurebase() {
-    # get IP for node
-    for i in {0..24}
-    do
-        IP=$(doAws ec2 describe-instances --instance-ids $INSTANCE_ID --filters 'Name=instance-state-name,Values=running' --query 'Reservations[*].Instances[*].PublicIpAddress' --output text)
-        if [ -n "$IP" ]; then
-            log "Public IP for EC2 instance: %s" "$IP"
-            break
-        fi
-
-        if [[ $? > 0 ]]; then
-            log "aws cli describe-instances command failed to find public IP"
-            return 1
-        fi
-
-        sleep 5
-    done
-
-    sleep 60 # to allow enough time for node to be ready for use
-
-    # copy featurebase binary and files to ec2 instance
-    doScp featurebase_linux_amd64 ./qa/scripts/featurebase.conf ./qa/scripts/featurebase.service ./qa/scripts/setup.sh ./qa/scripts/regression.sh ./qa/scripts/perf.sh
-    if [[ $? > 0 ]]; then
-        log "scp of featurebase binary, service and config files to EC2 instance failed"
-        return 1
-    fi
-
-    # execute script to configure featurebase on the EC2 node
-    aws ssm send-command --document-name "AWS-RunShellScript" --instance-ids $INSTANCE_ID --parameters commands="sudo ./setup.sh" --profile $PROFILE --region $REGION
-    if [[ $? > 0 ]]; then
-        echo "aws cli session manager send-command failed"
-        terminate_node
-        exit 1
-    fi
-
-    # doSsh bash ./setup.sh || return 1
-    doSsh bash ./regression.sh || return 1
-    doSsh bash ./perf.sh || return 1
-}
-
-# Pass variables to shell script
-PROFILE=$1
-shift
-
-# set some variables
-INSTANCE="t3a.large"
-REGION="us-east-2"
-
-# check for existing copies; no point in running if one's already up
-check_running || exit 1
-
-# Prep featurebase binary
-prep_binary || exit 1
-
-# Obtain subnet info, etc.
-get_config || exit 1
-
-# get AMI, security group and subnet for EC2 instance,
-# launch instance, save instance Id and run cloud-init to set up node env
-deploy_node || exit 1
-
-# Get IP for instance, scp featurebase binary, config and service files;
-# set up featurebase config in node
-initialize_featurebase || exit 1
--- a/qa/scripts/deploySingleNodeCluster.sh
+++ b/qa/scripts/deploySingleNodeCluster.sh
@ -0,0 +1,28 @@
+#!/bin/bash
+
+# To run script: ./deploySingleNodeCluster.sh
+# requires TF_VAR_gitlab_token env var to be set
+
+echo “$(pwd)”
+
+pushd ./qa/tf/ci/singlenode
+export TF_IN_AUTOMATION=1
+terraform init -input=false
+terraform apply -input=false -auto-approve
+popd
+
+# configure Featurebase
+
+# step 1a: get IPs of the cluster
+
+
+
+# step 1b: get IPs of the ingest nodes
+
+# step 2: write a featurebase.conf file
+
+# step 3: write featurebase.service
+
+# step 4: start featurebase
+
+# step 5: verify featurebase running
--- a/qa/scripts/ingestWorkload.sh
+++ b/qa/scripts/ingestWorkload.sh
@ -1,7 +1,14 @@
 #!/usr/bin/env bash

+# path for featurebase binary
+FEATUREBASE_PATH=/usr/local/bin
+
+# path for directory with csv directory files for all fields to be ingested 
+CSV_DIR_PATH=/data
+
+
 # To run:
-# ./ingestWorkload.sh {Path for featurebase binary} {Local host & port for featurebase} {Path for directory with csv files} {initialize flag}
+# ./ingestWorkload.sh {Local host & port for featurebase} {initialize flag}

 function delete_field {
    if (($INITIALIZE == 0));
@ -30,18 +37,10 @@ function ingest_set_field {
    $FEATUREBASE_PATH/featurebase import --host $HOST -i $INDEX -f $FIELD $CSV_FILE 
 }

-# path for featurebase binary
-FEATUREBASE_PATH=$1
-shift
-
 # featurebase host & port
 HOST=$1
 shift

-# path for directory with csv directory files for all fields to be ingested 
-CSV_DIR_PATH=$1
-shift 
-
 # intialize flag - 0:disabled, 1:enabled - creates the index and fields for testing
 INITIALIZE=$1
 shift 
--- a/qa/scripts/perf.sh
+++ b/qa/scripts/perf.sh
@ -1,2 +1,3 @@
 #!/bin/bash
 echo >&2 "performance testing"
+time ./simulacraData
--- a/qa/scripts/regression.sh
+++ b/qa/scripts/regression.sh
@ -1,2 +0,0 @@
-#!/bin/bash
-echo >&2 "regression testing"
--- a/qa/scripts/runSamsungGauntlet.sh
+++ b/qa/scripts/runSamsungGauntlet.sh
@ -0,0 +1,8 @@
+#!/bin/bash
+
+
+#openssl rand -base64 32 | tr -d /=+ | cut -c -16
+
+./setupSamsungGauntlet.sh
+./testSamsungGauntlet.sh
+./teardownSamsungGauntlet.sh
--- a/qa/scripts/setup.sh
+++ b/qa/scripts/setup.sh
@ -1,14 +0,0 @@
-#!/bin/bash
-mv /home/ec2-user/featurebase_linux_amd64 /usr/local/bin/featurebase
-mv /home/ec2-user/featurebase.conf /etc/
-mv /home/ec2-user/featurebase.service /etc/systemd/system/
-adduser molecula
-sudo mkdir /var/log/molecula
-sudo chown molecula /var/log/molecula
-sudo mkdir -p /opt/molecula/featurebase
-sudo chown molecula /opt/molecula/featurebase
-systemctl daemon-reload
-sudo systemctl start featurebase
-sudo systemctl enable featurebase
-sudo systemctl status featurebase
-curl localhost:10101
--- a/qa/scripts/setupSamsungGauntlet.sh
+++ b/qa/scripts/setupSamsungGauntlet.sh
@ -0,0 +1,41 @@
+#!/bin/bash
+
+# To run script: ./setupSamsungGauntlet.sh
+# requires TF_VAR_gitlab_token env var to be set
+
+pushd ./qa/tf/gauntlet/samsung
+export TF_IN_AUTOMATION=1
+echo "Running terraform init..."
+terraform init -input=false
+echo "Running terraform apply..."
+terraform apply -input=false -auto-approve
+terraform output -json > samsung-gauntlet.json
+popd
+
+# get the bastion host
+BASTION=$(cat ./qa/tf/gauntlet/samsung/samsung-gauntlet.json | jq -r '[.ingest_ips][0]["value"][0]')
+echo "using bastion ${BASTION}"
+
+NODE=$(cat ./qa/tf/gauntlet/samsung/samsung-gauntlet.json | jq -r '[.data_node_ips][0]["value"][0]')
+echo "using node ${NODE}"
+
+# remember that the nodes will take at least 2 mins to be up and going and finish cloud-init
+#while true
+#do
+#  nc -G 2 -w 1 $BASTION 22
+#  if [ $? -eq 0 ]
+#  then
+#    break
+#  fi
+#done
+sleep 150
+
+# verify featurebase running
+ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o "StrictHostKeyChecking no" ec2-user@${BASTION} "curl -s http://${NODE}:10101/status" 
+if (( $? != 0 )) 
+then 
+    echo "Featurebase cluster not running"
+    exit 1
+fi
+
+
--- a/qa/scripts/teardownSamsungGauntlet.sh
+++ b/qa/scripts/teardownSamsungGauntlet.sh
@ -0,0 +1,8 @@
+#!/bin/bash
+
+# To run script: ./teardownSamsungGauntlet.sh
+# requires TF_VAR_gitlab_token env var to be set
+
+cd qa/tf/gauntlet/samsung
+export TF_IN_AUTOMATION=1
+terraform destroy -auto-approve
--- a/qa/scripts/testSamsungGauntlet.sh
+++ b/qa/scripts/testSamsungGauntlet.sh
@ -0,0 +1,48 @@
+#!/bin/bash
+
+# get the bastion host
+BASTION=$(cat ./qa/tf/gauntlet/samsung/samsung-gauntlet.json | jq -r '[.ingest_ips][0]["value"][0]')
+echo "using bastion ${BASTION}"
+
+NODE=$(cat ./qa/tf/gauntlet/samsung/samsung-gauntlet.json | jq -r '[.data_node_ips][0]["value"][0]')
+echo "using node ${NODE}"
+
+# generate csv files
+GOOS=linux GOARCH=arm64 go build ./qa/simulacraData/...
+scp -i ~/.ssh/gitlab-featurebase-ci.pem simulacraData ec2-user@${BASTION}:/data
+if (( $? != 0 )) 
+then 
+    echo "Copy failed"
+    exit 1
+fi
+
+ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem ec2-user@${BASTION} "cd /data && /data/simulacraData" 
+if (( $? != 0 )) 
+then 
+    echo "Making big files failed"
+    exit 1
+fi
+
+# ingest these files the way that samsung does it
+scp -i ~/.ssh/gitlab-featurebase-ci.pem ./qa/scripts/testSamsungPayload.sh ec2-user@${BASTION}:
+if (( $? != 0 )) 
+then 
+    echo "Copy ingest script failed"
+    exit 1
+fi
+
+ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem ec2-user@${BASTION} "./testSamsungPayload.sh http://${NODE}:10101 1" 
+if (( $? != 0 )) 
+then 
+    echo "Running 1 testSamsungPayload.sh failed"
+    exit 1
+fi
+
+ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem ec2-user@${BASTION} "./testSamsungPayload.sh http://${NODE}:10101 0" 
+if (( $? != 0 )) 
+then 
+    echo "Running 0 testSamsungPayload.sh failed"
+    exit 1
+fi
+
+# query workload that runs
--- a/qa/scripts/testSamsungPayload.sh
+++ b/qa/scripts/testSamsungPayload.sh
@ -0,0 +1,84 @@
+#!/usr/bin/env bash
+
+# path for featurebase binary
+FEATUREBASE_PATH=/usr/local/bin
+
+# path for directory with csv directory files for all fields to be ingested 
+CSV_DIR_PATH=/data
+
+
+# To run:
+# ./testSamsungPayload.sh {Local host & port for featurebase} {initialize flag}
+
+function delete_field {
+    if (($INITIALIZE == 0));
+    then 
+        curl -XDELETE $HOST/index/$INDEX/field/$FIELD
+    fi
+}
+
+# Script to replicate samsung workload of deleting and re-ingesting fields every night 
+# outline delete and re-ingest workload 
+function ingest_int_field { 
+    delete_field
+    curl -XPOST $HOST/index/$INDEX/field/$FIELD -d '{"options": {"type": "int", "min": 0, "max":'$MAX'}}'
+    $FEATUREBASE_PATH/featurebase import --host $HOST -i $INDEX -f $FIELD $CSV_FILE 
+}
+
+function ingest_time_field {
+    delete_field
+    curl -XPOST $HOST/index/$INDEX/field/$FIELD -d '{"options": {"keys": true, "type": "time", "timeQuantum": "YMD"}}'
+    $FEATUREBASE_PATH/featurebase import --host $HOST -i $INDEX -f $FIELD $CSV_FILE 
+}
+
+function ingest_set_field {
+    delete_field
+    curl -XPOST $HOST/index/$INDEX/field/$FIELD -d '{"options": {"keys": true}}'
+    $FEATUREBASE_PATH/featurebase import --host $HOST -i $INDEX -f $FIELD $CSV_FILE 
+}
+
+# featurebase host & port
+HOST=$1
+shift
+
+# intialize flag - 0:disabled, 1:enabled - creates the index and fields for testing
+INITIALIZE=$1
+shift 
+
+# get a list of csv files in the directory 
+CSV_FILES=`ls $CSV_DIR_PATH/*.csv`
+
+# assign index name 
+INDEX="samsung"
+if (($INITIALIZE == 1));
+then 
+    curl -XPOST $HOST/index/$INDEX
+fi
+
+# perform delete and re-ingest for all fields 
+for CSV_FILE in ${CSV_FILES[@]}
+    do 
+        # get field name from csv file path 
+        FIELD="$(basename $CSV_FILE .csv)"
+        if [[ "$FIELD" == *"age"* ]];
+        then 
+            MAX=100 
+            ingest_int_field
+        elif [[ "$FIELD" == *"identifier"* ]];
+        then 
+            MAX=$((2**63 - 1)) # compute max value for 64bit
+            ingest_int_field
+        elif [[ "$FIELD" == *"ip"* ]];
+        then 
+            MAX=$((2**31 - 1)) # compute max value for 32bit
+            ingest_int_field
+        elif [[ "$FIELD" == *"time"* ]];
+        then 
+            ingest_time_field
+        else
+            ingest_set_field
+        fi
+    done
+
+
+
--- a/qa/tf/.modules/featurebase-cluster/README.md
+++ b/qa/tf/.modules/featurebase-cluster/README.md
@ -0,0 +1,37 @@
+# Summary
+
+This module provisions a VPC, subnets, instances, keys, and security groups needed for a basic featurebase cluster running in AWS. It is meant to be used as a module. For example:
+
+```hcl
+module "featurebase" {
+  source "/path/to/module/"
+  cluster_prefix = "sprockets"
+  azs = ["us-east-1a", "us-east-1b", "us-east-1c"]
+}
+```
+
+The path to the module is wherever the `featurebase-cloud` directory is. So if you have put it in `/var/opt/terraform/modules/featurebase-cloud` then calling the module would look like:
+
+```hcl
+module "featurebase" {
+  source "/var/opt/terraform/modules/featurebase-cloud"
+  cluster_prefix = "sprockets"
+}
+```
+
+Much more is configurable; for a complete list, look in `variables.tf`. Reasonable defaults have been set.
+
+## AWS Access
+
+Please make sure you have set up your AWS access in either environment variables, or in the credentials file.
+
+Some useful links for this are:
+
+AWS Environment Variables <https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-envvars.html>
+
+## State
+
+State is currently kept locally, for as this is intended for PoCs. It can be stored in a remote s3 or GCS bucket if desired.
+
+
+ 
--- a/qa/tf/.modules/featurebase-cluster/main.tf
+++ b/qa/tf/.modules/featurebase-cluster/main.tf
@ -0,0 +1,229 @@
+data "aws_ami" "amazon_linux_2" {
+  most_recent = true
+  owners      = ["amazon"]
+  filter {
+    name   = "name"
+    values = ["amzn2-ami-hvm-*"]
+  }
+
+  filter {
+    name   = "virtualization-type"
+    values = ["hvm"]
+  }
+
+  filter {
+    name   = "architecture"
+    values = ["arm64"]
+  }
+}
+
+resource "aws_instance" "fb_cluster_nodes" {
+  count                  = var.fb_data_node_count
+  ami                    = data.aws_ami.amazon_linux_2.id
+  instance_type          = var.fb_data_node_type
+  key_name               = aws_key_pair.gitlab-featurebase-ci.key_name
+  vpc_security_group_ids = [aws_security_group.featurebase.id]
+  monitoring             = true
+  subnet_id              = var.subnet != "" ? var.subnet : module.vpc.private_subnets[count.index % length(module.vpc.private_subnets)]
+  availability_zone      = var.zone != "" ? var.zone : var.azs[count.index % length(var.azs)]
+  iam_instance_profile   = "${aws_iam_instance_profile.fb_cluster_node_profile.name}"
+
+  root_block_device {
+    volume_type = "gp3"
+    volume_size = 20
+  }
+
+  ebs_block_device {
+    device_name = "/dev/sdb"
+    volume_type = var.fb_data_disk_type
+    volume_size = var.fb_data_disk_size_gb
+    iops        = var.fb_data_disk_iops
+  }
+
+  tags = {
+    Prefix = "${var.cluster_prefix}"
+    Name   = "${var.cluster_prefix}-featurebase-cluster-${count.index}"
+    Role   = "cluster_node"
+  }
+
+  user_data = base64encode(templatefile("${path.module}/setup_cluster_node.sh.tpl", { gitlab_token = var.gitlab_token, cluster_prefix = var.cluster_prefix, node_count = var.fb_data_node_count, fb_cluster_replica_count = var.fb_cluster_replica_count, region = var.region }))
+}
+
+resource "aws_instance" "fb_ingest" {
+  count                       = var.fb_ingest_node_count
+  ami                         = data.aws_ami.amazon_linux_2.id
+  key_name                    = aws_key_pair.gitlab-featurebase-ci.key_name
+  vpc_security_group_ids      = [aws_security_group.ingest.id]
+  instance_type               = var.fb_ingest_type
+  associate_public_ip_address = true
+  monitoring                  = true
+  subnet_id                   = var.subnet != "" ? var.subnet : module.vpc.public_subnets[count.index % length(module.vpc.public_subnets)]
+  availability_zone           = var.zone != "" ? var.zone : var.azs[count.index % length(var.azs)]
+  iam_instance_profile        = "${aws_iam_instance_profile.fb_cluster_node_profile.name}"
+
+  root_block_device {
+    volume_type = "gp3"
+    volume_size = 20
+  }
+
+  ebs_block_device {
+    device_name = "/dev/sdb"
+    volume_type = var.fb_ingest_disk_type
+    volume_size = var.fb_ingest_disk_size_gb
+    iops        = var.fb_ingest_disk_iops
+  }
+
+  tags = {
+    Prefix = "${var.cluster_prefix}"
+    Name   = "${var.cluster_prefix}-featurebase-ingest-${count.index}"
+    Role   = "ingest_node"
+  }
+
+  user_data = base64encode(templatefile("${path.module}/setup_ingest_node.sh.tpl", { gitlab_token = var.gitlab_token, cluster_prefix = var.cluster_prefix, node_count = var.fb_ingest_node_count, this_node = count.index, region = var.region }))
+}
+
+resource "aws_key_pair" "gitlab-featurebase-ci" {
+  key_name   = "gitlab-featurebase-ci"
+  public_key = "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC91hhpVHNonAG7ku2ugpxEskf9KHeyHJPQJT26OHrMUw7R+T5A8TjqSzTau07sXQ/E9SO3ebV8SJ5PqeaQOnQB8VEvVNK0DjQH7ppvNg1Rfs42FZT9ttzTMvOjsSbK3vZTHXdoKQEdC9NxBwSkFIRGQojK1HUOq9xGrw31fA1OjSwlpLcbx7yyg18lcqW6UOptnVR8U9Yy9qQ5jZF1HtkQ6L9J+gv4o1UyNAUK2bopeGiXpBc3PQ/CFaFT2h/aqLBP66qAHsHVyAFD3PIRtplC5EHa8jXDgLacEls0uF7Q3kRPxvzcuo4g4VkOn1rDy9qH3vd2hT3aKVnM73FIDUiL"
+}
+
+resource "aws_security_group" "featurebase" {
+  name        = "allow_featurebase"
+  description = "Allow featurebase inbound traffic"
+  vpc_id      = module.vpc.vpc_id
+
+  ingress {
+    description = "TLS from Internal"
+    from_port   = 10101
+    to_port     = 10101
+    protocol    = "tcp"
+    cidr_blocks = [module.vpc.vpc_cidr_block]
+  }
+
+  ingress {
+
+    description = "GRPC from Internal"
+    from_port   = 20101
+    to_port     = 20101
+    protocol    = "tcp"
+    cidr_blocks = [module.vpc.vpc_cidr_block]
+  }
+
+  ingress {
+    description = "PostgreSQL from Internal"
+    from_port   = 55432
+    to_port     = 55432
+    protocol    = "tcp"
+    cidr_blocks = [module.vpc.vpc_cidr_block]
+  }
+
+  ingress {
+    description = "etcd from internal"
+    from_port   = 10301
+    to_port     = 10301
+    protocol    = "tcp"
+    cidr_blocks = [module.vpc.vpc_cidr_block]
+  }
+
+  ingress {
+    description = "etcd from internal 2"
+    from_port   = 10401
+    to_port     = 10401
+    protocol    = "tcp"
+    cidr_blocks = [module.vpc.vpc_cidr_block]
+  }
+
+  ingress {
+    description      = "SSH"
+    from_port        = 22
+    to_port          = 22
+    protocol         = "tcp"
+    cidr_blocks      = ["0.0.0.0/0"]
+    ipv6_cidr_blocks = ["::/0"]
+  }
+
+  egress {
+    from_port        = 0
+    to_port          = 0
+    protocol         = "-1"
+    cidr_blocks      = ["0.0.0.0/0"]
+    ipv6_cidr_blocks = ["::/0"]
+  }
+
+  tags = {
+    Name = "allow_featurebase"
+  }
+}
+
+resource "aws_security_group" "ingest" {
+  name        = "allow_ingest"
+  description = "Allow ingest inbound traffic"
+  vpc_id      = module.vpc.vpc_id
+
+  ingress {
+    from_port        = 10101
+    to_port          = 10101
+    protocol         = "tcp"
+    cidr_blocks      = ["0.0.0.0/0"]
+    ipv6_cidr_blocks = ["::/0"]
+  }
+
+  ingress {
+    description      = "SSH"
+    from_port        = 22
+    to_port          = 22
+    protocol         = "tcp"
+    cidr_blocks      = ["0.0.0.0/0"]
+    ipv6_cidr_blocks = ["::/0"]
+  }
+
+  egress {
+    from_port        = 0
+    to_port          = 0
+    protocol         = "-1"
+    cidr_blocks      = ["0.0.0.0/0"]
+    ipv6_cidr_blocks = ["::/0"]
+  }
+
+  tags = {
+    Name = "allow_ingest"
+  }
+}
+
+resource "aws_iam_instance_profile" "fb_cluster_node_profile" {
+  name = "fb_cluster_node_profile"
+  role = aws_iam_role.fb_cluster_node_role.name
+}
+
+resource "aws_iam_role" "fb_cluster_node_role" {
+  name = "fb_cluster_node"
+
+  assume_role_policy = jsonencode({
+    Version = "2012-10-17"
+    Statement = [
+      {
+        Action = "sts:AssumeRole"
+        Effect = "Allow"
+        Sid    = ""
+        Principal = {
+          Service = "ec2.amazonaws.com"
+        }
+      },
+    ]
+  })
+
+  inline_policy {
+    name = "ec2_read_all"
+    policy = jsonencode({
+      Version = "2012-10-17"
+      Statement = [
+        {
+          Action   = ["ec2:Describe*"]
+          Effect   = "Allow"
+          Resource = "*"
+        },
+      ]
+    })
+  }
+
+}
--- a/qa/tf/.modules/featurebase-cluster/outputs.tf
+++ b/qa/tf/.modules/featurebase-cluster/outputs.tf
@ -0,0 +1,7 @@
+output "ingest_ips" {
+    value = aws_instance.fb_ingest.*.public_ip
+}
+
+output "data_node_ips" {
+    value = aws_instance.fb_cluster_nodes.*.private_ip
+}
--- a/qa/tf/.modules/featurebase-cluster/provider.tf
+++ b/qa/tf/.modules/featurebase-cluster/provider.tf
@ -0,0 +1,11 @@
+terraform {
+  required_version = ">= 0.13.1"
+
+  required_providers {
+    aws = {
+      source  = "hashicorp/aws"
+      version = ">= 3.38.0"
+    }
+  }
+}
+
--- a/qa/tf/.modules/featurebase-cluster/setup_cluster_node.sh.tpl
+++ b/qa/tf/.modules/featurebase-cluster/setup_cluster_node.sh.tpl
@ -0,0 +1,188 @@
+#!/bin/bash
+
+#path to the featurebase.conf file
+CONFIG_FILE_PATH="/etc/featurebase.conf"
+#path to the featurebase.service file
+SERVICE_FILE_PATH="/etc/systemd/system/featurebase.service"
+
+AWS_INSTANCE_ID=""
+#IP of this node
+PRIVATE_IP=""
+PRIVATE_IP_INDEX=-1
+#IPs of the cluster
+CLUSTER_IPS=""
+
+get_aws_instance_id() {
+    echo "Getting AWS instance ID..."
+    while true
+    do
+        curl -s http://169.254.169.254/latest/meta-data/instance-id > /dev/null
+        if [ $? -eq 0 ]
+        then
+            break
+        fi
+    done
+    AWS_INSTANCE_ID=`curl http://169.254.169.254/latest/meta-data/instance-id`
+    echo "AWS instance ID is: $${AWS_INSTANCE_ID}"
+}
+
+wait_on_all_cluster_ips() {
+    echo "Waiting on all cluster IPs..."
+    # get IP for node 
+    IPS=$(aws ec2 describe-instances --filters "Name=instance-state-name, Values=running" "Name=tag:Role, Values=cluster_node" "Name=tag:Prefix, Values=${cluster_prefix}" --query 'Reservations[*].Instances[*].PrivateIpAddress' --output text --region ${region})
+    IP_LENGTH=`echo "$IPS" | wc -l`
+
+    for i in {0..24}
+    do 
+        echo "Comparing $${IP_LENGTH} with ${node_count}"
+        if [ $IP_LENGTH == "${node_count}" ]; then
+            echo "Cluster is up after $${i} tries." 
+            break
+        fi
+        sleep 10s
+    done
+
+    if [ $IP_LENGTH != "${node_count}" ]; then
+        echo "Timed out waiting for cluster to be available $${IP_LENGTH} actual nodes compared with ${node_count} desire nodes."
+        exit 1
+    fi
+}
+
+get_private_ip() {
+    echo "Getting private IP address..."
+    PRIVATE_IP=$(aws ec2 describe-instances --filters "Name=instance-state-name, Values=running" "Name=instance-id,Values=$${AWS_INSTANCE_ID}" --query 'Reservations[*].Instances[*].PrivateIpAddress' --output text --region ${region})
+    echo "Private IP is $${PRIVATE_IP}"
+}
+
+get_cluster_ips() {
+    echo "Getting cluster IPs..."
+    # get IP for node 
+    IPS=$(aws ec2 describe-instances --filters "Name=instance-state-name, Values=running" "Name=tag:Role, Values=cluster_node" "Name=tag:Prefix, Values=${cluster_prefix}" --query 'Reservations[*].Instances[*].PrivateIpAddress' --output text --region ${region})
+    IP_LENGTH=`echo "$IPS" | wc -l`
+
+    IFS=$'\n'
+    cnt=0
+    for ip in $IPS
+    do
+        echo $cnt $ip
+        if (($cnt + 1 != $IP_LENGTH)) 
+        then
+            CLUSTER_IPS="$${CLUSTER_IPS}p$${cnt}=http://$ip:10301,"
+        else
+            CLUSTER_IPS="$${CLUSTER_IPS}p$${cnt}=http://$ip:10301"
+        fi
+        echo "comparing $ip to $PRIVATE_IP"
+        if [ "$ip" = "$PRIVATE_IP" ]; then
+            PRIVATE_IP_INDEX=$cnt
+        fi
+        cnt=$((cnt+1))
+    done
+
+    echo "CLUSTER_IPS are: $${CLUSTER_IPS}"
+}
+
+write_featurebase_config_file() {
+    echo "Writing featurebase.conf file..."
+    cat << EOT > $${CONFIG_FILE_PATH}
+name = "p$${PRIVATE_IP_INDEX}"
+bind = "0.0.0.0:10101"
+bind-grpc = "0.0.0.0:20101"
+
+data-dir = "/data/featurebase"
+log-path = "/var/log/molecula/featurebase.log"
+
+max-file-count=900000
+max-map-count=900000
+
+long-query-time = "10s"
+
+[postgres]
+
+    bind = "localhost:55432"
+
+[cluster]
+
+    name = "${cluster_prefix}"
+    replicas = ${fb_cluster_replica_count}
+
+[etcd]
+
+    listen-client-address = "http://$${PRIVATE_IP}:10401"
+    listen-peer-address = "http://$${PRIVATE_IP}:10301"
+    initial-cluster = "$${CLUSTER_IPS}"
+
+[metric]
+
+    service = "prometheus"
+EOT
+
+    echo "featurebase.conf written to $${CONFIG_FILE_PATH}."
+}
+
+write_featurebase_service_file() {
+    echo "Writing featurebase.service file..."
+    cat << EOT > $${SERVICE_FILE_PATH}
+# Not Ansible managed
+
+[Unit]
+Description="Service for FeatureBase"
+
+[Service]
+RestartSec=30
+Restart=on-failure
+EnvironmentFile=
+User=molecula
+ExecStart=/usr/local/bin/featurebase server -c /etc/featurebase.conf
+
+[Install]
+EOT
+
+    echo "featurebase.service written to $${SERVICE_FILE_PATH}."
+    
+}
+
+#get the instance id
+get_aws_instance_id
+
+#copy the script so we can look at it later if needed
+sudo cp /var/lib/cloud/instances/$${AWS_INSTANCE_ID}/user-data.txt /home/ec2-user/setup_cluster_node.sh
+
+#wait for the count of nodes to equal requested nodes
+wait_on_all_cluster_ips
+
+#get private ip
+get_private_ip
+
+#generate cluster ips
+get_cluster_ips
+
+#write the featurebase config file
+write_featurebase_config_file
+
+#write the featurebase service file
+write_featurebase_service_file
+
+#get the featurebase binary and put in in the right spot
+echo "Getting featurebase binary..."
+curl --header "PRIVATE-TOKEN: ${gitlab_token}" -o "/home/ec2-user/featurebase_linux_arm64" https://gitlab.com/api/v4/projects/molecula%2Ffeaturebase/jobs/artifacts/master/raw/featurebase_linux_arm64?job=build%20for%20linux%20arm64
+chown ec2-user:ec2-user "/home/ec2-user/featurebase_linux_arm64"
+chmod ugo+x "/home/ec2-user/featurebase_linux_arm64"
+
+mv /home/ec2-user/featurebase_linux_arm64 /usr/local/bin/featurebase
+echo "featurebase binary copied."
+
+sudo mkdir /data
+sudo mkfs.ext4 /dev/nvme1n1
+sudo mount /dev/nvme1n1 /data
+
+adduser molecula
+sudo mkdir /var/log/molecula
+sudo chown molecula /var/log/molecula
+sudo mkdir -p /data/featurebase
+sudo chown molecula /data/featurebase
+sudo systemctl daemon-reload
+sudo systemctl start featurebase
+sudo systemctl enable featurebase
+sudo systemctl status featurebase
+
+echo "Done!"
--- a/qa/tf/.modules/featurebase-cluster/setup_ingest_node.sh.tpl
+++ b/qa/tf/.modules/featurebase-cluster/setup_ingest_node.sh.tpl
@ -0,0 +1,66 @@
+#!/bin/bash
+
+AWS_INSTANCE_ID=""
+
+get_aws_instance_id() {
+    echo "Getting AWS instance ID..."
+    while true
+    do
+        curl -s http://169.254.169.254/latest/meta-data/instance-id > /dev/null
+        if [ $? -eq 0 ]
+        then
+            break
+        fi
+    done
+    AWS_INSTANCE_ID=`curl http://169.254.169.254/latest/meta-data/instance-id`
+    echo "AWS instance ID is: $${AWS_INSTANCE_ID}"
+}
+
+wait_on_all_ingest_ips() {
+    echo "Waiting on all cluster IPs..."
+    # get IP for node 
+    IPS=$(aws ec2 describe-instances --filters "Name=instance-state-name, Values=running" "Name=tag:Role, Values=ingest_node" "Name=tag:Prefix, Values=${cluster_prefix}" --query 'Reservations[*].Instances[*].PrivateIpAddress' --output text --region ${region})
+    IP_LENGTH=`echo "$IPS" | wc -l`
+
+    for i in {0..24}
+    do 
+        echo "Comparing $${IP_LENGTH} with ${node_count}"
+        if [ $IP_LENGTH == "${node_count}" ]; then
+            echo "Cluster is up after $${i} tries." 
+            break
+        fi
+        sleep 10s
+    done
+
+    if [ $IP_LENGTH != "${node_count}" ]; then
+        echo "Timed out waiting for cluster to be available $${IP_LENGTH} actual nodes compared with ${node_count} desire nodes."
+        exit 1
+    fi
+}
+
+#copy the script so we can look at it later if needed
+sudo cp /var/lib/cloud/instances/$${AWS_INSTANCE_ID}/user-data.txt ~/setup_ingest_node.sh
+
+#get the instance id
+get_aws_instance_id
+
+#wait for the count of nodes to equal requested nodes
+wait_on_all_ingest_ips
+
+echo "Getting featurebase binary..."
+curl --header "PRIVATE-TOKEN: ${gitlab_token}" -o "/home/ec2-user/featurebase_linux_arm64" https://gitlab.com/api/v4/projects/molecula%2Ffeaturebase/jobs/artifacts/master/raw/featurebase_linux_arm64?job=build%20for%20linux%20arm64
+chown ec2-user:ec2-user "/home/ec2-user/featurebase_linux_arm64"
+chmod ugo+x "/home/ec2-user/featurebase_linux_arm64"
+
+mv /home/ec2-user/featurebase_linux_arm64 /usr/local/bin/featurebase
+echo "featurebase binary copied."
+
+
+sudo mkdir /data
+sudo mkfs.ext4 /dev/nvme1n1
+sudo mount /dev/nvme1n1 /data
+
+sudo chown -R ec2-user /data
+
+
+
--- a/qa/tf/.modules/featurebase-cluster/variables.tf
+++ b/qa/tf/.modules/featurebase-cluster/variables.tf
@ -0,0 +1,93 @@
+variable "cluster_prefix" {
+  type        = string
+  description = "This is a identifier that will be prefixed to created resources"
+}
+
+variable "fb_ingest_type" {
+  type    = string
+  default = "c6g.2xlarge"
+}
+
+variable "fb_ingest_node_count" {
+  type    = number
+  default = 1
+}
+
+variable "fb_data_node_type" {
+  type    = string
+  default = "c6g.16xlarge"
+}
+
+variable "fb_data_node_count" {
+  type    = number
+  default = 3
+}
+
+variable "fb_cluster_replica_count" {
+  type    = number
+  default = 1
+}
+
+variable "subnet" {
+  default = ""
+}
+
+variable "zone" {
+  default = ""
+}
+
+variable "fb_data_disk_type" {
+  default = "gp3"
+}
+variable "fb_data_disk_iops" {
+  default = 1000
+}
+
+variable "fb_data_disk_size_gb" {
+  default = 100
+}
+
+variable "fb_ingest_disk_type" {
+  default = "gp3"
+}
+variable "fb_ingest_disk_iops" {
+  default = 1000
+}
+
+variable "fb_ingest_disk_size_gb" {
+  default = 100
+}
+
+variable "azs" {
+  type    = list(any)
+  default = ["us-east-2a", "us-east-2b", "us-east-2c"]
+}
+
+variable "private_subnets" {
+  type    = list(any)
+  default = ["10.0.1.0/24", "10.0.2.0/24", "10.0.3.0/24"]
+}
+
+variable "public_subnets" {
+  type    = list(any)
+  default = ["10.0.101.0/24", "10.0.102.0/24", "10.0.103.0/24"]
+}
+
+variable "vpc_cidr" {
+  default = "10.0.0.0/16"
+}
+
+variable "region" {
+  description = "Region to create AWS resources in"
+  type        = string
+}
+
+variable "profile" {
+  description = "Profile to use to authenticate with AWS"
+  type        = string
+}
+
+variable "gitlab_token" {
+  description = "Gitlab API token"
+  type        = string
+}
--- a/qa/tf/.modules/featurebase-cluster/vpc.tf
+++ b/qa/tf/.modules/featurebase-cluster/vpc.tf
@ -0,0 +1,16 @@
+module "vpc" {
+  source = "terraform-aws-modules/vpc/aws"
+
+  name = "${var.cluster_prefix}"
+  cidr = var.vpc_cidr
+  azs             = var.azs
+  private_subnets = var.private_subnets
+  public_subnets  = var.public_subnets
+
+  enable_nat_gateway = true
+  enable_vpn_gateway = false
+
+  tags = {
+    Name = "${var.cluster_prefix}"
+  }
+}
--- a/qa/tf/README.md
+++ b/qa/tf/README.md
@ -0,0 +1,16 @@
+# Deploy testing environments with this one wierd trick
+
+This directiory contains Terraform to deploy test environments both ad-hoc and as part of CI/CD pipelines.
+
+The .modules contains the guts of the operation, the things you probably want are in the other directories, each with a README.
+
+## How to Terraform
+
+With terraform installed (`brew install terraform` if not)...
+
+You can do `terraform plan` -> `terraform apply` to spin up a cluster, `terraform destroy` to tear one down.
+
+## Other prerequisites:
+Please read these carefully.
+
+
--- a/qa/tf/ci/singlenode/main.tf
+++ b/qa/tf/ci/singlenode/main.tf
@ -0,0 +1,10 @@
+
+module "ci-cluster" {
+    source = "../../.modules/featurebase-cluster"
+    cluster_prefix = "ci-single-node"
+    region = var.region
+    profile = var.profile
+    fb_data_node_type = "m6g.large"
+    fb_data_node_count = 1
+    gitlab_token = var.gitlab_token
+}
--- a/qa/tf/ci/singlenode/outputs.tf
+++ b/qa/tf/ci/singlenode/outputs.tf
@ -0,0 +1,9 @@
+output "ingest_ips" {
+    description = "List of ingest IPs"
+    value = module.ci-cluster.ingest_ips
+}
+
+output "data_node_ips" {
+  description = "List of data node IPs"
+  value = module.ci-cluster.data_node_ips
+}
--- a/qa/tf/ci/singlenode/provider.tf
+++ b/qa/tf/ci/singlenode/provider.tf
@ -0,0 +1,4 @@
+provider "aws" {
+  region  = var.region
+  profile = var.profile
+}
--- a/qa/tf/ci/singlenode/tf.auto.tfvars
+++ b/qa/tf/ci/singlenode/tf.auto.tfvars
@ -0,0 +1,2 @@
+region  = "us-east-2"
+profile = "service-terraform"
--- a/qa/tf/ci/singlenode/variables.tf
+++ b/qa/tf/ci/singlenode/variables.tf
@ -0,0 +1,14 @@
+variable "region" {
+  description = "The AWS region in which the VPC should be built"
+  type        = string
+}
+
+variable "profile" {
+  description = "The name of the AWS profile Terraform should use for auth."
+  type        = string
+}
+
+variable "gitlab_token" {
+  description = "The API token for taking to Gitlab API - expected to come from an env variable."
+  type        = string
+}
--- a/qa/tf/gauntlet/samsung/README.md
+++ b/qa/tf/gauntlet/samsung/README.md
@ -0,0 +1,35 @@
+With terraform installed (`brew install terraform` if not)...
+
+You can do `terraform plan` -> `terraform apply` to spin up a cluster, `terraform destroy` to tear one down.
+
+## Other prerequisites:
+Please read these carefully.
+
+Be in the `tf` directory (e.g., when you try to run a `terraform` command, the output of `pwd` should be `.../molecula/featurebase/qa/tf`)
+
+Currently, the path to the terraform module is using a local reference, i.e., in `main.tf`, the source line is assuming that you have `molecula-terraform` project installed locally, such that the `molecular-terraform` project and `featurebase` have the same parent directory (e.g., `...A/featurebase/qa/tf` and `...A/molecular-terraform/aws/.modules/featurebase-cluster` should both be valid paths).
+
+In addition, you must currently have a local copy of the `fb901` branch for the `molecular-terraform` project (located in the previously specified directory).
+
+Last thing, there is a key that is currently in 1Password (in the `Shared` vault, called `gitlab-featurebase-ci AWS key`) that must be in `~/.ssh/`, `chmod 400`, named `gitlab-featurebase-ci.pem`. You need this key to SSH to these instances. Assuming an `~/.ssh/config` like the following (append to the top of yours)
+```
+Host test_*
+  User ec2-user
+  IdentityFile ~/.ssh/gitlab-featurebase-ci.pem
+Host test_ingest
+  HostName 3.143.237.165
+Host test_node
+  HostName 10.0.1.142
+  ProxyJump test_ingest
+```
+except with the `test_ingest`'s `HostName` being the public, `ingest_ips` output from `terraform output` and `test_node`'s `HostName` being one of the private, `data_node_ips` output from `terraform output`. (Hopefully the rationale to use the ssh config to do the jumping like this makes sense; you can do `ssh test_ingest` or `ssh test_node` with minimal further fiddling.)
+
+OR specify cert to us directly thus:
+
+`ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem ec2-user@ip_address`
+
+-A is used to ensure key forwarding.
+
+### TODOs
+* We need a `user-data.sh` script which sets up/installs featurebase (possibly installs go, most likely pulls the artifacts from GitLab; sets up featurebase on both the node and data workers).
+* Logs get sent to DataDog?
--- a/qa/tf/gauntlet/samsung/main.tf
+++ b/qa/tf/gauntlet/samsung/main.tf
@ -0,0 +1,13 @@
+module "samsung-cluster" {
+    source = "../../.modules/featurebase-cluster"
+    cluster_prefix = "samsung-gauntlet"
+    region = var.region
+    profile = var.profile
+    fb_data_node_type = "m6g.xlarge"
+    fb_data_disk_iops = 10000
+    fb_data_node_count = 3
+    fb_ingest_type = "m6g.large"
+    fb_ingest_disk_iops = 10000
+    fb_ingest_node_count = 1
+    gitlab_token = var.gitlab_token
+}
--- a/qa/tf/gauntlet/samsung/outputs.tf
+++ b/qa/tf/gauntlet/samsung/outputs.tf
@ -0,0 +1,9 @@
+output "ingest_ips" {
+    description = "List of ingest IPs"
+    value = module.samsung-cluster.ingest_ips
+}
+
+output "data_node_ips" {
+  description = "List of data node IPs"
+  value = module.samsung-cluster.data_node_ips
+}
--- a/qa/tf/gauntlet/samsung/provider.tf
+++ b/qa/tf/gauntlet/samsung/provider.tf
@ -0,0 +1,4 @@
+provider "aws" {
+  region  = var.region
+  profile = var.profile
+}
--- a/qa/tf/gauntlet/samsung/samsung-gauntlet.json
+++ b/qa/tf/gauntlet/samsung/samsung-gauntlet.json
@ -0,0 +1,30 @@
+{
+  "data_node_ips": {
+    "sensitive": false,
+    "type": [
+      "tuple",
+      [
+        "string",
+        "string",
+        "string"
+      ]
+    ],
+    "value": [
+      "10.0.1.144",
+      "10.0.2.108",
+      "10.0.3.178"
+    ]
+  },
+  "ingest_ips": {
+    "sensitive": false,
+    "type": [
+      "tuple",
+      [
+        "string"
+      ]
+    ],
+    "value": [
+      "3.145.104.76"
+    ]
+  }
+}
--- a/qa/tf/gauntlet/samsung/tf.auto.tfvars
+++ b/qa/tf/gauntlet/samsung/tf.auto.tfvars
@ -0,0 +1,2 @@
+region  = "us-east-2"
+profile = "service-terraform"
--- a/qa/tf/gauntlet/samsung/variables.tf
+++ b/qa/tf/gauntlet/samsung/variables.tf
@ -0,0 +1,14 @@
+variable "region" {
+  description = "The AWS region in which the VPC should be built"
+  type        = string
+}
+
+variable "profile" {
+  description = "The name of the AWS profile Terraform should use for auth."
+  type        = string
+}
+
+variable "gitlab_token" {
+  description = "The API token for taking to Gitlab API - expected to come from an env variable."
+  type        = string
+}