WIP: start integration test setup

TODO:
- remove commented out gitlab jobs
- make it run only on merge to master

.gitlab/.gitlab-ci.yml

@@ -1,8 +1,8 @@
-include:
-  - template: Security/SAST.gitlab-ci.yml
-  - template: Security/License-Scanning.gitlab-ci.yml
-  - template: Security/Dependency-Scanning.gitlab-ci.yml
-
+#include:
+#  - template: Security/SAST.gitlab-ci.yml
+#  - template: Security/License-Scanning.gitlab-ci.yml
+#  - template: Security/Dependency-Scanning.gitlab-ci.yml
+#
 variables:
   GOVERSION: "1.17.7"
 
@@ -16,35 +16,35 @@ stages:
   - post build
   - nonblocking
 
-smoke build:
-  image: golang:$GOVERSION
-  stage: lint
-  allow_failure: false
-  rules:
-    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
-  script:
-    - echo "Let's just see if it compiles... (sometimes the linter gives unclear errors if it doesn't)"
-    - go build ./...
-
-golangci-lint:
-  image: golangci/golangci-lint:v1.39.0
-  stage: lint
-  allow_failure: false
-  rules:
-    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
-  script:
-    - echo "Checking for issues in new code"
-    - golangci-lint run
-
-go mod tidy:
-  stage: lint
-  image: golang:$GOVERSION
-  rules:
-    - if: '$CI_COMMIT_TAG == null && ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web")'
-  script:
-    - go mod tidy
-    - git diff --exit-code -- go.mod go.sum
-
+#smoke build:
+#  image: golang:$GOVERSION
+#  stage: lint
+#  allow_failure: false
+#  rules:
+#    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
+#  script:
+#    - echo "Let's just see if it compiles... (sometimes the linter gives unclear errors if it doesn't)"
+#    - go build ./...
+#
+#golangci-lint:
+#  image: golangci/golangci-lint:v1.39.0
+#  stage: lint
+#  allow_failure: false
+#  rules:
+#    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
+#  script:
+#    - echo "Checking for issues in new code"
+#    - golangci-lint run
+#
+#go mod tidy:
+#  stage: lint
+#  image: golang:$GOVERSION
+#  rules:
+#    - if: '$CI_COMMIT_TAG == null && ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web")'
+#  script:
+#    - go mod tidy
+#    - git diff --exit-code -- go.mod go.sum
+#
 build lattice:
   stage: test
   image: node:14
@@ -64,92 +64,92 @@ build lattice:
   artifacts:
     paths:
       - lattice.tar.gz
-
-run jest tests:
-  stage: test
-  image: node:14
-  variables:
-    CI: "true"
-  rules:
-    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
-  script:
-    - echo "Testing lattice..."
-    - cd lattice
-    - npm install --force
-    - npm test -- --coverage --testResultsProcessor=jest-sonar-reporter
-  artifacts:
-    paths:
-      - lattice/coverage/lcov.info
-
-run go tests:
-  stage: test
-  image: golang:$GOVERSION
-  rules:
-    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
-  retry: 1
-  script:
-    - echo "Running featurebase unit tests..."
-    - go test -timeout=30m ./...
-  tags:
-    - aws
-
-run go tests race:
-  stage: nonblocking # don't let this job block any other jobs because it takes much longer than the other tests.
-  image: golang:$GOVERSION
-  rules:
-    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
-  retry: 1
-  needs: [] # don't wait to start running this.
-  script:
-    - echo "Running featurebase race tests..."
-    - go test -race -v -timeout=90m ./...
-  tags:
-    - aws
-
-run go tests shardwidth22:
-  stage: test
-  image: golang:$GOVERSION
-  rules:
-    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
-  script:
-    - echo "Running featurebase shardwidth22 tests..."
-    - go test -timeout=30m -tags=shardwidth22 ./...
-  tags:
-    - aws
-    
-# we do coverage reporting from the future tests because the json
-# output is very difficult to human-read. The alternative would be to
-# run the regular tests twice and also run the future tests.
-run go tests future:
-  stage: test
-  image: golang:1.17.6
-  rules:
-    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
-  retry: 1
-  script:
-    - echo "Running featurebase unit tests..."
-    - PKG_LIST=$(go list ./... | grep -v internal/clustertests | paste -s -d, -)
-    - go test -timeout=30m -json -coverprofile=coverage.out -covermode=atomic -coverpkg=${PKG_LIST} ./... | tee test-report.out
-  artifacts:
-    paths:
-      - coverage.out
-      - test-report.out
-  tags:
-    - aws
-
-upload to sonarcloud:
-  stage: integration
-  image: sonarsource/sonar-scanner-cli:4.6
-  variables:
-    SONAR_TOKEN: $SONAR_TOKEN
-  rules:
-    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
-  script:
-    - sonar-scanner -Dsonar.projectKey=molecula_featurebase -Dsonar.organization=molecula -Dsonar.sources=. -Dsonar.host.url=https://sonarcloud.io -Dsonar.go.coverage.reportPaths=coverage.out,results/coverage*.out -Dsonar.go.tests.reportPaths=test-report.out,results/report*  -Dsonar.javascript.lcov.reportPaths=lattice/coverage/lcov.info
-  needs:
-    - job: run go tests future
-    - job: run jest tests 
-    - job: clustertests
+#
+#run jest tests:
+#  stage: test
+#  image: node:14
+#  variables:
+#    CI: "true"
+#  rules:
+#    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
+#  script:
+#    - echo "Testing lattice..."
+#    - cd lattice
+#    - npm install --force
+#    - npm test -- --coverage --testResultsProcessor=jest-sonar-reporter
+#  artifacts:
+#    paths:
+#      - lattice/coverage/lcov.info
+#
+#run go tests:
+#  stage: test
+#  image: golang:$GOVERSION
+#  rules:
+#    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
+#  retry: 1
+#  script:
+#    - echo "Running featurebase unit tests..."
+#    - go test -timeout=30m ./...
+#  tags:
+#    - aws
+#
+#run go tests race:
+#  stage: nonblocking # don't let this job block any other jobs because it takes much longer than the other tests.
+#  image: golang:$GOVERSION
+#  rules:
+#    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
+#  retry: 1
+#  needs: [] # don't wait to start running this.
+#  script:
+#    - echo "Running featurebase race tests..."
+#    - go test -race -v -timeout=90m ./...
+#  tags:
+#    - aws
+#
+#run go tests shardwidth22:
+#  stage: test
+#  image: golang:$GOVERSION
+#  rules:
+#    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
+#  script:
+#    - echo "Running featurebase shardwidth22 tests..."
+#    - go test -timeout=30m -tags=shardwidth22 ./...
+#  tags:
+#    - aws
+#    
+## we do coverage reporting from the future tests because the json
+## output is very difficult to human-read. The alternative would be to
+## run the regular tests twice and also run the future tests.
+#run go tests future:
+#  stage: test
+#  image: golang:1.17.6
+#  rules:
+#    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
+#  retry: 1
+#  script:
+#    - echo "Running featurebase unit tests..."
+#    - PKG_LIST=$(go list ./... | grep -v internal/clustertests | paste -s -d, -)
+#    - go test -timeout=30m -json -coverprofile=coverage.out -covermode=atomic -coverpkg=${PKG_LIST} ./... | tee test-report.out
+#  artifacts:
+#    paths:
+#      - coverage.out
+#      - test-report.out
+#  tags:
+#    - aws
+#
+#upload to sonarcloud:
+#  stage: integration
+#  image: sonarsource/sonar-scanner-cli:4.6
+#  variables:
+#    SONAR_TOKEN: $SONAR_TOKEN
+#  rules:
+#    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
+#  script:
+#    - sonar-scanner -Dsonar.projectKey=molecula_featurebase -Dsonar.organization=molecula -Dsonar.sources=. -Dsonar.host.url=https://sonarcloud.io -Dsonar.go.coverage.reportPaths=coverage.out,results/coverage*.out -Dsonar.go.tests.reportPaths=test-report.out,results/report*  -Dsonar.javascript.lcov.reportPaths=lattice/coverage/lcov.info
+#  needs:
+#    - job: run go tests future
+#    - job: run jest tests 
+#    - job: clustertests
     
 build for linux amd64:
   stage: build
@@ -500,6 +500,64 @@ perf_able:
   variables:
     PARENT_PIPELINE_ID: $CI_PIPELINE_ID
 
+perf_delete:
+  stage: integration
+  timeout: 2h
+  image: registry.gitlab.com/gitlab-org/cloud-deploy/aws-base:latest
+  variables:
+    PROFILE: "service-terraform"
+    INFRA_PROFILE: "service-gitlab"
+    AWS_SSH_PRIVATE_KEY: $AWS_FBCI_SSH_KEY
+    AWS_ACCESS_KEY_ID: $AWS_FBCI_ACCESS_KEY_ID
+    AWS_SECRET_ACCESS_KEY: $AWS_FBCI_SECRET_ACCESS_KEY
+    ASG_NAME: "gitlab-runners"
+    TF_VAR_cluster_prefix: ""
+  tags:
+    - aws
+    - docker
+    - fbsmoke
+  before_script:
+    - apt-get update && apt-get install -y gnupg software-properties-common curl git
+    - curl -fsSL https://apt.releases.hashicorp.com/gpg | apt-key add -
+    - apt-add-repository "deb [arch=amd64] https://apt.releases.hashicorp.com $(lsb_release -cs) main"
+    - apt-get update && apt-get install terraform 
+    - apt-get update && apt-get install python3.7
+    - aws configure set aws_access_key_id $AWS_FBCI_ACCESS_KEY_ID --profile $PROFILE
+    - aws configure set aws_secret_access_key $AWS_FBCI_SECRET_ACCESS_KEY --profile $PROFILE
+    - aws configure set region "us-east-2" --profile $PROFILE
+    - aws configure set aws_profile $PROFILE
+    - aws configure set aws_access_key_id $AWS_INFRA_ACCESS_KEY_ID --profile $INFRA_PROFILE
+    - aws configure set aws_secret_access_key $AWS_INFRA_SECRET_ACCESS_KEY --profile $INFRA_PROFILE
+    - aws configure set region "us-east-2" --profile $INFRA_PROFILE
+    - echo $AWS_FBCI_SSH_KEY > gitlab-featurebase-ci.pem
+    - chmod 400 gitlab-featurebase-ci.pem
+    - 'which ssh-agent || ( apt-get update -y && apt-get install openssh-client -y )'
+    - eval $(ssh-agent -s)
+    - mkdir -p ~/.ssh
+    - echo $AWS_FBCI_SSH_KEY > /root/.ssh/gitlab-featurebase-ci.pem
+    - chmod 400 /root/.ssh/gitlab-featurebase-ci.pem
+    - echo "$AWS_FBCI_SSH_KEY" | ssh-add -
+    - chmod 700 /root/.ssh
+    - '[[ -f /.dockerenv ]] && echo -e "Host *\n\tStrictHostKeyChecking no\n\n" > ~/.ssh/config'
+    - apt update && apt -y install jq wget
+    - wget -q https://go.dev/dl/go1.17.5.linux-amd64.tar.gz
+    - tar -C /usr/local -xzf go1.17.5.linux-amd64.tar.gz
+    - export PATH=$PATH:/usr/local/go/bin
+    - TF_VAR_cluster_prefix="delete-$(openssl rand -base64 12 | tr -d /=+ | cut -c -16)"
+    - echo "Cluster Prefix --> $TF_VAR_cluster_prefix"
+    - export INSTANCE_ID=$(curl --silent --fail "http://169.254.169.254/latest/meta-data/instance-id" | tee instance_id)
+    - aws autoscaling set-instance-protection --instance-ids "$INSTANCE_ID" --auto-scaling-group-name $ASG_NAME --protected-from-scale-in --profile $INFRA_PROFILE
+  script:
+    - ./qa/scripts/perf/delete/deleteSetup.sh
+    - ./qa/scripts/perf/delete/deleteTest.sh
+  after_script:
+    - ./qa/scripts/perf/delete/deleteTeardown.sh || true
+    - export INSTANCE_ID=$(cat instance_id)
+    - aws autoscaling set-instance-protection --instance-ids "$INSTANCE_ID" --auto-scaling-group-name $ASG_NAME --no-protected-from-scale-in --profile $INFRA_PROFILE
+  needs:
+    - job: build for linux arm64
+    - job: build for linux amd64
+
 s3 dump tag:
   stage: post build
   variables:

qa/scripts/perf/delete/confluent

@@ -0,0 +1,13 @@
+[Confluent.dist]
+name=Confluent repository (dist)
+baseurl=http://packages.confluent.io/rpm/3.1/6
+gpgcheck=1
+gpgkey=http://packages.confluent.io/rpm/3.1/archive.key
+enabled=1
+
+[Confluent]
+name=Confluent repository
+baseurl=http://packages.confluent.io/rpm/3.1
+gpgcheck=1
+gpgkey=http://packages.confluent.io/rpm/3.1/archive.key
+enabled=1

qa/scripts/perf/delete/deleteSetup.sh

@@ -0,0 +1,86 @@
+#!/bin/bash
+
+# To run script: ./deleteSetup.sh
+export TF_IN_AUTOMATION=1
+
+if [ -z ${TF_VAR_cluster_prefix+x} ]; then 
+    echo "TF_VAR_cluster_prefix is unset";
+    exit 1
+else 
+    echo "TF_VAR_cluster_prefix is set to '$TF_VAR_cluster_prefix'"; 
+fi
+
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+source $SCRIPT_DIR/../../utilCluster.sh
+
+pushd ./qa/tf/perf/delete
+echo "Running terraform init..."
+terraform init -input=false
+echo "Running terraform apply..."
+terraform apply -input=false -auto-approve
+terraform output -json > outputs.json
+popd
+
+# get the first ingest host
+INGESTNODE0=$(cat ./qa/tf/perf/delete/outputs.json | jq -r '[.ingest_ips][0]["value"][0]')
+echo "using INGESTNODE0 ${INGESTNODE0}"
+
+# get the first data host
+DATANODE0=$(cat ./qa/tf/perf/delete/outputs.json | jq -r '[.data_node_ips][0]["value"][0]')
+echo "using DATANODE0 ${DATANODE0}"
+
+
+DEPLOYED_CLUSTER_PREFIX=$(cat ./qa/tf/perf/delete/outputs.json | jq -r '[.cluster_prefix][0]["value"]')
+echo "Using DEPLOYED_CLUSTER_PREFIX: ${DEPLOYED_CLUSTER_PREFIX}"
+
+DEPLOYED_CLUSTER_REPLICA_COUNT=$(cat ./qa/tf/perf/delete/outputs.json | jq -r '[.fb_cluster_replica_count][0]["value"]')
+echo "Using DEPLOYED_CLUSTER_REPLICA_COUNT: ${DEPLOYED_CLUSTDEPLOYED_CLUSTER_REPLICA_COUNTER_PREFIX}"
+
+DEPLOYED_DATA_IPS=$(cat ./qa/tf/perf/delete/outputs.json | jq -r '[.data_node_ips][0]["value"][]')
+echo "DEPLOYED_DATA_IPS: {"
+echo "${DEPLOYED_DATA_IPS}"
+echo "}"
+
+DEPLOYED_DATA_IPS_LEN=`echo "$DEPLOYED_DATA_IPS" | wc -l`
+
+DEPLOYED_INGEST_IPS=$(cat ./qa/tf/perf/delete/outputs.json | jq -r '[.ingest_ips][0]["value"][]')
+echo "DEPLOYED_INGEST_IPS: {"
+echo "${DEPLOYED_INGEST_IPS}"
+echo "}"
+
+DEPLOYED_INGEST_IPS_LEN=`echo "$DEPLOYED_INGEST_IPS" | wc -l`
+
+#wait until we can connect to one of the hosts
+for i in {0..24}
+do 
+    ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no -o ConnectTimeout=10 ec2-user@${DATANODE0} "pwd"
+    if [ $? -eq 0 ]
+    then
+        echo "Cluster is up after ${i} tries." 
+        break
+    fi
+    sleep 10
+done
+
+ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no -o ConnectTimeout=10 ec2-user@${DATANODE0} "pwd"
+if [ $? -ne 0 ]
+then
+    echo "Unable to connect to cluster - giving up" 
+    exit 1
+fi
+
+setupClusterNodes
+
+# verify featurebase running
+echo "Verifying featurebase cluster running..."
+curl -s http://${DATANODE0}:10101/status
+if (( $? != 0 )) 
+then 
+    echo "Featurebase cluster not running"
+    exit 1
+fi
+
+echo "Cluster running."
+
+
+

qa/scripts/perf/delete/deleteTeardown.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+# To run script: ./deleteTeardown.sh
+
+cd qa/tf/perf/delete
+export TF_IN_AUTOMATION=1
+terraform destroy -auto-approve

qa/scripts/perf/delete/deleteTest.sh

@@ -0,0 +1,88 @@
+#!/bin/bash
+
+# get the first ingest host
+INGESTNODE0=$(cat ./qa/tf/perf/delete/outputs.json | jq -r '[.ingest_ips][0]["value"][0]')
+echo "using INGESTNODE0 ${INGESTNODE0}"
+
+# get the first data host
+DATANODE0=$(cat ./qa/tf/perf/delete/outputs.json | jq -r '[.data_node_ips][0]["value"][0]')
+echo "using DATANODE0 ${DATANODE0}"
+
+# download datagen
+aws s3 cp s3://molecula-artifact-storage/idk/master/_latest/idk-linux-arm64/datagen datagen
+if (( $? != 0 ))
+then
+    echo "datagen binary copy failed"
+    exit 1
+fi
+
+# make it executable
+chmod +x datagen 
+if (( $? != 0 ))
+then
+    echo "couldn't make datagen executable"
+    exit 1
+fi
+
+# copy it over to the ingest node
+scp -r -i ~/.ssh/gitlab-featurebase-ci.pem ./datagen ec2-user@${INGESTNODE0}:/data
+if (( $? != 0 )) 
+then 
+    echo "datagen copy failed"
+    exit 1
+fi
+
+# setup the yum repo needed for librdkafka onto the ingest node
+scp -r -i ~/.ssh/gitlab-featurebase-ci.pem ./qa/scripts/perf/delete/confluent ec2-user@${INGESTNODE0}:/data
+if (( $? != 0 )) 
+then 
+    echo "confluent repo setup copy failed"
+    exit 1
+fi
+
+echo "setting up confluent repo"
+ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o "StrictHostKeyChecking no" ec2-user@${INGESTNODE0} "sudo mv /data/confluent /etc/yum.repos.d" 
+if (( $? != 0 ))
+then 
+    echo "setting up confluent repo failed"
+    exit 1
+fi
+
+
+echo "installing librdkafka on ingest node"
+ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o "StrictHostKeyChecking no" ec2-user@${INGESTNODE0} "sudo rpm --import http://packages.confluent.io/rpm/3.1/archive.key && sudo yum clean all && sudo yum install librdkafka-devel -y"
+if (( $? != 0 ))
+then 
+    echo "librdkafka install failed"
+    exit 1
+fi
+
+# copy tremor.yaml over to the ingest node
+scp -r -i ~/.ssh/gitlab-featurebase-ci.pem ./qa/scripts/perf/delete/tremor.yaml ec2-user@${INGESTNODE0}:/data
+if (( $? != 0 )) 
+then 
+    echo "tremor.yaml copy failed"
+    exit 1
+fi
+
+# copy the tests over to ingest node
+scp -r -i ~/.ssh/gitlab-featurebase-ci.pem ./qa/scripts/perf/delete/test.py ec2-user@${INGESTNODE0}:/data
+if (( $? != 0 )) 
+then 
+    echo "test copy failed"
+    exit 1
+fi
+
+# run test on ingest node
+echo "running delete test"
+ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o "StrictHostKeyChecking no" ec2-user@${INGESTNODE0} "cd /data; python3 test.py ${DATANODE0}"
+TESTRESULT=$?
+
+if (( $TESTRESULT != 0 )) 
+then 
+    echo "delete test failed"
+else
+    echo "delete test complete"
+fi
+
+exit $TESTRESULT 

qa/scripts/perf/delete/test.py

@@ -0,0 +1,131 @@
+"""this script checks that memory usage does not increase when data is deleted and ingested again"""
+import subprocess as sp
+from time import sleep
+import requests
+import sys
+import os
+
+if len(sys.argv) != 2:
+    print("Usage:")
+    print("python {} <public ip of data node>".format(sys.argv[0]))
+    exit(1)
+
+ip = sys.argv[1]
+
+
+def datagen():
+    """run_datagen runs datagen using the configured tremor.yaml file and blocks until it has completed and returns whether or not the command was successful"""
+    try:
+        sp.check_call(
+            [
+                "./datagen",
+                "-s=custom",
+                "--custom-config=./tremor.yaml",
+                "--pilosa.index=tremor",
+                "--pilosa.batch-size=300000",
+                "--pilosa.hosts={}:10101".format(ip),
+            ],
+            stdout=sp.DEVNULL,
+            stderr=sp.STDOUT,
+        )
+    except Exception as e:
+        pass
+
+
+def mem_use():
+    """mem_use returns the memory usage of the data node and whether the request was successful"""
+    response = requests.get("http://" + ip + ":10101/internal/mem-usage")
+    if response.status_code != 200:
+        return 0
+    body = response.json()
+    return body["totalUsed"]
+
+
+def delete_all():
+    """delete_all runs a Delete(All()) query on the data node and returns whether the delete was successful"""
+    response = requests.post(
+        "http://" + ip + ":10101/index/tremor/query", data="Delete(All())"
+    )
+    count = 0
+    retry_period = 0.5
+    while try_again(response) and (count < 9):
+        print(response.text.strip())
+        print(
+            "There was a problem executing the Delete(All()) query. Trying again in {} seconds (attempt {}/9)".format(
+                retry_period,
+                count,
+            )
+        )
+        count += 1
+        sleep(retry_period)
+        retry_period *= 2
+        response = requests.post(
+            "http://" + ip + ":10101/index/tremor/query", data="Delete(All())"
+        )
+    return response.status_code == 200
+
+
+def try_again(response):
+    if response.ok:
+        # if we got a response but it was unsuccessful, try again
+        return not response.json()["results"][0]
+    # if we didn't get a good response, try again
+    return True
+
+
+def disk_use():
+    response = requests.get("http://" + ip + ":10101/internal/disk-usage")
+    if response.status_code != 200:
+        return 0
+    body = response.json()
+    return body["usage"]
+
+
+print("# memory usage tests")
+usage = new_usage = 0
+du = new_du = 0
+datagen()
+usage = mem_use()
+du = disk_use()
+print(
+    "{},{},{},{}".format(
+        "mem_usage", "disk_usage", "new_mem_greater", "new_disk_greater"
+    )
+)
+print("{},{},{},{}".format(usage, du, None, None))
+for i in range(5):
+    variance = 0.25
+    if delete_all():
+        datagen()
+        new_usage = mem_use()
+        new_du = disk_use()
+        print(
+            "{},{},{},{}".format(
+                usage,
+                du,
+                (new_usage / usage),
+                (new_du / du),
+            )
+        )
+        if (new_usage / usage) > (1 + variance):
+            print(
+                new_usage,
+                "is greater than",
+                usage,
+                "by at least {}%.".format(variance * 100),
+            )
+            print(new_usage - usage, "is a lot of bytes.")
+            print("Failing...")
+            exit(1)
+        if (new_du / du) > (1 + variance):
+            print(
+                new_du, "is greater than", du, "by at least {}%.".format(variance * 100)
+            )
+            print(new_du - du, "is a lot of bytes.")
+            print("Failing...")
+            exit(1)
+    else:
+        print("Couldn't execute Delete(All()) query in 9 tries. Failing...")
+        exit(1)
+print("Success")
+exit(0)

qa/scripts/perf/delete/tremor.yaml

@@ -0,0 +1,61 @@
+fields:
+  - name: "a_random_string"
+    type: "string" # (default StringField (non-mutex))
+    generator_type: "random-string" # used to generate random strings rather than pulling from known set
+    min_len: 8
+    max_len: 12
+    charset: "AB" # set of possible characters to pull from when generating random string
+  - name: "id"
+    type: "uint"
+    distribution: "sequential"
+    min: 0
+    max: 1000000 # 2%24
+    step: 1
+    repeat: false
+  - name: "type"
+    type: "int" # (default IntField)
+    min: 0
+    max: 3
+    distribution: "zipfian"
+    s: 1.1
+    v: 5.1
+  - name: "ts"
+    type: "timestamp"
+    min_date: 2006-01-02T15:04:05.001Z # RFC3339Nano
+    max_date: 2007-01-02T15:04:05.001Z # RFC3339Nano
+    distribution: "increasing" # only "increasing" is supported right now
+    min_step_duration: "10ns"
+    max_step_duration: "200ms" 
+  - name: "slice"
+    type: "uint-set" # (default IDArrayField)
+    min: 0
+    max: 35000
+    distribution: "zipfian"
+    s: 1.1
+    v: 5.1
+    min_num: 1
+    max_num: 50
+
+# idk_params describe how data from "fields" should be ingested by IDK
+idk_params:
+  primary_key_config:
+    field: "a_random_string" # if this is a single field named "id" then we'll use uint IDs, if it's empty we'll autogen ids, and if it's anything else we'll do string keys... yes this is a bit hacky, needs to be cleaned up.
+  # fields is keyed by names of fields from top level "fields". It is
+  # not required that all fields appear here, those that don't will
+  # use the default ingestion.
+  fields:
+    id:
+      - type: "ID"
+    type:
+      - type: "ID"
+    a_decimal_field:
+      - type: "Decimal"
+        scale: 4
+    ts:
+      - type: "RecordTime"
+        layout: "2006-01-02T15:04:05Z"
+        epoch: 1970-01-01T00:00:00.0Z
+        name: "na"
+    slice:
+      - type: "IDArray"
+        time_quantum: "YMD"

qa/scripts/utilCluster.sh

@@ -47,14 +47,13 @@ EOT
     #echo "featurebase.service <<"
 
     scp -i ~/.ssh/gitlab-featurebase-ci.pem -o "StrictHostKeyChecking no" featurebase.service ec2-user@${NODEIP}:
-    if (( $? != 0 )) 
-    then 
+    if (( $? != 0 ))
+    then
         echo "featurebase.service copy failed"
         exit 1
     fi
 
     rm -f featurebase.service
-    
     ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "sudo mv featurebase.service ${SERVICE_FILE_PATH}"
 }
 
@@ -127,8 +126,8 @@ executeGeneralNodeConfigCommands() {
     
     # TODO handle different archs
     scp -i ~/.ssh/gitlab-featurebase-ci.pem -o "StrictHostKeyChecking no" featurebase_linux_arm64 ec2-user@${NODEIP}:
-    if (( $? != 0 )) 
-    then 
+    if (( $? != 0 ))
+    then
         echo "featurebase binary copy failed"
         exit 1
     fi
@@ -136,7 +135,7 @@ executeGeneralNodeConfigCommands() {
     ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "chown ec2-user:ec2-user /home/ec2-user/featurebase_linux_arm64"
     ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "chmod ugo+x /home/ec2-user/featurebase_linux_arm64"
     ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "sudo mv /home/ec2-user/featurebase_linux_arm64 /usr/local/bin/featurebase"
-    
+
     echo "featurebase binary copied."
 }
 
@@ -229,4 +228,4 @@ setupClusterNodes() {
     #ingest nodes
     setupIngestNodes
 
-}
+}

qa/tf/perf/delete/main.tf

@@ -0,0 +1,17 @@
+module "delete-cluster" {
+  source                 = "../../.modules/featurebase-cluster"
+  cluster_prefix         = var.cluster_prefix
+  region                 = var.region
+  profile                = var.profile
+  fb_data_node_type      = "m6g.12xlarge"
+  fb_data_disk_iops      = 10000
+  fb_data_node_count     = 3
+  fb_ingest_type         = "m6g.2xlarge"
+  fb_ingest_disk_iops    = 10000
+  fb_ingest_disk_size_gb = 500
+  fb_ingest_node_count   = 1
+  vpc_id                 = "vpc-05a26a122f961dc2b"
+  vpc_cidr_block         = "10.0.0.0/16"
+  vpc_public_subnets     = ["subnet-066b4b922b54e51a2", "subnet-037b8884269a69025", "subnet-08482631514426210", ]
+  vpc_private_subnets    = ["subnet-0319dde319380326f", "subnet-0517ca9a646d80f88", "subnet-05a7b685ed27eb1cf", ]
+}

qa/tf/perf/delete/outputs.tf

@@ -0,0 +1,19 @@
+output "ingest_ips" {
+  description = "List of ingest IPs"
+  value       = module.delete-cluster.ingest_ips
+}
+
+output "data_node_ips" {
+  description = "List of data node IPs"
+  value       = module.delete-cluster.data_node_ips
+}
+
+output "cluster_prefix" {
+  description = "The cluster prefix used"
+  value       = module.delete-cluster.cluster_prefix
+}
+
+output "fb_cluster_replica_count" {
+  description = "The cluster replica count used"
+  value       = module.delete-cluster.fb_cluster_replica_count
+}

qa/tf/perf/delete/provider.tf

@@ -0,0 +1,4 @@
+provider "aws" {
+  region  = var.region
+  profile = var.profile
+}

qa/tf/perf/delete/tf.auto.tfvars

@@ -0,0 +1,2 @@
+region  = "us-east-2"
+profile = "service-terraform"

qa/tf/perf/delete/variables.tf

@@ -0,0 +1,14 @@
+variable "region" {
+  description = "The AWS region in which the VPC should be built"
+  type        = string
+}
+
+variable "profile" {
+  description = "The name of the AWS profile Terraform should use for auth."
+  type        = string
+}
+
+variable "cluster_prefix" {
+  type        = string
+  description = "This is a identifier that will be prefixed to created resources"
+}