Tremor delete test scripts (#1974)

- terraform scripts to set up cluster - cloud-formation scripts to set up cluster - set up ingest node with kafka server and datagen - set up second ingest node with molecula-consumer-kafka-static - set up datadog in all nodes (ingest + featurebase) - set up script to execute different queries - only run delete test on schedule
2026-08-28 10:54:59 +00:00 · 2022-05-05 10:25:35 -05:00 · 2022-05-05 10:25:35 -05:00 · b82375bca2
commit b82375bca2
parent a093099a8c
32 changed files with 1671 additions and 30 deletions
--- a/.gitlab/.gitlab-ci.yml
+++ b/.gitlab/.gitlab-ci.yml
@ -455,7 +455,85 @@ smoke test:
      - report.xml
    reports:
      junit: report.xml
+  

+tremor-delete-test:
+  stage: integration
+  image: registry.gitlab.com/gitlab-org/cloud-deploy/aws-base:latest
+  timeout: 6h
+  variables:
+    PROFILE: "service-terraform"
+    AWS_SSH_PRIVATE_KEY: $AWS_FBCI_SSH_KEY
+    AWS_ACCESS_KEY_ID: $AWS_FBCI_ACCESS_KEY_ID
+    AWS_SECRET_ACCESS_KEY: $AWS_FBCI_SECRET_ACCESS_KEY
+    STACK_PREFIX: $CI_JOB_NAME
+    FB_INSTANCE_COUNT: 5
+    REPLICA_COUNT: 1
+  tags:
+    - aws
+    - docker
+    - fbsmoke
+  rules:
+    - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && ($CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web")'
+  before_script:
+    - apt-get update && apt-get install -y gnupg software-properties-common curl git
+    - curl -fsSL https://apt.releases.hashicorp.com/gpg | apt-key add -
+    - apt-add-repository "deb [arch=amd64] https://apt.releases.hashicorp.com $(lsb_release -cs) main"
+    - apt-get update && apt-get install terraform
+    - aws configure set aws_access_key_id $AWS_FBCI_ACCESS_KEY_ID --profile $PROFILE
+    - aws configure set aws_secret_access_key $AWS_FBCI_SECRET_ACCESS_KEY --profile $PROFILE
+    - aws configure set region "us-east-2" --profile $PROFILE
+    - aws configure set aws_profile $PROFILE
+    - echo $AWS_FBCI_SSH_KEY > gitlab-featurebase-ci.pem
+    - chmod 400 gitlab-featurebase-ci.pem
+    - 'which ssh-agent || ( apt-get update -y && apt-get install openssh-client -y )'
+    - eval $(ssh-agent -s)
+    - mkdir -p ~/.ssh
+    - echo $AWS_FBCI_SSH_KEY > /root/.ssh/gitlab-featurebase-ci.pem
+    - chmod 400 /root/.ssh/gitlab-featurebase-ci.pem
+    - echo "$AWS_FBCI_SSH_KEY" | ssh-add -
+    - chmod 700 /root/.ssh
+    - '[[ -f /.dockerenv ]] && echo -e "Host *\n\tStrictHostKeyChecking no\n\n" > ~/.ssh/config'
+    - apt update && apt -y install jq wget
+    - wget -q https://go.dev/dl/go$GOVERSION.linux-amd64.tar.gz
+    - tar -C /usr/local -xzf go$GOVERSION.linux-amd64.tar.gz
+    - export PATH=$PATH:/usr/local/go/bin
+    - CF_STACK_NAME="${STACK_PREFIX}-$(openssl rand -base64 12 | tr -d /=+ | cut -c -16)"
+    - echo "Cloudformation Stack Name --> $CF_STACK_NAME"
+    - echo $CF_STACK_NAME > /tmp/stackname-{$CI_JOB_ID}
+  script:
+    - echo "Start CloudFormation deploy..."
+    - >
+        aws cloudformation deploy 
+        --stack-name $CF_STACK_NAME 
+        --template-file ./qa/cft/FeatureBaseClusterCFTTremor.yaml
+        --parameter-overrides
+        FBInstanceCount=$FB_INSTANCE_COUNT 
+        EBSVolumeSize=1200 
+        TestName=$CI_JOB_NAME 
+        TestID=$CI_JOB_ID
+        TestCommitSHA=$CI_COMMIT_SHA
+        --capabilities CAPABILITY_NAMED_IAM
+        --profile $PROFILE
+    - echo "CloudFormation deploy done."
+    - echo "Start configuring and start featurebase..."
+    - ./qa/scripts/setupTremorDeleteCFT.sh $CF_STACK_NAME $PROFILE $REPLICA_COUNT $FB_INSTANCE_COUNT $CI_COMMIT_BRANCH
+    # producer.log, consumer.log, queries.log are generated when runTremorDeleteCFT.sh is ran
+    - ./qa/scripts/runTremorDeleteCFT.sh $CF_STACK_NAME $PROFILE > tremor-delete-test.log 2>&1
+    - echo "Featurebase configuration done."        
+    - echo "Start running tests..."    
+    - echo "Done with tests!"
+  artifacts:
+    when: always 
+    paths:
+      - tremor-delete-test.log 
+  after_script:
+    # note that we have to read the file to get the $CF_STACK_NAME
+    # because variables from the previous script & before_script do not get passed to after_script
+    - CF_STACK_NAME=$(cat /tmp/stackname-{$CI_JOB_ID})
+    - echo "Deleting stack after test, stack name = ${CF_STACK_NAME}"
+    - aws cloudformation delete-stack --stack-name $CF_STACK_NAME --profile $PROFILE
+    - echo "Delete stack complete!"

 samsung-gauntlet:
  stage: gauntlet
@ -715,4 +793,4 @@ s3 dump tag:
    - job: build for darwin amd64
    - job: build for darwin arm64
    - job: build for linux amd64
-    - job: build for linux arm64
+    - job: build for linux arm64
--- a/.gitlab/.perf-able-gitlab-ci.yml
+++ b/.gitlab/.perf-able-gitlab-ci.yml
@ -1,3 +1,6 @@
+variables:
+  GOVERSION: "1.17.8"
+  
 stages:
  - performance

@ -40,8 +43,8 @@ perf_able:
    - chmod 700 /root/.ssh
    - '[[ -f /.dockerenv ]] && echo -e "Host *\n\tStrictHostKeyChecking no\n\n" > ~/.ssh/config'
    - apt update && apt -y install jq wget
-    - wget -q https://go.dev/dl/go1.17.5.linux-amd64.tar.gz
-    - tar -C /usr/local -xzf go1.17.5.linux-amd64.tar.gz
+    - wget -q https://go.dev/dl/go$GOVERSION.linux-amd64.tar.gz
+    - tar -C /usr/local -xzf go$GOVERSION.linux-amd64.tar.gz
    - export PATH=$PATH:/usr/local/go/bin
    - TF_VAR_cluster_prefix="able-$(openssl rand -base64 12 | tr -d /=+ | cut -c -16)"
    - echo "Cluster Prefix --> $TF_VAR_cluster_prefix"
--- a/qa/cft/FeatureBaseClusterCFTTremor.yaml
+++ b/qa/cft/FeatureBaseClusterCFTTremor.yaml
@ -0,0 +1,309 @@
+AWSTemplateFormatVersion: 2010-09-09
+Description: FeatureBase Cluster for testing
+
+Parameters:
+  # Network params
+  PrivateSubnetIds:
+    Type: String
+    Default: 'subnet-0319dde319380326f,subnet-0517ca9a646d80f88,subnet-05a7b685ed27eb1cf'
+    Description: private subnet provisioned for integration test.
+  PublicSubnetIds:
+    Type: String
+    Default: 'subnet-066b4b922b54e51a2,subnet-037b8884269a69025,subnet-08482631514426210'
+    Description: public subnet provisioned for integration test.
+  FeatureBasePort:
+    Type: Number
+    Default: 10101
+    Description: The port where FeatureBase is listening on.
+  # basic ec2 instance params
+  SecurityGroupID:
+    Type: String
+    Default: sg-000ac63a3e020c470
+    Description: security group id
+  KeyPairName:
+    Type: String
+    Default: smoke-0fjLX8lrTQpONO2-gitlab-ci
+    Description: key pair for ec2 instances
+  InstanceImageId:
+    Type: 'AWS::SSM::Parameter::Value<String>'
+    Description: The ID of the AMI to use for the deployment EC2 instances.
+    Default: /aws/service/ami-amazon-linux-latest/amzn2-ami-hvm-x86_64-gp2  
+  # basic root device volume params
+  VolumeSize:
+    Type: Number
+    Default: 20
+    Description: The volume size in gigabytes for the root volume of deployment instances.
+  Iops:
+    Type: Number
+    Default: 3000
+    Description: The IOPS for the root EBS volume of deployment instances.
+  # FeatureBase Params
+  FBInstanceCount:
+    Type: Number
+    Default: 5
+    Description: The number of EC2 instances for the FeatureBase cluster  
+  FBInstanceSize:
+    Type: String
+    Default: m5ad.16xlarge
+    Description: Class of EC2 instance used to host the deployment.  
+  # note that we are using a producer, which is also the kafka host
+  # and the consumer uses the kafka-static binary for ingesting data
+  # to featurebase
+  # producer/consumer related params  
+  ProducerInstanceSize:
+    Type: String
+    Default: m4.16xlarge
+  ConsumerInstanceSize:
+    Type: String
+    Default: m4.16xlarge
+  ProducerInstanceCount:
+    Type: Number
+    Default: 1
+    Description: number of producer instances.
+  ConsumerInstanceCount:
+    Type: Number
+    Default: 1
+    Description: number of consumer instances.
+  # producer/consumer ebs volume params  
+  EBSVolumeSize:
+    Type: Number
+    Default: 1200    
+  EBSIops:
+    Type: Number
+    Default: 3000    
+  # test info
+  TestCommitSHA:
+    Type: String
+    Default: the-current-featurebase-commit-sha
+    Description: FeatureBase repo gitlab commit SHA. 
+  TestName:
+    Type: String
+    Default: some-test-name
+    Description: Test name (gitlab job name).
+  TestID:
+    Type: String
+    Default: some-test-id
+    Description: Test run id (gitlab CI job id).
+
+Resources:
+
+####################### For provisioning the Featurebase Cluster #########################
+# Autoscaling group. This launches the actual EC2 instances containing FeatureBase
+# we are not really auto scaling anything. this is just a way to launch multiple instances
+  DeploymentAutoScalingGroup:
+    Type: AWS::AutoScaling::AutoScalingGroup
+    Properties:
+      AutoScalingGroupName: !Sub '${AWS::StackName}-asg'
+      VPCZoneIdentifier:
+        !Split [ ",", !Ref PrivateSubnetIds ]
+      LaunchConfigurationName: !Ref 'DeploymentInstances'
+      MinSize: !Ref 'FBInstanceCount'
+      MaxSize: !Ref 'FBInstanceCount'
+      DesiredCapacity: !Ref 'FBInstanceCount'
+      HealthCheckGracePeriod: 120
+      Tags:      
+        - Key: TestCommitSHA
+          Value: !Ref TestCommitSHA
+          PropagateAtLaunch: "true"
+        - Key: TestName
+          Value: !Sub '${TestName}-featurebase-cluster'
+          PropagateAtLaunch: "true"
+        - Key: TestID
+          Value: !Ref TestID
+          PropagateAtLaunch: "true"        
+    UpdatePolicy:
+      AutoScalingRollingUpdate:
+        MaxBatchSize: 1
+
+# Metadata describing the EC2 instance
+  DeploymentInstances:
+    Type: AWS::AutoScaling::LaunchConfiguration
+    Properties:
+      LaunchConfigurationName: !Sub '${AWS::StackName}-launch-cfg'
+      KeyName: !Ref KeyPairName
+      ImageId: !Ref InstanceImageId
+      SecurityGroups:
+      - !Ref SecurityGroupID
+      InstanceType: !Ref FBInstanceSize
+      BlockDeviceMappings:
+        - DeviceName: /dev/xvda
+          Ebs:
+            Encrypted: true
+            VolumeSize: !Ref 'VolumeSize'
+            VolumeType: "gp3"
+            Iops: !Ref 'Iops'
+      IamInstanceProfile: !Ref DeploymentInstanceProfile
+      UserData:
+        Fn::Base64: !Sub |
+          #!/bin/bash
+          set +x
+          set -e
+          sudo mdadm --create --verbose /dev/md0 --level=0 --raid-devices=4 /dev/nvme1n1 /dev/nvme2n1 /dev/nvme3n1 /dev/nvme4n1
+          sudo mkfs.ext4 /dev/md0
+          sudo mkdir data
+          sudo mount /dev/md0 /data
+          sudo chown -R ec2-user:ec2-user /data
+
+####################### For provisioning the Producer Cluster #########################
+  ProducerAutoScalingGroup:
+    Type: AWS::AutoScaling::AutoScalingGroup
+    Properties:
+      AutoScalingGroupName: !Sub '${AWS::StackName}-producer-asg'
+      VPCZoneIdentifier:
+        !Split [ ",", !Ref PrivateSubnetIds ]
+      LaunchConfigurationName: !Ref 'ProducerInstances'
+      MinSize: !Ref 'ProducerInstanceCount'
+      MaxSize: !Ref 'ProducerInstanceCount'
+      DesiredCapacity: !Ref 'ProducerInstanceCount'
+      HealthCheckGracePeriod: 120
+      Tags:      
+        - Key: TestCommitSHA
+          Value: !Ref TestCommitSHA
+          PropagateAtLaunch: "true"
+        - Key: TestName
+          Value: !Sub '${TestName}-ingest-producer'
+          PropagateAtLaunch: "true"
+        - Key: TestID
+          Value: !Ref TestID
+          PropagateAtLaunch: "true"        
+    UpdatePolicy:
+      AutoScalingRollingUpdate:
+        MaxBatchSize: 1
+
+# Metadata describing the EC2 instance
+  ProducerInstances:
+    Type: AWS::AutoScaling::LaunchConfiguration
+    Properties:
+      LaunchConfigurationName: !Sub '${AWS::StackName}-producer-launch-cfg'
+      KeyName: !Ref KeyPairName
+      ImageId: !Ref InstanceImageId
+      SecurityGroups:
+      - !Ref SecurityGroupID
+      InstanceType: !Ref ProducerInstanceSize
+      BlockDeviceMappings:
+        # root device volume
+        - DeviceName: /dev/xvda
+          Ebs:
+            Encrypted: true
+            VolumeSize: !Ref 'VolumeSize'
+            VolumeType: "gp3"
+            Iops: !Ref 'Iops'
+        - DeviceName: /dev/sdb
+          Ebs:
+            Encrypted: true
+            VolumeSize: !Ref 'EBSVolumeSize'
+            VolumeType: "gp3"
+            Iops: !Ref 'EBSIops'
+      IamInstanceProfile: !Ref DeploymentInstanceProfile
+
+####################### For provisioning the Consumer Cluster #########################
+  ConsumerAutoScalingGroup:
+    Type: AWS::AutoScaling::AutoScalingGroup
+    Properties:
+      AutoScalingGroupName: !Sub '${AWS::StackName}-consumer-asg'
+      VPCZoneIdentifier:
+        !Split [ ",", !Ref PrivateSubnetIds ]
+      LaunchConfigurationName: !Ref 'ConsumerInstances'
+      MinSize: !Ref 'ConsumerInstanceCount'
+      MaxSize: !Ref 'ConsumerInstanceCount'
+      DesiredCapacity: !Ref 'ConsumerInstanceCount'
+      HealthCheckGracePeriod: 120
+      Tags:      
+        - Key: TestCommitSHA
+          Value: !Ref TestCommitSHA
+          PropagateAtLaunch: "true"
+        - Key: TestName
+          Value: !Sub '${TestName}-ingest-consumer'
+          PropagateAtLaunch: "true"
+        - Key: TestID
+          Value: !Ref TestID
+          PropagateAtLaunch: "true"        
+    UpdatePolicy:
+      AutoScalingRollingUpdate:
+        MaxBatchSize: 1
+
+# Metadata describing the EC2 instance
+  ConsumerInstances:
+    Type: AWS::AutoScaling::LaunchConfiguration
+    Properties:
+      LaunchConfigurationName: !Sub '${AWS::StackName}-consumer-launch-cfg'
+      KeyName: !Ref KeyPairName
+      ImageId: !Ref InstanceImageId
+      SecurityGroups:
+      - !Ref SecurityGroupID
+      InstanceType: !Ref ConsumerInstanceSize
+      BlockDeviceMappings:
+        # root device volume
+        - DeviceName: /dev/xvda
+          Ebs:
+            Encrypted: true
+            VolumeSize: !Ref 'VolumeSize'
+            VolumeType: "gp3"
+            Iops: !Ref 'Iops'
+        - DeviceName: /dev/sdb
+          Ebs:
+            Encrypted: true
+            VolumeSize: !Ref 'EBSVolumeSize'
+            VolumeType: "gp3"
+            Iops: !Ref 'EBSIops'
+      IamInstanceProfile: !Ref DeploymentInstanceProfile
+
+####################### IAM roles and policies #################################
+# Role for the deployment's EC2 hosts. Adds policies for SSM-based access and CloudWatchAgent.
+  DeploymentEC2Role:
+    Type: AWS::IAM::Role
+    Properties:
+      RoleName: !Sub '${AWS::StackName}'
+      AssumeRolePolicyDocument:
+        Statement:
+        - Effect: Allow
+          Principal:
+            Service: [ec2.amazonaws.com]
+          Action: ['sts:AssumeRole']
+      Path: /
+      ManagedPolicyArns:
+        - 'arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore'
+        - 'arn:aws:iam::aws:policy/CloudWatchAgentServerPolicy'
+        - !Ref EC2InstanceS3PermissionsPolicy
+
+# Attach roles to the deployment instances
+  DeploymentInstanceProfile:
+    Type: AWS::IAM::InstanceProfile
+    Properties:
+      InstanceProfileName: !Sub '${AWS::StackName}'
+      Path: /
+      Roles: [!Ref 'DeploymentEC2Role']
+
+  EC2InstanceS3PermissionsPolicy:
+    Type: AWS::IAM::ManagedPolicy
+    Properties:
+      PolicyDocument:
+        Version: 2012-10-17
+        Statement:
+          - Effect: Allow
+            Action:
+              - 's3:GetObject'              
+              - 's3:PutObject*'
+            Resource:               
+              - !Sub 'arn:aws:s3:::*/*'
+          - Effect: Allow
+            Action: 's3:ListBucket'            
+            Resource: 
+              - !Sub 'arn:aws:s3:::*'
+
+# "Action": [
+#     "s3:PutObject",
+#     "s3:GetObject"
+# ],
+# "Resource": "arn:aws:s3:::tremor-data-backup/*"
+
+Outputs:
+  DeploymentAutoscalingGroupName:
+    Description: Deployment Auto Scaling Group
+    Value: !Ref DeploymentAutoScalingGroup
+  ProducerAutoscalingGroupName:
+    Description: Producer Auto Scaling Group
+    Value: !Ref ProducerAutoScalingGroup
+  ConsumerAutoscalingGroupName:
+    Description: Consumer Auto Scaling Group
+    Value: !Ref ConsumerAutoScalingGroup
--- a/qa/cft/README.md
+++ b/qa/cft/README.md
@ -0,0 +1,135 @@
+# How to deploy featurebase cluster with cloudformation?
+
+- first login to your AWS profile for the FeatureBase-CI AWS account and then cd to featurebase repo root directory. here we assume that your profile is called `fb-ci`
+```
+aws sso login --profile fb-ci
+cd <feature base repo root dir>
+```
+- then run cloudformation deploy. You can customize stuff by overriding the parameters using the `--parameter-overrides` flag. Please see the `FeatureBaseClusterCFT.yaml` for the complete list of parameters that can be customized.
+```
+aws cloudformation deploy \
+--stack-name jon-test-stack \
+--template-file qa/cft/FeatureBaseClusterCFTTremor.yaml \
+--parameter-overrides \
+FBInstanceCount=1 \
+ConsumerInstanceCount=1 \
+ProducerEBSVolumeSize=2400 \
+ProducerEBSIops=3000 \
+TestName=some-test-name \
+--capabilities CAPABILITY_NAMED_IAM \
+--profile fb-ci
+```
+
+## How to manually ssh to the FeatureBase Cluster, Consumer, or Producer nodes?
+
+You will need to setup the aws profile with the credentials to access the FeatureBase-CI AWS account in the job, and then call the above cloudformation command in the job.
+
+After deploying you will need to get one of the ips of the featurebase nodes for ssh. The script `getASGInstanceIPs.sh` takes 2 arguments:
+- auto scaling group name, which is 
+  - for featurebase nodes: `${STACK_NAME}-asg`
+  - for producer nodes: `${STACK_NAME}-producer-asg` (given our current architecture, we can only have 1 producer node, which also serves as the kafka host)
+  - for consumer nodes: `${STACK_NAME}-consumer-asg`
+- aws profile name
+
+The output of the script is the list of ips of the instances in the auto scaling group
+```
+.qa/scriptsgetASGInstanceIPs.sh jon-test-stack2-asg fb-ci
+```
+Now you will need to use VPN to ssh to one of the nodes. To setup VPN, follow this [link](https://molecula.atlassian.net/wiki/spaces/EN/pages/697892918/How+to+Setup+VPN+Acccess+to+MCloud).
+After setting up the VPN, you can then ssh to the ec2 instance with its private IPv4 address
+```
+ssh -i <the "gitlab-featurebase-ci AWS key" pem file in 1Password> ec2-user@<the private IPv4 address of the ec2 instance>
+```
+
+## How to setup producer (& kafka) and consumers?
+go to the top level directory of this featurebase repo
+run the following command (assuming you have already run the previous cloudformation command)
+- arguments for setupFeatureBaseClusterCFTTremor.sh:
+  1) stack name
+  2) aws profile name
+  3) feature base replica count
+  4) feature base instance count
+  5) datadog tag for the ticket
+example use of the script:
+```
+./qa/scripts/setupFeatureBaseClusterCFTTremor.sh some-test-stack fb-ci 3 5 some-datadog-tag
+```
+
+
+
+## How to restore data from the ebs volume?
+As of now 4/25/2022, i made an ebs volume (vol-0f9af1a7bc5dd63e0, tremor-integration-test-restore-records) in *us-east-2a* and backed up ~1B tremor records in the /tremor_backup directory. FYI, you cannot attach this ebs volume outside of us-east-2a.
+
+After setting up featurebase, and ensuring that there is a featurebase data node in the us-east-2a AZ, then we can start the following process to restore those billions of tremor records stored in ebs volume to featurebase for testing.
+- First, attach the ebs volume to the us-east-2a featurebase data node. Note that this ebs volume must be available (not attached to any other instances) for the following command to work.
+```
+aws ec2 attach-volume --instance-id <instance in us-east-2a>  --volume-id vol-0f9af1a7bc5dd63e0 --device /dev/sdf --region us-east-2 --profile fb-ci
+```
+- then, ssh to the featurebase data node and mount the ebs volume to the directory /restore, like so:
+```
+sudo mkdir /restore
+sudo mount /dev/sdf /restore
+sudo chown -R ec2-user:ec2-user /restore
+```
+- now we can perform the featurebase restore, this may take ~20min for 1B tremor records
+```
+featurebase restore --concurrency 64 --source /restore/tremor_backup
+```
+- after restore, please unmount the ebs device 
+```
+sudo umount /restore
+```
+- Exit your ssh session, and then Please detach the ebs volume, so that other processes can use it!!!
+```
+aws ec2 detach-volume --instance-id i-015fca53e6a50c5bf --volume-id vol-0f9af1a7bc5dd63e0 --region us-east-2 --profile fb-ci
+```
+
+## How to destroy old backup and create new backup data to this ebs volume
+In the future, we may want to remove the obsolete back up data and store a new dataset in the ebs volume, we can do the following steps:
+- First, attach the ebs volume to the us-east-2a featurebase data node. Note that this ebs volume must be available (not attached to any other instances) for the following command to work.
+```
+aws ec2 attach-volume --instance-id <instance in us-east-2a>  --volume-id vol-0f9af1a7bc5dd63e0 --device /dev/sdf --region us-east-2 --profile fb-ci
+```
+- then, ssh to the featurebase data node and mount the ebs volume to the directory /restore, like so: (note that the `mkfs.ext4` command will destroy all data in the ebs volume!)
+```
+sudo mkdir /restore
+sudo mkfs.ext4 /dev/sdf
+sudo mount /dev/sdf /restore
+sudo chown -R ec2-user:ec2-user /restore
+```
+- now we can perform the featurebase backup, this may take ~56min for 1B tremor records & a 3 node cluster
+```
+featurebase backup --index tremor -o /restore/tremor-backup --concurrency 64 
+```
+- after backup, please unmount the ebs device 
+```
+sudo umount /restore
+```
+- Exit your ssh session, and then Please detach the ebs volume, so that other processes can use it!!!
+```
+aws ec2 detach-volume --instance-id i-015fca53e6a50c5bf --volume-id vol-0f9af1a7bc5dd63e0 --region us-east-2 --profile fb-ci
+```
+
+## How to launch multiple consumers?
+You can just specify ConsumerInstanceCount to be 2 when you run cloudformation deploy, like so:
+```
+aws cloudformation deploy \
+--stack-name jon-test-stack \
+--template-file qa/cft/FeatureBaseClusterCFTTremor.yaml \
+--parameter-overrides \
+FBInstanceCount=1 \
+ConsumerInstanceCount=1 \
+ProducerEBSVolumeSize=2400 \
+ProducerEBSIops=3000 \
+TestName=some-test-name \
+--capabilities CAPABILITY_NAMED_IAM \
+--profile fb-ci
+```
+The setup scripts are written to support multiple consumers, so you can just run the same setup script as mentioned previously.
+But the performance is not better than just using 1 consumer as of now, so some fine tuning is required i think...
+
+## Other things to be mindful of:
+- If you are creating multiple featurebase clusters to parallelize testing, make sure you give them different stack names!
+- Note that this cloudformation template FeatureBaseClusterCFT.yaml is still using the same smoke vpc and subnets.
+
+
--- a/qa/scripts/auth-smoke/setup.sh
+++ b/qa/scripts/auth-smoke/setup.sh
@ -18,6 +18,12 @@ terraform apply -input=false -auto-approve
 terraform output -json > outputs.json
 popd

+# path for ebs device
+EBS_DEVICE_NAME=/dev/nvme1n1
+
+# featurebase architecture
+FB_BINARY=featurebase_linux_arm64
+
 # get the first ingest host
 INGESTNODE0=$(cat ./qa/tf/ci/auth-smoke/outputs.json | jq -r '[.ingest_ips][0]["value"][0]')
 echo "using INGESTNODE0 ${INGESTNODE0}"
--- a/qa/scripts/delete/README.md
+++ b/qa/scripts/delete/README.md
@ -0,0 +1,7 @@
+
+
+# backup command
+featurebase backup --index tremor -o tremor-backup --concurrency 64 
+
+# data was tar in parallel with this command
+time tar cfv - ./tremor-backup/ | pigz -1 -p 64 > tremor.tar.gz
--- a/qa/scripts/delete/runConsumer.sh
+++ b/qa/scripts/delete/runConsumer.sh
@ -0,0 +1,52 @@
+#!/bin/bash
+set -e
+set +x
+
+runConsumer(){
+    echo "Started running molecula-consumer-kafka-static on consumer node"
+    START_CONSUMER=$(date +%s%N) # in nano seconds
+
+    # start molecula consumer kafka static - read messages from kafka server and ingest into FeatureBase 
+    ./molecula-consumer-kafka-static --pilosa-hosts=$DATANODE --kafka-hosts $INGESTNODE0:9092 --index $INDEX --primary-key-fields uuid --header ./schema.json --batch-size $BATCH_SIZE --topics $TOPIC --max-msgs $INGEST_COUNT --concurrency $CONCURRENCY &
+
+    # wait for all background processes to finish
+    for job in `jobs -p`
+    do
+        wait $job 
+    done
+
+    END_CONSUMER=$(date +%s%N) # in nano seconds
+    DURATION_CONSUMER=$(($(($END_CONSUMER-$START_CONSUMER))/1000000000/60)) # convert from ns to min
+
+    # send metrics to featurebase 
+    echo -n "consumer_duration_min:$DURATION_CONSUMER|g|#test_name:$TEST_NAME" | nc -4u -w1 127.0.0.1 8125
+
+    echo "finished consumer - duration: $DURATION_CONSUMER min"   
+}
+
+
+DATANODE=$1
+shift
+
+INGESTNODE0=$1
+shift 
+
+INGEST_COUNT=$1
+shift
+
+BATCH_SIZE=$1
+shift
+
+CONCURRENCY=$1
+shift
+
+INDEX=$1
+shift
+
+TOPIC=$1
+shift 
+
+TEST_NAME=$1
+shift 
+
+runConsumer
--- a/qa/scripts/delete/runDeleteTest.sh
+++ b/qa/scripts/delete/runDeleteTest.sh
@ -0,0 +1,71 @@
+#!/bin/bash
+
+# before running make sure runQueries.sh script is in the same directory as runTremorDeleteTest.sh
+# and ./runProducer.sh is copied to the producer node, ./runConsumer.sh copied to the consumer node
+# assumes setupTremorDeleteTestCFT.sh has been run to set up data nodes in the cluster and producer/consumer nodes 
+# To run script: ./runTremorDeleteTest.sh <IP for data node> <IP for consumer node> <IP for producer node>
+
+INGEST_COUNT=1000000000 
+MAX_MSGS=5000000000
+BATCH_SIZE=500000
+CONCURRENCY=128
+PARTITIONS=128
+INDEX=tremor
+TOPIC=tremor
+TEST_NAME="tremor-delete-ci"
+
+producer(){
+    SEED=$1 
+
+    # run kafka ingest in the first ingest node - kafka was installed and started in this node 
+    ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${PRODUCER0} "bash runProducer.sh ${DATANODE0} ${PRODUCER0} ${INGEST_COUNT} ${BATCH_SIZE} ${PARTITIONS} ${INDEX} ${TOPIC} ${TEST_NAME} ${SEED}"
+}
+
+consumer(){
+    sleep 120 # sleep 2min, allow enough messages to be generated in kafka 
+
+    # run ingest to featurebase in the second ingest node 
+    ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${CONSUMER0} "bash runConsumer.sh ${DATANODE0} ${PRODUCER0} ${MAX_MSGS} ${BATCH_SIZE} ${CONCURRENCY} ${INDEX} ${TOPIC} ${TEST_NAME}"
+
+}
+
+ingest(){
+    # many producers - produce messages to kafka     
+    for i in {1..5}
+    do
+        echo "launching producer with seed $i"
+        ( producer $i & )
+    done
+
+    # 1 consumer  - consume messages from kafka to featurebase
+    echo "launching consumer" 
+    consumer & 
+}
+
+runQueries() {
+    echo "running queries" 
+    # run ingest to featurebase in the second ingest node 
+    ssh -A -o StrictHostKeyChecking=no ec2-user@${DATANODE0} "bash runQueries.sh ${DATANODE0} ${INDEX} ${TEST_NAME} ${DEPLOYED_DATA_IPS[@]}"
+}
+
+runIngestAndQueries() {
+    DATANODE0=$1
+    PRODUCER0=$2 # producer node is expected to have kafka running
+    CONSUMER0=$3 
+    DEPLOYED_DATA_IPS=("${@:4}")
+
+    echo "Producer Node: ${PRODUCER0}, Consumer Node: ${CONSUMER0}, Data Node: ${DATANODE0}"
+    echo "DEPLOYED_DATA_IPS: ${DEPLOYED_DATA_IPS[@]}"
+
+    # perform ingest 
+    ingest
+    
+    # wait for all background processes to finish
+    for job in `jobs -p`
+    do
+        wait $job 
+    done
+
+    # perform queries 
+    runQueries
+}
--- a/qa/scripts/delete/runProducer.sh
+++ b/qa/scripts/delete/runProducer.sh
@ -0,0 +1,56 @@
+#!/bin/bash
+set -e
+set +x
+
+runDatagen(){
+    echo "Started datagen on producer node"
+    START_DATAGEN=$(date +%s%N) # in nano seconds
+
+    # start datagen - produce messages to kafka server
+    ./datagen --kafka.confluent-command.kafka-bootstrap-servers $INGESTNODE0:9092 --source custom --custom-config ./tremor_keys.yaml --pilosa.index $INDEX --pilosa.hosts=$DATANODE --target kafkastatic --kafka.topic $TOPIC --end-at $INGEST_COUNT --kafka.num-partitions $PARTITIONS --track-progress --seed $SEED
+
+    # wait for all background processes to finish
+    for job in `jobs -p`
+    do
+        wait $job 
+    done
+
+    END_DATAGEN=$(date +%s%N) # in nano seconds
+
+    # compute duration for ingest
+    DURATION_DATAGEN=$(($(($END_DATAGEN-$START_DATAGEN))/1000000000/60)) # convert from ns to min
+
+    # send metrics to datagen 
+    echo -n "datagen_duration_min:$DURATION_DATAGEN|g|#test_name:$TEST_NAME" | nc -4u -w1 127.0.0.1 8125   
+
+    echo "finished datagen - duration: $DURATION_DATAGEN min"    
+}
+
+DATANODE=$1
+shift
+
+INGESTNODE0=$1
+shift 
+
+INGEST_COUNT=$1
+shift
+
+BATCH_SIZE=$1
+shift
+
+PARTITIONS=$1
+shift
+
+INDEX=$1
+shift
+
+TOPIC=$1
+shift 
+
+TEST_NAME=$1
+shift 
+
+SEED=$1
+shift 
+
+runDatagen
--- a/qa/scripts/delete/runQueries.sh
+++ b/qa/scripts/delete/runQueries.sh
@ -0,0 +1,179 @@
+#!/bin/bash
+
+# To run script: ./runQueries.sh <IP for data node> <index name> <test name for datadog tag>
+
+DATANODE0=""
+INDEX=""
+TEST_NAME=""
+
+getEpochTime() {
+    DATE=$1
+    # epoch time since Jan 1, 1970 in days 
+    # DATE format is expected to be %Y%M%D
+    EPOCH_DATE=$(echo $(( ($(date --date="$DATE" +%s) - $(date --date="700101" +%s)) /(60*60*24))))
+}
+
+
+delete(){
+    # get shards for index
+    SHARDS=$(curl -s ${DATANODE0}:10101/internal/index/${INDEX}/shards | jq '.shards[]')
+
+    OLDEST_DATE="220102"
+    getEpochTime $OLDEST_DATE
+
+    echo "running delete query for data older than ${OLDEST_DATE} - epoch in days: ${EPOCH_DATE}"
+
+    # increment epoch by 1 before creating query string to exclude oldest date 
+    ((EPOCH_DATE=$EPOCH_DATE+1))
+    
+    UNION_STRING=""
+    for day in {0..29}
+    do 
+        CURR_STRING="Row(lastupdated=$EPOCH_DATE)"
+
+        # concatenate row query string 
+        UNION_STRING="$UNION_STRING$CURR_STRING,"
+
+        # increment date by 1
+        ((EPOCH_DATE=$EPOCH_DATE+1))
+    done 
+
+    # perform delete per shard - get all records that have not been updated in the last 30 days
+    for SHARD in ${SHARDS}
+    do
+        SHARD_COUNT=$(curl -s ${DATANODE0}:10101/index/${INDEX}/query -d "Options(Count(Not(Union($UNION_STRING))), shards=[${SHARD}])" | jq -r  '.results[0]')
+        if [[ $SHARD_COUNT -gt "0" ]]; then 
+            START=$(date +%s%N) # in nano seconds
+            RES=$(curl -s ${DATANODE0}:10101/index/${INDEX}/query -d "Options(Delete(Not(Union($UNION_STRING))), shards=[${SHARD}])")
+            END=$(date +%s%N) # in nano seconds
+            DURATION=$(($(($END-$START))/1000000)) # convert from ns to ms
+            echo "deleting records older than ${OLDEST_DATE} from shard ${SHARD}, count: ${SHARD_COUNT}, duration: ${DURATION}ms, result: ${RES}"   
+            echo -n "delete_count_per_shard:$SHARD_COUNT|g|#shard:$SHARD,test_name:$TEST_NAME,delete_cycle:$DELETE_CYCLE" | nc -4u -w1 127.0.0.1 8125
+            echo -n "delete_duration_per_shard_ms:$DURATION|g|#shard:$SHARD,test_name:$TEST_NAME,delete_cycle:$DELETE_CYCLE" | nc -4u -w1 127.0.0.1 8125
+        # commented since it was spamming the logs, only needed for debugging 
+        # else
+        #     echo "no records found to delete for index $INDEX for query: Options(Count(Not(Union($UNION_STRING))),shards=[${SHARD}])"
+        fi 
+    done
+}
+
+countAll(){
+    echo "running countAll query: $i"
+    QUERY_STRING=countAll
+    START_COUNT=$(date +%s%N)  # in nano seconds
+    TOTAL_COUNT=$(curl -s "${DATANODE0}:10101/index/${INDEX}/query" -d "Count(all())" | jq -r  '.results[0]')
+    END_COUNT=$(date +%s%N)  # in nano seconds
+    DURATION_COUNT=$(($(($END_COUNT-$START_COUNT))/1000000)) # convert from ns to ms
+
+    echo "count for $QUERY_STRING is ${TOTAL_COUNT}, query duration is ${DURATION_COUNT}s"
+
+    echo -n "featurebase_count:$TOTAL_COUNT|g|#test_name:$TEST_NAME,query_string:$QUERY_STRING" | nc -4u -w1 127.0.0.1 8125
+    echo -n "featurebase_count_duration_ms:$DURATION_COUNT|g|#test_name:$TEST_NAME,query_string:$QUERY_STRING" | nc -4u -w1 127.0.0.1 8125  
+}
+
+countIntersect(){
+    echo "running countIntersect query: $i"
+    SEGID_1=34999
+    SEGID_2=33999
+    QUERY_STRING=countIntersect_segid\(${SEGID_1}_${SEGID_2}\)
+    START_COUNT=$(date +%s%N)  # in nano seconds
+    TOTAL_COUNT=$(curl -s "${DATANODE0}:10101/index/${INDEX}/query" -d"Count(Intersect(Row(segid=$SEGID_1), Row(segid=$SEGID_2)))" | jq -r  '.results[0]')
+    END_COUNT=$(date +%s%N)  # in nano seconds
+    DURATION_COUNT=$(($(($END_COUNT-$START_COUNT))/1000000)) # convert from ns to ms
+    
+
+    echo -n "featurebase_count:$TOTAL_COUNT|g|#test_name:$TEST_NAME,query_string:$QUERY_STRING" | nc -4u -w1 127.0.0.1 8125
+    echo -n "featurebase_count_duration_ms:$DURATION_COUNT|g|#test_name:$TEST_NAME,query_string:$QUERY_STRING" | nc -4u -w1 127.0.0.1 8125 
+    echo "count for $QUERY_STRING is ${TOTAL_COUNT}, query duration is ${DURATION_COUNT}s"
+}
+
+countRowDate(){
+    echo "running countRowDate query: $i"
+    SEGID=34999
+    DATE_FROM=2022-01-15
+    DATE_TO=2022-02-02
+    QUERY_STRING=countRowDate_segid\(${SEGID}\)_date\(${DATE_FROM}_${DATE_TO}\)
+    START_COUNT=$(date +%s%N)  # in nano seconds
+    TOTAL_COUNT=$(curl -s "${DATANODE0}:10101/index/${INDEX}/query" -d"Count(Row(segid=$SEGID, from='$DATE_FROM', to='$DATE_TO'))" | jq -r  '.results[0]')
+    END_COUNT=$(date +%s%N)  # in nano seconds
+    DURATION_COUNT=$(($(($END_COUNT-$START_COUNT))/1000000)) # convert from ns to ms
+
+    echo -n "featurebase_count:$TOTAL_COUNT|g|#test_name:$TEST_NAME,query_string:$QUERY_STRING" | nc -4u -w1 127.0.0.1 8125
+    echo -n "featurebase_count_duration_ms:$DURATION_COUNT|g|#test_name:$TEST_NAME,query_string:$QUERY_STRING" | nc -4u -w1 127.0.0.1 8125 
+    echo "count for $QUERY_STRING is ${TOTAL_COUNT}, query duration is ${DURATION_COUNT}s"
+}
+
+extract(){
+    echo "running extract query: $i"
+    FIELD=lastupdated
+    LIMIT=3
+    QUERY_STRING=extract_row\(${FIELD}\)_limit\(${LIMIT}\)
+    START_COUNT=$(date +%s%N)  # in nano seconds
+    TOTAL_COUNT=$(curl -s "${DATANODE0}:10101/index/${INDEX}/query" -d"Extract(Limit(All( ), limit=$LIMIT), Rows(lastupdated))"| jq -r  '[..|scalars]|length')
+    END_COUNT=$(date +%s%N)  # in nano seconds
+    DURATION_COUNT=$(($(($END_COUNT-$START_COUNT))/1000000000)) # convert from ns to s
+
+    echo -n "featurebase_extract:$TOTAL_COUNT|g|#test_name:$TEST_NAME,query_string:$QUERY_STRING" | nc -4u -w1 127.0.0.1 8125
+    echo -n "featurebase_extract_duration_s:$DURATION_COUNT|g|#test_name:$TEST_NAME,query_string:$QUERY_STRING" | nc -4u -w1 127.0.0.1 8125 
+    echo "count for $QUERY_STRING is ${TOTAL_COUNT}, query duration is ${DURATION_COUNT}s"
+}
+
+disk_usage() {
+    TOTAL_DISK=0
+    for IP in ${DEPLOYED_DATA_IPS[@]}
+    do
+        DISK_PER_NODE=$(curl -s $IP:10101/internal/disk-usage | jq -r '.usage')
+        DISK_PER_NODE=$(($DISK_PER_NODE/1000000000)) # conver to GB 
+        echo "disk usage is ${DISK_PER_NODE}GB for data node ${IP}"
+        TOTAL_DISK=$(($TOTAL_DISK+$DISK_PER_NODE))
+    done 
+    echo -n "featurebase_disk_usage_GB:$TOTAL_DISK|g|#test_name:$TEST_NAME" | nc -4u -w1 127.0.0.1 8125
+    echo "total disk usage for all nodes is ${TOTAL_DISK}GB"
+}
+
+memory_usage() {
+    TOTAL_MEMORY=0
+    for IP in ${DEPLOYED_DATA_IPS[@]}
+    do
+        MEMORY_PER_NODE=$(curl -s localhost:10101/internal/mem-usage | jq -r '.totalUsed')
+        MEMORY_PER_NODE=$(($MEMORY_PER_NODE/1000000000)) # conver to GB 
+        echo "memory usage is ${MEMORY_PER_NODE}GB for data node ${IP}"
+        TOTAL_MEMORY=$(($TOTAL_MEMORY+$MEMORY_PER_NODE))
+    done 
+    echo -n "featurebase_memory_usage_GB:$TOTAL_MEMORY|g|#test_name:$TEST_NAME" | nc -4u -w1 127.0.0.1 8125
+    echo "total memory usage for all nodes is ${TOTAL_MEMORY}GB"
+}
+
+runAllQueries(){
+    echo "Start running queries"
+    echo "run multiple queries 10 times"
+    for i in {1..10}
+    do
+        countAll 
+        countIntersect 
+        countRowDate 
+        extract 
+    done
+
+    # get count, disk and memory prior to delete 
+    countAll 
+    disk_usage 
+    memory_usage 
+
+    # perform delete 
+    delete 
+
+    # get count, disk and memory after delete
+    countAll
+    disk_usage 
+    memory_usage   
+}
+
+DATANODE0=$1 
+INDEX=$2
+TEST_NAME=$3
+DEPLOYED_DATA_IPS=("${@:4}")
+
+echo "Data node IPs: ${DEPLOYED_DATA_IPS[@]}"
+
+runAllQueries
--- a/qa/scripts/delete/schema.json
+++ b/qa/scripts/delete/schema.json
@ -0,0 +1,44 @@
+[
+    {
+        "name": "uuid",
+        "path": [
+            "uuid"
+        ],
+        "type": "string"
+    },
+    {
+        "name": "type",
+        "path": [
+            "type"
+        ],
+        "type": "id"
+    },
+    {
+        "name": "segid",
+        "path": [
+            "slice"
+        ],
+        "type": "ids",
+        "config": {
+            "quantum": "D"
+        }
+    },
+    {
+        "name": "timestamp",
+        "path": [
+            "ts"
+        ],
+        "type": "recordTime",
+        "config": {
+            "granularity": "s",
+            "unit": "ms"
+        }
+    },
+    {
+        "name": "lastupdated",
+        "path": [
+            "lastupdated"
+        ],
+        "type": "id"
+    }
+ ]
--- a/qa/scripts/delete/tremor_keys.yaml
+++ b/qa/scripts/delete/tremor_keys.yaml
@ -0,0 +1,65 @@
+fields:
+  - name: "uuid"
+    type: string
+    distribution: "shifting" # zipfian distruction and shifting the window over time.
+    step: 2 # step for "shifting" distribution means how often do we shift. Every time we generate a value, we'll add an amount to it, every <step> generations, we'll increase the amount by one. 
+    cardinality: 4000000000
+    max_len: 32
+    s: 1.0001
+    v: 1
+    null_chance: 0  
+  - name: "type"
+    type: "int" # (default IntField)
+    min: 0
+    max: 3
+    distribution: "zipfian"
+    s: 1.1
+    v: 5.1
+  - name: "ts"
+    type: "timestamp"
+    min_date: 2022-01-02T15:04:05.001Z # RFC3339Nano
+    max_date: 2022-02-02T15:04:05.001Z # RFC3339Nano
+    distribution: "increasing" # only "increasing" is supported right now
+    min_step_duration: "1ns"
+    max_step_duration: "2.50ms" 
+    time_format: "unix"
+    time_unit: "ms"
+  - name: "lastupdated"
+    type: "timestamp"
+    min_date: 2022-01-02T15:04:05.001Z # RFC3339Nano
+    max_date: 2022-02-02T15:04:05.001Z # RFC3339Nano
+    distribution: "increasing" # only "increasing" is supported right now
+    min_step_duration: "1ns"
+    max_step_duration: "2.50ms" 
+    time_format: "unix"
+    time_unit: "d"
+  - name: "slice"
+    type: "uint-set" # (default IDArrayField)
+    min: 0
+    max: 35000
+    distribution: "zipfian"
+    s: 1.1
+    v: 5.1
+    min_num: 1
+    max_num: 50
+
+# idk_params describe how data from "fields" should be ingested by IDK
+idk_params:
+  primary_key_config:
+    field: "uuid" # if this is a single field named "id" then we'll use uint IDs, if it's empty we'll autogen ids, and if it's anything else we'll do string keys... yes this is a bit hacky, needs to be cleaned up.
+  # fields is keyed by names of fields from top level "fields". It is
+  # not required that all fields appear here, those that don't will
+  # use the default ingestion.
+  fields:
+    lastupdated:
+      - type: "ID"
+    ts:
+      - type: "RecordTime"
+        unit: "ms"
+        name: "timestamp"
+    slice:
+      - type: "IDArray"
+        quantum: "D"
+        name: "segid"
+    type:
+      - type: "ID"
--- a/qa/scripts/getASGInstanceIPs.sh
+++ b/qa/scripts/getASGInstanceIPs.sh
@ -0,0 +1,21 @@
+#!/bin/bash
+
+# usage of this script:
+# ./getInstanceIPs.sh my-asg-name my-aws-profile-name
+
+# if you have used the FeatureBaseCLusterCFTTremor.yaml cloudoformation template, 
+# then you can do the following to save the IPs of the auto scaling groups:
+
+# for the featurebase data nodes ips:
+# DEPLOYED_DATA_IPS=$(./getASGInstanceIPs.sh ${STACK_NAME}-asg ${AWS_PROFILE})
+
+# for the producer nodes:
+# PRODUCER_IPS=$(./getASGInstanceIPs.sh ${STACK_NAME}-producer-asg ${AWS_PROFILE})
+
+# for the consumer nodes:
+# CONSUMER_IPS=$(./getASGInstanceIPs.sh ${STACK_NAME}-consumer-asg ${AWS_PROFILE})
+
+for i in `aws autoscaling describe-auto-scaling-groups --auto-scaling-group-name $1 --profile $2 --output=json | grep -i instanceid  | awk '{ print $2}' | cut -d',' -f1| sed -e 's/"//g'`
+do
+    aws ec2 describe-instances --output=json --instance-ids $i --profile $2 | grep -i PrivateIpAddress | awk '{ print $2 }' | head -1 | cut -d"," -f1| sed -e 's/"//g'
+done;
--- a/qa/scripts/runTremorDeleteCFT.sh
+++ b/qa/scripts/runTremorDeleteCFT.sh
@ -0,0 +1,35 @@
+#!/bin/bash
+set -e
+set +x
+
+# before running, it is expected that ./setupTremorDeleteTestCFT.sh has been run in a running cluster
+# To run script: ./runTremorDeleteTestCFT.sh <10.0.105.100-name> <aws profile>
+
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+
+STACK_NAME=$1
+AWS_PROFILE=$2
+
+# get the data node ips
+DEPLOYED_DATA_IPS=($($SCRIPT_DIR/getASGInstanceIPs.sh ${STACK_NAME}-asg ${AWS_PROFILE}))
+echo "DEPLOYED_DATA_IPS: {"
+echo "${DEPLOYED_DATA_IPS[@]}"
+echo "}"
+
+DATANODE0=${DEPLOYED_DATA_IPS[0]}
+echo "using DATANODE0 ${DATANODE0}"
+
+# get producer node IP
+PRODUCER_IPS=$($SCRIPT_DIR/getASGInstanceIPs.sh ${STACK_NAME}-producer-asg ${AWS_PROFILE})
+PRODUCER0=$(echo $PRODUCER_IPS | head -n1 | cut -d " " -f1)
+
+# get consumer node IP
+CONSUMER_IPS=$($SCRIPT_DIR/getASGInstanceIPs.sh ${STACK_NAME}-consumer-asg ${AWS_PROFILE})
+CONSUMER0=$(echo $CONSUMER_IPS | head -n1 | cut -d " " -f1)
+
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+source $SCRIPT_DIR/delete/runDeleteTest.sh
+
+# run test
+runIngestAndQueries ${DATANODE0} ${PRODUCER0} ${CONSUMER0} "${DEPLOYED_DATA_IPS[@]}"
+
--- a/qa/scripts/setupSmokeTest.sh
+++ b/qa/scripts/setupSmokeTest.sh
@ -15,6 +15,12 @@ terraform apply -input=false -auto-approve
 terraform output -json > outputs.json
 popd

+# path for ebs device
+EBS_DEVICE_NAME=/dev/nvme1n1
+
+# featurebase architecture
+FB_BINARY=featurebase_linux_arm64
+
 # get the first ingest host
 INGESTNODE0=$(cat ./qa/tf/ci/smoketest/outputs.json | jq -r '[.ingest_ips][0]["value"][0]')
 echo "using INGESTNODE0 ${INGESTNODE0}"
--- a/qa/scripts/setupTLS.sh
+++ b/qa/scripts/setupTLS.sh
@ -20,6 +20,9 @@ echo "installing go"
 sudo yum install wget -y
 ifErr "installing wget"

+sudo yum install git -y
+ifErr "installing git"
+
 sudo wget -q https://go.dev/dl/go1.17.8.linux-arm64.tar.gz
 ifErr "downloading golang"

--- a/qa/scripts/setupTremorDeleteCFT.sh
+++ b/qa/scripts/setupTremorDeleteCFT.sh
@ -0,0 +1,116 @@
+#!/bin/bash
+
+# Make sure you have already deployed your cloudformation stack
+# before you run this script
+# To run script: ./setupTremorDeleteTestCFT.sh <stack-name> <aws profile> <replica count> <fb instance count> <branch-name>
+
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+source $SCRIPT_DIR/utilCluster.sh
+
+# store arguments
+STACK_NAME=$1
+AWS_PROFILE=$2
+DEPLOYED_CLUSTER_REPLICA_COUNT=$3
+DEPLOYED_CLUSTER_INSTANCE_COUNT=$4
+BRANCH_NAME=$5
+
+# update ebs device name for ingest nodes 
+EBS_DEVICE_NAME="/dev/sdb"
+
+# get producer ip (assume only 1 instance for now)
+PRODUCER_IPS=$($SCRIPT_DIR/getASGInstanceIPs.sh ${STACK_NAME}-producer-asg ${AWS_PROFILE})
+PRODUCER0=$(echo $PRODUCER_IPS | head -n1 | cut -d " " -f1)
+echo "setting up producer node, producer_ip: $PRODUCER0, EBS device: $EBS_DEVICE_NAME"
+setupProducerNode $PRODUCER0
+
+# setup consumer
+CONSUMER_IPS=$($SCRIPT_DIR/getASGInstanceIPs.sh ${STACK_NAME}-consumer-asg ${AWS_PROFILE})
+CONSUMER0=$(echo $CONSUMER_IPS | head -n1 | cut -d " " -f1)
+echo "setting up consumer node, consumer_ip: $CONSUMER0, EBS device: $EBS_DEVICE_NAME"
+setupConsumerNode $CONSUMER0
+
+# no ebs device, using nvme instead, we can skip ebs mount in executeGeneralNodeConfigCommands()
+# nvme is used for data nodes
+EBS_DEVICE_NAME="nvme"
+
+# get featurebase binary from the current branch 
+FB_BINARY=featurebase_linux_amd64
+echo "Downloading FeatureBase binary"
+aws s3 cp s3://molecula-artifact-storage/featurebase/${BRANCH_NAME}/_latest/featurebase_linux_amd64 ./ --profile=$AWS_PROFILE
+
+# get the data node ips
+DEPLOYED_DATA_IPS=$($SCRIPT_DIR/getASGInstanceIPs.sh ${STACK_NAME}-asg ${AWS_PROFILE})
+echo "DEPLOYED_DATA_IPS: {"
+echo "${DEPLOYED_DATA_IPS}"
+echo "}"
+DEPLOYED_DATA_IPS_LEN=`echo "$DEPLOYED_DATA_IPS" | wc -l`
+
+# poll for autoscaling instance counts, until it matches the desired count
+ASG_NOT_STABLE="true"
+for i in {0..24}
+do
+    if [ $DEPLOYED_DATA_IPS_LEN -eq $DEPLOYED_CLUSTER_INSTANCE_COUNT ]
+    then 
+        echo "Autoscaling group is ready after ${i} tries."
+        ASG_NOT_STABLE="false"
+        break
+    fi 
+    echo "waiting for autoscaling group to reach steady state..."
+    sleep 10
+    DEPLOYED_DATA_IPS=$($SCRIPT_DIR/getASGInstanceIPs.sh ${STACK_NAME}-asg ${AWS_PROFILE})
+    echo "DEPLOYED_DATA_IPS: {"
+    echo "${DEPLOYED_DATA_IPS}"
+    echo "}"
+    DEPLOYED_DATA_IPS_LEN=`echo "$DEPLOYED_DATA_IPS" | wc -l`    
+done
+
+if [[ "$ASG_NOT_STABLE" == "true" ]]
+then
+    echo "Autoscaling group still unstable - giving up" 
+    exit 1
+fi
+
+DATANODE0=$(echo $DEPLOYED_DATA_IPS | head -n1 | cut -d " " -f1)
+echo "using DATANODE0 ${DATANODE0}"
+
+# define some env var and print them
+DEPLOYED_CLUSTER_PREFIX=${STACK_NAME}
+echo "Using DEPLOYED_CLUSTER_PREFIX: ${DEPLOYED_CLUSTER_PREFIX}"
+echo "Using DEPLOYED_CLUSTER_REPLICA_COUNT: ${DEPLOYED_CLUSTER_REPLICA_COUNT}"
+
+#wait until we can connect to one of the hosts
+for i in {0..24}
+do 
+    ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no -o ConnectTimeout=10 ec2-user@${DATANODE0} "pwd"
+    if [ $? -eq 0 ]
+    then
+        echo "Cluster is up after ${i} tries." 
+        break
+    fi
+    sleep 10
+done
+
+ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no -o ConnectTimeout=10 ec2-user@${DATANODE0} "pwd"
+if [ $? -ne 0 ]
+then
+    echo "Unable to connect to cluster - giving up" 
+    exit 1
+fi
+
+#setup data nodes
+generateInitialClusterString
+setupDataNodes
+startDataNodes
+
+# copy script for test 
+scp -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ./qa/scripts/delete/runQueries.sh ec2-user@${DATANODE0}:
+
+# verify featurebase running
+echo "Verifying featurebase cluster running..."
+curl -s http://${DATANODE0}:10101/status
+if (( $? != 0 )) 
+then 
+    echo "Featurebase cluster not running"
+    exit 1
+fi
+echo "FeatureBase Cluster running."
--- a/qa/scripts/setupTremorDeleteTest.sh
+++ b/qa/scripts/setupTremorDeleteTest.sh
@ -0,0 +1,120 @@
+#!/bin/bash
+set -e 
+set +x 
+
+# To run script: ./setupTremorDeleteTest.sh
+export TF_IN_AUTOMATION=1
+
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+source $SCRIPT_DIR/utilCluster.sh
+
+# path for ebs device
+EBS_DEVICE_NAME=nvme
+
+# featurebase architecture
+FB_BINARY=featurebase_linux_amd64
+echo "Downloading FeatureBase binary"
+aws s3 cp s3://molecula-artifact-storage/featurebase/master/_latest/featurebase_linux_amd64 ./ --profile=$TF_VAR_profile
+
+# datagen binary
+DATAGEN_BINARY=datagen
+echo "Downloading Datagen binary"
+aws s3 cp s3://molecula-artifact-storage/idk/master/_latest/idk-linux-amd64/datagen $DATAGEN_BINARY --profile=$TF_VAR_profile
+
+KAFKA_STATIC=molecula-consumer-kafka-static
+echo "Downloading Molecular Consumer Kafka Static binary"
+aws s3 cp s3://molecula-artifact-storage/idk/master/_latest/idk-linux-amd64/molecula-consumer-kafka-static ./$KAFKA_STATIC --profile=$TF_VAR_profile
+
+# get the first ingest host
+INGESTNODE0=$(cat ./qa/tf/ci/deletetest/outputs.json | jq -r '[.ingest_ips][0]["value"][0]')
+echo "using INGESTNODE0 ${INGESTNODE0}"
+
+# get the first data host
+DATANODE0=$(cat ./qa/tf/ci/deletetest/outputs.json | jq -r '[.data_node_ips][0]["value"][0]')
+echo "using DATANODE0 ${DATANODE0}"
+
+DEPLOYED_CLUSTER_PREFIX=$(cat ./qa/tf/ci/deletetest/outputs.json | jq -r '[.cluster_prefix][0]["value"]')
+echo "Using DEPLOYED_CLUSTER_PREFIX: ${DEPLOYED_CLUSTER_PREFIX}"
+
+DEPLOYED_CLUSTER_REPLICA_COUNT=$(cat ./qa/tf/ci/deletetest/outputs.json | jq -r '[.fb_cluster_replica_count][0]["value"]')
+echo "Using DEPLOYED_CLUSTER_REPLICA_COUNT: ${DEPLOYED_CLUSTER_REPLICA_COUNT}"
+
+DEPLOYED_DATA_IPS=($(cat ./qa/tf/ci/deletetest/outputs.json | jq -r '[.data_node_ips][0]["value"][]'))
+echo "DEPLOYED_DATA_IPS: {"
+echo "${DEPLOYED_DATA_IPS[@]}"
+echo "}"
+
+DEPLOYED_DATA_IPS_LEN=${#DEPLOYED_DATA_IPS[@]}
+
+DEPLOYED_INGEST_IPS=($(cat ./qa/tf/ci/deletetest/outputs.json | jq -r '[.ingest_ips][0]["value"][]'))
+echo "DEPLOYED_INGEST_IPS: {"
+echo "${DEPLOYED_INGEST_IPS[@]}"
+echo "}"
+
+DEPLOYED_INGEST_IPS_LEN=${#DEPLOYED_INGEST_IPS[@]}
+
+
+#wait until we can connect to one of the hosts
+for i in {0..24}
+do 
+    ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no -o ConnectTimeout=10 ec2-user@${DATANODE0} "pwd"
+    if [ $? -eq 0 ]
+    then
+        echo "Cluster is up after ${i} tries." 
+        break
+    fi
+    sleep 10
+done
+
+ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no -o ConnectTimeout=10 ec2-user@${DATANODE0} "pwd"
+if [ $? -ne 0 ]
+then
+    echo "Unable to connect to cluster - giving up" 
+    exit 1
+fi
+
+setupClusterNodes
+rm $FB_BINARY
+
+# verify featurebase running
+echo "Verifying featurebase cluster running..."
+curl -s http://${DATANODE0}:10101/status
+if (( $? != 0 )) 
+then 
+    echo "Featurebase cluster not running"
+    exit 1
+fi
+
+echo "Cluster running."
+
+# path for ebs device
+EBS_DEVICE_NAME=/dev/sdb
+
+# copy datagen to ingest nodes 
+setupIngestNode1(){
+    IP=$1
+    echo "Setting up first ingest node: ${IP}"
+    scp -i ~/.ssh/gitlab-featurebase-ci.pem -o "StrictHostKeyChecking no" ${DATAGEN_BINARY} ec2-user@${IP}:
+    ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${IP} "chmod ugo+x /home/ec2-user/${DATAGEN_BINARY}"
+    scp -i ~/.ssh/gitlab-featurebase-ci.pem -o "StrictHostKeyChecking no" ./qa/scripts/delete/tremor_keys.yaml ./qa/scripts/delete/kafkaIngest.sh ec2-user@${IP}:
+    setupKafkaServer ${IP}
+    ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${IP} "sudo yum install jq htop tmux -y"
+}
+
+setupIngestNode2(){
+    IP=$1
+    echo "Setting up second ingest node: ${IP}"
+    scp -i ~/.ssh/gitlab-featurebase-ci.pem -o "StrictHostKeyChecking no" ./qa/scripts/delete/schema.json ./qa/scripts/delete/featurebaseIngest.sh ec2-user@${IP}:
+    ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${IP} "sudo yum install librdkafka -y"
+    scp -i ~/.ssh/gitlab-featurebase-ci.pem -o "StrictHostKeyChecking no" ${KAFKA_STATIC} ec2-user@${IP}:
+    ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${IP} "chmod ugo+x /home/ec2-user/${KAFKA_STATIC}"
+    ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${IP} "sudo yum install jq htop tmux nc -y"
+}
+
+if [[ $DEPLOYED_INGEST_IPS_LEN -ne "2" ]]; then 
+    echo "error expected 2 ingest nodes, got $DEPLOYED_INGEST_IPS_LEN"
+    exit 1
+fi 
+
+setupIngestNode1 ${DEPLOYED_INGEST_IPS[0]}
+setupIngestNode2 ${DEPLOYED_INGEST_IPS[1]}
--- a/qa/scripts/teardownTremorDeleteTest.sh
+++ b/qa/scripts/teardownTremorDeleteTest.sh
@ -0,0 +1,7 @@
+#!/bin/bash
+
+# To run script: ./teardownTremorDeleteTest.sh
+
+cd qa/tf/ci/deletetest
+export TF_IN_AUTOMATION=1
+terraform destroy -auto-approve
--- a/qa/scripts/utilCluster.sh
+++ b/qa/scripts/utilCluster.sh
@ -22,6 +22,17 @@ DEPLOYED_INGEST_IPS_LEN=0
 #Initial cluster string
 INITIAL_CLUSTER=""

+# ebs device name
+EBS_DEVICE_NAME=""
+
+# featurebase binary
+FB_BINARY=""
+
+# branch name
+BRANCH_NAME=""
+
+# AWS Profile
+AWS_PROFILE=""
 ifErr() {
    res=$?
    if (( res != 0 )); then
@ -55,13 +66,14 @@ EOT
    #echo "featurebase.service <<"

    scp -i ~/.ssh/gitlab-featurebase-ci.pem -o "StrictHostKeyChecking no" featurebase.service ec2-user@${NODEIP}:
-    if (( $? != 0 ))
-    then
+    if (( $? != 0 )) 
+    then 
        echo "featurebase.service copy failed"
        exit 1
    fi

    rm -f featurebase.service
+    
    ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "sudo mv featurebase.service ${SERVICE_FILE_PATH}"
 }

@ -155,6 +167,7 @@ long-query-time = "10s"

 [metric]
    service = "prometheus"
+
 EOT
    fi

@ -210,14 +223,26 @@ setupTLS() {
    ifErr "error setting up TLS"
 }

+mountEBSVolume() {
+    NODEIP=$1
+    echo "in mountEBSVolume, EBS_DEVICE_NAME = $EBS_DEVICE_NAME, NODEIP = $NODEIP"
+    # if it is nvme, then skip the following 3 lines, because we mount the device in UserData
+    # in the CloudFormation template
+    if [[ "$EBS_DEVICE_NAME" != "nvme" ]]; then
+        echo "mounting EBS device"
+        ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "sudo mkdir /data"
+        ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "sudo mkfs.ext4 ${EBS_DEVICE_NAME}"
+        ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "sudo mount  ${EBS_DEVICE_NAME} /data"
+        ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "sudo chown -R ec2-user:ec2-user /data"
+    fi  
+}
+
 executeGeneralNodeConfigCommands() {
    echo "Executing node config...index: $1, ip:$2"
    NODEIDX=$1
    NODEIP=$2

-    ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "sudo mkdir -p /data"
-    ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "sudo mkfs.ext4 /dev/nvme1n1"
-    ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "sudo mount /dev/nvme1n1 /data"
+    mountEBSVolume $NODEIP

    ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "sudo adduser molecula"
    ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "sudo mkdir /var/log/molecula"
@ -226,18 +251,17 @@ executeGeneralNodeConfigCommands() {
    ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "sudo chown molecula /data/featurebase"
    
    # TODO handle different archs
-    scp -i ~/.ssh/gitlab-featurebase-ci.pem -o "StrictHostKeyChecking no" featurebase_linux_arm64 ec2-user@${NODEIP}:
-    if (( $? != 0 ))
-    then
+    scp -i ~/.ssh/gitlab-featurebase-ci.pem -o "StrictHostKeyChecking no" ${FB_BINARY} ec2-user@${NODEIP}:
+    if (( $? != 0 )) 
+    then 
        echo "featurebase binary copy failed"
        exit 1
    fi

-    ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "chown ec2-user:ec2-user /home/ec2-user/featurebase_linux_arm64"
-    ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "chmod ugo+x /home/ec2-user/featurebase_linux_arm64"
-    ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "sudo mv /home/ec2-user/featurebase_linux_arm64 /usr/local/bin/featurebase"
-    ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "sudo yum install git -y"
-
+    ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "chown ec2-user:ec2-user /home/ec2-user/${FB_BINARY}"
+    ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "chmod ugo+x /home/ec2-user/${FB_BINARY}"
+    ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "sudo mv /home/ec2-user/${FB_BINARY} /usr/local/bin/featurebase"
+    ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "sudo yum install jq htop tmux nc -y"
    echo "featurebase binary copied."

    if [[ "$AUTH_ENABLED" = "1" ]]; then
@ -272,7 +296,7 @@ executeDataStartCommands() {
 startDataNodes() {
    #now go thru loop again to start up each node
    cnt=0
-    for ip in $DEPLOYED_DATA_IPS
+    for ip in ${DEPLOYED_DATA_IPS[@]}
    do
        executeDataStartCommands $cnt $ip
        cnt=$((cnt+1))
@ -294,7 +318,6 @@ setupIngestNode() {

    executeGeneralNodeConfigCommands $1 $2

-    ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "sudo chown -R ec2-user /data"
    ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "pip3 install -U pytest"
    ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "pip3 install -U requests"
    ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "pip3 install -U json"
@ -304,18 +327,21 @@ setupIngestNode() {

 setupDataNodes() {
    cnt=0
-    for ip in $DEPLOYED_DATA_IPS
+    for ip in ${DEPLOYED_DATA_IPS[@]}
    do
        setupDataNode $cnt $ip
+        setupDatadog $ip "featurebase"
        cnt=$((cnt+1))
    done
 }

 setupIngestNodes() {
    cnt=0
-    for ip in $DEPLOYED_INGEST_IPS
+    echo "setupIngestNodes: " ${DEPLOYED_INGEST_IPS[@]}
+    for ip in ${DEPLOYED_INGEST_IPS[@]}
    do
        setupIngestNode $cnt $ip
+        setupDatadog $ip "ingest"
        cnt=$((cnt+1))
    done
 }
@ -323,7 +349,7 @@ setupIngestNodes() {
 generateInitialClusterString() {
    IFS=$'\n'
    cnt=0
-    for ip in $DEPLOYED_DATA_IPS
+    for ip in ${DEPLOYED_DATA_IPS[@]}
    do
        if (($cnt + 1 != $DEPLOYED_DATA_IPS_LEN)) 
        then
@ -355,6 +381,166 @@ setupClusterNodes() {

 }

+writeKafkaServerConfig() {
+    echo "Writing server.properties file"
+    NODEIP=$1
+
+    cat << EOT > server.properties
+process.roles=broker,controller
+node.id=1
+controller.quorum.voters=1@localhost:9093
+listeners=PLAINTEXT://${NODEIP}:9092,CONTROLLER://:9093
+inter.broker.listener.name=PLAINTEXT
+advertised.listeners=PLAINTEXT://${NODEIP}:9092
+controller.listener.names=CONTROLLER
+listener.security.protocol.map=CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT,SSL:SSL,SASL_PLAINTEXT:SASL_PLAINTEXT,SASL_SSL:SASL_SSL
+num.network.threads=3
+num.io.threads=8
+socket.send.buffer.bytes=102400
+socket.receive.buffer.bytes=102400
+socket.request.max.bytes=104857600
+log.dirs=/data/kraft-combined-logs
+num.partitions=1
+num.recovery.threads.per.data.dir=1
+offsets.topic.replication.factor=1
+transaction.state.log.replication.factor=1
+transaction.state.log.min.isr=1
+log.retention.hours=168
+log.segment.bytes=1073741824
+log.retention.check.interval.ms=300000
+group.initial.rebalance.delay.ms=0
+confluent.license.topic.replication.factor=1
+confluent.metadata.topic.replication.factor=1
+confluent.security.event.logger.exporter.kafka.topic.replicas=1
+confluent.balancer.enable=true
+confluent.balancer.topic.replication.factor=1
+EOT
+
+    scp -i ~/.ssh/gitlab-featurebase-ci.pem -o "StrictHostKeyChecking no" server.properties ec2-user@${NODEIP}:/home/ec2-user/kafka/etc/kafka/kraft/server.properties
+    if (( $? != 0 )) 
+    then 
+        echo "kafka config file: server.properties copy failed"
+        exit 1
+    fi
+    rm -f server.properties
+}
+
+setupKafkaServer() {
+    NODEIP=$1
+
+    # install kafka 
+    echo "setting up kafka server for: $NODEIP"
+    ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "sudo yum install java-1.8.0-openjdk nc -y"
+    ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "mkdir kafka && cd kafka && curl https://packages.confluent.io/archive/7.0/confluent-community-7.0.1.tar.gz -o kafka.tgz && tar -xvzf kafka.tgz --strip 1"
+
+    # update kafka config 
+    writeKafkaServerConfig $NODEIP
+
+cat << 'EOF' >> runKafka.sh 
+/home/ec2-user/kafka/bin/kafka-storage format --config /home/ec2-user/kafka/etc/kafka/kraft/server.properties --cluster-id $(/home/ec2-user/kafka/bin/kafka-storage random-uuid)
+sudo /home/ec2-user/kafka/bin/kafka-server-start /home/ec2-user/kafka/etc/kafka/kraft/server.properties > /tmp/kafka.log &
+echo "checking kafka server status"
+EOF
+
+    cat runKafka.sh
+    # start zookeeper and kafka server
+    echo "starting kafka server"
+    scp -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ./runKafka.sh ec2-user@${NODEIP}:.
+    ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "bash runKafka.sh"
+    ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "timeout 60s nc -z ${NODEIP} 9092"
+    # rm ./runKafka.sh
+}
+
+writeDatadogConfigForIngest() {
+    # general datadog config 
+    cat << EOT > datadog.yaml
+api_key: 6ab706d12de9cb46db25a0aabab6a004
+site: datadoghq.com
+tags:
+  - team:core
+  - app:${NODE_TYPE}
+  - branch_name:${BRANCH_NAME}
+process_config:
+enabled: "true"
+EOT
+
+    # openmetrics config 
+cat << EOT > conf.yaml
+# The prometheus endpoint to query from
+instances:
+  - openmetrics_endpoint: "http://localhost:9093/metrics"
+    namespace: "ingest"
+    metrics:
+      - "ingester_kafka_static_.+"
+      - go*
+EOT
+}
+
+writeDatadogConfigForFeaturebase() {
+    # general datadog config 
+    cat << EOT > datadog.yaml
+api_key: 6ab706d12de9cb46db25a0aabab6a004
+site: datadoghq.com
+tags:
+  - team:core
+  - app:${NODE_TYPE}
+  - branch_name:${BRANCH_NAME}
+process_config:
+  enabled: "true"
+EOT
+
+    # openmetrics config 
+cat << EOT > conf.yaml 
+# The prometheus endpoint to query from
+instances:
+  - prometheus_url: http://localhost:10101/metrics
+    namespace: "featurebase"
+    metrics:
+    - prometheus_target_interval_length_seconds: target_interval_length
+    - http_requests_total
+    - http*
+    - etcd*
+    - pilosa*
+    - go*
+    - process*
+    - os*
+EOT
+}
+
+setupDatadog() {
+    NODEIP=$1
+    NODE_TYPE=$2
+cat << 'EOF' > runDatadog.sh 
+DD_AGENT_MAJOR_VERSION=7 DD_API_KEY=6ab706d12de9cb46db25a0aabab6a004 DD_SITE="datadoghq.com" bash -c "$(curl -L https://s3.amazonaws.com/dd-agent/scripts/install_script.sh)"
+EOF
+
+    if [[ "$NODE_TYPE" == "featurebase" ]]
+    then
+        echo "writing datadog config for featurebase node: $NODEIP, $NODE_TYPE, $BRANCH_NAME"
+        writeDatadogConfigForFeaturebase
+    else
+        echo "writing datadog config for ingest node: $NODEIP, $NODE_TYPE, $BRANCH_NAME"
+        writeDatadogConfigForIngest
+    fi 
+
+    echo "setting up datadog for $NODE_TYPE node: $NODEIP"
+    echo "copying datadog config"
+    scp -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ./runDatadog.sh ./datadog.yaml ./conf.yaml ec2-user@${NODEIP}:.
+    echo "installing datadog"
+    cat ./runDatadog.sh
+    ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "bash ./runDatadog.sh"
+    echo "restarting datadog and updating config"
+    ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "sudo systemctl stop datadog-agent"
+    # update configuration and openmetrics for datadog 
+    ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "sudo mv ./datadog.yaml /etc/datadog-agent/datadog.yaml"
+    ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "sudo mv ./conf.yaml /etc/datadog-agent/conf.d/openmetrics.d/conf.yaml"
+    ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "sudo systemctl start datadog-agent"
+    echo "checking datadog status"
+    ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "sudo systemctl status datadog-agent"
+    ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "sudo yum install nc htop -y"
+    rm runDatadog.sh datadog.yaml conf.yaml
+}
+
 installDatagen() {
    INGESTNODE0=$1
    # download datagen
@ -406,3 +592,39 @@ installDatagen() {
        exit 1
    fi
 }
+
+# copy datagen to ingest nodes 
+setupProducerNode(){
+    IP=$1
+
+    # download producer(datagen binary) from S3 
+    DATAGEN_BINARY=datagen
+    echo "Downloading Datagen binary"
+    aws s3 cp s3://molecula-artifact-storage/idk/master/_latest/idk-linux-amd64/datagen $DATAGEN_BINARY --profile $AWS_PROFILE
+
+    mountEBSVolume $IP
+    scp -i ~/.ssh/gitlab-featurebase-ci.pem -o "StrictHostKeyChecking no" ${DATAGEN_BINARY} ec2-user@${IP}:
+    ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${IP} "chmod ugo+x /home/ec2-user/${DATAGEN_BINARY}"
+    scp -i ~/.ssh/gitlab-featurebase-ci.pem -o "StrictHostKeyChecking no" ./qa/scripts/delete/tremor_keys.yaml ./qa/scripts/delete/runProducer.sh ec2-user@${IP}:
+    ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${IP} "sudo yum install jq htop tmux -y"
+    setupKafkaServer ${IP}
+    setupDatadog $IP "ingest"
+}
+
+setupConsumerNode(){
+    IP=$1
+
+    # download consumer(kafka-static binary) from S3
+    KAFKA_STATIC=molecula-consumer-kafka-static
+    echo "Downloading Molecular Consumer Kafka Static binary"
+    aws s3 cp s3://molecula-artifact-storage/idk/master/_latest/idk-linux-amd64/molecula-consumer-kafka-static ./$KAFKA_STATIC --profile $AWS_PROFILE
+
+    mountEBSVolume $IP
+    
+    scp -i ~/.ssh/gitlab-featurebase-ci.pem -o "StrictHostKeyChecking no" ./qa/scripts/delete/schema.json ./qa/scripts/delete/runConsumer.sh ec2-user@${IP}:
+    ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${IP} "sudo yum install librdkafka -y"
+    scp -i ~/.ssh/gitlab-featurebase-ci.pem -o "StrictHostKeyChecking no" ${KAFKA_STATIC} ec2-user@${IP}:
+    ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${IP} "chmod ugo+x /home/ec2-user/${KAFKA_STATIC}"
+    ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${IP} "sudo yum install jq htop tmux nc -y"
+    setupDatadog $IP "ingest"
+}
--- a/qa/tf/.modules/featurebase-cluster/cloud-init.sh
+++ b/qa/tf/.modules/featurebase-cluster/cloud-init.sh
@ -0,0 +1,4 @@
+#!/bin/bash 
+set +x # get debug log
+set -e # exit on errors 
+
--- a/qa/tf/.modules/featurebase-cluster/main.tf
+++ b/qa/tf/.modules/featurebase-cluster/main.tf
@ -27,18 +27,22 @@ resource "aws_instance" "fb_cluster_nodes" {
  subnet_id              = var.subnet != "" ? var.subnet : var.vpc_private_subnets[count.index % length(var.vpc_private_subnets)]
  availability_zone      = var.zone != "" ? var.zone : var.azs[count.index % length(var.azs)]
  iam_instance_profile   = "${aws_iam_instance_profile.fb_cluster_node_profile.name}"
-
+  user_data = var.user_data != "" ? file("${var.user_data}") : file("${path.module}/cloud-init.sh") 
+  
  root_block_device {
    volume_type = "gp3"
    volume_size = 20
  }

-  ebs_block_device {
-    device_name = "/dev/sdb"
-    volume_type = var.fb_data_disk_type
-    volume_size = var.fb_data_disk_size_gb
-    iops        = var.fb_data_disk_iops
-    encrypted   = true 
+  dynamic "ebs_block_device" {
+    for_each = var.ebs_volumes
+    content {
+      device_name = "/dev/sdb"
+      volume_type = var.fb_data_disk_type
+      volume_size = var.fb_data_disk_size_gb
+      iops        = var.fb_data_disk_iops
+      encrypted   = true 
+    }
  }

  tags = {
@ -46,7 +50,6 @@ resource "aws_instance" "fb_cluster_nodes" {
    Name   = "${var.cluster_prefix}-featurebase-cluster-${count.index}"
    Role   = "cluster_node"
  }
-
 }

 resource "aws_instance" "fb_ingest" {
@ -191,6 +194,14 @@ resource "aws_security_group" "ingest" {
    ipv6_cidr_blocks = ["::/0"]
  }

+  ingress {
+    description = "HTTP from Internal"
+    from_port   = 9092
+    to_port     = 9092
+    protocol    = "tcp"
+    cidr_blocks = ["10.0.0.0/8", "172.31.0.0/16"]
+  }
+  
  ingress {
    description      = "SSH"
    from_port        = 22
--- a/qa/tf/.modules/featurebase-cluster/outputs.tf
+++ b/qa/tf/.modules/featurebase-cluster/outputs.tf
@ -1,5 +1,5 @@
 output "ingest_ips" {
-    value = aws_instance.fb_ingest.*.public_ip
+    value = aws_instance.fb_ingest.*.private_ip
 }

 output "data_node_ips" {
--- a/qa/tf/.modules/featurebase-cluster/variables.tf
+++ b/qa/tf/.modules/featurebase-cluster/variables.tf
@ -112,3 +112,14 @@ variable "vpc_private_subnets" {
  description = "A private net underneath in the VPC in which we will build the cluster"
  type        = list(string)
 }
+
+variable "user_data" {
+  description = "Cloud init script"
+  type = string 
+}
+
+variable "ebs_volumes" {
+  type = list(string)
+  default = ["/dev/sdb"]
+}
+
--- a/qa/tf/ci/auth-smoke/main.tf
+++ b/qa/tf/ci/auth-smoke/main.tf
@ -10,4 +10,5 @@ module "ci-cluster" {
  vpc_cidr_block      = "10.0.0.0/16"
  vpc_public_subnets  = ["subnet-066b4b922b54e51a2", "subnet-037b8884269a69025", "subnet-08482631514426210", ]
  vpc_private_subnets = ["subnet-0319dde319380326f", "subnet-0517ca9a646d80f88", "subnet-05a7b685ed27eb1cf", ]
+  user_data = "../../.modules/featurebase-cluster/cloud-init.sh"
 }
--- a/qa/tf/ci/deletetest/delete_cloud_init.sh
+++ b/qa/tf/ci/deletetest/delete_cloud_init.sh
@ -0,0 +1,20 @@
+#!/bin/bash 
+set +x # get debug log
+set -e # exit on errors 
+
+echo "----------------------------"
+echo "Running delete_cloud_init.sh"
+
+# create and initialize a raid0 array 
+# for m5ad.16xlarge instance type, there are 4 NVMe SSDs. 
+sudo mdadm --create --verbose /dev/md0 --level=0 --raid-devices=4 /dev/nvme1n1 /dev/nvme2n1 /dev/nvme3n1 /dev/nvme4n1
+
+# format filesystem 
+sudo mkfs.ext4 /dev/md0
+
+# create a mount directory & mount the raid drive to this directory
+sudo mkdir data
+sudo mount /dev/md0 /data
+
+# change permissions from root to ec2-user for data directory 
+sudo chown -R ec2-user:ec2-user /data
--- a/qa/tf/ci/deletetest/main.tf
+++ b/qa/tf/ci/deletetest/main.tf
@ -0,0 +1,18 @@
+module "ci-cluster" {
+    source = "../../.modules/featurebase-cluster"
+    cluster_prefix = var.cluster_prefix
+    region = var.region
+    profile = var.profile
+    fb_cluster_arch = ["x86_64"]
+    fb_data_node_type = "m5ad.16xlarge" 
+    fb_data_node_count = 5
+    ebs_volumes = []
+    fb_ingest_type = "m4.16xlarge"
+    fb_ingest_node_count = 2
+    fb_ingest_disk_size_gb = 1200
+    vpc_id = "vpc-05a26a122f961dc2b"
+    vpc_cidr_block = "10.0.0.0/16"
+    vpc_public_subnets = ["subnet-066b4b922b54e51a2","subnet-037b8884269a69025","subnet-08482631514426210",]
+    vpc_private_subnets = ["subnet-0319dde319380326f","subnet-0517ca9a646d80f88","subnet-05a7b685ed27eb1cf",]
+    user_data = "./delete_cloud_init.sh"
+}
--- a/qa/tf/ci/deletetest/outputs.tf
+++ b/qa/tf/ci/deletetest/outputs.tf
@ -0,0 +1,19 @@
+output "ingest_ips" {
+    description = "List of ingest IPs"
+    value = module.ci-cluster.ingest_ips
+}
+
+output "data_node_ips" {
+  description = "List of data node IPs"
+  value = module.ci-cluster.data_node_ips
+}
+
+output "cluster_prefix" {
+  description = "The cluster prefix used"
+  value = module.ci-cluster.cluster_prefix
+}
+
+output "fb_cluster_replica_count" {
+  description = "The cluster replica count used"
+  value = module.ci-cluster.fb_cluster_replica_count
+}
--- a/qa/tf/ci/deletetest/provider.tf
+++ b/qa/tf/ci/deletetest/provider.tf
@ -0,0 +1,4 @@
+provider "aws" {
+  region  = var.region
+  profile = var.profile
+}
--- a/qa/tf/ci/deletetest/tf.auto.tfvars
+++ b/qa/tf/ci/deletetest/tf.auto.tfvars
@ -0,0 +1,2 @@
+region  = "us-east-2"
+profile = "service-terraform"
--- a/qa/tf/ci/deletetest/variables.tf
+++ b/qa/tf/ci/deletetest/variables.tf
@ -0,0 +1,15 @@
+variable "region" {
+  description = "The AWS region in which the VPC should be built"
+  type        = string
+}
+
+variable "profile" {
+  description = "The name of the AWS profile Terraform should use for auth."
+  type        = string
+}
+
+variable "cluster_prefix" {
+  type        = string
+  description = "This is a identifier that will be prefixed to created resources"
+}
+
--- a/qa/tf/ci/smoketest/main.tf
+++ b/qa/tf/ci/smoketest/main.tf
@ -11,4 +11,5 @@ module "ci-cluster" {
    vpc_cidr_block = "10.0.0.0/16"
    vpc_public_subnets = ["subnet-066b4b922b54e51a2","subnet-037b8884269a69025","subnet-08482631514426210",]
    vpc_private_subnets = ["subnet-0319dde319380326f","subnet-0517ca9a646d80f88","subnet-05a7b685ed27eb1cf",]
+    user_data = "../../.modules/featurebase-cluster/cloud-init.sh"
 }