added 'able' perf testing environment

.gitlab/.gitlab-ci.yml

@@ -328,7 +328,6 @@ smoke test:
     AWS_ACCESS_KEY_ID: $AWS_FBCI_ACCESS_KEY_ID
     AWS_SECRET_ACCESS_KEY: $AWS_FBCI_SECRET_ACCESS_KEY
     TF_VAR_cluster_prefix: ""
-    TF_VAR_branch: ""
   tags:
     - aws
     - docker
@@ -360,8 +359,6 @@ smoke test:
     - export PATH=$PATH:/usr/local/go/bin
     - TF_VAR_cluster_prefix="smoke-$(openssl rand -base64 12 | tr -d /=+ | cut -c -16)"
     - echo "Cluster Prefix --> $TF_VAR_cluster_prefix"
-    - TF_VAR_branch=$CI_COMMIT_BRANCH
-    - echo "Branch --> $TF_VAR_branch"
   script:
     - ./qa/scripts/setupSmokeTest.sh
     - ./qa/scripts/testSmokeTest.sh
@@ -387,7 +384,6 @@ gauntlet:
     AWS_ACCESS_KEY_ID: $AWS_FBCI_ACCESS_KEY_ID
     AWS_SECRET_ACCESS_KEY: $AWS_FBCI_SECRET_ACCESS_KEY
     TF_VAR_cluster_prefix: ""
-    TF_VAR_branch: ""
   tags:
     - aws
     - docker
@@ -419,8 +415,6 @@ gauntlet:
     - export PATH=$PATH:/usr/local/go/bin
     - TF_VAR_cluster_prefix="gauntlet-$(openssl rand -base64 12 | tr -d /=+ | cut -c -16)"
     - echo "Cluster Prefix --> $TF_VAR_cluster_prefix"
-    - TF_VAR_branch=$CI_COMMIT_BRANCH
-    - echo "Branch --> $TF_VAR_branch"
   script:
     - ./qa/scripts/setupSamsungGauntlet.sh
     - ./qa/scripts/testSamsungGauntlet.sh

qa/scripts/ableRunGauntlet.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+
+# requires TF_VAR_cluster_prefix env var to be set
+if [ -z ${TF_VAR_cluster_prefix+x} ]; then 
+    echo "setting TF_VAR_cluster_prefix";
+    export TF_VAR_cluster_prefix="able-$(openssl rand -base64 12 | tr -d /=+ | cut -c -16)"
+    echo "TF_VAR_cluster_prefix is set to '$TF_VAR_cluster_prefix'";  
+else 
+    echo "TF_VAR_cluster_prefix is set to '$TF_VAR_cluster_prefix'"; 
+fi
+
+$SCRIPT_DIR/ableSetupGauntlet.sh
+$SCRIPT_DIR/ableTestGauntlet.sh
+$SCRIPT_DIR/ableTeardownGauntlet.sh

qa/scripts/ableSetupGauntlet.sh

@@ -0,0 +1,86 @@
+#!/bin/bash
+
+# To run script: ./SetupGauntlet.sh
+export TF_IN_AUTOMATION=1
+
+if [ -z ${TF_VAR_cluster_prefix+x} ]; then 
+    echo "TF_VAR_cluster_prefix is unset";
+    exit 1
+else 
+    echo "TF_VAR_cluster_prefix is set to '$TF_VAR_cluster_prefix'"; 
+fi
+
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+source $SCRIPT_DIR/utilCluster.sh
+
+pushd ./qa/tf/gauntlet/able
+echo "Running terraform init..."
+terraform init -input=false
+echo "Running terraform apply..."
+terraform apply -input=false -auto-approve
+terraform output -json > outputs.json
+popd
+
+# get the first ingest host
+INGESTNODE0=$(cat ./qa/tf/gauntlet/able/outputs.json | jq -r '[.ingest_ips][0]["value"][0]')
+echo "using INGESTNODE0 ${INGESTNODE0}"
+
+# get the first data host
+DATANODE0=$(cat ./qa/tf/gauntlet/able/outputs.json | jq -r '[.data_node_ips][0]["value"][0]')
+echo "using DATANODE0 ${DATANODE0}"
+
+
+DEPLOYED_CLUSTER_PREFIX=$(cat ./qa/tf/gauntlet/able/outputs.json | jq -r '[.cluster_prefix][0]["value"]')
+echo "Using DEPLOYED_CLUSTER_PREFIX: ${DEPLOYED_CLUSTER_PREFIX}"
+
+DEPLOYED_CLUSTER_REPLICA_COUNT=$(cat ./qa/tf/gauntlet/able/outputs.json | jq -r '[.fb_cluster_replica_count][0]["value"]')
+echo "Using DEPLOYED_CLUSTER_REPLICA_COUNT: ${DEPLOYED_CLUSTDEPLOYED_CLUSTER_REPLICA_COUNTER_PREFIX}"
+
+DEPLOYED_DATA_IPS=$(cat ./qa/tf/gauntlet/able/outputs.json | jq -r '[.data_node_ips][0]["value"][]')
+echo "DEPLOYED_DATA_IPS: {"
+echo "${DEPLOYED_DATA_IPS}"
+echo "}"
+
+DEPLOYED_DATA_IPS_LEN=`echo "$DEPLOYED_DATA_IPS" | wc -l`
+
+DEPLOYED_INGEST_IPS=$(cat ./qa/tf/gauntlet/able/outputs.json | jq -r '[.ingest_ips][0]["value"][]')
+echo "DEPLOYED_INGEST_IPS: {"
+echo "${DEPLOYED_INGEST_IPS}"
+echo "}"
+
+DEPLOYED_INGEST_IPS_LEN=`echo "$DEPLOYED_INGEST_IPS" | wc -l`
+
+#wait until we can connect to one of the hosts
+for i in {0..24}
+do 
+    ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no -o ConnectTimeout=10 ec2-user@${DATANODE0} "pwd"
+    if [ $? -eq 0 ]
+    then
+        echo "Cluster is up after ${i} tries." 
+        break
+    fi
+    sleep 10
+done
+
+ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no -o ConnectTimeout=10 ec2-user@${DATANODE0} "pwd"
+if [ $? -ne 0 ]
+then
+    echo "Unable to connect to cluster - giving up" 
+    exit 1
+fi
+
+setupClusterNodes
+
+# verify featurebase running
+echo "Verifying featurebase cluster running..."
+curl -s http://${DATANODE0}:10101/status
+if (( $? != 0 )) 
+then 
+    echo "Featurebase cluster not running"
+    exit 1
+fi
+
+echo "Cluster running."
+
+
+

qa/scripts/ableTeardownGauntlet.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+# To run script: ./ableTeardownGauntlet.sh
+
+cd qa/tf/gauntlet/able
+export TF_IN_AUTOMATION=1
+terraform destroy -auto-approve

qa/scripts/ableTestGauntlet.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+# get the first ingest host
+INGESTNODE0=$(cat ./qa/tf/gauntlet/able/outputs.json | jq -r '[.ingest_ips][0]["value"][0]')
+echo "using INGESTNODE0 ${INGESTNODE0}"
+
+# get the first data host
+DATANODE0=$(cat ./qa/tf/gauntlet/able/outputs.json | jq -r '[.data_node_ips][0]["value"][0]')
+echo "using DATANODE0 ${DATANODE0}"
+
+
+echo "Done."

qa/scripts/runSamsungGauntlet.sh

@@ -2,9 +2,6 @@
 
 SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
 
-# requires TF_VAR_branch env var to be set
-if [ -z ${TF_VAR_branch+x} ]; then echo "TF_VAR_branch is unset"; else echo "TF_VAR_branch is set to '$TF_VAR_branch'"; fi
-
 # requires TF_VAR_cluster_prefix env var to be set
 if [ -z ${TF_VAR_cluster_prefix+x} ]; then 
     echo "setting TF_VAR_cluster_prefix";

qa/scripts/runSmokeTest.sh

@@ -2,10 +2,6 @@
 
 SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
 
-
-# requires TF_VAR_branch env var to be set
-if [ -z ${TF_VAR_branch+x} ]; then echo "TF_VAR_branch is unset"; else echo "TF_VAR_branch is set to '$TF_VAR_branch'"; fi
-
 # requires TF_VAR_cluster_prefix env var to be set
 if [ -z ${TF_VAR_cluster_prefix+x} ]; then 
     echo "setting TF_VAR_cluster_prefix";

qa/scripts/setupSamsungGauntlet.sh

@@ -3,9 +3,6 @@
 # To run script: ./setupSamsungGauntlet.sh
 export TF_IN_AUTOMATION=1
 
-# requires TF_VAR_branch env var to be set
-if [ -z ${TF_VAR_branch+x} ]; then echo "TF_VAR_branch is unset"; else echo "TF_VAR_branch is set to '$TF_VAR_branch'"; fi
-
 SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
 source $SCRIPT_DIR/utilCluster.sh
 

qa/scripts/setupSmokeTest.sh

@@ -3,9 +3,6 @@
 # To run script: ./setupSmokeTest.sh
 export TF_IN_AUTOMATION=1
 
-# requires TF_VAR_branch env var to be set
-if [ -z ${TF_VAR_branch+x} ]; then echo "TF_VAR_branch is unset"; else echo "TF_VAR_branch is set to '$TF_VAR_branch'"; fi
-
 SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
 source $SCRIPT_DIR/utilCluster.sh
 

qa/scripts/teardownSmokeTest.sh

@@ -2,9 +2,6 @@
 
 # To run script: ./teardownSmokeTest.sh
 
-# requires TF_VAR_branch env var to be set
-if [ -z ${TF_VAR_branch+x} ]; then echo "TF_VAR_branch is unset"; else echo "TF_VAR_branch is set to '$TF_VAR_branch'"; fi
-
 cd qa/tf/ci/smoketest
 export TF_IN_AUTOMATION=1
 terraform destroy -auto-approve

qa/scripts/testSmokeTest.sh

@@ -1,8 +1,5 @@
 #!/bin/bash
 
-# requires TF_VAR_branch env var to be set
-if [ -z ${TF_VAR_branch+x} ]; then echo "TF_VAR_branch is unset"; else echo "TF_VAR_branch is set to '$TF_VAR_branch'"; fi
-
 SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
 source $SCRIPT_DIR/utilCluster.sh
 

qa/tf/ci/smoketest/variables.tf

@@ -13,7 +13,3 @@ variable "cluster_prefix" {
   description = "This is a identifier that will be prefixed to created resources"
 }
 
-variable "branch" {
-  type        = string
-  description = "The branch we are on"
-}

qa/tf/gauntlet/able/main.tf

@@ -0,0 +1,16 @@
+module "samsung-cluster" {
+    source = "../../.modules/featurebase-cluster"
+    cluster_prefix = var.cluster_prefix
+    region = var.region
+    profile = var.profile
+    fb_data_node_type = "m6g.xlarge"
+    fb_data_disk_iops = 10000
+    fb_data_node_count = 5
+    fb_ingest_type = "m6g.large"
+    fb_ingest_disk_iops = 10000
+    fb_ingest_node_count = 5
+    vpc_id = "vpc-05a26a122f961dc2b"
+    vpc_cidr_block = "10.0.0.0/16"
+    vpc_public_subnets = ["subnet-066b4b922b54e51a2","subnet-037b8884269a69025","subnet-08482631514426210",]
+    vpc_private_subnets = ["subnet-0319dde319380326f","subnet-0517ca9a646d80f88","subnet-05a7b685ed27eb1cf",]
+}

qa/tf/gauntlet/able/outputs.tf

@@ -0,0 +1,19 @@
+output "ingest_ips" {
+    description = "List of ingest IPs"
+    value = module.samsung-cluster.ingest_ips
+}
+
+output "data_node_ips" {
+  description = "List of data node IPs"
+  value = module.samsung-cluster.data_node_ips
+}
+
+output "cluster_prefix" {
+  description = "The cluster prefix used"
+  value = module.samsung-cluster.cluster_prefix
+}
+
+output "fb_cluster_replica_count" {
+  description = "The cluster replica count used"
+  value = module.samsung-cluster.fb_cluster_replica_count
+}

qa/tf/gauntlet/able/provider.tf

@@ -0,0 +1,4 @@
+provider "aws" {
+  region  = var.region
+  profile = var.profile
+}

qa/tf/gauntlet/able/tf.auto.tfvars

@@ -0,0 +1,2 @@
+region  = "us-east-2"
+profile = "service-terraform"

qa/tf/gauntlet/able/variables.tf

@@ -0,0 +1,14 @@
+variable "region" {
+  description = "The AWS region in which the VPC should be built"
+  type        = string
+}
+
+variable "profile" {
+  description = "The name of the AWS profile Terraform should use for auth."
+  type        = string
+}
+
+variable "cluster_prefix" {
+  type        = string
+  description = "This is a identifier that will be prefixed to created resources"
+}

qa/tf/gauntlet/samsung/variables.tf

@@ -12,8 +12,3 @@ variable "cluster_prefix" {
   type        = string
   description = "This is a identifier that will be prefixed to created resources"
 }
-
-variable "branch" {
-  type        = string
-  description = "The branch we are on"
-}