Merge pull request #1923 from molecula/cicd-will-it-never-end

Cicd will it never end

.gitlab/.gitlab-ci.yml

@@ -12,6 +12,7 @@ stages:
   - build
   - integration
   - gauntlet
+  - performance
   - post build
 
 smoke build:
@@ -337,7 +338,6 @@ smoke test:
     AWS_ACCESS_KEY_ID: $AWS_FBCI_ACCESS_KEY_ID
     AWS_SECRET_ACCESS_KEY: $AWS_FBCI_SECRET_ACCESS_KEY
     TF_VAR_cluster_prefix: ""
-    TF_VAR_branch: ""
   tags:
     - aws
     - docker
@@ -369,8 +369,6 @@ smoke test:
     - export PATH=$PATH:/usr/local/go/bin
     - TF_VAR_cluster_prefix="smoke-$(openssl rand -base64 12 | tr -d /=+ | cut -c -16)"
     - echo "Cluster Prefix --> $TF_VAR_cluster_prefix"
-    - TF_VAR_branch=$CI_COMMIT_BRANCH
-    - echo "Branch --> $TF_VAR_branch"
   script:
     - ./qa/scripts/setupSmokeTest.sh
     - ./qa/scripts/testSmokeTest.sh
@@ -398,7 +396,6 @@ gauntlet:
     AWS_SECRET_ACCESS_KEY: $AWS_FBCI_SECRET_ACCESS_KEY
     ASG_NAME: "gitlab-runners"
     TF_VAR_cluster_prefix: ""
-    TF_VAR_branch: ""
   tags:
     - aws
     - docker
@@ -433,8 +430,6 @@ gauntlet:
     - export PATH=$PATH:/usr/local/go/bin
     - TF_VAR_cluster_prefix="gauntlet-$(openssl rand -base64 12 | tr -d /=+ | cut -c -16)"
     - echo "Cluster Prefix --> $TF_VAR_cluster_prefix"
-    - TF_VAR_branch=$CI_COMMIT_BRANCH
-    - echo "Branch --> $TF_VAR_branch"
     - export INSTANCE_ID=$(curl --silent --fail "http://169.254.169.254/latest/meta-data/instance-id" | tee instance_id)
     - aws autoscaling set-instance-protection --instance-ids "$INSTANCE_ID" --auto-scaling-group-name $ASG_NAME --protected-from-scale-in --profile $INFRA_PROFILE
   script:
@@ -482,3 +477,11 @@ s3 dump:
     - job: build for darwin arm64
     - job: build for linux amd64
     - job: build for linux arm64
+
+perf_able:
+  stage: performance
+  trigger:
+    include: .gitlab/.perf-able-gitlab-ci.yml
+  rules:
+    - changes:
+      - featurebase/*

.gitlab/.perf-able-gitlab-ci.yml

@@ -0,0 +1,11 @@
+stages:
+  - loadtest
+
+loadtest:
+  image:
+    name: loadimpact/k6:latest
+    entrypoint: ['']
+  stage: loadtest
+  script:
+    - echo "executing local k6 in k6 container..."
+    - k6 run ./qa/scripts/perf/able/script.js

qa/scripts/perf/able/ableRun.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+
+# requires TF_VAR_cluster_prefix env var to be set
+if [ -z ${TF_VAR_cluster_prefix+x} ]; then 
+    echo "setting TF_VAR_cluster_prefix";
+    export TF_VAR_cluster_prefix="able-$(openssl rand -base64 12 | tr -d /=+ | cut -c -16)"
+    echo "TF_VAR_cluster_prefix is set to '$TF_VAR_cluster_prefix'";  
+else 
+    echo "TF_VAR_cluster_prefix is set to '$TF_VAR_cluster_prefix'"; 
+fi
+
+$SCRIPT_DIR/ableSetup.sh
+$SCRIPT_DIR/ableTest.sh
+$SCRIPT_DIR/ableTeardown.sh

qa/scripts/perf/able/ableSetup.sh

@@ -0,0 +1,86 @@
+#!/bin/bash
+
+# To run script: ./ableSetup.sh
+export TF_IN_AUTOMATION=1
+
+if [ -z ${TF_VAR_cluster_prefix+x} ]; then 
+    echo "TF_VAR_cluster_prefix is unset";
+    exit 1
+else 
+    echo "TF_VAR_cluster_prefix is set to '$TF_VAR_cluster_prefix'"; 
+fi
+
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+source $SCRIPT_DIR/../../utilCluster.sh
+
+pushd ./qa/tf/gauntlet/able
+echo "Running terraform init..."
+terraform init -input=false
+echo "Running terraform apply..."
+terraform apply -input=false -auto-approve
+terraform output -json > outputs.json
+popd
+
+# get the first ingest host
+INGESTNODE0=$(cat ./qa/tf/gauntlet/able/outputs.json | jq -r '[.ingest_ips][0]["value"][0]')
+echo "using INGESTNODE0 ${INGESTNODE0}"
+
+# get the first data host
+DATANODE0=$(cat ./qa/tf/gauntlet/able/outputs.json | jq -r '[.data_node_ips][0]["value"][0]')
+echo "using DATANODE0 ${DATANODE0}"
+
+
+DEPLOYED_CLUSTER_PREFIX=$(cat ./qa/tf/gauntlet/able/outputs.json | jq -r '[.cluster_prefix][0]["value"]')
+echo "Using DEPLOYED_CLUSTER_PREFIX: ${DEPLOYED_CLUSTER_PREFIX}"
+
+DEPLOYED_CLUSTER_REPLICA_COUNT=$(cat ./qa/tf/gauntlet/able/outputs.json | jq -r '[.fb_cluster_replica_count][0]["value"]')
+echo "Using DEPLOYED_CLUSTER_REPLICA_COUNT: ${DEPLOYED_CLUSTDEPLOYED_CLUSTER_REPLICA_COUNTER_PREFIX}"
+
+DEPLOYED_DATA_IPS=$(cat ./qa/tf/gauntlet/able/outputs.json | jq -r '[.data_node_ips][0]["value"][]')
+echo "DEPLOYED_DATA_IPS: {"
+echo "${DEPLOYED_DATA_IPS}"
+echo "}"
+
+DEPLOYED_DATA_IPS_LEN=`echo "$DEPLOYED_DATA_IPS" | wc -l`
+
+DEPLOYED_INGEST_IPS=$(cat ./qa/tf/gauntlet/able/outputs.json | jq -r '[.ingest_ips][0]["value"][]')
+echo "DEPLOYED_INGEST_IPS: {"
+echo "${DEPLOYED_INGEST_IPS}"
+echo "}"
+
+DEPLOYED_INGEST_IPS_LEN=`echo "$DEPLOYED_INGEST_IPS" | wc -l`
+
+#wait until we can connect to one of the hosts
+for i in {0..24}
+do 
+    ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no -o ConnectTimeout=10 ec2-user@${DATANODE0} "pwd"
+    if [ $? -eq 0 ]
+    then
+        echo "Cluster is up after ${i} tries." 
+        break
+    fi
+    sleep 10
+done
+
+ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no -o ConnectTimeout=10 ec2-user@${DATANODE0} "pwd"
+if [ $? -ne 0 ]
+then
+    echo "Unable to connect to cluster - giving up" 
+    exit 1
+fi
+
+setupClusterNodes
+
+# verify featurebase running
+echo "Verifying featurebase cluster running..."
+curl -s http://${DATANODE0}:10101/status
+if (( $? != 0 )) 
+then 
+    echo "Featurebase cluster not running"
+    exit 1
+fi
+
+echo "Cluster running."
+
+
+

qa/scripts/perf/able/ableTeardown.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+# To run script: ./ableTeardown.sh
+
+cd qa/tf/gauntlet/able
+export TF_IN_AUTOMATION=1
+terraform destroy -auto-approve

qa/scripts/perf/able/ableTest.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+# get the first ingest host
+INGESTNODE0=$(cat ./qa/tf/gauntlet/able/outputs.json | jq -r '[.ingest_ips][0]["value"][0]')
+echo "using INGESTNODE0 ${INGESTNODE0}"
+
+# get the first data host
+DATANODE0=$(cat ./qa/tf/gauntlet/able/outputs.json | jq -r '[.data_node_ips][0]["value"][0]')
+echo "using DATANODE0 ${DATANODE0}"
+
+
+echo "Done."

qa/scripts/perf/able/script.js

@@ -0,0 +1,7 @@
+import http from 'k6/http';
+import { sleep } from 'k6';
+	
+export default function () {
+    http.get('https://test.k6.io');
+    sleep(1);
+}

qa/scripts/runSamsungGauntlet.sh

@@ -2,9 +2,6 @@
 
 SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
 
-# requires TF_VAR_branch env var to be set
-if [ -z ${TF_VAR_branch+x} ]; then echo "TF_VAR_branch is unset"; else echo "TF_VAR_branch is set to '$TF_VAR_branch'"; fi
-
 # requires TF_VAR_cluster_prefix env var to be set
 if [ -z ${TF_VAR_cluster_prefix+x} ]; then 
     echo "setting TF_VAR_cluster_prefix";

qa/scripts/runSmokeTest.sh

@@ -2,10 +2,6 @@
 
 SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
 
-
-# requires TF_VAR_branch env var to be set
-if [ -z ${TF_VAR_branch+x} ]; then echo "TF_VAR_branch is unset"; else echo "TF_VAR_branch is set to '$TF_VAR_branch'"; fi
-
 # requires TF_VAR_cluster_prefix env var to be set
 if [ -z ${TF_VAR_cluster_prefix+x} ]; then 
     echo "setting TF_VAR_cluster_prefix";

qa/scripts/setupSamsungGauntlet.sh

@@ -3,9 +3,6 @@
 # To run script: ./setupSamsungGauntlet.sh
 export TF_IN_AUTOMATION=1
 
-# requires TF_VAR_branch env var to be set
-if [ -z ${TF_VAR_branch+x} ]; then echo "TF_VAR_branch is unset"; else echo "TF_VAR_branch is set to '$TF_VAR_branch'"; fi
-
 SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
 source $SCRIPT_DIR/utilCluster.sh
 

qa/scripts/setupSmokeTest.sh

@@ -3,9 +3,6 @@
 # To run script: ./setupSmokeTest.sh
 export TF_IN_AUTOMATION=1
 
-# requires TF_VAR_branch env var to be set
-if [ -z ${TF_VAR_branch+x} ]; then echo "TF_VAR_branch is unset"; else echo "TF_VAR_branch is set to '$TF_VAR_branch'"; fi
-
 SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
 source $SCRIPT_DIR/utilCluster.sh
 

qa/scripts/teardownSmokeTest.sh

@@ -2,9 +2,6 @@
 
 # To run script: ./teardownSmokeTest.sh
 
-# requires TF_VAR_branch env var to be set
-if [ -z ${TF_VAR_branch+x} ]; then echo "TF_VAR_branch is unset"; else echo "TF_VAR_branch is set to '$TF_VAR_branch'"; fi
-
 cd qa/tf/ci/smoketest
 export TF_IN_AUTOMATION=1
 terraform destroy -auto-approve

qa/scripts/testSmokeTest.sh

@@ -1,8 +1,5 @@
 #!/bin/bash
 
-# requires TF_VAR_branch env var to be set
-if [ -z ${TF_VAR_branch+x} ]; then echo "TF_VAR_branch is unset"; else echo "TF_VAR_branch is set to '$TF_VAR_branch'"; fi
-
 SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
 source $SCRIPT_DIR/utilCluster.sh
 

qa/tf/.modules/featurebase-cluster/main.tf

@@ -13,7 +13,7 @@ data "aws_ami" "amazon_linux_2" {
 
   filter {
     name   = "architecture"
-    values = ["arm64"]
+    values = var.fb_cluster_arch
   }
 }
 
@@ -38,6 +38,7 @@ resource "aws_instance" "fb_cluster_nodes" {
     volume_type = var.fb_data_disk_type
     volume_size = var.fb_data_disk_size_gb
     iops        = var.fb_data_disk_iops
+    encrypted   = true 
   }
 
   tags = {
@@ -70,6 +71,7 @@ resource "aws_instance" "fb_ingest" {
     volume_type = var.fb_ingest_disk_type
     volume_size = var.fb_ingest_disk_size_gb
     iops        = var.fb_ingest_disk_iops
+    encrypted   = true
   }
 
   tags = {

qa/tf/.modules/featurebase-cluster/variables.tf

@@ -3,6 +3,11 @@ variable "cluster_prefix" {
   description = "This is a identifier that will be prefixed to created resources"
 }
 
+variable "fb_cluster_arch" {
+  type    = list(string)
+  default = ["arm64"]
+}
+
 variable "fb_ingest_type" {
   type    = string
   default = "c6g.2xlarge"

qa/tf/ci/smoketest/variables.tf

@@ -13,7 +13,3 @@ variable "cluster_prefix" {
   description = "This is a identifier that will be prefixed to created resources"
 }
 
-variable "branch" {
-  type        = string
-  description = "The branch we are on"
-}

qa/tf/gauntlet/able/main.tf

@@ -0,0 +1,17 @@
+module "able-cluster" {
+    source = "../../.modules/featurebase-cluster"
+    cluster_prefix = var.cluster_prefix
+    region = var.region
+    profile = var.profile
+    fb_data_node_type = "m6g.12xlarge"
+    fb_data_disk_iops = 10000
+    fb_data_node_count = 3
+    fb_ingest_type = "m6g.2xlarge"
+    fb_ingest_disk_iops = 10000
+    fb_ingest_disk_size_gb = 500
+    fb_ingest_node_count = 1
+    vpc_id = "vpc-05a26a122f961dc2b"
+    vpc_cidr_block = "10.0.0.0/16"
+    vpc_public_subnets = ["subnet-066b4b922b54e51a2","subnet-037b8884269a69025","subnet-08482631514426210",]
+    vpc_private_subnets = ["subnet-0319dde319380326f","subnet-0517ca9a646d80f88","subnet-05a7b685ed27eb1cf",]
+}

qa/tf/gauntlet/able/outputs.tf

@@ -0,0 +1,19 @@
+output "ingest_ips" {
+    description = "List of ingest IPs"
+    value = module.able-cluster.ingest_ips
+}
+
+output "data_node_ips" {
+  description = "List of data node IPs"
+  value = module.able-cluster.data_node_ips
+}
+
+output "cluster_prefix" {
+  description = "The cluster prefix used"
+  value = module.able-cluster.cluster_prefix
+}
+
+output "fb_cluster_replica_count" {
+  description = "The cluster replica count used"
+  value = module.able-cluster.fb_cluster_replica_count
+}

qa/tf/gauntlet/able/provider.tf

@@ -0,0 +1,4 @@
+provider "aws" {
+  region  = var.region
+  profile = var.profile
+}

qa/tf/gauntlet/able/tf.auto.tfvars

@@ -0,0 +1,2 @@
+region  = "us-east-2"
+profile = "service-terraform"

qa/tf/gauntlet/able/variables.tf

@@ -0,0 +1,14 @@
+variable "region" {
+  description = "The AWS region in which the VPC should be built"
+  type        = string
+}
+
+variable "profile" {
+  description = "The name of the AWS profile Terraform should use for auth."
+  type        = string
+}
+
+variable "cluster_prefix" {
+  type        = string
+  description = "This is a identifier that will be prefixed to created resources"
+}

qa/tf/gauntlet/samsung/variables.tf

@@ -12,8 +12,3 @@ variable "cluster_prefix" {
   type        = string
   description = "This is a identifier that will be prefixed to created resources"
 }
-
-variable "branch" {
-  type        = string
-  description = "The branch we are on"
-}