From b7516eacc5a1da3a6ad3fd5abf3fb1adbd107f83 Mon Sep 17 00:00:00 2001
From: Bruce Baranowski <92940816+bruce-b-molecula@users.noreply.github.com>
Date: Tue, 14 Jun 2022 11:49:25 -0400
Subject: [PATCH] CICD: fix Cleanup_Build_Job inconsistent failures (#2098)

* add aws profile to cleanup job

* Split cleanup into bash script

* change spot instances to one-tme
---
 .gitlab/.gitlab-ci.yml                     | 18 ++++++++---
 qa/scripts/gitlabCleanupBuild.sh           | 37 ++++++++++++++++++++++
 qa/tf/.modules/featurebase-cluster/main.tf |  2 ++
 3 files changed, 52 insertions(+), 5 deletions(-)
 create mode 100755 qa/scripts/gitlabCleanupBuild.sh

diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml
index ee96dd10d..b6e1fba77 100644
--- a/.gitlab/.gitlab-ci.yml
+++ b/.gitlab/.gitlab-ci.yml
@@ -975,12 +975,20 @@ s3 dump plg tag:
 
 cleanup_build_job:
   stage: cleanup_build
+  image: registry.gitlab.com/gitlab-org/cloud-deploy/aws-base:latest
+  variables:
+    FBCI_PROFILE: "service-terraform"
+    AWS_ACCESS_KEY_ID: $AWS_FBCI_ACCESS_KEY_ID
+    AWS_SECRET_ACCESS_KEY: $AWS_FBCI_SECRET_ACCESS_KEY
   tags:
-    - shell
+    - aws
+    - docker
+    - fbsmoke
   script:
-    - CLEANUP_EC2_IDS=$(aws ec2 describe-instances --region us-east-2 --filters Name=instance-state-name,Values=pending,running "Name=tag:Name,Values=pipeline-${CI_PIPELINE_ID}-*" --query 'Reservations[*].Instances[*].[InstanceId]' --output text | tr '\n' ' ')
-    - '[[ ! -z "$CLEANUP_EC2_IDS" ]] && aws ec2 terminate-instances --region us-east-2 --instance-ids $CLEANUP_IDS || echo "No instances to cleanup"'
-    - CLEANUP_CF_ID=$(aws cloudformation list-stacks --region us-east-2 --stack-status-filter CREATE_IN_PROGRESS CREATE_COMPLETE --query  "StackSummaries[?contains(StackName, \`pipeline-${CI_PIPELINE_ID}\`) == \`true\`].[StackName]" --output text )
-    - '[[ ! -z "$CLEANUP_CF_ID" ]] && aws cloudformation delete-stack --region us-east-2 --stack-name $CLEANUP_CF_ID --retain-resources DeploymentEC2Role || echo "No CF stack to cleanup"'
+    - aws configure set aws_access_key_id $AWS_FBCI_ACCESS_KEY_ID --profile $FBCI_PROFILE
+    - aws configure set aws_secret_access_key $AWS_FBCI_SECRET_ACCESS_KEY --profile $FBCI_PROFILE
+    - aws configure set region "us-east-2" --profile $FBCI_PROFILE
+    - aws configure set aws_profile $FBCI_PROFILE
+    - ./qa/scripts/gitlabCleanupBuild.sh
   when: always
 
diff --git a/qa/scripts/gitlabCleanupBuild.sh b/qa/scripts/gitlabCleanupBuild.sh
new file mode 100755
index 000000000..adda9e8e5
--- /dev/null
+++ b/qa/scripts/gitlabCleanupBuild.sh
@@ -0,0 +1,37 @@
+#!/bin/bash
+
+CLEANUP_EC2_IDS=$(aws ec2 describe-instances --profile service-terraform --region us-east-2 --filters Name=instance-state-name,Values=pending,running "Name=tag:Name,Values=pipeline-${CI_PIPELINE_ID}-*" --query 'Reservations[*].Instances[*].[InstanceId]' --output text | tr '\n' ' ')
+if [ -n "${CLEANUP_EC2_IDS}" ]
+then
+    echo "ec2 Instances found, terminating."
+    echo $CLEANUP_EC2_IDS
+    IFS=', ' read -r -a array <<< "$CLEANUP_EC2_IDS"
+    aws ec2 terminate-instances --profile service-terraform --region us-east-2 --instance-ids "${array[@]}"
+    
+    res=$?
+    if (( $res != 0 )); then 
+        echo "Error: instances not terminated - ${CLEANUP_EC2_IDS}"
+        exit $res
+    fi
+else
+    echo 'No hanging ec2 instances'
+fi
+
+CLEANUP_CF_ID=$(aws cloudformation list-stacks --region us-east-2 --stack-status-filter CREATE_IN_PROGRESS CREATE_COMPLETE --query  "StackSummaries[?contains(StackName, \`pipeline-${CI_PIPELINE_ID}\`) == \`true\`].[StackName]" --output text  | tr '\n' ' ')
+if [ -n "${CLEANUP_CF_ID}" ]
+then
+    echo "CF stacks found, deleting."
+    echo $CLEANUP_CF_ID
+    IFS=', ' read -r -a array <<< "$CLEANUP_CF_ID"
+    for element in "${array[@]}"
+    do
+        aws cloudformation delete-stack --region us-east-2 --stack-name $element --retain-resources DeploymentEC2Role
+        res=$?
+        if (( $res != 0 )); then 
+            echo "Error: stacks not deleted - ${CLEANUP_CF_ID}"
+            exit $res
+        fi
+    done
+else
+    echo "No CF stacks to cleanup"
+fi
\ No newline at end of file
diff --git a/qa/tf/.modules/featurebase-cluster/main.tf b/qa/tf/.modules/featurebase-cluster/main.tf
index 141190037..8aafa4be7 100644
--- a/qa/tf/.modules/featurebase-cluster/main.tf
+++ b/qa/tf/.modules/featurebase-cluster/main.tf
@@ -86,6 +86,7 @@ resource "aws_instance" "fb_ingest" {
 }
 resource "aws_spot_instance_request" "fb_cluster_nodes" {
   wait_for_fulfillment   = true
+  spot_type              = "one-time"
   count                  = var.use_spot_instances ? var.fb_data_node_count : 0
   ami                    = data.aws_ami.amazon_linux_2.id
   instance_type          = var.fb_data_node_type
@@ -127,6 +128,7 @@ resource "aws_spot_instance_request" "fb_cluster_nodes" {
 
 resource "aws_spot_instance_request" "fb_ingest" {
   wait_for_fulfillment        = true
+  spot_type                   = "one-time"
   count                       = var.use_spot_instances ? var.fb_ingest_node_count : 0
   ami                         = data.aws_ami.amazon_linux_2.id
   key_name                    = aws_key_pair.gitlab-featurebase-ci.key_name