From a6ef3dabe28245ea66546aa995aa0b08b7c7b0a6 Mon Sep 17 00:00:00 2001
From: Seebs <seebs@molecula.com>
Date: Thu, 22 Sep 2022 12:27:35 -0500
Subject: [PATCH] improve resilience of smoke test setup

We want to retry our terraform setup if it fails, so let's check whether
it worked and possibly retry.

This loop is awful because I'm trying to both check the exit status
and the reported IPs. Once I know whether the exit status predicts the
reported IPs that should go away.
---
 qa/scripts/setupSmokeTest.sh | 32 +++++++++++++++++++++++++++++++-
 qa/scripts/utilCluster.sh    | 11 ++++++++++-
 2 files changed, 41 insertions(+), 2 deletions(-)

diff --git a/qa/scripts/setupSmokeTest.sh b/qa/scripts/setupSmokeTest.sh
index 91e00650c..4df2ca052 100755
--- a/qa/scripts/setupSmokeTest.sh
+++ b/qa/scripts/setupSmokeTest.sh
@@ -1,5 +1,7 @@
 #!/bin/bash
 set -x
+# how often should we try to do our terraform setup?
+max_tries=3
 
 # To run script: ./setupSmokeTest.sh <BRANCH_NAME>
 ADMIN_TOKEN=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJuYW1lIjoiYWRtaW4ifQ.I1iCgk1VU7m6e-En4ACTHIs6V2dZpy_8j2blSSo7K3U
@@ -19,7 +21,35 @@ pushd ./qa/tf/ci/smoketest
 echo "Running terraform init..."
 terraform init -input=false
 echo "Running terraform apply..."
-terraform apply -input=false -auto-approve
+
+okay=false
+# This logic is gratuitously complicated because I want visibility
+# into how it's working, or not-working. The chances are this should
+# just be a test against the exit status of terraform apply.
+tries=1
+while ! $okay && [ $tries -le $max_tries ] ; do
+	echo "Try $tries/$max_tries, running terraform..."
+	terraform apply -input=false -auto-approve
+	tf=$?
+	terraform output -json > outputs.json
+	echo "Outputs:"
+	cat outputs.json
+	must_get_value outputs.json TMP_I .ingest_ips 0 '"value"' 0
+	must_get_value outputs.json TMP_D .data_node_ips 0 '"value"' 0
+	echo "TF status: $tf, ingest_ips $TMP_I, data_node_ips $TMP_D"
+	case $TMP_I.$TMP_D in
+	*null*)	echo >&2 "looks like we failed, null in IPs."
+		tries=$(expr $tries + 1)
+		;;
+	*)	okay=true
+		;;
+	esac
+done
+echo "okay $okay, tries $tries"
+if ! $okay; then
+	echo >&2 "didn't start terraform successfully, giving up"
+	exit 1
+fi
 terraform output -json > outputs.json
 echo "Outputs:"
 cat outputs.json
diff --git a/qa/scripts/utilCluster.sh b/qa/scripts/utilCluster.sh
index d7f51a787..819c26579 100755
--- a/qa/scripts/utilCluster.sh
+++ b/qa/scripts/utilCluster.sh
@@ -649,6 +649,12 @@ setupConsumerNode(){
 # do so and returns a non-zero status. Note the quoting to get the double
 # quotes around "value".
 get_value() {
+    flags=$-
+    set +x
+    case $flags in
+    *x*)	restore="set -x";;
+    *)		restore="";;
+    esac
     file=$1
     var=$2
     shift 2
@@ -663,15 +669,18 @@ get_value() {
     tmp=$(jq -r "$expr" < $file)
     if [ $? -ne 0 ]; then
         echo >&2 "jq failed parsing input file"
+	$restore
         return 1
     fi
     case $tmp in
     null)
       echo >&2 "reading IPs for ${node}: got null"
+      $restore
       return 1
       ;;
     *)
       eval $var=\$tmp
+      $restore
       return 0
       ;;
     esac
@@ -686,4 +695,4 @@ must_get_value() {
     if ! get_value "$@"; then
         exit 1
     fi
-}
\ No newline at end of file
+}