diff --git a/qa/scripts/setupSmokeTest.sh b/qa/scripts/setupSmokeTest.sh index 91e00650c..4df2ca052 100755 --- a/qa/scripts/setupSmokeTest.sh +++ b/qa/scripts/setupSmokeTest.sh @@ -1,5 +1,7 @@ #!/bin/bash set -x +# how often should we try to do our terraform setup? +max_tries=3 # To run script: ./setupSmokeTest.sh ADMIN_TOKEN=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJuYW1lIjoiYWRtaW4ifQ.I1iCgk1VU7m6e-En4ACTHIs6V2dZpy_8j2blSSo7K3U @@ -19,7 +21,35 @@ pushd ./qa/tf/ci/smoketest echo "Running terraform init..." terraform init -input=false echo "Running terraform apply..." -terraform apply -input=false -auto-approve + +okay=false +# This logic is gratuitously complicated because I want visibility +# into how it's working, or not-working. The chances are this should +# just be a test against the exit status of terraform apply. +tries=1 +while ! $okay && [ $tries -le $max_tries ] ; do + echo "Try $tries/$max_tries, running terraform..." + terraform apply -input=false -auto-approve + tf=$? + terraform output -json > outputs.json + echo "Outputs:" + cat outputs.json + must_get_value outputs.json TMP_I .ingest_ips 0 '"value"' 0 + must_get_value outputs.json TMP_D .data_node_ips 0 '"value"' 0 + echo "TF status: $tf, ingest_ips $TMP_I, data_node_ips $TMP_D" + case $TMP_I.$TMP_D in + *null*) echo >&2 "looks like we failed, null in IPs." + tries=$(expr $tries + 1) + ;; + *) okay=true + ;; + esac +done +echo "okay $okay, tries $tries" +if ! $okay; then + echo >&2 "didn't start terraform successfully, giving up" + exit 1 +fi terraform output -json > outputs.json echo "Outputs:" cat outputs.json diff --git a/qa/scripts/utilCluster.sh b/qa/scripts/utilCluster.sh index d7f51a787..819c26579 100755 --- a/qa/scripts/utilCluster.sh +++ b/qa/scripts/utilCluster.sh @@ -649,6 +649,12 @@ setupConsumerNode(){ # do so and returns a non-zero status. Note the quoting to get the double # quotes around "value". get_value() { + flags=$- + set +x + case $flags in + *x*) restore="set -x";; + *) restore="";; + esac file=$1 var=$2 shift 2 @@ -663,15 +669,18 @@ get_value() { tmp=$(jq -r "$expr" < $file) if [ $? -ne 0 ]; then echo >&2 "jq failed parsing input file" + $restore return 1 fi case $tmp in null) echo >&2 "reading IPs for ${node}: got null" + $restore return 1 ;; *) eval $var=\$tmp + $restore return 0 ;; esac @@ -686,4 +695,4 @@ must_get_value() { if ! get_value "$@"; then exit 1 fi -} \ No newline at end of file +}