diff --git a/qa/scripts/perf/able/ableSetup.sh b/qa/scripts/perf/able/ableSetup.sh index ae180011a..d642937a9 100755 --- a/qa/scripts/perf/able/ableSetup.sh +++ b/qa/scripts/perf/able/ableSetup.sh @@ -20,7 +20,35 @@ pushd ./qa/tf/perf/able echo "Running terraform init..." terraform init -input=false echo "Running terraform apply..." -terraform apply -input=false -auto-approve + +okay=false +# This logic is gratuitously complicated because I want visibility +# into how it's working, or not-working. The chances are this should +# just be a test against the exit status of terraform apply. +tries=1 +while ! $okay && [ $tries -le $max_tries ] ; do + echo "Try $tries/$max_tries, running terraform..." + terraform apply -input=false -auto-approve + tf=$? + terraform output -json > outputs.json + echo "Outputs:" + cat outputs.json + must_get_value outputs.json TMP_I .ingest_ips 0 '"value"' 0 + must_get_value outputs.json TMP_D .data_node_ips 0 '"value"' 0 + echo "TF status: $tf, ingest_ips $TMP_I, data_node_ips $TMP_D" + case $TMP_I.$TMP_D in + *null*) echo >&2 "looks like we failed, null in IPs." + tries=$(expr $tries + 1) + ;; + *) okay=true + ;; + esac +done +echo "okay $okay, tries $tries" +if ! $okay; then + echo >&2 "didn't start terraform successfully, giving up" + exit 1 +fi terraform output -json > outputs.json popd @@ -31,28 +59,33 @@ EBS_DEVICE_NAME=/dev/nvme1n1 FB_BINARY=featurebase_linux_arm64 # get the first ingest host -INGESTNODE0=$(cat ./qa/tf/perf/able/outputs.json | jq -r '[.ingest_ips][0]["value"][0]') +must_get_value ./qa/tf/perf/able/outputs.json INGESTNODE0 .ingest_ips 0 '"value"' 0 echo "using INGESTNODE0 ${INGESTNODE0}" # get the first data host -DATANODE0=$(cat ./qa/tf/perf/able/outputs.json | jq -r '[.data_node_ips][0]["value"][0]') +must_get_value ./qa/tf/perf/able/outputs.json DATANODE0 .data_node_ips 0 '"value"' 0 echo "using DATANODE0 ${DATANODE0}" +case ${INGESTNODE0}${DATANODE0} in +*null*) echo >&2 "didn't get nodes, giving up early" + exit 1 + ;; +esac -DEPLOYED_CLUSTER_PREFIX=$(cat ./qa/tf/perf/able/outputs.json | jq -r '[.cluster_prefix][0]["value"]') +must_get_value ./qa/tf/perf/able/outputs.json DEPLOYED_CLUSTER_PREFIX .cluster_prefix 0 '"value"' echo "Using DEPLOYED_CLUSTER_PREFIX: ${DEPLOYED_CLUSTER_PREFIX}" -DEPLOYED_CLUSTER_REPLICA_COUNT=$(cat ./qa/tf/perf/able/outputs.json | jq -r '[.fb_cluster_replica_count][0]["value"]') +must_get_value ./qa/tf/perf/able/outputs.json DEPLOYED_CLUSTER_REPLICA_COUNT .fb_cluster_replica_count 0 '"value"' echo "Using DEPLOYED_CLUSTER_REPLICA_COUNT: ${DEPLOYED_CLUSTDEPLOYED_CLUSTER_REPLICA_COUNTER_PREFIX}" -DEPLOYED_DATA_IPS=$(cat ./qa/tf/perf/able/outputs.json | jq -r '[.data_node_ips][0]["value"][]') +must_get_value ./qa/tf/perf/able/outputs.json DEPLOYED_DATA_IPS .data_node_ips 0 '"value"' "" echo "DEPLOYED_DATA_IPS: {" echo "${DEPLOYED_DATA_IPS}" echo "}" DEPLOYED_DATA_IPS_LEN=`echo "$DEPLOYED_DATA_IPS" | wc -l` -DEPLOYED_INGEST_IPS=$(cat ./qa/tf/perf/able/outputs.json | jq -r '[.ingest_ips][0]["value"][]') +must_get_value ./qa/tf/perf/able/outputs.json DEPLOYED_INGEST_IPS .ingest_ips 0 '"value"' "" echo "DEPLOYED_INGEST_IPS: {" echo "${DEPLOYED_INGEST_IPS}" echo "}" diff --git a/qa/scripts/perf/able/ableTest.sh b/qa/scripts/perf/able/ableTest.sh index 282bf3d7a..d001b89d2 100755 --- a/qa/scripts/perf/able/ableTest.sh +++ b/qa/scripts/perf/able/ableTest.sh @@ -1,12 +1,13 @@ #!/bin/bash # get the first ingest host -INGESTNODE0=$(cat ./qa/tf/perf/able/outputs.json | jq -r '[.ingest_ips][0]["value"][0]') +must_get_value ./qa/tf/perf/able/outputs.json INGESTNODE0 .ingest_ips 0 '"value"' 0 echo "using INGESTNODE0 ${INGESTNODE0}" # get the first data host -DATANODE0=$(cat ./qa/tf/perf/able/outputs.json | jq -r '[.data_node_ips][0]["value"][0]') +must_get_value ./qa/tf/perf/able/outputs.json DATANODE0 .data_node_ips 0 '"value"' 0 echo "using DATANODE0 ${DATANODE0}" +must_get_value ./qa/tf/perf/able/outputs.json DEPLOYED_DATA_IPS .data_node_ips 0 '"value"' "" # leaving this here because K6 is timing out and need to work out why # ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o "StrictHostKeyChecking no" ec2-user@${INGESTNODE0} "wget https://github.com/grafana/k6/releases/download/v0.36.0/k6-v0.36.0-linux-arm64.tar.gz" @@ -45,7 +46,13 @@ echo "Restoring data" ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o "StrictHostKeyChecking no" ec2-user@${INGESTNODE0} "cd /data; featurebase restore --host http://${DATANODE0}:10101 -s /data/data/backup > restore.out" if (( $? != 0 )) then - echo "Restoring failed" + echo "restoring failed. trying to dump server logs in case server crashed:" + for i in $DEPLOYED_DATA_IPS; do + echo "BEGIN LOGS $i:" + ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o "StrictHostKeyChecking no" ec2-user@${i} "cat /var/log/molecula/featurebase.log" + echo "END LOGS $i" + + done exit 1 fi diff --git a/qa/tf/perf/able/main.tf b/qa/tf/perf/able/main.tf index a86e6dc4f..508153d21 100644 --- a/qa/tf/perf/able/main.tf +++ b/qa/tf/perf/able/main.tf @@ -3,10 +3,10 @@ module "able-cluster" { cluster_prefix = var.cluster_prefix region = var.region profile = var.profile - fb_data_node_type = "m6g.12xlarge" + fb_data_node_type = "c6g.16xlarge" fb_data_disk_iops = 10000 fb_data_node_count = 3 - fb_ingest_type = "m6g.2xlarge" + fb_ingest_type = "c6g.2xlarge" fb_ingest_disk_iops = 10000 fb_ingest_disk_size_gb = 500 fb_ingest_node_count = 1