make perf_able run at all, make it debug a bit better

switch perf-able to using same node type we use for other spot instances,
because otherwise it never finds any available capacity.

we switch the perf-able script to use the standard get_value function
instead of direct jq calls.

we try to grab server logs if the restore fails in the hopes of finding
out why the restore very occasionally fails.

qa/scripts/perf/able/ableSetup.sh

@@ -20,7 +20,35 @@ pushd ./qa/tf/perf/able
 echo "Running terraform init..."
 terraform init -input=false
 echo "Running terraform apply..."
-terraform apply -input=false -auto-approve
+
+okay=false
+# This logic is gratuitously complicated because I want visibility
+# into how it's working, or not-working. The chances are this should
+# just be a test against the exit status of terraform apply.
+tries=1
+while ! $okay && [ $tries -le $max_tries ] ; do
+	echo "Try $tries/$max_tries, running terraform..."
+	terraform apply -input=false -auto-approve
+	tf=$?
+	terraform output -json > outputs.json
+	echo "Outputs:"
+	cat outputs.json
+	must_get_value outputs.json TMP_I .ingest_ips 0 '"value"' 0
+	must_get_value outputs.json TMP_D .data_node_ips 0 '"value"' 0
+	echo "TF status: $tf, ingest_ips $TMP_I, data_node_ips $TMP_D"
+	case $TMP_I.$TMP_D in
+	*null*)	echo >&2 "looks like we failed, null in IPs."
+		tries=$(expr $tries + 1)
+		;;
+	*)	okay=true
+		;;
+	esac
+done
+echo "okay $okay, tries $tries"
+if ! $okay; then
+	echo >&2 "didn't start terraform successfully, giving up"
+	exit 1
+fi
 terraform output -json > outputs.json
 popd
 
@@ -31,28 +59,33 @@ EBS_DEVICE_NAME=/dev/nvme1n1
 FB_BINARY=featurebase_linux_arm64
 
 # get the first ingest host
-INGESTNODE0=$(cat ./qa/tf/perf/able/outputs.json | jq -r '[.ingest_ips][0]["value"][0]')
+must_get_value ./qa/tf/perf/able/outputs.json INGESTNODE0 .ingest_ips 0 '"value"' 0
 echo "using INGESTNODE0 ${INGESTNODE0}"
 
 # get the first data host
-DATANODE0=$(cat ./qa/tf/perf/able/outputs.json | jq -r '[.data_node_ips][0]["value"][0]')
+must_get_value ./qa/tf/perf/able/outputs.json DATANODE0 .data_node_ips 0 '"value"' 0
 echo "using DATANODE0 ${DATANODE0}"
 
+case ${INGESTNODE0}${DATANODE0} in
+*null*)	echo >&2 "didn't get nodes, giving up early"
+	exit 1
+	;;
+esac
 
-DEPLOYED_CLUSTER_PREFIX=$(cat ./qa/tf/perf/able/outputs.json | jq -r '[.cluster_prefix][0]["value"]')
+must_get_value ./qa/tf/perf/able/outputs.json DEPLOYED_CLUSTER_PREFIX .cluster_prefix 0 '"value"'
 echo "Using DEPLOYED_CLUSTER_PREFIX: ${DEPLOYED_CLUSTER_PREFIX}"
 
-DEPLOYED_CLUSTER_REPLICA_COUNT=$(cat ./qa/tf/perf/able/outputs.json | jq -r '[.fb_cluster_replica_count][0]["value"]')
+must_get_value ./qa/tf/perf/able/outputs.json DEPLOYED_CLUSTER_REPLICA_COUNT .fb_cluster_replica_count 0 '"value"'
 echo "Using DEPLOYED_CLUSTER_REPLICA_COUNT: ${DEPLOYED_CLUSTDEPLOYED_CLUSTER_REPLICA_COUNTER_PREFIX}"
 
-DEPLOYED_DATA_IPS=$(cat ./qa/tf/perf/able/outputs.json | jq -r '[.data_node_ips][0]["value"][]')
+must_get_value ./qa/tf/perf/able/outputs.json DEPLOYED_DATA_IPS .data_node_ips 0 '"value"' ""
 echo "DEPLOYED_DATA_IPS: {"
 echo "${DEPLOYED_DATA_IPS}"
 echo "}"
 
 DEPLOYED_DATA_IPS_LEN=`echo "$DEPLOYED_DATA_IPS" | wc -l`
 
-DEPLOYED_INGEST_IPS=$(cat ./qa/tf/perf/able/outputs.json | jq -r '[.ingest_ips][0]["value"][]')
+must_get_value ./qa/tf/perf/able/outputs.json DEPLOYED_INGEST_IPS .ingest_ips 0 '"value"' ""
 echo "DEPLOYED_INGEST_IPS: {"
 echo "${DEPLOYED_INGEST_IPS}"
 echo "}"

qa/scripts/perf/able/ableTest.sh

@@ -1,12 +1,13 @@
 #!/bin/bash
 
 # get the first ingest host
-INGESTNODE0=$(cat ./qa/tf/perf/able/outputs.json | jq -r '[.ingest_ips][0]["value"][0]')
+must_get_value ./qa/tf/perf/able/outputs.json INGESTNODE0 .ingest_ips 0 '"value"' 0
 echo "using INGESTNODE0 ${INGESTNODE0}"
 
 # get the first data host
-DATANODE0=$(cat ./qa/tf/perf/able/outputs.json | jq -r '[.data_node_ips][0]["value"][0]')
+must_get_value ./qa/tf/perf/able/outputs.json DATANODE0 .data_node_ips 0 '"value"' 0
 echo "using DATANODE0 ${DATANODE0}"
+must_get_value ./qa/tf/perf/able/outputs.json DEPLOYED_DATA_IPS .data_node_ips 0 '"value"' ""
 
 # leaving this here because K6 is timing out and need to work out why
 # ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o "StrictHostKeyChecking no" ec2-user@${INGESTNODE0} "wget https://github.com/grafana/k6/releases/download/v0.36.0/k6-v0.36.0-linux-arm64.tar.gz"
@@ -45,7 +46,13 @@ echo "Restoring data"
 ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o "StrictHostKeyChecking no" ec2-user@${INGESTNODE0} "cd /data; featurebase restore --host http://${DATANODE0}:10101 -s /data/data/backup > restore.out"
 if (( $? != 0 ))
 then
-    echo "Restoring failed"
+    echo "restoring failed. trying to dump server logs in case server crashed:"
+    for i in $DEPLOYED_DATA_IPS; do
+    	echo "BEGIN LOGS $i:"
+	ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o "StrictHostKeyChecking no" ec2-user@${i} "cat /var/log/molecula/featurebase.log"
+    	echo "END LOGS $i"
+
+    done
     exit 1
 fi
 

qa/tf/perf/able/main.tf

@@ -3,10 +3,10 @@ module "able-cluster" {
     cluster_prefix = var.cluster_prefix
     region = var.region
     profile = var.profile
-    fb_data_node_type = "m6g.12xlarge"
+    fb_data_node_type = "c6g.16xlarge"
     fb_data_disk_iops = 10000
     fb_data_node_count = 3
-    fb_ingest_type = "m6g.2xlarge"
+    fb_ingest_type = "c6g.2xlarge"
     fb_ingest_disk_iops = 10000
     fb_ingest_disk_size_gb = 500
     fb_ingest_node_count = 1