From 782faa3354ff64c4245567882dd38d6a557c7341 Mon Sep 17 00:00:00 2001
From: Seebs <seebs@molecula.com>
Date: Tue, 6 Sep 2022 12:11:46 -0500
Subject: [PATCH] squash auth tests into regular smoke tests, improve smoke
 tests

This is a bit complicated and entangled, sorry.

First, we squash the auth-based smoke tests into the regular smoke
tests; we just run all the tests with auth on and that way we don't
need to spin up an entire separate cluster of machines just to run
a single query against them.

We improve the error detection, and standardize the jq-to-get-config
code. The purpose of this is to try to make sure that, if we actually
hit a failure and get "null" for a host name, we report *that*
as an error, rather than running ahead and producing 20+ separate
reports that ssh failed because it couldn't find a host named null.
---
 .gitlab/.gitlab-ci.yml                        | 62 +------------
 qa/scripts/auth-smoke/setup.sh                | 91 -------------------
 qa/scripts/auth-smoke/teardown.sh             |  7 --
 qa/scripts/auth-smoke/test.sh                 | 37 --------
 qa/scripts/auth-smoke/tests/README            | 16 ----
 qa/scripts/auth-smoke/tests/run-all.sh        |  8 --
 qa/scripts/auth-smoke/tests/sup218-test.sh    | 33 -------
 .../auth-smoke/tests/sup218_datagen.yaml      | 61 -------------
 qa/scripts/bug_repro_tests.sh                 | 16 ++--
 qa/scripts/setupSmokeTest.sh                  | 60 ++++++++----
 qa/scripts/testSmokeTest.sh                   | 22 ++---
 qa/scripts/utilCluster.sh                     | 48 ++++++++++
 qa/testcases/bug-repros/fb-1287-test.sh       |  6 +-
 qa/testcases/bug-repros/fb-1332.sh            |  8 +-
 qa/testcases/bug-repros/fb-1371-test.sh       |  8 +-
 qa/testcases/bug-repros/fb1270-test.sh        |  9 +-
 qa/testcases/bug-repros/run-all.sh            |  4 +-
 qa/testcases/smoketest/test_smoke.py          | 17 ++--
 18 files changed, 134 insertions(+), 379 deletions(-)
 delete mode 100755 qa/scripts/auth-smoke/setup.sh
 delete mode 100755 qa/scripts/auth-smoke/teardown.sh
 delete mode 100755 qa/scripts/auth-smoke/test.sh
 delete mode 100644 qa/scripts/auth-smoke/tests/README
 delete mode 100755 qa/scripts/auth-smoke/tests/run-all.sh
 delete mode 100755 qa/scripts/auth-smoke/tests/sup218-test.sh
 delete mode 100644 qa/scripts/auth-smoke/tests/sup218_datagen.yaml

diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml
index 195a8286d..438a1a4a2 100644
--- a/.gitlab/.gitlab-ci.yml
+++ b/.gitlab/.gitlab-ci.yml
@@ -809,65 +809,6 @@ external lookup tests:
     - apt-get install -y postgresql-client
     - go test . -run "^TestExternalLookup" -externalLookupDSN postgresql://$POSTGRES_USER:$POSTGRES_PASSWORD@postgres/$POSTGRES_DB?sslmode=disable
 
-smoke test auth:
-  stage: integration
-  image: registry.gitlab.com/gitlab-org/cloud-deploy/aws-base:latest
-  variables:
-    PROFILE: "service-terraform"
-    AWS_SSH_PRIVATE_KEY: $AWS_FBCI_SSH_KEY
-    AWS_ACCESS_KEY_ID: $AWS_FBCI_ACCESS_KEY_ID
-    AWS_SECRET_ACCESS_KEY: $AWS_FBCI_SECRET_ACCESS_KEY
-    TF_VAR_cluster_prefix: ""
-  tags:
-    - aws
-    - docker
-    - fbsmoke
-  rules:
-    - if: '$CI_PIPELINE_SOURCE == "push"'
-  before_script:
-    - apt-get update && apt-get install -y gnupg software-properties-common curl git
-    - curl -fsSL https://apt.releases.hashicorp.com/gpg | apt-key add -
-    - apt-add-repository "deb [arch=amd64] https://apt.releases.hashicorp.com $(lsb_release -cs) main"
-    - apt-get update && apt-get install terraform
-    - aws configure set aws_access_key_id $AWS_FBCI_ACCESS_KEY_ID --profile $PROFILE
-    - aws configure set aws_secret_access_key $AWS_FBCI_SECRET_ACCESS_KEY --profile $PROFILE
-    - aws configure set region "us-east-2" --profile $PROFILE
-    - aws configure set aws_profile $PROFILE
-    - echo $AWS_FBCI_SSH_KEY > gitlab-featurebase-ci.pem
-    - chmod 400 gitlab-featurebase-ci.pem
-    - "which ssh-agent || ( apt-get update -y && apt-get install openssh-client -y )"
-    - eval $(ssh-agent -s)
-    - mkdir -p ~/.ssh
-    - echo $AWS_FBCI_SSH_KEY > /root/.ssh/gitlab-featurebase-ci.pem
-    - chmod 400 /root/.ssh/gitlab-featurebase-ci.pem
-    - echo "$AWS_FBCI_SSH_KEY" | ssh-add -
-    - chmod 700 /root/.ssh
-    - '[[ -f /.dockerenv ]] && echo -e "Host *\n\tStrictHostKeyChecking no\n\n" > ~/.ssh/config'
-    - apt update && apt -y install jq wget git libnss3-tools
-    - wget -q https://go.dev/dl/go$GOVERSION.linux-amd64.tar.gz
-    - tar -C /usr/local -xzf go$GOVERSION.linux-amd64.tar.gz
-    - export PATH=$PATH:/usr/local/go/bin
-    - TF_VAR_cluster_prefix="pipeline-$CI_PIPELINE_ID-auth-smoke-$CI_JOB_ID"
-    - echo "Cluster Prefix --> $TF_VAR_cluster_prefix"
-    # download datagen for FB-1270 repro test.
-    - aws s3 cp s3://molecula-artifact-storage/idk/${CI_COMMIT_BRANCH}/_latest/idk-linux-arm64/datagen ./datagen_linux_arm64
-    - aws s3 cp s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/_latest/featurebase_linux_arm64 ./
-    - chmod +x ./datagen_linux_arm64 ./featurebase_linux_arm64
-  script:
-    - ./qa/scripts/auth-smoke/setup.sh $CI_COMMIT_BRANCH
-    - ./qa/scripts/auth-smoke/test.sh
-  after_script:
-    - ./qa/scripts/auth-smoke/teardown.sh
-  needs:
-    - job: s3 dump
-    - job: idk s3 dump
-  artifacts:
-    when: always
-    paths:
-      - report.xml
-    reports:
-      junit: report.xml
-
 smoke test:
   stage: integration
   image: registry.gitlab.com/gitlab-org/cloud-deploy/aws-base:latest
@@ -902,7 +843,7 @@ smoke test:
     - echo "$AWS_FBCI_SSH_KEY" | ssh-add -
     - chmod 700 /root/.ssh
     - '[[ -f /.dockerenv ]] && echo -e "Host *\n\tStrictHostKeyChecking no\n\n" > ~/.ssh/config'
-    - apt update && apt -y install jq wget
+    - apt update && apt -y install jq wget git libnss3-tools
     - wget -q https://go.dev/dl/go$GOVERSION.linux-amd64.tar.gz
     - tar -C /usr/local -xzf go$GOVERSION.linux-amd64.tar.gz
     - export PATH=$PATH:/usr/local/go/bin
@@ -1359,4 +1300,3 @@ cleanup_build_job:
   when: always
   needs:
     - job: smoke test
-    - job: smoke test auth
diff --git a/qa/scripts/auth-smoke/setup.sh b/qa/scripts/auth-smoke/setup.sh
deleted file mode 100755
index 8243ece95..000000000
--- a/qa/scripts/auth-smoke/setup.sh
+++ /dev/null
@@ -1,91 +0,0 @@
-#!/bin/bash
-
-# To run script: ./setup.sh <BRANCH_NAME>
-BRANCH_NAME=$1
-
-echo "Running tests for branch ${BRANCH_NAME}"
-
-ADMIN_TOKEN=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJuYW1lIjoiYWRtaW4ifQ.I1iCgk1VU7m6e-En4ACTHIs6V2dZpy_8j2blSSo7K3U
-READER_TOKEN=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJuYW1lIjoicmVhZGVyIn0.QcHy_W6oAYFgdBWy1CqLr55HcOyymn5zAXPJUKCvQE4
-WRITER_TOKEN=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJuYW1lIjoid3JpdGVyIn0.aEk-12xP9RJeXog4MHO8LhuQFEjNNG2BcWDcMzSX_HI
-export TF_IN_AUTOMATION=1
-export AUTH_ENABLED=1
-
-SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
-source ./qa/scripts/utilCluster.sh
-
-pushd ./qa/tf/ci/auth-smoke
-echo "Running terraform init..."
-terraform init -input=false
-echo "Running terraform apply..."
-terraform apply -input=false -auto-approve
-terraform output -json > outputs.json
-popd
-
-# path for ebs device
-EBS_DEVICE_NAME=/dev/nvme1n1
-
-# featurebase architecture
-FB_BINARY=featurebase_linux_arm64
-
-# get the first ingest host
-INGESTNODE0=$(cat ./qa/tf/ci/auth-smoke/outputs.json | jq -r '[.ingest_ips][0]["value"][0]')
-echo "using INGESTNODE0 ${INGESTNODE0}"
-
-# get the first data host
-DATANODE0=$(cat ./qa/tf/ci/auth-smoke/outputs.json | jq -r '[.data_node_ips][0]["value"][0]')
-echo "using DATANODE0 ${DATANODE0}"
-
-DEPLOYED_CLUSTER_PREFIX=$(cat ./qa/tf/ci/auth-smoke/outputs.json | jq -r '[.cluster_prefix][0]["value"]')
-echo "Using DEPLOYED_CLUSTER_PREFIX: ${DEPLOYED_CLUSTER_PREFIX}"
-
-DEPLOYED_CLUSTER_REPLICA_COUNT=$(cat ./qa/tf/ci/auth-smoke/outputs.json | jq -r '[.fb_cluster_replica_count][0]["value"]')
-echo "Using DEPLOYED_CLUSTER_REPLICA_COUNT: ${DEPLOYED_CLUSTDEPLOYED_CLUSTER_REPLICA_COUNTER_PREFIX}"
-
-DEPLOYED_DATA_IPS=$(cat ./qa/tf/ci/auth-smoke/outputs.json | jq -r '[.data_node_ips][0]["value"][]')
-echo "DEPLOYED_DATA_IPS: {"
-echo "${DEPLOYED_DATA_IPS}"
-echo "}"
-
-DEPLOYED_DATA_IPS_LEN=`echo "$DEPLOYED_DATA_IPS" | wc -l`
-
-DEPLOYED_INGEST_IPS=$(cat ./qa/tf/ci/auth-smoke/outputs.json | jq -r '[.ingest_ips][0]["value"][]')
-echo "DEPLOYED_INGEST_IPS: {"
-echo "${DEPLOYED_INGEST_IPS}"
-echo "}"
-
-DEPLOYED_INGEST_IPS_LEN=`echo "$DEPLOYED_INGEST_IPS" | wc -l`
-
-
-#wait until we can connect to one of the hosts
-for i in {0..24}
-do
-    ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no -o ConnectTimeout=10 ec2-user@${DATANODE0} "pwd"
-    if [ $? -eq 0 ]
-    then
-        echo "Cluster is up after ${i} tries."
-        break
-    fi
-    sleep 10
-done
-
-ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no -o ConnectTimeout=10 ec2-user@${DATANODE0} "pwd"
-if [ $? -ne 0 ]
-then
-    echo "Unable to connect to cluster - giving up"
-    exit 1
-fi
-
-setupClusterNodes $BRANCH_NAME
-
-# verify featurebase running
-echo "Verifying featurebase cluster running..."
-for i in {0..24}; do
-    curl -k -v https://${DATANODE0}:10101/status -H "Authorization: Bearer ${ADMIN_TOKEN}"
-    if [ $? -eq 0 ]; then
-        echo "Cluster is up after ${i} tries"
-        exit 0
-    fi
-    sleep 1
-done
-exit $?
diff --git a/qa/scripts/auth-smoke/teardown.sh b/qa/scripts/auth-smoke/teardown.sh
deleted file mode 100755
index 40665efd9..000000000
--- a/qa/scripts/auth-smoke/teardown.sh
+++ /dev/null
@@ -1,7 +0,0 @@
-#!/bin/bash
-
-# To run script: ./teardown.sh
-
-cd qa/tf/ci/auth-smoke
-export TF_IN_AUTOMATION=1
-terraform destroy -auto-approve
diff --git a/qa/scripts/auth-smoke/test.sh b/qa/scripts/auth-smoke/test.sh
deleted file mode 100755
index 082d81429..000000000
--- a/qa/scripts/auth-smoke/test.sh
+++ /dev/null
@@ -1,37 +0,0 @@
-#!/bin/bash
-
-source ./qa/scripts/utilCluster.sh
-
-# get the first ingest host
-INGESTNODE0=$(cat ./qa/tf/ci/auth-smoke/outputs.json | jq -r '[.ingest_ips][0]["value"][0]')
-echo "using INGESTNODE0 ${INGESTNODE0}"
-
-# get the first data host
-DATANODE0=$(cat ./qa/tf/ci/auth-smoke/outputs.json | jq -r '[.data_node_ips][0]["value"][0]')
-echo "using DATANODE0 ${DATANODE0}"
-
-HOSTS=($( cat ./qa/tf/ci/auth-smoke/outputs.json | jq -r '.data_node_ips.value' | tr -d '[],"'))
-
-echo "Copying tests to remote"
-scp -r -i ~/.ssh/gitlab-featurebase-ci.pem ./qa/scripts/auth-smoke/tests/ ec2-user@${INGESTNODE0}:/data
-scp -r -i ~/.ssh/gitlab-featurebase-ci.pem ./datagen_linux_arm64 ec2-user@${INGESTNODE0}:/data
-if (( $? != 0 ))
-then
-    echo "Copy failed"
-    exit 1
-fi
-
-# run all repros
-echo "Running smoke tests..."
-ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o "StrictHostKeyChecking no" ec2-user@${INGESTNODE0} "cd /data/tests; ./run-all.sh ${HOSTS[@]}"
-SMOKETESTRESULT=$?
-
-
-if (( $SMOKETESTRESULT != 0 ))
-then
-    echo "smoke tests complete with test failures"
-else
-    echo "smoke tests complete"
-fi
-
-exit $SMOKETESTRESULT
diff --git a/qa/scripts/auth-smoke/tests/README b/qa/scripts/auth-smoke/tests/README
deleted file mode 100644
index c9ebf1977..000000000
--- a/qa/scripts/auth-smoke/tests/README
+++ /dev/null
@@ -1,16 +0,0 @@
-adding a test case to the auth-smoke directory??? no problem!!! its a snap!!!
-
-just make a shell script that does the test you want and name it some thing like:
-
-fb42069-test.sh
-
-this will get picked up by run-all.sh and get run automatically!!!
-
-# THINGS TO NOTE
-- the datanode0 ip will be passed to your script in $1 via qa/scripts/bug_repro_tests.sh
-
-# ENTHUSIASM
-WOW
-
-shout out to computers for making our lives easier! :) 👍
-
diff --git a/qa/scripts/auth-smoke/tests/run-all.sh b/qa/scripts/auth-smoke/tests/run-all.sh
deleted file mode 100755
index ac27d85ce..000000000
--- a/qa/scripts/auth-smoke/tests/run-all.sh
+++ /dev/null
@@ -1,8 +0,0 @@
-#!/usr/bin/env bash
-
-set -eou pipefail
-
-for file in `ls *-test.sh`; do
-    echo "running $file";
-    ./$file "$@"
-done
diff --git a/qa/scripts/auth-smoke/tests/sup218-test.sh b/qa/scripts/auth-smoke/tests/sup218-test.sh
deleted file mode 100755
index d09a0fce4..000000000
--- a/qa/scripts/auth-smoke/tests/sup218-test.sh
+++ /dev/null
@@ -1,33 +0,0 @@
-#!/bin/bash
-
-ifErr() {
-    res=$?
-    if (( res != 0 )); then
-        echo "error: $1"
-        exit $res
-    fi
-}
-
-HOSTS=($@)
-
-for host in ${HOSTS[@]}; do
-    echo $host;
-done
-
-HOST=${HOSTS[2]}
-
-# ingest string key data to user index
-/data/datagen --source custom --custom-config /data/tests/sup218_datagen.yaml --pilosa.index=user --pilosa.hosts=https://$HOST:10101 --pilosa.batch-size=1000
-ifErr "running datagen on $HOST"
-
-# install grpcurl
-wget https://github.com/fullstorydev/grpcurl/releases/download/v1.8.6/grpcurl_1.8.6_linux_arm64.tar.gz
-tar -xvf grpcurl_1.8.6_linux_arm64.tar.gz
-chmod +x grpcurl
-
-for ip in ${HOSTS[@]}; do
-    # then make a `select distinct test_field from user`
-    ./grpcurl -H 'Authorization: Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJuYW1lIjoiYWRtaW4ifQ.I1iCgk1VU7m6e-En4ACTHIs6V2dZpy_8j2blSSo7K3U' -d '{"sql": "select * from user limit 1"}' $ip:20101 pilosa.Pilosa.QuerySQL
-    # make sure it doesn't fail
-    ifErr "select * from user failed when it shouldn't have!"
-done
diff --git a/qa/scripts/auth-smoke/tests/sup218_datagen.yaml b/qa/scripts/auth-smoke/tests/sup218_datagen.yaml
deleted file mode 100644
index 9dafe7913..000000000
--- a/qa/scripts/auth-smoke/tests/sup218_datagen.yaml
+++ /dev/null
@@ -1,61 +0,0 @@
-fields:
-  - name: "a_random_string"
-    type: "string" # (default StringField (non-mutex))
-    generator_type: "random-string" # used to generate random strings rather than pulling from known set
-    min_len: 8
-    max_len: 12
-    charset: "AB" # set of possible characters to pull from when generating random string
-  - name: "id"
-    type: "uint"
-    distribution: "sequential"
-    min: 0
-    max: 1000 # 2%24
-    step: 1
-    repeat: false
-  - name: "type"
-    type: "int" # (default IntField)
-    min: 0
-    max: 3
-    distribution: "zipfian"
-    s: 1.1
-    v: 5.1
-  - name: "ts"
-    type: "timestamp"
-    min_date: 2006-01-02T15:04:05.001Z # RFC3339Nano
-    max_date: 2007-01-02T15:04:05.001Z # RFC3339Nano
-    distribution: "increasing" # only "increasing" is supported right now
-    min_step_duration: "10ns"
-    max_step_duration: "200ms"
-  - name: "slice"
-    type: "uint-set" # (default IDArrayField)
-    min: 0
-    max: 35000
-    distribution: "zipfian"
-    s: 1.1
-    v: 5.1
-    min_num: 1
-    max_num: 50
-
-# idk_params describe how data from "fields" should be ingested by IDK
-idk_params:
-  primary_key_config:
-    field: "a_random_string" # if this is a single field named "id" then we'll use uint IDs, if it's empty we'll autogen ids, and if it's anything else we'll do string keys... yes this is a bit hacky, needs to be cleaned up.
-  # fields is keyed by names of fields from top level "fields". It is
-  # not required that all fields appear here, those that don't will
-  # use the default ingestion.
-  fields:
-    id:
-      - type: "ID"
-    type:
-      - type: "ID"
-    a_decimal_field:
-      - type: "Decimal"
-        scale: 4
-    ts:
-      - type: "RecordTime"
-        layout: "2006-01-02T15:04:05Z"
-        epoch: 1970-01-01T00:00:00.0Z
-        name: "na"
-    slice:
-      - type: "IDArray"
-        time_quantum: "YMD"
diff --git a/qa/scripts/bug_repro_tests.sh b/qa/scripts/bug_repro_tests.sh
index d97fcc28c..1ade540a4 100755
--- a/qa/scripts/bug_repro_tests.sh
+++ b/qa/scripts/bug_repro_tests.sh
@@ -1,18 +1,16 @@
 #!/usr/bin/env bash
 
-
 SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
-source $SCRIPT_DIR/utilCluster.sh
+. $SCRIPT_DIR/utilCluster.sh
 
 # get the first ingest host
-INGESTNODE0=$(cat ./qa/tf/ci/smoketest/outputs.json | jq -r '[.ingest_ips][0]["value"][0]')
+must_get_value ./qa/tf/ci/smoketest/outputs.json INGESTNODE0 .ingest_ips 0 '"value"' 0
 echo "using INGESTNODE0 ${INGESTNODE0}"
 
 # get the first data host
-DATANODE0=$(cat ./qa/tf/ci/smoketest/outputs.json | jq -r '[.data_node_ips][0]["value"][0]')
+must_get_value ./qa/tf/ci/smoketest/outputs.json DATANODE0 .data_node_ips 0 '"value"' 0
 echo "using DATANODE0 ${DATANODE0}"
 
-
 # install librdkafka ... workaround until we have static datagen builds for arm
 ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o "StrictHostKeyChecking no" ec2-user@${INGESTNODE0} "sudo yum -y install librdkafka"
 
@@ -26,16 +24,16 @@ then
 fi
 
 # run all repros
-echo "Running smoke tests..."
-ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o "StrictHostKeyChecking no" ec2-user@${INGESTNODE0} "cd /data/bug-repros; ./run-all.sh ${DATANODE0}:10101"
+echo "Running bug-repro tests..."
+ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o "StrictHostKeyChecking no" ec2-user@${INGESTNODE0} "cd /data/bug-repros; ./run-all.sh 'https://${DATANODE0}:10101'"
 SMOKETESTRESULT=$?
 
 
 if (( $SMOKETESTRESULT != 0 ))
 then
-    echo "smoke tests complete with test failures"
+    echo "bug-repro tests complete with test failures"
 else
-    echo "smoke tests complete"
+    echo "bug-repro tests complete"
 fi
 
 exit $SMOKETESTRESULT
diff --git a/qa/scripts/setupSmokeTest.sh b/qa/scripts/setupSmokeTest.sh
index ab86f05dd..91e00650c 100755
--- a/qa/scripts/setupSmokeTest.sh
+++ b/qa/scripts/setupSmokeTest.sh
@@ -1,15 +1,19 @@
 #!/bin/bash
+set -x
 
 # To run script: ./setupSmokeTest.sh <BRANCH_NAME>
+ADMIN_TOKEN=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJuYW1lIjoiYWRtaW4ifQ.I1iCgk1VU7m6e-En4ACTHIs6V2dZpy_8j2blSSo7K3U
+READER_TOKEN=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJuYW1lIjoicmVhZGVyIn0.QcHy_W6oAYFgdBWy1CqLr55HcOyymn5zAXPJUKCvQE4
+WRITER_TOKEN=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJuYW1lIjoid3JpdGVyIn0.aEk-12xP9RJeXog4MHO8LhuQFEjNNG2BcWDcMzSX_HI
 export TF_IN_AUTOMATION=1
-export AUTH_ENABLED=0
+export AUTH_ENABLED=1
 
 BRANCH_NAME=$1
 
 echo "Running tests for branch ${BRANCH_NAME}"
 
 SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
-source $SCRIPT_DIR/utilCluster.sh
+. $SCRIPT_DIR/utilCluster.sh
 
 pushd ./qa/tf/ci/smoketest
 echo "Running terraform init..."
@@ -17,6 +21,8 @@ terraform init -input=false
 echo "Running terraform apply..."
 terraform apply -input=false -auto-approve
 terraform output -json > outputs.json
+echo "Outputs:"
+cat outputs.json
 popd
 
 # path for ebs device
@@ -26,33 +32,52 @@ EBS_DEVICE_NAME=/dev/nvme1n1
 FB_BINARY=featurebase_linux_arm64
 
 # get the first ingest host
-INGESTNODE0=$(cat ./qa/tf/ci/smoketest/outputs.json | jq -r '[.ingest_ips][0]["value"][0]')
+must_get_value ./qa/tf/ci/smoketest/outputs.json INGESTNODE0 .ingest_ips 0 '"value"' 0
 echo "using INGESTNODE0 ${INGESTNODE0}"
 
 # get the first data host
-DATANODE0=$(cat ./qa/tf/ci/smoketest/outputs.json | jq -r '[.data_node_ips][0]["value"][0]')
+must_get_value ./qa/tf/ci/smoketest/outputs.json DATANODE0 .data_node_ips 0 '"value"' 0
 echo "using DATANODE0 ${DATANODE0}"
 
-DEPLOYED_CLUSTER_PREFIX=$(cat ./qa/tf/ci/smoketest/outputs.json | jq -r '[.cluster_prefix][0]["value"]')
+must_get_value ./qa/tf/ci/smoketest/outputs.json DEPLOYED_CLUSTER_PREFIX .cluster_prefix 0 '"value"'
 echo "Using DEPLOYED_CLUSTER_PREFIX: ${DEPLOYED_CLUSTER_PREFIX}"
 
-DEPLOYED_CLUSTER_REPLICA_COUNT=$(cat ./qa/tf/ci/smoketest/outputs.json | jq -r '[.fb_cluster_replica_count][0]["value"]')
+must_get_value ./qa/tf/ci/smoketest/outputs.json DEPLOYED_CLUSTER_REPLICA_COUNT .fb_cluster_replica_count 0 '"value"'
 echo "Using DEPLOYED_CLUSTER_REPLICA_COUNT: ${DEPLOYED_CLUSTDEPLOYED_CLUSTER_REPLICA_COUNTER_PREFIX}"
 
-DEPLOYED_DATA_IPS=$(cat ./qa/tf/ci/smoketest/outputs.json | jq -r '[.data_node_ips][0]["value"][]')
+must_get_value ./qa/tf/ci/smoketest/outputs.json DEPLOYED_DATA_IPS .data_node_ips 0 '"value"' ""
 echo "DEPLOYED_DATA_IPS: {"
 echo "${DEPLOYED_DATA_IPS}"
 echo "}"
 
 DEPLOYED_DATA_IPS_LEN=`echo "$DEPLOYED_DATA_IPS" | wc -l`
 
-DEPLOYED_INGEST_IPS=$(cat ./qa/tf/ci/smoketest/outputs.json | jq -r '[.ingest_ips][0]["value"][]')
+must_get_value ./qa/tf/ci/smoketest/outputs.json DEPLOYED_INGEST_IPS .ingest_ips 0 '"value"' ""
 echo "DEPLOYED_INGEST_IPS: {"
 echo "${DEPLOYED_INGEST_IPS}"
 echo "}"
 
 DEPLOYED_INGEST_IPS_LEN=`echo "$DEPLOYED_INGEST_IPS" | wc -l`
 
+echo "Writing config.py file..."
+cat << EOT > config.py
+datanode0="${DATANODE0}"
+admin_token="${ADMIN_TOKEN}"
+reader_token="${READER_TOKEN}"
+writer_token="${WRITER_TOKEN}"
+EOT
+cp config.py ./qa/testcases/smoketest/config.py
+echo "Writing config.sh file..."
+cat << EOT > config.sh
+export DATANODE0="${DATANODE0}"
+export ADMIN_TOKEN="${ADMIN_TOKEN}"
+export READER_TOKEN="${READER_TOKEN}"
+export WRITER_TOKEN="${WRITER_TOKEN}"
+authcurl() {
+    curl -H "Authorization: Bearer ${ADMIN_TOKEN}" -k "\$@"
+}
+EOT
+cp config.sh ./qa/testcases/bug-repros/config.sh
 
 #wait until we can connect to one of the hosts
 for i in {0..24}
@@ -76,14 +101,15 @@ fi
 setupClusterNodes $BRANCH_NAME
 
 # verify featurebase running
-echo "Verifying featurebase cluster running..."
-curl -s http://${DATANODE0}:10101/status
-if (( $? != 0 ))
-then
-    echo "Featurebase cluster not running"
-    exit 1
-fi
-
-echo "Cluster running."
+for i in {0..24}; do
+    sleep 1
+    curl -k -v https://${DATANODE0}:10101/status -H "Authorization: Bearer ${ADMIN_TOKEN}"
+    if [ $? -eq 0 ]; then
+        echo "Cluster is up after ${i} tries"
+        exit 0
+    fi
+done
+echo >&2 "Featurebase cluster not running."
+exit 1
 
 
diff --git a/qa/scripts/testSmokeTest.sh b/qa/scripts/testSmokeTest.sh
index a9b66a53c..d8232a18a 100755
--- a/qa/scripts/testSmokeTest.sh
+++ b/qa/scripts/testSmokeTest.sh
@@ -4,24 +4,14 @@ SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
 source $SCRIPT_DIR/utilCluster.sh
 
 # get the first ingest host
-INGESTNODE0=$(cat ./qa/tf/ci/smoketest/outputs.json | jq -r '[.ingest_ips][0]["value"][0]')
+must_get_value ./qa/tf/ci/smoketest/outputs.json INGESTNODE0 .ingest_ips 0 '"value"' 0
 echo "using INGESTNODE0 ${INGESTNODE0}"
 
-# get the first data host
-DATANODE0=$(cat ./qa/tf/ci/smoketest/outputs.json | jq -r '[.data_node_ips][0]["value"][0]')
-echo "using DATANODE0 ${DATANODE0}"
-
-echo "Writing config.py file..."
-cat << EOT > config.py
-datanode0="${DATANODE0}"
-EOT
-mv config.py ./qa/testcases/smoketest/config.py
+HOSTS=($( cat ./qa/tf/ci/smoketest/outputs.json | jq -r '.data_node_ips.value' | tr -d '[],"'))
 
 echo "Copying tests to remote"
-scp -r -i ~/.ssh/gitlab-featurebase-ci.pem ./qa/testcases/smoketest/*.py ec2-user@${INGESTNODE0}:/data
-if (( $? != 0 ))
-then
-    echo "Copy failed"
+if ! scp -r -i ~/.ssh/gitlab-featurebase-ci.pem ./qa/testcases/smoketest/* ./datagen_linux_arm64 ec2-user@${INGESTNODE0}:/data; then
+    echo >&2 "Copy failed"
     exit 1
 fi
 
@@ -29,6 +19,10 @@ fi
 echo "Running smoke test..."
 ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o "StrictHostKeyChecking no" ec2-user@${INGESTNODE0} "cd /data; ~/.local/bin/pytest --junitxml=report.xml"
 SMOKETESTRESULT=$?
+if [ $SMOKETESTRESULT -ne 0 ]; then
+    echo >&2 "initial smoke tests failed"
+    exit 1
+fi
 
 echo "Copying test report to local"
 scp -r -i ~/.ssh/gitlab-featurebase-ci.pem ec2-user@${INGESTNODE0}:/data/report.xml report.xml
diff --git a/qa/scripts/utilCluster.sh b/qa/scripts/utilCluster.sh
index 4acd08c2b..d7f51a787 100755
--- a/qa/scripts/utilCluster.sh
+++ b/qa/scripts/utilCluster.sh
@@ -639,3 +639,51 @@ setupConsumerNode(){
     ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${IP} "sudo yum install jq htop tmux nc -y"
     setupDatadog $IP "ingest" $BRANCH_NAME
 }
+
+# get_value [file] [var] [nodes] checks for a value stored in [nodes];
+# for instance, `get_value file foo a b` checks for [a][b] in the JSON
+# in file, and stores the result in $foo. it exits with a non-zero status
+# if this fails.
+# e.g., `get_value output.json INGESTNODE0 .ingest_ips 0 '"value"' 0` sets 
+# $INGESTNODE0 to the value of [.ingest_ips][0]["value"][0], or fails to
+# do so and returns a non-zero status. Note the quoting to get the double
+# quotes around "value".
+get_value() {
+    file=$1
+    var=$2
+    shift 2
+    # assemble [a][b][c] from "a" "b" "c"
+    expr="["
+    infix=""
+    for word; do
+        expr="$expr$infix$word"
+        infix="]["
+    done
+    expr="$expr]"
+    tmp=$(jq -r "$expr" < $file)
+    if [ $? -ne 0 ]; then
+        echo >&2 "jq failed parsing input file"
+        return 1
+    fi
+    case $tmp in
+    null)
+      echo >&2 "reading IPs for ${node}: got null"
+      return 1
+      ;;
+    *)
+      eval $var=\$tmp
+      return 0
+      ;;
+    esac
+}
+
+# must_get_value [file] [var] [nodes] checks for the contents of [nodes], and stores
+# the result in $var. if it fails, it causes the *calling script* to exit. do not
+# use this if you need to handle the error more gracefully.
+# e.g., `must_get_value output.json INGESTNODE0 ingest_ips 0 "value" 0` sets $INGESTNODE0
+# to the value of [.ingest_ips][0]["value"][0].
+must_get_value() {
+    if ! get_value "$@"; then
+        exit 1
+    fi
+}
\ No newline at end of file
diff --git a/qa/testcases/bug-repros/fb-1287-test.sh b/qa/testcases/bug-repros/fb-1287-test.sh
index 8d275d087..2428902e5 100755
--- a/qa/testcases/bug-repros/fb-1287-test.sh
+++ b/qa/testcases/bug-repros/fb-1287-test.sh
@@ -1,9 +1,9 @@
 #!/usr/bin/env bash
-
-/data/datagen_linux_arm64 -s custom --custom-config=./fb-1287-datagen.yaml --pilosa.index=fb1287 --pilosa.batch-size=100 --pilosa.hosts=$1
+. ./config.sh
+$DATAGEN -s custom --custom-config=./fb-1287-datagen.yaml --pilosa.index=fb1287 --pilosa.batch-size=100 --pilosa.hosts=$1
 
 # if we get an error, exit 1
-if [[ $( curl $1/index/fb1287/query -d 'Rows(segid,from="2022-01-02T15:04",to="2022-04-02T15:04")' | jq '.error' ) != "null" ]]; then
+if [[ $( authcurl $1/index/fb1287/query -d 'Rows(segid,from="2022-01-02T15:04",to="2022-04-02T15:04")' | jq '.error' ) != "null" ]]; then
     exit 1;
 else
     exit 0;
diff --git a/qa/testcases/bug-repros/fb-1332.sh b/qa/testcases/bug-repros/fb-1332.sh
index d98ba0513..a569a7986 100755
--- a/qa/testcases/bug-repros/fb-1332.sh
+++ b/qa/testcases/bug-repros/fb-1332.sh
@@ -1,7 +1,9 @@
 #!/bin/bash
 
+. ./config.sh
+
 # datagen some timestamps onto DATANODE0 in index fb_1332_test
-/data/datagen_linux_arm64 -s custom --custom-config=./fb-1332-datagen.yaml --pilosa.index=fb1332 --pilosa.batch-size=100 --pilosa.hosts=$1
+$DATAGEN -s custom --custom-config=./fb-1332-datagen.yaml --pilosa.index=fb1332 --pilosa.batch-size=100 --pilosa.hosts=$1
 if (( $? != 0 )); then
     echo "couldn't datagen"
     exit 1
@@ -15,7 +17,7 @@ if (( $? != 0 )); then
 fi
 
 # delete that index
-if [[ $( curl -XDELETE $1/index/fb1332| jq '.error' ) != "null" ]]; then
+if [[ $( authcurl -XDELETE $1/index/fb1332| jq '.error' ) != "null" ]]; then
     echo "couldn't delete index"
     exit 1
 fi
@@ -29,7 +31,7 @@ fi
 
 
 # datagen some more
-/data/datagen_linux_arm64 -s custom --custom-config=./fb-1332-datagen.yaml --pilosa.index=fb1332 --pilosa.batch-size=100 --pilosa.hosts=$1
+$DATAGEN -s custom --custom-config=./fb-1332-datagen.yaml --pilosa.index=fb1332 --pilosa.batch-size=100 --pilosa.hosts=$1
 if (( $? != 0 )); then
     echo "couldn't ingest a second time"
     exit 1
diff --git a/qa/testcases/bug-repros/fb-1371-test.sh b/qa/testcases/bug-repros/fb-1371-test.sh
index 2c4fb4cef..320cfc633 100755
--- a/qa/testcases/bug-repros/fb-1371-test.sh
+++ b/qa/testcases/bug-repros/fb-1371-test.sh
@@ -1,5 +1,7 @@
 #!/bin/bash
 
+. ./config.sh
+
 ifErr() {
     RESCODE=$?
     if [[ $RESCODE != 0 ]]; then
@@ -8,13 +10,13 @@ ifErr() {
     fi
 }
 
-/data/datagen_linux_arm64 -s custom --custom-config=./fb-1371-datagen.yaml --pilosa.index=fb1371 --pilosa.batch-size=100 --pilosa.hosts=$1
+$DATAGEN -s custom --custom-config=./fb-1371-datagen.yaml --pilosa.index=fb1371 --pilosa.batch-size=100 --pilosa.hosts=$1
 ifErr "datagenning fb1371"
 
-RES1=$( curl $1/index/fb1371/query -d 'Max(ts)' | jq '.results[0].timestampValue' )
+RES1=$( authcurl $1/index/fb1371/query -d 'Max(ts)' | jq '.results[0].timestampValue' )
 ifErr "getting Max 1"
 
-RES2=$( curl $1/index/fb1371/query -d 'Max(ts)' | jq '.results[0].timestampValue' )
+RES2=$( authcurl $1/index/fb1371/query -d 'Max(ts)' | jq '.results[0].timestampValue' )
 ifErr "getting Max 2"
 
 if [[ $RES1 != $RES2 ]]; then
diff --git a/qa/testcases/bug-repros/fb1270-test.sh b/qa/testcases/bug-repros/fb1270-test.sh
index 36e778fd5..b4c175aba 100755
--- a/qa/testcases/bug-repros/fb1270-test.sh
+++ b/qa/testcases/bug-repros/fb1270-test.sh
@@ -1,12 +1,7 @@
 #!/usr/bin/env bash
 
 set -e
-
-if [ -f /data/datagen_linux_arm64 ]; then
-    datagen_loc=/data/datagen_linux_arm64
-else
-    datagen_loc=`which datagen`
-fi
+. ./config.sh
 
 declare -i end=${2:-44} # end at 44 or whatever the second argument is
 
@@ -14,5 +9,5 @@ for (( c=41; c<=$end; c++ )); do
     echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
     echo "ROUND $c"
     echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
-    $datagen_loc -s texas_health --pilosa.index thr --end-at 1048575 --pilosa.batch-size 1048576 --concurrency 1 --seed=$c --pilosa.hosts=$1
+    $DATAGEN -s texas_health --pilosa.index thr --end-at 1048575 --pilosa.batch-size 1048576 --concurrency 1 --seed=$c --pilosa.hosts=$1
 done
diff --git a/qa/testcases/bug-repros/run-all.sh b/qa/testcases/bug-repros/run-all.sh
index ac27d85ce..d8983d088 100755
--- a/qa/testcases/bug-repros/run-all.sh
+++ b/qa/testcases/bug-repros/run-all.sh
@@ -2,7 +2,9 @@
 
 set -eou pipefail
 
+. ./config.sh
+export DATAGEN="/data/datagen_linux_arm64 --auth-token ${ADMIN_TOKEN}"
 for file in `ls *-test.sh`; do
     echo "running $file";
-    ./$file "$@"
+    sh -x ./$file "$@"
 done
diff --git a/qa/testcases/smoketest/test_smoke.py b/qa/testcases/smoketest/test_smoke.py
index 2d38b686c..acea92c5c 100644
--- a/qa/testcases/smoketest/test_smoke.py
+++ b/qa/testcases/smoketest/test_smoke.py
@@ -5,17 +5,18 @@ import requests
 
 createIndex = { 'options': { 'keys': False } }
 createField = { 'options': { 'type': 'int', 'min': 0, 'max': 100000 } }
+headers = {"Authorization": "Bearer " + config.admin_token}
 
 def setup_module(module):
     data_to_send = json.dumps(createIndex).encode("utf-8")
-    response = requests.post("http://" + config.datanode0 + ":10101/index/user", data = data_to_send)
+    response = requests.post("https://" + config.datanode0 + ":10101/index/user", data = data_to_send, verify = False, headers = headers)
     assert response.status_code == 200
     assert response.headers["Content-Type"] == "application/json"
     resp_body = response.json()
     assert resp_body['success'] == True
 
     data_to_send = json.dumps(createField).encode("utf-8")
-    response = requests.post("http://" + config.datanode0 + ":10101/index/user/field/stats", data = data_to_send)
+    response = requests.post("https://" + config.datanode0 + ":10101/index/user/field/stats", data = data_to_send, verify = False, headers = headers)
     assert response.status_code == 200
     assert response.headers["Content-Type"] == "application/json"
     resp_body = response.json()
@@ -23,7 +24,7 @@ def setup_module(module):
 
 
 def teardown_module(module):
-    response = requests.delete("http://" + config.datanode0 + ":10101/index/user")
+    response = requests.delete("https://" + config.datanode0 + ":10101/index/user", verify = False, headers = headers)
     assert response.status_code == 200
     assert response.headers["Content-Type"] == "application/json"
     resp_body = response.json()
@@ -31,7 +32,7 @@ def teardown_module(module):
 
 
 def test_api_is_responding():
-    response = requests.get("http://" + config.datanode0 + ":10101/status")
+    response = requests.get("https://" + config.datanode0 + ":10101/status", verify = False, headers = headers)
     assert response.status_code == 200
     assert response.headers["Content-Type"] == "application/json"
     resp_body = response.json()
@@ -39,7 +40,7 @@ def test_api_is_responding():
     
 
 def test_sql3_is_responding():
-    response = requests.post("http://" + config.datanode0 + ":10101/sql", data = "select 1")
+    response = requests.post("https://" + config.datanode0 + ":10101/sql", data = "select 1", verify = False, headers = headers)
     assert response.status_code == 200
     assert response.headers["Content-Type"] == "application/json"
     resp_body = response.json()
@@ -47,7 +48,7 @@ def test_sql3_is_responding():
     assert resp_body['data'][0][0] == 1
 
 def test_get_index_api():
-    response = requests.get("http://" + config.datanode0 + ":10101/index/user")
+    response = requests.get("https://" + config.datanode0 + ":10101/index/user", verify = False, headers = headers)
     assert response.status_code == 200
     assert response.headers["Content-Type"] == "application/json"
     resp_body = response.json()
@@ -55,13 +56,13 @@ def test_get_index_api():
     
     
 def test_set_and_read_query_api():
-    response = requests.post("http://" + config.datanode0 + ":10101/index/user/query", data = "Set(10, stats=1)")
+    response = requests.post("https://" + config.datanode0 + ":10101/index/user/query", data = "Set(10, stats=1)", verify = False, headers = headers)
     assert response.status_code == 200
     assert response.headers["Content-Type"] == "application/json"
     resp_body = response.json()
     assert resp_body['results'][0] == True
 
-    response = requests.post("http://" + config.datanode0 + ":10101/index/user/query", data = "Row(stats=1)")
+    response = requests.post("https://" + config.datanode0 + ":10101/index/user/query", data = "Row(stats=1)", verify = False, headers = headers)
     assert response.status_code == 200
     assert response.headers["Content-Type"] == "application/json"
     resp_body = response.json()