integration test for backup restore sev 2!

Co-authored-by: tgruben <tgruben@gmail.com>

qa/scripts/backupRestoreTest.sh

@@ -0,0 +1,112 @@
+#!/bin/bash
+
+HOSTS=($@)
+/data/datagen --source custom --custom-config /data/backup_test_datagen.yaml --pilosa.index=backup_test --pilosa.hosts=$HOSTS --pilosa.batch-size=1000
+# make sure backup doesn't fail 
+if (( $? != 0 )); then
+    echo "datagen failed"
+    exit 1
+fi
+
+# kill a node somehow 
+echo "hosts we've got"
+for host in ${HOSTS[@]}; do
+    echo $host;
+done
+
+KILLNODE=${HOSTS[1]}
+echo "getting checksum from $KILLNODE"
+firstCheckSum=$(featurebase chksum --host $KILLNODE)
+echo "first check sum: $firstCheckSum"
+if (( $? != 0 ))
+then 
+    echo "getting checksum from $KILLNODE failed"
+    exit 1
+fi
+
+echo "pausing node $KILLNODE"
+ssh -A -o "StrictHostKeyChecking no" ec2-user@${KILLNODE} "sudo systemctl stop featurebase" 
+if (( $? != 0 ))
+then 
+    echo "pausing node $KILLNODE failed"
+    exit 1
+fi
+
+# backup from ingest node
+featurebase backup --host=${HOSTS[2]} --retry-period=0s --output=backupWOOO
+# make sure backup doesn't fail 
+if (( $? != 0 )); then
+    echo "backup failed!!!!!!!!!!"
+    exit 1
+fi
+
+echo "stopping all featurebase nodes"
+# kill all featurebase nodes and wipe the data directory
+for host in ${HOSTS[@]}; do
+    echo "stopping featurebase on ${host}"
+    ssh -A -o "StrictHostKeyChecking no" ec2-user@${host} "sudo systemctl stop featurebase && sudo find /data/featurebase/ -mindepth 1 -delete" 
+    if (( $? != 0 )); then 
+        echo "emptying and stopping node failed"
+        exit 1
+    fi
+done
+
+echo "starting all featurebase nodes"
+# start all featurebase nodes - systemd
+for host in ${HOSTS[@]}; do
+    echo "starting featurebase on ${host}"
+    ssh -A -o "StrictHostKeyChecking no" ec2-user@${host} "sudo systemctl restart featurebase" 
+    if (( $? != 0 )); then 
+        echo "restarting featurebase failed"
+        exit 1
+    fi
+done
+
+echo "waiting for featurebase to start"
+# wait until we can connect to one of the hosts
+for i in {0..24}; do 
+    echo "checking ${HOSTS[1]}:10101/status"
+    curl -v ${HOSTS[1]}:10101/status
+    S=$(curl -s ${HOSTS[1]}:10101/status | jq -r ".state"); 
+    if [[ $S == "NORMAL" ]]; then 
+        echo "hosts up after $i tries"; 
+        break;
+    fi
+    echo "attempt $i resulted in $S"
+    sleep 5
+done
+
+
+if [[ $S != "NORMAL" ]]; then
+    echo "couldn't connect, featurebase never stable"
+    exit 1
+fi
+
+echo "restoring featurebase"
+# featurebase restore 
+featurebase restore -s=backupWOOO --host=${HOSTS[1]}
+
+# make sure it doesn't fail
+if (( $? != 0 )); then
+    echo "restore failed!!!!"
+    exit 1
+fi
+
+# make sure it's the same data
+echo "getting checksum from $KILLNODE"
+secondChkSum=$(featurebase chksum --host $KILLNODE)
+echo "second checkSum $secondChkSum"
+if (( $? != 0 )); then 
+    echo "getting checksum from $KILLNODE failed"
+    exit 1
+fi
+
+if [[ $firstCheckSum == $secondChkSum ]]; then 
+    echo "checksums match"
+    exit 0
+else
+    echo "first check sum $firstCheckSum"
+    echo "second check sum $secondCheckSum"
+    echo "check sums don't match"
+    exit 1
+fi 

qa/scripts/backup_test_datagen.yaml

@@ -0,0 +1,61 @@
+fields:
+  - name: "a_random_string"
+    type: "string" # (default StringField (non-mutex))
+    generator_type: "random-string" # used to generate random strings rather than pulling from known set
+    min_len: 8
+    max_len: 12
+    charset: "AB" # set of possible characters to pull from when generating random string
+  - name: "id"
+    type: "uint"
+    distribution: "sequential"
+    min: 0
+    max: 1000 # 2%24
+    step: 1
+    repeat: false
+  - name: "type"
+    type: "int" # (default IntField)
+    min: 0
+    max: 3
+    distribution: "zipfian"
+    s: 1.1
+    v: 5.1
+  - name: "ts"
+    type: "timestamp"
+    min_date: 2006-01-02T15:04:05.001Z # RFC3339Nano
+    max_date: 2007-01-02T15:04:05.001Z # RFC3339Nano
+    distribution: "increasing" # only "increasing" is supported right now
+    min_step_duration: "10ns"
+    max_step_duration: "200ms" 
+  - name: "slice"
+    type: "uint-set" # (default IDArrayField)
+    min: 0
+    max: 35000
+    distribution: "zipfian"
+    s: 1.1
+    v: 5.1
+    min_num: 1
+    max_num: 50
+
+# idk_params describe how data from "fields" should be ingested by IDK
+idk_params:
+  primary_key_config:
+    field: "a_random_string" # if this is a single field named "id" then we'll use uint IDs, if it's empty we'll autogen ids, and if it's anything else we'll do string keys... yes this is a bit hacky, needs to be cleaned up.
+  # fields is keyed by names of fields from top level "fields". It is
+  # not required that all fields appear here, those that don't will
+  # use the default ingestion.
+  fields:
+    id:
+      - type: "ID"
+    type:
+      - type: "ID"
+    a_decimal_field:
+      - type: "Decimal"
+        scale: 4
+    ts:
+      - type: "RecordTime"
+        layout: "2006-01-02T15:04:05Z"
+        epoch: 1970-01-01T00:00:00.0Z
+        name: "na"
+    slice:
+      - type: "IDArray"
+        time_quantum: "YMD"

qa/scripts/perf/delete/deleteSetup.sh

@@ -50,7 +50,7 @@ echo "}"
 
 DEPLOYED_INGEST_IPS_LEN=`echo "$DEPLOYED_INGEST_IPS" | wc -l`
 
-#wait until we can connect to one of the hosts
+# wait until we can connect to one of the hosts
 for i in {0..24}
 do 
     ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no -o ConnectTimeout=10 ec2-user@${DATANODE0} "pwd"

qa/scripts/setupBackupRestoreGauntlet.sh

@@ -0,0 +1,79 @@
+#!/bin/bash
+
+# To run script: ./setupBackupRestoreGauntlet.sh
+export TF_IN_AUTOMATION=1
+
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+source $SCRIPT_DIR/utilCluster.sh
+
+pushd ./qa/tf/gauntlet/backuprestore
+echo "Running terraform init..."
+terraform init -input=false
+echo "Running terraform apply..."
+terraform apply -input=false -auto-approve
+terraform output -json > outputs.json
+popd
+
+# get the first ingest host
+INGESTNODE0=$(cat ./qa/tf/gauntlet/backuprestore/outputs.json | jq -r '[.ingest_ips][0]["value"][0]')
+echo "using INGESTNODE0 ${INGESTNODE0}"
+
+# get the first data host
+DATANODE0=$(cat ./qa/tf/gauntlet/backuprestore/outputs.json | jq -r '[.data_node_ips][0]["value"][0]')
+echo "using DATANODE0 ${DATANODE0}"
+
+
+DEPLOYED_CLUSTER_PREFIX=$(cat ./qa/tf/gauntlet/backuprestore/outputs.json | jq -r '[.cluster_prefix][0]["value"]')
+echo "Using DEPLOYED_CLUSTER_PREFIX: ${DEPLOYED_CLUSTER_PREFIX}"
+
+DEPLOYED_CLUSTER_REPLICA_COUNT=$(cat ./qa/tf/gauntlet/backuprestore/outputs.json | jq -r '[.fb_cluster_replica_count][0]["value"]')
+echo "Using DEPLOYED_CLUSTER_REPLICA_COUNT: ${DEPLOYED_CLUSTER_REPLICA_COUNT}"
+
+DEPLOYED_DATA_IPS=$(cat ./qa/tf/gauntlet/backuprestore/outputs.json | jq -r '[.data_node_ips][0]["value"][]')
+echo "DEPLOYED_DATA_IPS: {"
+echo "${DEPLOYED_DATA_IPS}"
+echo "}"
+
+DEPLOYED_DATA_IPS_LEN=`echo "$DEPLOYED_DATA_IPS" | wc -l`
+
+DEPLOYED_INGEST_IPS=$(cat ./qa/tf/gauntlet/backuprestore/outputs.json | jq -r '[.ingest_ips][0]["value"][]')
+echo "DEPLOYED_INGEST_IPS: {"
+echo "${DEPLOYED_INGEST_IPS}"
+echo "}"
+
+DEPLOYED_INGEST_IPS_LEN=`echo "$DEPLOYED_INGEST_IPS" | wc -l`
+
+#wait until we can connect to one of the hosts
+for i in {0..24}
+do 
+    ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no -o ConnectTimeout=10 ec2-user@${DATANODE0} "pwd"
+    if [ $? -eq 0 ]
+    then
+        echo "Cluster is up after ${i} tries." 
+        break
+    fi
+    sleep 10
+done
+
+ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no -o ConnectTimeout=10 ec2-user@${DATANODE0} "pwd"
+if [ $? -ne 0 ]
+then
+    echo "Unable to connect to cluster - giving up" 
+    exit 1
+fi
+
+setupClusterNodes
+
+# verify featurebase running
+echo "Verifying featurebase cluster running..."
+curl -s http://${DATANODE0}:10101/status
+if (( $? != 0 )) 
+then 
+    echo "Featurebase cluster not running"
+    exit 1
+fi
+
+echo "Cluster running."
+
+
+

qa/scripts/teardownBackupRestoreGauntlet.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+# To run script: ./teardownBackupRestoreGauntlet.sh
+
+cd qa/tf/gauntlet/backuprestore
+export TF_IN_AUTOMATION=1
+terraform destroy -auto-approve

qa/scripts/testBackupRestoreGauntlet.sh

@@ -0,0 +1,51 @@
+#!/bin/bash
+
+source ./qa/scripts/utilCluster.sh
+
+# get the first ingest host
+INGESTNODE0=$(cat ./qa/tf/gauntlet/backuprestore/outputs.json | jq -r '[.ingest_ips][0]["value"][0]')
+echo "using INGESTNODE0 ${INGESTNODE0}"
+
+# get the first data host
+DATANODE0=$(cat ./qa/tf/gauntlet/backuprestore/outputs.json | jq -r '[.data_node_ips][0]["value"][0]')
+echo "using DATANODE0 ${DATANODE0}"
+
+# get the data node host ips
+HOSTS=($( cat ./qa/tf/gauntlet/backuprestore/outputs.json | jq -r '.data_node_ips.value' | tr -d '[],"'))
+
+echo "using hosts:"
+for host in ${HOSTS[@]}; do
+    echo $host;
+done
+
+installDatagen $INGESTNODE0
+
+# copy the tests over to ingest node
+scp -r -i ~/.ssh/gitlab-featurebase-ci.pem ./qa/scripts/backupRestoreTest.sh ec2-user@${INGESTNODE0}:/data
+if (( $? != 0 )) 
+then 
+    echo "test copy failed"
+    exit 1
+fi
+
+# copy the datagen over to ingest node
+scp -r -i ~/.ssh/gitlab-featurebase-ci.pem ./qa/scripts/backup_test_datagen.yaml ec2-user@${INGESTNODE0}:/data
+if (( $? != 0 )) 
+then 
+    echo "test copy failed"
+    exit 1
+fi
+
+
+echo "running backup test with hosts: ${HOSTS[@]}"
+ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o "StrictHostKeyChecking no" ec2-user@${INGESTNODE0} "cd /data; ./backupRestoreTest.sh ${HOSTS[@]}"
+TESTRESULT=$?
+
+if (( $TESTRESULT != 0 )) 
+then 
+    echo "backup test failed"
+else
+    echo "backup test complete"
+fi
+
+exit $TESTRESULT 

qa/scripts/utilCluster.sh

@@ -230,3 +230,55 @@ setupClusterNodes() {
     setupIngestNodes
 
 }
+
+installDatagen() {
+    INGESTNODE0=$1
+    # download datagen
+    aws s3 cp s3://molecula-artifact-storage/idk/master/_latest/idk-linux-arm64/datagen datagen
+    if (( $? != 0 ))
+    then
+        echo "datagen binary copy failed"
+        exit 1
+    fi
+
+    # make it executable
+    chmod +x datagen 
+    if (( $? != 0 ))
+    then
+        echo "couldn't make datagen executable"
+        exit 1
+    fi
+
+    # copy it over to the ingest node
+    scp -r -i ~/.ssh/gitlab-featurebase-ci.pem ./datagen ec2-user@${INGESTNODE0}:/data
+    if (( $? != 0 )) 
+    then 
+        echo "datagen copy failed"
+        exit 1
+    fi
+
+    # setup the yum repo needed for librdkafka onto the ingest node
+    scp -r -i ~/.ssh/gitlab-featurebase-ci.pem ./qa/scripts/perf/delete/confluent ec2-user@${INGESTNODE0}:/data
+    if (( $? != 0 )) 
+    then 
+        echo "confluent repo setup copy failed"
+        exit 1
+    fi
+
+    echo "setting up confluent repo"
+    ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o "StrictHostKeyChecking no" ec2-user@${INGESTNODE0} "sudo mv /data/confluent /etc/yum.repos.d" 
+    if (( $? != 0 ))
+    then 
+        echo "setting up confluent repo failed"
+        exit 1
+    fi
+
+
+    echo "installing librdkafka on ingest node"
+    ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o "StrictHostKeyChecking no" ec2-user@${INGESTNODE0} "sudo rpm --import http://packages.confluent.io/rpm/3.1/archive.key && sudo yum clean all && sudo yum install librdkafka-devel -y"
+    if (( $? != 0 ))
+    then 
+        echo "librdkafka install failed"
+        exit 1
+    fi
+}

qa/tf/gauntlet/backuprestore/README.md

@@ -0,0 +1,35 @@
+With terraform installed (`brew install terraform` if not)...
+
+You can do `terraform plan` -> `terraform apply` to spin up a cluster, `terraform destroy` to tear one down.
+
+## Other prerequisites:
+Please read these carefully.
+
+Be in the `tf` directory (e.g., when you try to run a `terraform` command, the output of `pwd` should be `.../featurebase/qa/tf`)
+
+Currently, the path to the terraform module is using a local reference, i.e., in `main.tf`, the source line is assuming that you have `molecula-terraform` project installed locally, such that the `molecular-terraform` project and `featurebase` have the same parent directory (e.g., `...A/featurebase/qa/tf` and `...A/molecular-terraform/aws/.modules/featurebase-cluster` should both be valid paths).
+
+In addition, you must currently have a local copy of the `fb901` branch for the `molecular-terraform` project (located in the previously specified directory).
+
+Last thing, there is a key that is currently in 1Password (in the `Shared` vault, called `gitlab-featurebase-ci AWS key`) that must be in `~/.ssh/`, `chmod 400`, named `gitlab-featurebase-ci.pem`. You need this key to SSH to these instances. Assuming an `~/.ssh/config` like the following (append to the top of yours)
+```
+Host test_*
+  User ec2-user
+  IdentityFile ~/.ssh/gitlab-featurebase-ci.pem
+Host test_ingest
+  HostName 3.143.237.165
+Host test_node
+  HostName 10.0.1.142
+  ProxyJump test_ingest
+```
+except with the `test_ingest`'s `HostName` being the public, `ingest_ips` output from `terraform output` and `test_node`'s `HostName` being one of the private, `data_node_ips` output from `terraform output`. (Hopefully the rationale to use the ssh config to do the jumping like this makes sense; you can do `ssh test_ingest` or `ssh test_node` with minimal further fiddling.)
+
+OR specify cert to us directly thus:
+
+`ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem ec2-user@ip_address`
+
+-A is used to ensure key forwarding.
+
+### TODOs
+* We need a `user-data.sh` script which sets up/installs featurebase (possibly installs go, most likely pulls the artifacts from GitLab; sets up featurebase on both the node and data workers).
+* Logs get sent to DataDog?

qa/tf/gauntlet/backuprestore/main.tf

@@ -0,0 +1,17 @@
+module "samsung-cluster" {
+  source                   = "../../.modules/featurebase-cluster"
+  cluster_prefix           = var.cluster_prefix
+  region                   = var.region
+  profile                  = var.profile
+  fb_data_node_type        = "m6g.xlarge"
+  fb_data_disk_iops        = 10000
+  fb_data_node_count       = 3
+  fb_ingest_type           = "m6g.large"
+  fb_cluster_replica_count = 3
+  fb_ingest_disk_iops      = 10000
+  fb_ingest_node_count     = 1
+  vpc_id                   = "vpc-05a26a122f961dc2b"
+  vpc_cidr_block           = "10.0.0.0/16"
+  vpc_public_subnets       = ["subnet-066b4b922b54e51a2", "subnet-037b8884269a69025", "subnet-08482631514426210", ]
+  vpc_private_subnets      = ["subnet-0319dde319380326f", "subnet-0517ca9a646d80f88", "subnet-05a7b685ed27eb1cf", ]
+}

qa/tf/gauntlet/backuprestore/outputs.tf

@@ -0,0 +1,19 @@
+output "ingest_ips" {
+    description = "List of ingest IPs"
+    value = module.samsung-cluster.ingest_ips
+}
+
+output "data_node_ips" {
+  description = "List of data node IPs"
+  value = module.samsung-cluster.data_node_ips
+}
+
+output "cluster_prefix" {
+  description = "The cluster prefix used"
+  value = module.samsung-cluster.cluster_prefix
+}
+
+output "fb_cluster_replica_count" {
+  description = "The cluster replica count used"
+  value = module.samsung-cluster.fb_cluster_replica_count
+}

qa/tf/gauntlet/backuprestore/provider.tf

@@ -0,0 +1,4 @@
+provider "aws" {
+  region  = var.region
+  profile = var.profile
+}

qa/tf/gauntlet/backuprestore/tf.auto.tfvars

@@ -0,0 +1,2 @@
+region  = "us-east-2"
+profile = "service-terraform"

qa/tf/gauntlet/backuprestore/variables.tf

@@ -0,0 +1,14 @@
+variable "region" {
+  description = "The AWS region in which the VPC should be built"
+  type        = string
+}
+
+variable "profile" {
+  description = "The name of the AWS profile Terraform should use for auth."
+  type        = string
+}
+
+variable "cluster_prefix" {
+  type        = string
+  description = "This is a identifier that will be prefixed to created resources"
+}