diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 729ff7f20..48e92cf3a 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -12,6 +12,7 @@ stages: - build - integration - gauntlet + - performance - post build smoke build: @@ -337,7 +338,6 @@ smoke test: AWS_ACCESS_KEY_ID: $AWS_FBCI_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY: $AWS_FBCI_SECRET_ACCESS_KEY TF_VAR_cluster_prefix: "" - TF_VAR_branch: "" tags: - aws - docker @@ -369,8 +369,6 @@ smoke test: - export PATH=$PATH:/usr/local/go/bin - TF_VAR_cluster_prefix="smoke-$(openssl rand -base64 12 | tr -d /=+ | cut -c -16)" - echo "Cluster Prefix --> $TF_VAR_cluster_prefix" - - TF_VAR_branch=$CI_COMMIT_BRANCH - - echo "Branch --> $TF_VAR_branch" script: - ./qa/scripts/setupSmokeTest.sh - ./qa/scripts/testSmokeTest.sh @@ -398,7 +396,6 @@ gauntlet: AWS_SECRET_ACCESS_KEY: $AWS_FBCI_SECRET_ACCESS_KEY ASG_NAME: "gitlab-runners" TF_VAR_cluster_prefix: "" - TF_VAR_branch: "" tags: - aws - docker @@ -433,8 +430,6 @@ gauntlet: - export PATH=$PATH:/usr/local/go/bin - TF_VAR_cluster_prefix="gauntlet-$(openssl rand -base64 12 | tr -d /=+ | cut -c -16)" - echo "Cluster Prefix --> $TF_VAR_cluster_prefix" - - TF_VAR_branch=$CI_COMMIT_BRANCH - - echo "Branch --> $TF_VAR_branch" - export INSTANCE_ID=$(curl --silent --fail "http://169.254.169.254/latest/meta-data/instance-id" | tee instance_id) - aws autoscaling set-instance-protection --instance-ids "$INSTANCE_ID" --auto-scaling-group-name $ASG_NAME --protected-from-scale-in --profile $INFRA_PROFILE script: @@ -482,3 +477,11 @@ s3 dump: - job: build for darwin arm64 - job: build for linux amd64 - job: build for linux arm64 + +perf_able: + stage: performance + trigger: + include: .gitlab/.perf-able-gitlab-ci.yml + rules: + - changes: + - featurebase/* \ No newline at end of file diff --git a/.gitlab/.perf-able-gitlab-ci.yml b/.gitlab/.perf-able-gitlab-ci.yml new file mode 100644 index 000000000..25b590f3c --- /dev/null +++ b/.gitlab/.perf-able-gitlab-ci.yml @@ -0,0 +1,11 @@ +stages: + - loadtest + +loadtest: + image: + name: loadimpact/k6:latest + entrypoint: [''] + stage: loadtest + script: + - echo "executing local k6 in k6 container..." + - k6 run ./qa/scripts/perf/able/script.js diff --git a/qa/scripts/perf/able/ableRun.sh b/qa/scripts/perf/able/ableRun.sh new file mode 100644 index 000000000..92ca26987 --- /dev/null +++ b/qa/scripts/perf/able/ableRun.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) + +# requires TF_VAR_cluster_prefix env var to be set +if [ -z ${TF_VAR_cluster_prefix+x} ]; then + echo "setting TF_VAR_cluster_prefix"; + export TF_VAR_cluster_prefix="able-$(openssl rand -base64 12 | tr -d /=+ | cut -c -16)" + echo "TF_VAR_cluster_prefix is set to '$TF_VAR_cluster_prefix'"; +else + echo "TF_VAR_cluster_prefix is set to '$TF_VAR_cluster_prefix'"; +fi + +$SCRIPT_DIR/ableSetup.sh +$SCRIPT_DIR/ableTest.sh +$SCRIPT_DIR/ableTeardown.sh diff --git a/qa/scripts/perf/able/ableSetup.sh b/qa/scripts/perf/able/ableSetup.sh new file mode 100755 index 000000000..30a868b69 --- /dev/null +++ b/qa/scripts/perf/able/ableSetup.sh @@ -0,0 +1,86 @@ +#!/bin/bash + +# To run script: ./ableSetup.sh +export TF_IN_AUTOMATION=1 + +if [ -z ${TF_VAR_cluster_prefix+x} ]; then + echo "TF_VAR_cluster_prefix is unset"; + exit 1 +else + echo "TF_VAR_cluster_prefix is set to '$TF_VAR_cluster_prefix'"; +fi + +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +source $SCRIPT_DIR/../../utilCluster.sh + +pushd ./qa/tf/gauntlet/able +echo "Running terraform init..." +terraform init -input=false +echo "Running terraform apply..." +terraform apply -input=false -auto-approve +terraform output -json > outputs.json +popd + +# get the first ingest host +INGESTNODE0=$(cat ./qa/tf/gauntlet/able/outputs.json | jq -r '[.ingest_ips][0]["value"][0]') +echo "using INGESTNODE0 ${INGESTNODE0}" + +# get the first data host +DATANODE0=$(cat ./qa/tf/gauntlet/able/outputs.json | jq -r '[.data_node_ips][0]["value"][0]') +echo "using DATANODE0 ${DATANODE0}" + + +DEPLOYED_CLUSTER_PREFIX=$(cat ./qa/tf/gauntlet/able/outputs.json | jq -r '[.cluster_prefix][0]["value"]') +echo "Using DEPLOYED_CLUSTER_PREFIX: ${DEPLOYED_CLUSTER_PREFIX}" + +DEPLOYED_CLUSTER_REPLICA_COUNT=$(cat ./qa/tf/gauntlet/able/outputs.json | jq -r '[.fb_cluster_replica_count][0]["value"]') +echo "Using DEPLOYED_CLUSTER_REPLICA_COUNT: ${DEPLOYED_CLUSTDEPLOYED_CLUSTER_REPLICA_COUNTER_PREFIX}" + +DEPLOYED_DATA_IPS=$(cat ./qa/tf/gauntlet/able/outputs.json | jq -r '[.data_node_ips][0]["value"][]') +echo "DEPLOYED_DATA_IPS: {" +echo "${DEPLOYED_DATA_IPS}" +echo "}" + +DEPLOYED_DATA_IPS_LEN=`echo "$DEPLOYED_DATA_IPS" | wc -l` + +DEPLOYED_INGEST_IPS=$(cat ./qa/tf/gauntlet/able/outputs.json | jq -r '[.ingest_ips][0]["value"][]') +echo "DEPLOYED_INGEST_IPS: {" +echo "${DEPLOYED_INGEST_IPS}" +echo "}" + +DEPLOYED_INGEST_IPS_LEN=`echo "$DEPLOYED_INGEST_IPS" | wc -l` + +#wait until we can connect to one of the hosts +for i in {0..24} +do + ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no -o ConnectTimeout=10 ec2-user@${DATANODE0} "pwd" + if [ $? -eq 0 ] + then + echo "Cluster is up after ${i} tries." + break + fi + sleep 10 +done + +ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no -o ConnectTimeout=10 ec2-user@${DATANODE0} "pwd" +if [ $? -ne 0 ] +then + echo "Unable to connect to cluster - giving up" + exit 1 +fi + +setupClusterNodes + +# verify featurebase running +echo "Verifying featurebase cluster running..." +curl -s http://${DATANODE0}:10101/status +if (( $? != 0 )) +then + echo "Featurebase cluster not running" + exit 1 +fi + +echo "Cluster running." + + + diff --git a/qa/scripts/perf/able/ableTeardown.sh b/qa/scripts/perf/able/ableTeardown.sh new file mode 100755 index 000000000..5ca1a2dde --- /dev/null +++ b/qa/scripts/perf/able/ableTeardown.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +# To run script: ./ableTeardown.sh + +cd qa/tf/gauntlet/able +export TF_IN_AUTOMATION=1 +terraform destroy -auto-approve diff --git a/qa/scripts/perf/able/ableTest.sh b/qa/scripts/perf/able/ableTest.sh new file mode 100755 index 000000000..d414545e4 --- /dev/null +++ b/qa/scripts/perf/able/ableTest.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +# get the first ingest host +INGESTNODE0=$(cat ./qa/tf/gauntlet/able/outputs.json | jq -r '[.ingest_ips][0]["value"][0]') +echo "using INGESTNODE0 ${INGESTNODE0}" + +# get the first data host +DATANODE0=$(cat ./qa/tf/gauntlet/able/outputs.json | jq -r '[.data_node_ips][0]["value"][0]') +echo "using DATANODE0 ${DATANODE0}" + + +echo "Done." \ No newline at end of file diff --git a/qa/scripts/perf/able/script.js b/qa/scripts/perf/able/script.js new file mode 100644 index 000000000..77b293c66 --- /dev/null +++ b/qa/scripts/perf/able/script.js @@ -0,0 +1,7 @@ +import http from 'k6/http'; +import { sleep } from 'k6'; + +export default function () { + http.get('https://test.k6.io'); + sleep(1); +} \ No newline at end of file diff --git a/qa/scripts/runSamsungGauntlet.sh b/qa/scripts/runSamsungGauntlet.sh index 37cc0a930..6c1822c91 100644 --- a/qa/scripts/runSamsungGauntlet.sh +++ b/qa/scripts/runSamsungGauntlet.sh @@ -2,9 +2,6 @@ SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) -# requires TF_VAR_branch env var to be set -if [ -z ${TF_VAR_branch+x} ]; then echo "TF_VAR_branch is unset"; else echo "TF_VAR_branch is set to '$TF_VAR_branch'"; fi - # requires TF_VAR_cluster_prefix env var to be set if [ -z ${TF_VAR_cluster_prefix+x} ]; then echo "setting TF_VAR_cluster_prefix"; diff --git a/qa/scripts/runSmokeTest.sh b/qa/scripts/runSmokeTest.sh index 6b598995e..8d4ff1923 100755 --- a/qa/scripts/runSmokeTest.sh +++ b/qa/scripts/runSmokeTest.sh @@ -2,10 +2,6 @@ SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) - -# requires TF_VAR_branch env var to be set -if [ -z ${TF_VAR_branch+x} ]; then echo "TF_VAR_branch is unset"; else echo "TF_VAR_branch is set to '$TF_VAR_branch'"; fi - # requires TF_VAR_cluster_prefix env var to be set if [ -z ${TF_VAR_cluster_prefix+x} ]; then echo "setting TF_VAR_cluster_prefix"; diff --git a/qa/scripts/setupSamsungGauntlet.sh b/qa/scripts/setupSamsungGauntlet.sh index e7929ea9f..1e6060f2e 100755 --- a/qa/scripts/setupSamsungGauntlet.sh +++ b/qa/scripts/setupSamsungGauntlet.sh @@ -3,9 +3,6 @@ # To run script: ./setupSamsungGauntlet.sh export TF_IN_AUTOMATION=1 -# requires TF_VAR_branch env var to be set -if [ -z ${TF_VAR_branch+x} ]; then echo "TF_VAR_branch is unset"; else echo "TF_VAR_branch is set to '$TF_VAR_branch'"; fi - SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) source $SCRIPT_DIR/utilCluster.sh diff --git a/qa/scripts/setupSmokeTest.sh b/qa/scripts/setupSmokeTest.sh index 1059e5f29..997974494 100755 --- a/qa/scripts/setupSmokeTest.sh +++ b/qa/scripts/setupSmokeTest.sh @@ -3,9 +3,6 @@ # To run script: ./setupSmokeTest.sh export TF_IN_AUTOMATION=1 -# requires TF_VAR_branch env var to be set -if [ -z ${TF_VAR_branch+x} ]; then echo "TF_VAR_branch is unset"; else echo "TF_VAR_branch is set to '$TF_VAR_branch'"; fi - SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) source $SCRIPT_DIR/utilCluster.sh diff --git a/qa/scripts/teardownSmokeTest.sh b/qa/scripts/teardownSmokeTest.sh index 76eeb564b..21e9f390a 100755 --- a/qa/scripts/teardownSmokeTest.sh +++ b/qa/scripts/teardownSmokeTest.sh @@ -2,9 +2,6 @@ # To run script: ./teardownSmokeTest.sh -# requires TF_VAR_branch env var to be set -if [ -z ${TF_VAR_branch+x} ]; then echo "TF_VAR_branch is unset"; else echo "TF_VAR_branch is set to '$TF_VAR_branch'"; fi - cd qa/tf/ci/smoketest export TF_IN_AUTOMATION=1 terraform destroy -auto-approve diff --git a/qa/scripts/testSmokeTest.sh b/qa/scripts/testSmokeTest.sh index b6ba34e7f..b27b12931 100755 --- a/qa/scripts/testSmokeTest.sh +++ b/qa/scripts/testSmokeTest.sh @@ -1,8 +1,5 @@ #!/bin/bash -# requires TF_VAR_branch env var to be set -if [ -z ${TF_VAR_branch+x} ]; then echo "TF_VAR_branch is unset"; else echo "TF_VAR_branch is set to '$TF_VAR_branch'"; fi - SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) source $SCRIPT_DIR/utilCluster.sh diff --git a/qa/tf/.modules/featurebase-cluster/main.tf b/qa/tf/.modules/featurebase-cluster/main.tf index d21fbd4df..325516bfc 100644 --- a/qa/tf/.modules/featurebase-cluster/main.tf +++ b/qa/tf/.modules/featurebase-cluster/main.tf @@ -13,7 +13,7 @@ data "aws_ami" "amazon_linux_2" { filter { name = "architecture" - values = ["arm64"] + values = var.fb_cluster_arch } } @@ -38,6 +38,7 @@ resource "aws_instance" "fb_cluster_nodes" { volume_type = var.fb_data_disk_type volume_size = var.fb_data_disk_size_gb iops = var.fb_data_disk_iops + encrypted = true } tags = { @@ -70,6 +71,7 @@ resource "aws_instance" "fb_ingest" { volume_type = var.fb_ingest_disk_type volume_size = var.fb_ingest_disk_size_gb iops = var.fb_ingest_disk_iops + encrypted = true } tags = { diff --git a/qa/tf/.modules/featurebase-cluster/variables.tf b/qa/tf/.modules/featurebase-cluster/variables.tf index 4d7762d7e..7f7bf2e95 100644 --- a/qa/tf/.modules/featurebase-cluster/variables.tf +++ b/qa/tf/.modules/featurebase-cluster/variables.tf @@ -3,6 +3,11 @@ variable "cluster_prefix" { description = "This is a identifier that will be prefixed to created resources" } +variable "fb_cluster_arch" { + type = list(string) + default = ["arm64"] +} + variable "fb_ingest_type" { type = string default = "c6g.2xlarge" diff --git a/qa/tf/ci/smoketest/variables.tf b/qa/tf/ci/smoketest/variables.tf index 578305341..a327ea4ff 100644 --- a/qa/tf/ci/smoketest/variables.tf +++ b/qa/tf/ci/smoketest/variables.tf @@ -13,7 +13,3 @@ variable "cluster_prefix" { description = "This is a identifier that will be prefixed to created resources" } -variable "branch" { - type = string - description = "The branch we are on" -} \ No newline at end of file diff --git a/qa/tf/gauntlet/able/main.tf b/qa/tf/gauntlet/able/main.tf new file mode 100644 index 000000000..514f52173 --- /dev/null +++ b/qa/tf/gauntlet/able/main.tf @@ -0,0 +1,17 @@ +module "able-cluster" { + source = "../../.modules/featurebase-cluster" + cluster_prefix = var.cluster_prefix + region = var.region + profile = var.profile + fb_data_node_type = "m6g.12xlarge" + fb_data_disk_iops = 10000 + fb_data_node_count = 3 + fb_ingest_type = "m6g.2xlarge" + fb_ingest_disk_iops = 10000 + fb_ingest_disk_size_gb = 500 + fb_ingest_node_count = 1 + vpc_id = "vpc-05a26a122f961dc2b" + vpc_cidr_block = "10.0.0.0/16" + vpc_public_subnets = ["subnet-066b4b922b54e51a2","subnet-037b8884269a69025","subnet-08482631514426210",] + vpc_private_subnets = ["subnet-0319dde319380326f","subnet-0517ca9a646d80f88","subnet-05a7b685ed27eb1cf",] +} \ No newline at end of file diff --git a/qa/tf/gauntlet/able/outputs.tf b/qa/tf/gauntlet/able/outputs.tf new file mode 100644 index 000000000..43acb92ab --- /dev/null +++ b/qa/tf/gauntlet/able/outputs.tf @@ -0,0 +1,19 @@ +output "ingest_ips" { + description = "List of ingest IPs" + value = module.able-cluster.ingest_ips +} + +output "data_node_ips" { + description = "List of data node IPs" + value = module.able-cluster.data_node_ips +} + +output "cluster_prefix" { + description = "The cluster prefix used" + value = module.able-cluster.cluster_prefix +} + +output "fb_cluster_replica_count" { + description = "The cluster replica count used" + value = module.able-cluster.fb_cluster_replica_count +} diff --git a/qa/tf/gauntlet/able/provider.tf b/qa/tf/gauntlet/able/provider.tf new file mode 100644 index 000000000..c0fc95d9d --- /dev/null +++ b/qa/tf/gauntlet/able/provider.tf @@ -0,0 +1,4 @@ +provider "aws" { + region = var.region + profile = var.profile +} \ No newline at end of file diff --git a/qa/tf/gauntlet/able/tf.auto.tfvars b/qa/tf/gauntlet/able/tf.auto.tfvars new file mode 100644 index 000000000..ac6de62a6 --- /dev/null +++ b/qa/tf/gauntlet/able/tf.auto.tfvars @@ -0,0 +1,2 @@ +region = "us-east-2" +profile = "service-terraform" \ No newline at end of file diff --git a/qa/tf/gauntlet/able/variables.tf b/qa/tf/gauntlet/able/variables.tf new file mode 100644 index 000000000..e55c7936d --- /dev/null +++ b/qa/tf/gauntlet/able/variables.tf @@ -0,0 +1,14 @@ +variable "region" { + description = "The AWS region in which the VPC should be built" + type = string +} + +variable "profile" { + description = "The name of the AWS profile Terraform should use for auth." + type = string +} + +variable "cluster_prefix" { + type = string + description = "This is a identifier that will be prefixed to created resources" +} diff --git a/qa/tf/gauntlet/samsung/variables.tf b/qa/tf/gauntlet/samsung/variables.tf index 578305341..e55c7936d 100644 --- a/qa/tf/gauntlet/samsung/variables.tf +++ b/qa/tf/gauntlet/samsung/variables.tf @@ -12,8 +12,3 @@ variable "cluster_prefix" { type = string description = "This is a identifier that will be prefixed to created resources" } - -variable "branch" { - type = string - description = "The branch we are on" -} \ No newline at end of file