diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index d5ecc1817..9cde8e07a 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -292,7 +292,6 @@ build arm container fb: # 3. make sure docker/docker-compose is installed # 4. make sure the git config is done `git config --global --add url."ssh://git@github.com/".insteadOf "https://github.com/"` # 5. Add deploy key github.com/molecula/featurebase/settings/keys and add public key in .ssh folder of gitlab-runner user -# TODO: (I think) get clustertests coverage added to coverage report clustertests: variables: PROJECT: clustertests_${CI_CONCURRENT_ID} @@ -500,6 +499,67 @@ perf_able: variables: PARENT_PIPELINE_ID: $CI_PIPELINE_ID +# only run on merge to master, the whole process takes about 10 minutes in total +# (which is a long time) +perf_delete: + stage: performance + rules: + - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $CI_PIPELINE_SOURCE == "push"' + timeout: 2h + image: registry.gitlab.com/gitlab-org/cloud-deploy/aws-base:latest + variables: + PROFILE: "service-terraform" + INFRA_PROFILE: "service-gitlab" + AWS_SSH_PRIVATE_KEY: $AWS_FBCI_SSH_KEY + AWS_ACCESS_KEY_ID: $AWS_FBCI_ACCESS_KEY_ID + AWS_SECRET_ACCESS_KEY: $AWS_FBCI_SECRET_ACCESS_KEY + ASG_NAME: "gitlab-runners" + TF_VAR_cluster_prefix: "" + tags: + - aws + - docker + - fbsmoke + before_script: + - apt-get update && apt-get install -y gnupg software-properties-common curl git + - curl -fsSL https://apt.releases.hashicorp.com/gpg | apt-key add - + - apt-add-repository "deb [arch=amd64] https://apt.releases.hashicorp.com $(lsb_release -cs) main" + - apt-get update && apt-get install terraform + - apt-get update && apt-get install python3.7 + - aws configure set aws_access_key_id $AWS_FBCI_ACCESS_KEY_ID --profile $PROFILE + - aws configure set aws_secret_access_key $AWS_FBCI_SECRET_ACCESS_KEY --profile $PROFILE + - aws configure set region "us-east-2" --profile $PROFILE + - aws configure set aws_profile $PROFILE + - aws configure set aws_access_key_id $AWS_INFRA_ACCESS_KEY_ID --profile $INFRA_PROFILE + - aws configure set aws_secret_access_key $AWS_INFRA_SECRET_ACCESS_KEY --profile $INFRA_PROFILE + - aws configure set region "us-east-2" --profile $INFRA_PROFILE + - echo $AWS_FBCI_SSH_KEY > gitlab-featurebase-ci.pem + - chmod 400 gitlab-featurebase-ci.pem + - 'which ssh-agent || ( apt-get update -y && apt-get install openssh-client -y )' + - eval $(ssh-agent -s) + - mkdir -p ~/.ssh + - echo $AWS_FBCI_SSH_KEY > /root/.ssh/gitlab-featurebase-ci.pem + - chmod 400 /root/.ssh/gitlab-featurebase-ci.pem + - echo "$AWS_FBCI_SSH_KEY" | ssh-add - + - chmod 700 /root/.ssh + - '[[ -f /.dockerenv ]] && echo -e "Host *\n\tStrictHostKeyChecking no\n\n" > ~/.ssh/config' + - apt update && apt -y install jq wget + - wget -q https://go.dev/dl/go1.17.5.linux-amd64.tar.gz + - tar -C /usr/local -xzf go1.17.5.linux-amd64.tar.gz + - export PATH=$PATH:/usr/local/go/bin + - TF_VAR_cluster_prefix="delete-$(openssl rand -base64 12 | tr -d /=+ | cut -c -16)" + - echo "Cluster Prefix --> $TF_VAR_cluster_prefix" + - export INSTANCE_ID=$(curl --silent --fail "http://169.254.169.254/latest/meta-data/instance-id" | tee instance_id) + - aws autoscaling set-instance-protection --instance-ids "$INSTANCE_ID" --auto-scaling-group-name $ASG_NAME --protected-from-scale-in --profile $INFRA_PROFILE + script: + - ./qa/scripts/perf/delete/deleteSetup.sh + - ./qa/scripts/perf/delete/deleteTest.sh + after_script: + - ./qa/scripts/perf/delete/deleteTeardown.sh || true + - export INSTANCE_ID=$(cat instance_id) + - aws autoscaling set-instance-protection --instance-ids "$INSTANCE_ID" --auto-scaling-group-name $ASG_NAME --no-protected-from-scale-in --profile $INFRA_PROFILE + needs: + - job: build for linux arm64 + s3 dump tag: stage: post build variables: diff --git a/qa/scripts/perf/delete/confluent b/qa/scripts/perf/delete/confluent new file mode 100644 index 000000000..8e00040b8 --- /dev/null +++ b/qa/scripts/perf/delete/confluent @@ -0,0 +1,13 @@ +[Confluent.dist] +name=Confluent repository (dist) +baseurl=http://packages.confluent.io/rpm/3.1/6 +gpgcheck=1 +gpgkey=http://packages.confluent.io/rpm/3.1/archive.key +enabled=1 + +[Confluent] +name=Confluent repository +baseurl=http://packages.confluent.io/rpm/3.1 +gpgcheck=1 +gpgkey=http://packages.confluent.io/rpm/3.1/archive.key +enabled=1 diff --git a/qa/scripts/perf/delete/deleteSetup.sh b/qa/scripts/perf/delete/deleteSetup.sh new file mode 100755 index 000000000..6cadbcd6f --- /dev/null +++ b/qa/scripts/perf/delete/deleteSetup.sh @@ -0,0 +1,86 @@ +#!/bin/bash + +# To run script: ./deleteSetup.sh +export TF_IN_AUTOMATION=1 + +if [ -z ${TF_VAR_cluster_prefix+x} ]; then + echo "TF_VAR_cluster_prefix is unset"; + exit 1 +else + echo "TF_VAR_cluster_prefix is set to '$TF_VAR_cluster_prefix'"; +fi + +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +source $SCRIPT_DIR/../../utilCluster.sh + +pushd ./qa/tf/perf/delete +echo "Running terraform init..." +terraform init -input=false +echo "Running terraform apply..." +terraform apply -input=false -auto-approve +terraform output -json > outputs.json +popd + +# get the first ingest host +INGESTNODE0=$(cat ./qa/tf/perf/delete/outputs.json | jq -r '[.ingest_ips][0]["value"][0]') +echo "using INGESTNODE0 ${INGESTNODE0}" + +# get the first data host +DATANODE0=$(cat ./qa/tf/perf/delete/outputs.json | jq -r '[.data_node_ips][0]["value"][0]') +echo "using DATANODE0 ${DATANODE0}" + + +DEPLOYED_CLUSTER_PREFIX=$(cat ./qa/tf/perf/delete/outputs.json | jq -r '[.cluster_prefix][0]["value"]') +echo "Using DEPLOYED_CLUSTER_PREFIX: ${DEPLOYED_CLUSTER_PREFIX}" + +DEPLOYED_CLUSTER_REPLICA_COUNT=$(cat ./qa/tf/perf/delete/outputs.json | jq -r '[.fb_cluster_replica_count][0]["value"]') +echo "Using DEPLOYED_CLUSTER_REPLICA_COUNT: ${DEPLOYED_CLUSTDEPLOYED_CLUSTER_REPLICA_COUNTER_PREFIX}" + +DEPLOYED_DATA_IPS=$(cat ./qa/tf/perf/delete/outputs.json | jq -r '[.data_node_ips][0]["value"][]') +echo "DEPLOYED_DATA_IPS: {" +echo "${DEPLOYED_DATA_IPS}" +echo "}" + +DEPLOYED_DATA_IPS_LEN=`echo "$DEPLOYED_DATA_IPS" | wc -l` + +DEPLOYED_INGEST_IPS=$(cat ./qa/tf/perf/delete/outputs.json | jq -r '[.ingest_ips][0]["value"][]') +echo "DEPLOYED_INGEST_IPS: {" +echo "${DEPLOYED_INGEST_IPS}" +echo "}" + +DEPLOYED_INGEST_IPS_LEN=`echo "$DEPLOYED_INGEST_IPS" | wc -l` + +#wait until we can connect to one of the hosts +for i in {0..24} +do + ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no -o ConnectTimeout=10 ec2-user@${DATANODE0} "pwd" + if [ $? -eq 0 ] + then + echo "Cluster is up after ${i} tries." + break + fi + sleep 10 +done + +ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no -o ConnectTimeout=10 ec2-user@${DATANODE0} "pwd" +if [ $? -ne 0 ] +then + echo "Unable to connect to cluster - giving up" + exit 1 +fi + +setupClusterNodes + +# verify featurebase running +echo "Verifying featurebase cluster running..." +curl -s http://${DATANODE0}:10101/status +if (( $? != 0 )) +then + echo "Featurebase cluster not running" + exit 1 +fi + +echo "Cluster running." + + + diff --git a/qa/scripts/perf/delete/deleteTeardown.sh b/qa/scripts/perf/delete/deleteTeardown.sh new file mode 100755 index 000000000..fc1c4faa0 --- /dev/null +++ b/qa/scripts/perf/delete/deleteTeardown.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +# To run script: ./deleteTeardown.sh + +cd qa/tf/perf/delete +export TF_IN_AUTOMATION=1 +terraform destroy -auto-approve diff --git a/qa/scripts/perf/delete/deleteTest.sh b/qa/scripts/perf/delete/deleteTest.sh new file mode 100755 index 000000000..9046a2095 --- /dev/null +++ b/qa/scripts/perf/delete/deleteTest.sh @@ -0,0 +1,88 @@ +#!/bin/bash + +# get the first ingest host +INGESTNODE0=$(cat ./qa/tf/perf/delete/outputs.json | jq -r '[.ingest_ips][0]["value"][0]') +echo "using INGESTNODE0 ${INGESTNODE0}" + +# get the first data host +DATANODE0=$(cat ./qa/tf/perf/delete/outputs.json | jq -r '[.data_node_ips][0]["value"][0]') +echo "using DATANODE0 ${DATANODE0}" + +# download datagen +aws s3 cp s3://molecula-artifact-storage/idk/master/_latest/idk-linux-arm64/datagen datagen +if (( $? != 0 )) +then + echo "datagen binary copy failed" + exit 1 +fi + +# make it executable +chmod +x datagen +if (( $? != 0 )) +then + echo "couldn't make datagen executable" + exit 1 +fi + +# copy it over to the ingest node +scp -r -i ~/.ssh/gitlab-featurebase-ci.pem ./datagen ec2-user@${INGESTNODE0}:/data +if (( $? != 0 )) +then + echo "datagen copy failed" + exit 1 +fi + +# setup the yum repo needed for librdkafka onto the ingest node +scp -r -i ~/.ssh/gitlab-featurebase-ci.pem ./qa/scripts/perf/delete/confluent ec2-user@${INGESTNODE0}:/data +if (( $? != 0 )) +then + echo "confluent repo setup copy failed" + exit 1 +fi + +echo "setting up confluent repo" +ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o "StrictHostKeyChecking no" ec2-user@${INGESTNODE0} "sudo mv /data/confluent /etc/yum.repos.d" +if (( $? != 0 )) +then + echo "setting up confluent repo failed" + exit 1 +fi + + +echo "installing librdkafka on ingest node" +ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o "StrictHostKeyChecking no" ec2-user@${INGESTNODE0} "sudo rpm --import http://packages.confluent.io/rpm/3.1/archive.key && sudo yum clean all && sudo yum install librdkafka-devel -y" +if (( $? != 0 )) +then + echo "librdkafka install failed" + exit 1 +fi + +# copy tremor.yaml over to the ingest node +scp -r -i ~/.ssh/gitlab-featurebase-ci.pem ./qa/scripts/perf/delete/tremor.yaml ec2-user@${INGESTNODE0}:/data +if (( $? != 0 )) +then + echo "tremor.yaml copy failed" + exit 1 +fi + +# copy the tests over to ingest node +scp -r -i ~/.ssh/gitlab-featurebase-ci.pem ./qa/scripts/perf/delete/test.py ec2-user@${INGESTNODE0}:/data +if (( $? != 0 )) +then + echo "test copy failed" + exit 1 +fi + +# run test on ingest node +echo "running delete test" +ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o "StrictHostKeyChecking no" ec2-user@${INGESTNODE0} "cd /data; python3 test.py ${DATANODE0}" +TESTRESULT=$? + +if (( $TESTRESULT != 0 )) +then + echo "delete test failed" +else + echo "delete test complete" +fi + +exit $TESTRESULT diff --git a/qa/scripts/perf/delete/test.py b/qa/scripts/perf/delete/test.py new file mode 100644 index 000000000..57bdebd19 --- /dev/null +++ b/qa/scripts/perf/delete/test.py @@ -0,0 +1,131 @@ +"""this script checks that memory usage does not increase when data is deleted and ingested again""" +import subprocess as sp +from time import sleep +import requests +import sys +import os + +if len(sys.argv) != 2: + print("Usage:") + print("python {} ".format(sys.argv[0])) + exit(1) + +ip = sys.argv[1] + + +def datagen(): + """run_datagen runs datagen using the configured tremor.yaml file and blocks until it has completed and returns whether or not the command was successful""" + try: + sp.check_call( + [ + "./datagen", + "-s=custom", + "--custom-config=./tremor.yaml", + "--pilosa.index=tremor", + "--pilosa.batch-size=300000", + "--pilosa.hosts={}:10101".format(ip), + ], + stdout=sp.DEVNULL, + stderr=sp.STDOUT, + ) + except Exception as e: + pass + + +def mem_use(): + """mem_use returns the memory usage of the data node and whether the request was successful""" + response = requests.get("http://" + ip + ":10101/internal/mem-usage") + if response.status_code != 200: + return 0 + body = response.json() + return body["totalUsed"] + + +def delete_all(): + """delete_all runs a Delete(All()) query on the data node and returns whether the delete was successful""" + response = requests.post( + "http://" + ip + ":10101/index/tremor/query", data="Delete(All())" + ) + count = 0 + retry_period = 0.5 + while try_again(response) and (count < 9): + print(response.text.strip()) + print( + "There was a problem executing the Delete(All()) query. Trying again in {} seconds (attempt {}/9)".format( + retry_period, + count, + ) + ) + count += 1 + sleep(retry_period) + retry_period *= 2 + response = requests.post( + "http://" + ip + ":10101/index/tremor/query", data="Delete(All())" + ) + return response.status_code == 200 + + +def try_again(response): + if response.ok: + # if we got a response but it was unsuccessful, try again + return not response.json()["results"][0] + # if we didn't get a good response, try again + return True + + +def disk_use(): + response = requests.get("http://" + ip + ":10101/internal/disk-usage") + if response.status_code != 200: + return 0 + body = response.json() + return body["usage"] + + +print("# memory usage tests") +usage = new_usage = 0 +du = new_du = 0 +datagen() +usage = mem_use() +du = disk_use() +print( + "{},{},{},{}".format( + "mem_usage", "disk_usage", "new_mem_greater", "new_disk_greater" + ) +) +print("{},{},{},{}".format(usage, du, None, None)) +for i in range(5): + variance = 0.25 + if delete_all(): + datagen() + new_usage = mem_use() + new_du = disk_use() + print( + "{},{},{},{}".format( + usage, + du, + (new_usage / usage), + (new_du / du), + ) + ) + if (new_usage / usage) > (1 + variance): + print( + new_usage, + "is greater than", + usage, + "by at least {}%.".format(variance * 100), + ) + print(new_usage - usage, "is a lot of bytes.") + print("Failing...") + exit(1) + if (new_du / du) > (1 + variance): + print( + new_du, "is greater than", du, "by at least {}%.".format(variance * 100) + ) + print(new_du - du, "is a lot of bytes.") + print("Failing...") + exit(1) + else: + print("Couldn't execute Delete(All()) query in 9 tries. Failing...") + exit(1) +print("Success") +exit(0) diff --git a/qa/scripts/perf/delete/tremor.yaml b/qa/scripts/perf/delete/tremor.yaml new file mode 100644 index 000000000..45000a4ee --- /dev/null +++ b/qa/scripts/perf/delete/tremor.yaml @@ -0,0 +1,61 @@ +fields: + - name: "a_random_string" + type: "string" # (default StringField (non-mutex)) + generator_type: "random-string" # used to generate random strings rather than pulling from known set + min_len: 8 + max_len: 12 + charset: "AB" # set of possible characters to pull from when generating random string + - name: "id" + type: "uint" + distribution: "sequential" + min: 0 + max: 1000000 # 2%24 + step: 1 + repeat: false + - name: "type" + type: "int" # (default IntField) + min: 0 + max: 3 + distribution: "zipfian" + s: 1.1 + v: 5.1 + - name: "ts" + type: "timestamp" + min_date: 2006-01-02T15:04:05.001Z # RFC3339Nano + max_date: 2007-01-02T15:04:05.001Z # RFC3339Nano + distribution: "increasing" # only "increasing" is supported right now + min_step_duration: "10ns" + max_step_duration: "200ms" + - name: "slice" + type: "uint-set" # (default IDArrayField) + min: 0 + max: 35000 + distribution: "zipfian" + s: 1.1 + v: 5.1 + min_num: 1 + max_num: 50 + +# idk_params describe how data from "fields" should be ingested by IDK +idk_params: + primary_key_config: + field: "a_random_string" # if this is a single field named "id" then we'll use uint IDs, if it's empty we'll autogen ids, and if it's anything else we'll do string keys... yes this is a bit hacky, needs to be cleaned up. + # fields is keyed by names of fields from top level "fields". It is + # not required that all fields appear here, those that don't will + # use the default ingestion. + fields: + id: + - type: "ID" + type: + - type: "ID" + a_decimal_field: + - type: "Decimal" + scale: 4 + ts: + - type: "RecordTime" + layout: "2006-01-02T15:04:05Z" + epoch: 1970-01-01T00:00:00.0Z + name: "na" + slice: + - type: "IDArray" + time_quantum: "YMD" diff --git a/qa/scripts/utilCluster.sh b/qa/scripts/utilCluster.sh index 629a63aeb..95215c93c 100644 --- a/qa/scripts/utilCluster.sh +++ b/qa/scripts/utilCluster.sh @@ -47,14 +47,13 @@ EOT #echo "featurebase.service <<" scp -i ~/.ssh/gitlab-featurebase-ci.pem -o "StrictHostKeyChecking no" featurebase.service ec2-user@${NODEIP}: - if (( $? != 0 )) - then + if (( $? != 0 )) + then echo "featurebase.service copy failed" exit 1 fi rm -f featurebase.service - ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "sudo mv featurebase.service ${SERVICE_FILE_PATH}" } @@ -127,8 +126,8 @@ executeGeneralNodeConfigCommands() { # TODO handle different archs scp -i ~/.ssh/gitlab-featurebase-ci.pem -o "StrictHostKeyChecking no" featurebase_linux_arm64 ec2-user@${NODEIP}: - if (( $? != 0 )) - then + if (( $? != 0 )) + then echo "featurebase binary copy failed" exit 1 fi @@ -136,7 +135,7 @@ executeGeneralNodeConfigCommands() { ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "chown ec2-user:ec2-user /home/ec2-user/featurebase_linux_arm64" ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "chmod ugo+x /home/ec2-user/featurebase_linux_arm64" ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "sudo mv /home/ec2-user/featurebase_linux_arm64 /usr/local/bin/featurebase" - + echo "featurebase binary copied." } @@ -229,4 +228,4 @@ setupClusterNodes() { #ingest nodes setupIngestNodes -} \ No newline at end of file +} diff --git a/qa/tf/perf/delete/main.tf b/qa/tf/perf/delete/main.tf new file mode 100644 index 000000000..4033ad1d8 --- /dev/null +++ b/qa/tf/perf/delete/main.tf @@ -0,0 +1,17 @@ +module "delete-cluster" { + source = "../../.modules/featurebase-cluster" + cluster_prefix = var.cluster_prefix + region = var.region + profile = var.profile + fb_data_node_type = "m6g.12xlarge" + fb_data_disk_iops = 10000 + fb_data_node_count = 3 + fb_ingest_type = "m6g.2xlarge" + fb_ingest_disk_iops = 10000 + fb_ingest_disk_size_gb = 500 + fb_ingest_node_count = 1 + vpc_id = "vpc-05a26a122f961dc2b" + vpc_cidr_block = "10.0.0.0/16" + vpc_public_subnets = ["subnet-066b4b922b54e51a2", "subnet-037b8884269a69025", "subnet-08482631514426210", ] + vpc_private_subnets = ["subnet-0319dde319380326f", "subnet-0517ca9a646d80f88", "subnet-05a7b685ed27eb1cf", ] +} diff --git a/qa/tf/perf/delete/outputs.tf b/qa/tf/perf/delete/outputs.tf new file mode 100644 index 000000000..77fb2e9fc --- /dev/null +++ b/qa/tf/perf/delete/outputs.tf @@ -0,0 +1,19 @@ +output "ingest_ips" { + description = "List of ingest IPs" + value = module.delete-cluster.ingest_ips +} + +output "data_node_ips" { + description = "List of data node IPs" + value = module.delete-cluster.data_node_ips +} + +output "cluster_prefix" { + description = "The cluster prefix used" + value = module.delete-cluster.cluster_prefix +} + +output "fb_cluster_replica_count" { + description = "The cluster replica count used" + value = module.delete-cluster.fb_cluster_replica_count +} diff --git a/qa/tf/perf/delete/provider.tf b/qa/tf/perf/delete/provider.tf new file mode 100644 index 000000000..f6f28cca8 --- /dev/null +++ b/qa/tf/perf/delete/provider.tf @@ -0,0 +1,4 @@ +provider "aws" { + region = var.region + profile = var.profile +} diff --git a/qa/tf/perf/delete/tf.auto.tfvars b/qa/tf/perf/delete/tf.auto.tfvars new file mode 100644 index 000000000..d2b94a97a --- /dev/null +++ b/qa/tf/perf/delete/tf.auto.tfvars @@ -0,0 +1,2 @@ +region = "us-east-2" +profile = "service-terraform" diff --git a/qa/tf/perf/delete/variables.tf b/qa/tf/perf/delete/variables.tf new file mode 100644 index 000000000..e55c7936d --- /dev/null +++ b/qa/tf/perf/delete/variables.tf @@ -0,0 +1,14 @@ +variable "region" { + description = "The AWS region in which the VPC should be built" + type = string +} + +variable "profile" { + description = "The name of the AWS profile Terraform should use for auth." + type = string +} + +variable "cluster_prefix" { + type = string + description = "This is a identifier that will be prefixed to created resources" +}