From c9c36f5fc6a818aa07fb653212780b79c71cdce4 Mon Sep 17 00:00:00 2001 From: reesporte Date: Wed, 2 Mar 2022 11:43:06 -0600 Subject: [PATCH 1/3] WIP: start integration test setup TODO: - remove commented out gitlab jobs - make it run only on merge to master --- .gitlab/.gitlab-ci.yml | 298 ++++++++++++++--------- qa/scripts/perf/delete/confluent | 13 + qa/scripts/perf/delete/deleteSetup.sh | 86 +++++++ qa/scripts/perf/delete/deleteTeardown.sh | 7 + qa/scripts/perf/delete/deleteTest.sh | 88 +++++++ qa/scripts/perf/delete/test.py | 131 ++++++++++ qa/scripts/perf/delete/tremor.yaml | 61 +++++ qa/scripts/utilCluster.sh | 13 +- qa/tf/perf/delete/main.tf | 17 ++ qa/tf/perf/delete/outputs.tf | 19 ++ qa/tf/perf/delete/provider.tf | 4 + qa/tf/perf/delete/tf.auto.tfvars | 2 + qa/tf/perf/delete/variables.tf | 14 ++ 13 files changed, 626 insertions(+), 127 deletions(-) create mode 100644 qa/scripts/perf/delete/confluent create mode 100755 qa/scripts/perf/delete/deleteSetup.sh create mode 100755 qa/scripts/perf/delete/deleteTeardown.sh create mode 100755 qa/scripts/perf/delete/deleteTest.sh create mode 100644 qa/scripts/perf/delete/test.py create mode 100644 qa/scripts/perf/delete/tremor.yaml create mode 100644 qa/tf/perf/delete/main.tf create mode 100644 qa/tf/perf/delete/outputs.tf create mode 100644 qa/tf/perf/delete/provider.tf create mode 100644 qa/tf/perf/delete/tf.auto.tfvars create mode 100644 qa/tf/perf/delete/variables.tf diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index d5ecc1817..4feb3ce5a 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -1,8 +1,8 @@ -include: - - template: Security/SAST.gitlab-ci.yml - - template: Security/License-Scanning.gitlab-ci.yml - - template: Security/Dependency-Scanning.gitlab-ci.yml - +#include: +# - template: Security/SAST.gitlab-ci.yml +# - template: Security/License-Scanning.gitlab-ci.yml +# - template: Security/Dependency-Scanning.gitlab-ci.yml +# variables: GOVERSION: "1.17.7" @@ -16,35 +16,35 @@ stages: - post build - nonblocking -smoke build: - image: golang:$GOVERSION - stage: lint - allow_failure: false - rules: - - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"' - script: - - echo "Let's just see if it compiles... (sometimes the linter gives unclear errors if it doesn't)" - - go build ./... - -golangci-lint: - image: golangci/golangci-lint:v1.39.0 - stage: lint - allow_failure: false - rules: - - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"' - script: - - echo "Checking for issues in new code" - - golangci-lint run - -go mod tidy: - stage: lint - image: golang:$GOVERSION - rules: - - if: '$CI_COMMIT_TAG == null && ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web")' - script: - - go mod tidy - - git diff --exit-code -- go.mod go.sum - +#smoke build: +# image: golang:$GOVERSION +# stage: lint +# allow_failure: false +# rules: +# - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"' +# script: +# - echo "Let's just see if it compiles... (sometimes the linter gives unclear errors if it doesn't)" +# - go build ./... +# +#golangci-lint: +# image: golangci/golangci-lint:v1.39.0 +# stage: lint +# allow_failure: false +# rules: +# - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"' +# script: +# - echo "Checking for issues in new code" +# - golangci-lint run +# +#go mod tidy: +# stage: lint +# image: golang:$GOVERSION +# rules: +# - if: '$CI_COMMIT_TAG == null && ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web")' +# script: +# - go mod tidy +# - git diff --exit-code -- go.mod go.sum +# build lattice: stage: test image: node:14 @@ -64,92 +64,92 @@ build lattice: artifacts: paths: - lattice.tar.gz - -run jest tests: - stage: test - image: node:14 - variables: - CI: "true" - rules: - - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"' - script: - - echo "Testing lattice..." - - cd lattice - - npm install --force - - npm test -- --coverage --testResultsProcessor=jest-sonar-reporter - artifacts: - paths: - - lattice/coverage/lcov.info - -run go tests: - stage: test - image: golang:$GOVERSION - rules: - - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"' - retry: 1 - script: - - echo "Running featurebase unit tests..." - - go test -timeout=30m ./... - tags: - - aws - -run go tests race: - stage: nonblocking # don't let this job block any other jobs because it takes much longer than the other tests. - image: golang:$GOVERSION - rules: - - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"' - retry: 1 - needs: [] # don't wait to start running this. - script: - - echo "Running featurebase race tests..." - - go test -race -v -timeout=90m ./... - tags: - - aws - -run go tests shardwidth22: - stage: test - image: golang:$GOVERSION - rules: - - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"' - script: - - echo "Running featurebase shardwidth22 tests..." - - go test -timeout=30m -tags=shardwidth22 ./... - tags: - - aws - -# we do coverage reporting from the future tests because the json -# output is very difficult to human-read. The alternative would be to -# run the regular tests twice and also run the future tests. -run go tests future: - stage: test - image: golang:1.17.6 - rules: - - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"' - retry: 1 - script: - - echo "Running featurebase unit tests..." - - PKG_LIST=$(go list ./... | grep -v internal/clustertests | paste -s -d, -) - - go test -timeout=30m -json -coverprofile=coverage.out -covermode=atomic -coverpkg=${PKG_LIST} ./... | tee test-report.out - artifacts: - paths: - - coverage.out - - test-report.out - tags: - - aws - -upload to sonarcloud: - stage: integration - image: sonarsource/sonar-scanner-cli:4.6 - variables: - SONAR_TOKEN: $SONAR_TOKEN - rules: - - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"' - script: - - sonar-scanner -Dsonar.projectKey=molecula_featurebase -Dsonar.organization=molecula -Dsonar.sources=. -Dsonar.host.url=https://sonarcloud.io -Dsonar.go.coverage.reportPaths=coverage.out,results/coverage*.out -Dsonar.go.tests.reportPaths=test-report.out,results/report* -Dsonar.javascript.lcov.reportPaths=lattice/coverage/lcov.info - needs: - - job: run go tests future - - job: run jest tests - - job: clustertests +# +#run jest tests: +# stage: test +# image: node:14 +# variables: +# CI: "true" +# rules: +# - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"' +# script: +# - echo "Testing lattice..." +# - cd lattice +# - npm install --force +# - npm test -- --coverage --testResultsProcessor=jest-sonar-reporter +# artifacts: +# paths: +# - lattice/coverage/lcov.info +# +#run go tests: +# stage: test +# image: golang:$GOVERSION +# rules: +# - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"' +# retry: 1 +# script: +# - echo "Running featurebase unit tests..." +# - go test -timeout=30m ./... +# tags: +# - aws +# +#run go tests race: +# stage: nonblocking # don't let this job block any other jobs because it takes much longer than the other tests. +# image: golang:$GOVERSION +# rules: +# - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"' +# retry: 1 +# needs: [] # don't wait to start running this. +# script: +# - echo "Running featurebase race tests..." +# - go test -race -v -timeout=90m ./... +# tags: +# - aws +# +#run go tests shardwidth22: +# stage: test +# image: golang:$GOVERSION +# rules: +# - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"' +# script: +# - echo "Running featurebase shardwidth22 tests..." +# - go test -timeout=30m -tags=shardwidth22 ./... +# tags: +# - aws +# +## we do coverage reporting from the future tests because the json +## output is very difficult to human-read. The alternative would be to +## run the regular tests twice and also run the future tests. +#run go tests future: +# stage: test +# image: golang:1.17.6 +# rules: +# - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"' +# retry: 1 +# script: +# - echo "Running featurebase unit tests..." +# - PKG_LIST=$(go list ./... | grep -v internal/clustertests | paste -s -d, -) +# - go test -timeout=30m -json -coverprofile=coverage.out -covermode=atomic -coverpkg=${PKG_LIST} ./... | tee test-report.out +# artifacts: +# paths: +# - coverage.out +# - test-report.out +# tags: +# - aws +# +#upload to sonarcloud: +# stage: integration +# image: sonarsource/sonar-scanner-cli:4.6 +# variables: +# SONAR_TOKEN: $SONAR_TOKEN +# rules: +# - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"' +# script: +# - sonar-scanner -Dsonar.projectKey=molecula_featurebase -Dsonar.organization=molecula -Dsonar.sources=. -Dsonar.host.url=https://sonarcloud.io -Dsonar.go.coverage.reportPaths=coverage.out,results/coverage*.out -Dsonar.go.tests.reportPaths=test-report.out,results/report* -Dsonar.javascript.lcov.reportPaths=lattice/coverage/lcov.info +# needs: +# - job: run go tests future +# - job: run jest tests +# - job: clustertests build for linux amd64: stage: build @@ -500,6 +500,64 @@ perf_able: variables: PARENT_PIPELINE_ID: $CI_PIPELINE_ID +perf_delete: + stage: integration + timeout: 2h + image: registry.gitlab.com/gitlab-org/cloud-deploy/aws-base:latest + variables: + PROFILE: "service-terraform" + INFRA_PROFILE: "service-gitlab" + AWS_SSH_PRIVATE_KEY: $AWS_FBCI_SSH_KEY + AWS_ACCESS_KEY_ID: $AWS_FBCI_ACCESS_KEY_ID + AWS_SECRET_ACCESS_KEY: $AWS_FBCI_SECRET_ACCESS_KEY + ASG_NAME: "gitlab-runners" + TF_VAR_cluster_prefix: "" + tags: + - aws + - docker + - fbsmoke + before_script: + - apt-get update && apt-get install -y gnupg software-properties-common curl git + - curl -fsSL https://apt.releases.hashicorp.com/gpg | apt-key add - + - apt-add-repository "deb [arch=amd64] https://apt.releases.hashicorp.com $(lsb_release -cs) main" + - apt-get update && apt-get install terraform + - apt-get update && apt-get install python3.7 + - aws configure set aws_access_key_id $AWS_FBCI_ACCESS_KEY_ID --profile $PROFILE + - aws configure set aws_secret_access_key $AWS_FBCI_SECRET_ACCESS_KEY --profile $PROFILE + - aws configure set region "us-east-2" --profile $PROFILE + - aws configure set aws_profile $PROFILE + - aws configure set aws_access_key_id $AWS_INFRA_ACCESS_KEY_ID --profile $INFRA_PROFILE + - aws configure set aws_secret_access_key $AWS_INFRA_SECRET_ACCESS_KEY --profile $INFRA_PROFILE + - aws configure set region "us-east-2" --profile $INFRA_PROFILE + - echo $AWS_FBCI_SSH_KEY > gitlab-featurebase-ci.pem + - chmod 400 gitlab-featurebase-ci.pem + - 'which ssh-agent || ( apt-get update -y && apt-get install openssh-client -y )' + - eval $(ssh-agent -s) + - mkdir -p ~/.ssh + - echo $AWS_FBCI_SSH_KEY > /root/.ssh/gitlab-featurebase-ci.pem + - chmod 400 /root/.ssh/gitlab-featurebase-ci.pem + - echo "$AWS_FBCI_SSH_KEY" | ssh-add - + - chmod 700 /root/.ssh + - '[[ -f /.dockerenv ]] && echo -e "Host *\n\tStrictHostKeyChecking no\n\n" > ~/.ssh/config' + - apt update && apt -y install jq wget + - wget -q https://go.dev/dl/go1.17.5.linux-amd64.tar.gz + - tar -C /usr/local -xzf go1.17.5.linux-amd64.tar.gz + - export PATH=$PATH:/usr/local/go/bin + - TF_VAR_cluster_prefix="delete-$(openssl rand -base64 12 | tr -d /=+ | cut -c -16)" + - echo "Cluster Prefix --> $TF_VAR_cluster_prefix" + - export INSTANCE_ID=$(curl --silent --fail "http://169.254.169.254/latest/meta-data/instance-id" | tee instance_id) + - aws autoscaling set-instance-protection --instance-ids "$INSTANCE_ID" --auto-scaling-group-name $ASG_NAME --protected-from-scale-in --profile $INFRA_PROFILE + script: + - ./qa/scripts/perf/delete/deleteSetup.sh + - ./qa/scripts/perf/delete/deleteTest.sh + after_script: + - ./qa/scripts/perf/delete/deleteTeardown.sh || true + - export INSTANCE_ID=$(cat instance_id) + - aws autoscaling set-instance-protection --instance-ids "$INSTANCE_ID" --auto-scaling-group-name $ASG_NAME --no-protected-from-scale-in --profile $INFRA_PROFILE + needs: + - job: build for linux arm64 + - job: build for linux amd64 + s3 dump tag: stage: post build variables: diff --git a/qa/scripts/perf/delete/confluent b/qa/scripts/perf/delete/confluent new file mode 100644 index 000000000..8e00040b8 --- /dev/null +++ b/qa/scripts/perf/delete/confluent @@ -0,0 +1,13 @@ +[Confluent.dist] +name=Confluent repository (dist) +baseurl=http://packages.confluent.io/rpm/3.1/6 +gpgcheck=1 +gpgkey=http://packages.confluent.io/rpm/3.1/archive.key +enabled=1 + +[Confluent] +name=Confluent repository +baseurl=http://packages.confluent.io/rpm/3.1 +gpgcheck=1 +gpgkey=http://packages.confluent.io/rpm/3.1/archive.key +enabled=1 diff --git a/qa/scripts/perf/delete/deleteSetup.sh b/qa/scripts/perf/delete/deleteSetup.sh new file mode 100755 index 000000000..6cadbcd6f --- /dev/null +++ b/qa/scripts/perf/delete/deleteSetup.sh @@ -0,0 +1,86 @@ +#!/bin/bash + +# To run script: ./deleteSetup.sh +export TF_IN_AUTOMATION=1 + +if [ -z ${TF_VAR_cluster_prefix+x} ]; then + echo "TF_VAR_cluster_prefix is unset"; + exit 1 +else + echo "TF_VAR_cluster_prefix is set to '$TF_VAR_cluster_prefix'"; +fi + +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +source $SCRIPT_DIR/../../utilCluster.sh + +pushd ./qa/tf/perf/delete +echo "Running terraform init..." +terraform init -input=false +echo "Running terraform apply..." +terraform apply -input=false -auto-approve +terraform output -json > outputs.json +popd + +# get the first ingest host +INGESTNODE0=$(cat ./qa/tf/perf/delete/outputs.json | jq -r '[.ingest_ips][0]["value"][0]') +echo "using INGESTNODE0 ${INGESTNODE0}" + +# get the first data host +DATANODE0=$(cat ./qa/tf/perf/delete/outputs.json | jq -r '[.data_node_ips][0]["value"][0]') +echo "using DATANODE0 ${DATANODE0}" + + +DEPLOYED_CLUSTER_PREFIX=$(cat ./qa/tf/perf/delete/outputs.json | jq -r '[.cluster_prefix][0]["value"]') +echo "Using DEPLOYED_CLUSTER_PREFIX: ${DEPLOYED_CLUSTER_PREFIX}" + +DEPLOYED_CLUSTER_REPLICA_COUNT=$(cat ./qa/tf/perf/delete/outputs.json | jq -r '[.fb_cluster_replica_count][0]["value"]') +echo "Using DEPLOYED_CLUSTER_REPLICA_COUNT: ${DEPLOYED_CLUSTDEPLOYED_CLUSTER_REPLICA_COUNTER_PREFIX}" + +DEPLOYED_DATA_IPS=$(cat ./qa/tf/perf/delete/outputs.json | jq -r '[.data_node_ips][0]["value"][]') +echo "DEPLOYED_DATA_IPS: {" +echo "${DEPLOYED_DATA_IPS}" +echo "}" + +DEPLOYED_DATA_IPS_LEN=`echo "$DEPLOYED_DATA_IPS" | wc -l` + +DEPLOYED_INGEST_IPS=$(cat ./qa/tf/perf/delete/outputs.json | jq -r '[.ingest_ips][0]["value"][]') +echo "DEPLOYED_INGEST_IPS: {" +echo "${DEPLOYED_INGEST_IPS}" +echo "}" + +DEPLOYED_INGEST_IPS_LEN=`echo "$DEPLOYED_INGEST_IPS" | wc -l` + +#wait until we can connect to one of the hosts +for i in {0..24} +do + ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no -o ConnectTimeout=10 ec2-user@${DATANODE0} "pwd" + if [ $? -eq 0 ] + then + echo "Cluster is up after ${i} tries." + break + fi + sleep 10 +done + +ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no -o ConnectTimeout=10 ec2-user@${DATANODE0} "pwd" +if [ $? -ne 0 ] +then + echo "Unable to connect to cluster - giving up" + exit 1 +fi + +setupClusterNodes + +# verify featurebase running +echo "Verifying featurebase cluster running..." +curl -s http://${DATANODE0}:10101/status +if (( $? != 0 )) +then + echo "Featurebase cluster not running" + exit 1 +fi + +echo "Cluster running." + + + diff --git a/qa/scripts/perf/delete/deleteTeardown.sh b/qa/scripts/perf/delete/deleteTeardown.sh new file mode 100755 index 000000000..fc1c4faa0 --- /dev/null +++ b/qa/scripts/perf/delete/deleteTeardown.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +# To run script: ./deleteTeardown.sh + +cd qa/tf/perf/delete +export TF_IN_AUTOMATION=1 +terraform destroy -auto-approve diff --git a/qa/scripts/perf/delete/deleteTest.sh b/qa/scripts/perf/delete/deleteTest.sh new file mode 100755 index 000000000..9046a2095 --- /dev/null +++ b/qa/scripts/perf/delete/deleteTest.sh @@ -0,0 +1,88 @@ +#!/bin/bash + +# get the first ingest host +INGESTNODE0=$(cat ./qa/tf/perf/delete/outputs.json | jq -r '[.ingest_ips][0]["value"][0]') +echo "using INGESTNODE0 ${INGESTNODE0}" + +# get the first data host +DATANODE0=$(cat ./qa/tf/perf/delete/outputs.json | jq -r '[.data_node_ips][0]["value"][0]') +echo "using DATANODE0 ${DATANODE0}" + +# download datagen +aws s3 cp s3://molecula-artifact-storage/idk/master/_latest/idk-linux-arm64/datagen datagen +if (( $? != 0 )) +then + echo "datagen binary copy failed" + exit 1 +fi + +# make it executable +chmod +x datagen +if (( $? != 0 )) +then + echo "couldn't make datagen executable" + exit 1 +fi + +# copy it over to the ingest node +scp -r -i ~/.ssh/gitlab-featurebase-ci.pem ./datagen ec2-user@${INGESTNODE0}:/data +if (( $? != 0 )) +then + echo "datagen copy failed" + exit 1 +fi + +# setup the yum repo needed for librdkafka onto the ingest node +scp -r -i ~/.ssh/gitlab-featurebase-ci.pem ./qa/scripts/perf/delete/confluent ec2-user@${INGESTNODE0}:/data +if (( $? != 0 )) +then + echo "confluent repo setup copy failed" + exit 1 +fi + +echo "setting up confluent repo" +ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o "StrictHostKeyChecking no" ec2-user@${INGESTNODE0} "sudo mv /data/confluent /etc/yum.repos.d" +if (( $? != 0 )) +then + echo "setting up confluent repo failed" + exit 1 +fi + + +echo "installing librdkafka on ingest node" +ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o "StrictHostKeyChecking no" ec2-user@${INGESTNODE0} "sudo rpm --import http://packages.confluent.io/rpm/3.1/archive.key && sudo yum clean all && sudo yum install librdkafka-devel -y" +if (( $? != 0 )) +then + echo "librdkafka install failed" + exit 1 +fi + +# copy tremor.yaml over to the ingest node +scp -r -i ~/.ssh/gitlab-featurebase-ci.pem ./qa/scripts/perf/delete/tremor.yaml ec2-user@${INGESTNODE0}:/data +if (( $? != 0 )) +then + echo "tremor.yaml copy failed" + exit 1 +fi + +# copy the tests over to ingest node +scp -r -i ~/.ssh/gitlab-featurebase-ci.pem ./qa/scripts/perf/delete/test.py ec2-user@${INGESTNODE0}:/data +if (( $? != 0 )) +then + echo "test copy failed" + exit 1 +fi + +# run test on ingest node +echo "running delete test" +ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o "StrictHostKeyChecking no" ec2-user@${INGESTNODE0} "cd /data; python3 test.py ${DATANODE0}" +TESTRESULT=$? + +if (( $TESTRESULT != 0 )) +then + echo "delete test failed" +else + echo "delete test complete" +fi + +exit $TESTRESULT diff --git a/qa/scripts/perf/delete/test.py b/qa/scripts/perf/delete/test.py new file mode 100644 index 000000000..57bdebd19 --- /dev/null +++ b/qa/scripts/perf/delete/test.py @@ -0,0 +1,131 @@ +"""this script checks that memory usage does not increase when data is deleted and ingested again""" +import subprocess as sp +from time import sleep +import requests +import sys +import os + +if len(sys.argv) != 2: + print("Usage:") + print("python {} ".format(sys.argv[0])) + exit(1) + +ip = sys.argv[1] + + +def datagen(): + """run_datagen runs datagen using the configured tremor.yaml file and blocks until it has completed and returns whether or not the command was successful""" + try: + sp.check_call( + [ + "./datagen", + "-s=custom", + "--custom-config=./tremor.yaml", + "--pilosa.index=tremor", + "--pilosa.batch-size=300000", + "--pilosa.hosts={}:10101".format(ip), + ], + stdout=sp.DEVNULL, + stderr=sp.STDOUT, + ) + except Exception as e: + pass + + +def mem_use(): + """mem_use returns the memory usage of the data node and whether the request was successful""" + response = requests.get("http://" + ip + ":10101/internal/mem-usage") + if response.status_code != 200: + return 0 + body = response.json() + return body["totalUsed"] + + +def delete_all(): + """delete_all runs a Delete(All()) query on the data node and returns whether the delete was successful""" + response = requests.post( + "http://" + ip + ":10101/index/tremor/query", data="Delete(All())" + ) + count = 0 + retry_period = 0.5 + while try_again(response) and (count < 9): + print(response.text.strip()) + print( + "There was a problem executing the Delete(All()) query. Trying again in {} seconds (attempt {}/9)".format( + retry_period, + count, + ) + ) + count += 1 + sleep(retry_period) + retry_period *= 2 + response = requests.post( + "http://" + ip + ":10101/index/tremor/query", data="Delete(All())" + ) + return response.status_code == 200 + + +def try_again(response): + if response.ok: + # if we got a response but it was unsuccessful, try again + return not response.json()["results"][0] + # if we didn't get a good response, try again + return True + + +def disk_use(): + response = requests.get("http://" + ip + ":10101/internal/disk-usage") + if response.status_code != 200: + return 0 + body = response.json() + return body["usage"] + + +print("# memory usage tests") +usage = new_usage = 0 +du = new_du = 0 +datagen() +usage = mem_use() +du = disk_use() +print( + "{},{},{},{}".format( + "mem_usage", "disk_usage", "new_mem_greater", "new_disk_greater" + ) +) +print("{},{},{},{}".format(usage, du, None, None)) +for i in range(5): + variance = 0.25 + if delete_all(): + datagen() + new_usage = mem_use() + new_du = disk_use() + print( + "{},{},{},{}".format( + usage, + du, + (new_usage / usage), + (new_du / du), + ) + ) + if (new_usage / usage) > (1 + variance): + print( + new_usage, + "is greater than", + usage, + "by at least {}%.".format(variance * 100), + ) + print(new_usage - usage, "is a lot of bytes.") + print("Failing...") + exit(1) + if (new_du / du) > (1 + variance): + print( + new_du, "is greater than", du, "by at least {}%.".format(variance * 100) + ) + print(new_du - du, "is a lot of bytes.") + print("Failing...") + exit(1) + else: + print("Couldn't execute Delete(All()) query in 9 tries. Failing...") + exit(1) +print("Success") +exit(0) diff --git a/qa/scripts/perf/delete/tremor.yaml b/qa/scripts/perf/delete/tremor.yaml new file mode 100644 index 000000000..45000a4ee --- /dev/null +++ b/qa/scripts/perf/delete/tremor.yaml @@ -0,0 +1,61 @@ +fields: + - name: "a_random_string" + type: "string" # (default StringField (non-mutex)) + generator_type: "random-string" # used to generate random strings rather than pulling from known set + min_len: 8 + max_len: 12 + charset: "AB" # set of possible characters to pull from when generating random string + - name: "id" + type: "uint" + distribution: "sequential" + min: 0 + max: 1000000 # 2%24 + step: 1 + repeat: false + - name: "type" + type: "int" # (default IntField) + min: 0 + max: 3 + distribution: "zipfian" + s: 1.1 + v: 5.1 + - name: "ts" + type: "timestamp" + min_date: 2006-01-02T15:04:05.001Z # RFC3339Nano + max_date: 2007-01-02T15:04:05.001Z # RFC3339Nano + distribution: "increasing" # only "increasing" is supported right now + min_step_duration: "10ns" + max_step_duration: "200ms" + - name: "slice" + type: "uint-set" # (default IDArrayField) + min: 0 + max: 35000 + distribution: "zipfian" + s: 1.1 + v: 5.1 + min_num: 1 + max_num: 50 + +# idk_params describe how data from "fields" should be ingested by IDK +idk_params: + primary_key_config: + field: "a_random_string" # if this is a single field named "id" then we'll use uint IDs, if it's empty we'll autogen ids, and if it's anything else we'll do string keys... yes this is a bit hacky, needs to be cleaned up. + # fields is keyed by names of fields from top level "fields". It is + # not required that all fields appear here, those that don't will + # use the default ingestion. + fields: + id: + - type: "ID" + type: + - type: "ID" + a_decimal_field: + - type: "Decimal" + scale: 4 + ts: + - type: "RecordTime" + layout: "2006-01-02T15:04:05Z" + epoch: 1970-01-01T00:00:00.0Z + name: "na" + slice: + - type: "IDArray" + time_quantum: "YMD" diff --git a/qa/scripts/utilCluster.sh b/qa/scripts/utilCluster.sh index 629a63aeb..95215c93c 100644 --- a/qa/scripts/utilCluster.sh +++ b/qa/scripts/utilCluster.sh @@ -47,14 +47,13 @@ EOT #echo "featurebase.service <<" scp -i ~/.ssh/gitlab-featurebase-ci.pem -o "StrictHostKeyChecking no" featurebase.service ec2-user@${NODEIP}: - if (( $? != 0 )) - then + if (( $? != 0 )) + then echo "featurebase.service copy failed" exit 1 fi rm -f featurebase.service - ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "sudo mv featurebase.service ${SERVICE_FILE_PATH}" } @@ -127,8 +126,8 @@ executeGeneralNodeConfigCommands() { # TODO handle different archs scp -i ~/.ssh/gitlab-featurebase-ci.pem -o "StrictHostKeyChecking no" featurebase_linux_arm64 ec2-user@${NODEIP}: - if (( $? != 0 )) - then + if (( $? != 0 )) + then echo "featurebase binary copy failed" exit 1 fi @@ -136,7 +135,7 @@ executeGeneralNodeConfigCommands() { ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "chown ec2-user:ec2-user /home/ec2-user/featurebase_linux_arm64" ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "chmod ugo+x /home/ec2-user/featurebase_linux_arm64" ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "sudo mv /home/ec2-user/featurebase_linux_arm64 /usr/local/bin/featurebase" - + echo "featurebase binary copied." } @@ -229,4 +228,4 @@ setupClusterNodes() { #ingest nodes setupIngestNodes -} \ No newline at end of file +} diff --git a/qa/tf/perf/delete/main.tf b/qa/tf/perf/delete/main.tf new file mode 100644 index 000000000..4033ad1d8 --- /dev/null +++ b/qa/tf/perf/delete/main.tf @@ -0,0 +1,17 @@ +module "delete-cluster" { + source = "../../.modules/featurebase-cluster" + cluster_prefix = var.cluster_prefix + region = var.region + profile = var.profile + fb_data_node_type = "m6g.12xlarge" + fb_data_disk_iops = 10000 + fb_data_node_count = 3 + fb_ingest_type = "m6g.2xlarge" + fb_ingest_disk_iops = 10000 + fb_ingest_disk_size_gb = 500 + fb_ingest_node_count = 1 + vpc_id = "vpc-05a26a122f961dc2b" + vpc_cidr_block = "10.0.0.0/16" + vpc_public_subnets = ["subnet-066b4b922b54e51a2", "subnet-037b8884269a69025", "subnet-08482631514426210", ] + vpc_private_subnets = ["subnet-0319dde319380326f", "subnet-0517ca9a646d80f88", "subnet-05a7b685ed27eb1cf", ] +} diff --git a/qa/tf/perf/delete/outputs.tf b/qa/tf/perf/delete/outputs.tf new file mode 100644 index 000000000..77fb2e9fc --- /dev/null +++ b/qa/tf/perf/delete/outputs.tf @@ -0,0 +1,19 @@ +output "ingest_ips" { + description = "List of ingest IPs" + value = module.delete-cluster.ingest_ips +} + +output "data_node_ips" { + description = "List of data node IPs" + value = module.delete-cluster.data_node_ips +} + +output "cluster_prefix" { + description = "The cluster prefix used" + value = module.delete-cluster.cluster_prefix +} + +output "fb_cluster_replica_count" { + description = "The cluster replica count used" + value = module.delete-cluster.fb_cluster_replica_count +} diff --git a/qa/tf/perf/delete/provider.tf b/qa/tf/perf/delete/provider.tf new file mode 100644 index 000000000..f6f28cca8 --- /dev/null +++ b/qa/tf/perf/delete/provider.tf @@ -0,0 +1,4 @@ +provider "aws" { + region = var.region + profile = var.profile +} diff --git a/qa/tf/perf/delete/tf.auto.tfvars b/qa/tf/perf/delete/tf.auto.tfvars new file mode 100644 index 000000000..d2b94a97a --- /dev/null +++ b/qa/tf/perf/delete/tf.auto.tfvars @@ -0,0 +1,2 @@ +region = "us-east-2" +profile = "service-terraform" diff --git a/qa/tf/perf/delete/variables.tf b/qa/tf/perf/delete/variables.tf new file mode 100644 index 000000000..e55c7936d --- /dev/null +++ b/qa/tf/perf/delete/variables.tf @@ -0,0 +1,14 @@ +variable "region" { + description = "The AWS region in which the VPC should be built" + type = string +} + +variable "profile" { + description = "The name of the AWS profile Terraform should use for auth." + type = string +} + +variable "cluster_prefix" { + type = string + description = "This is a identifier that will be prefixed to created resources" +} From f690694e777c520e848ce32380a497922c6f3f9a Mon Sep 17 00:00:00 2001 From: reesporte Date: Tue, 15 Mar 2022 08:57:21 -0500 Subject: [PATCH 2/3] remove commented gitlab jobs, run only on merge to master --- .gitlab/.gitlab-ci.yml | 247 +++++++++++++++++++++-------------------- 1 file changed, 125 insertions(+), 122 deletions(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 4feb3ce5a..65c3d7254 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -1,8 +1,8 @@ -#include: -# - template: Security/SAST.gitlab-ci.yml -# - template: Security/License-Scanning.gitlab-ci.yml -# - template: Security/Dependency-Scanning.gitlab-ci.yml -# +include: + - template: Security/SAST.gitlab-ci.yml + - template: Security/License-Scanning.gitlab-ci.yml + - template: Security/Dependency-Scanning.gitlab-ci.yml + variables: GOVERSION: "1.17.7" @@ -16,35 +16,35 @@ stages: - post build - nonblocking -#smoke build: -# image: golang:$GOVERSION -# stage: lint -# allow_failure: false -# rules: -# - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"' -# script: -# - echo "Let's just see if it compiles... (sometimes the linter gives unclear errors if it doesn't)" -# - go build ./... -# -#golangci-lint: -# image: golangci/golangci-lint:v1.39.0 -# stage: lint -# allow_failure: false -# rules: -# - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"' -# script: -# - echo "Checking for issues in new code" -# - golangci-lint run -# -#go mod tidy: -# stage: lint -# image: golang:$GOVERSION -# rules: -# - if: '$CI_COMMIT_TAG == null && ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web")' -# script: -# - go mod tidy -# - git diff --exit-code -- go.mod go.sum -# +smoke build: + image: golang:$GOVERSION + stage: lint + allow_failure: false + rules: + - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"' + script: + - echo "Let's just see if it compiles... (sometimes the linter gives unclear errors if it doesn't)" + - go build ./... + +golangci-lint: + image: golangci/golangci-lint:v1.39.0 + stage: lint + allow_failure: false + rules: + - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"' + script: + - echo "Checking for issues in new code" + - golangci-lint run + +go mod tidy: + stage: lint + image: golang:$GOVERSION + rules: + - if: '$CI_COMMIT_TAG == null && ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web")' + script: + - go mod tidy + - git diff --exit-code -- go.mod go.sum + build lattice: stage: test image: node:14 @@ -64,92 +64,92 @@ build lattice: artifacts: paths: - lattice.tar.gz -# -#run jest tests: -# stage: test -# image: node:14 -# variables: -# CI: "true" -# rules: -# - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"' -# script: -# - echo "Testing lattice..." -# - cd lattice -# - npm install --force -# - npm test -- --coverage --testResultsProcessor=jest-sonar-reporter -# artifacts: -# paths: -# - lattice/coverage/lcov.info -# -#run go tests: -# stage: test -# image: golang:$GOVERSION -# rules: -# - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"' -# retry: 1 -# script: -# - echo "Running featurebase unit tests..." -# - go test -timeout=30m ./... -# tags: -# - aws -# -#run go tests race: -# stage: nonblocking # don't let this job block any other jobs because it takes much longer than the other tests. -# image: golang:$GOVERSION -# rules: -# - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"' -# retry: 1 -# needs: [] # don't wait to start running this. -# script: -# - echo "Running featurebase race tests..." -# - go test -race -v -timeout=90m ./... -# tags: -# - aws -# -#run go tests shardwidth22: -# stage: test -# image: golang:$GOVERSION -# rules: -# - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"' -# script: -# - echo "Running featurebase shardwidth22 tests..." -# - go test -timeout=30m -tags=shardwidth22 ./... -# tags: -# - aws -# -## we do coverage reporting from the future tests because the json -## output is very difficult to human-read. The alternative would be to -## run the regular tests twice and also run the future tests. -#run go tests future: -# stage: test -# image: golang:1.17.6 -# rules: -# - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"' -# retry: 1 -# script: -# - echo "Running featurebase unit tests..." -# - PKG_LIST=$(go list ./... | grep -v internal/clustertests | paste -s -d, -) -# - go test -timeout=30m -json -coverprofile=coverage.out -covermode=atomic -coverpkg=${PKG_LIST} ./... | tee test-report.out -# artifacts: -# paths: -# - coverage.out -# - test-report.out -# tags: -# - aws -# -#upload to sonarcloud: -# stage: integration -# image: sonarsource/sonar-scanner-cli:4.6 -# variables: -# SONAR_TOKEN: $SONAR_TOKEN -# rules: -# - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"' -# script: -# - sonar-scanner -Dsonar.projectKey=molecula_featurebase -Dsonar.organization=molecula -Dsonar.sources=. -Dsonar.host.url=https://sonarcloud.io -Dsonar.go.coverage.reportPaths=coverage.out,results/coverage*.out -Dsonar.go.tests.reportPaths=test-report.out,results/report* -Dsonar.javascript.lcov.reportPaths=lattice/coverage/lcov.info -# needs: -# - job: run go tests future -# - job: run jest tests -# - job: clustertests + +run jest tests: + stage: test + image: node:14 + variables: + CI: "true" + rules: + - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"' + script: + - echo "Testing lattice..." + - cd lattice + - npm install --force + - npm test -- --coverage --testResultsProcessor=jest-sonar-reporter + artifacts: + paths: + - lattice/coverage/lcov.info + +run go tests: + stage: test + image: golang:$GOVERSION + rules: + - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"' + retry: 1 + script: + - echo "Running featurebase unit tests..." + - go test -timeout=30m ./... + tags: + - aws + +run go tests race: + stage: nonblocking # don't let this job block any other jobs because it takes much longer than the other tests. + image: golang:$GOVERSION + rules: + - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"' + retry: 1 + needs: [] # don't wait to start running this. + script: + - echo "Running featurebase race tests..." + - go test -race -v -timeout=90m ./... + tags: + - aws + +run go tests shardwidth22: + stage: test + image: golang:$GOVERSION + rules: + - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"' + script: + - echo "Running featurebase shardwidth22 tests..." + - go test -timeout=30m -tags=shardwidth22 ./... + tags: + - aws + +# we do coverage reporting from the future tests because the json +# output is very difficult to human-read. The alternative would be to +# run the regular tests twice and also run the future tests. +run go tests future: + stage: test + image: golang:1.17.6 + rules: + - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"' + retry: 1 + script: + - echo "Running featurebase unit tests..." + - PKG_LIST=$(go list ./... | grep -v internal/clustertests | paste -s -d, -) + - go test -timeout=30m -json -coverprofile=coverage.out -covermode=atomic -coverpkg=${PKG_LIST} ./... | tee test-report.out + artifacts: + paths: + - coverage.out + - test-report.out + tags: + - aws + +upload to sonarcloud: + stage: integration + image: sonarsource/sonar-scanner-cli:4.6 + variables: + SONAR_TOKEN: $SONAR_TOKEN + rules: + - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"' + script: + - sonar-scanner -Dsonar.projectKey=molecula_featurebase -Dsonar.organization=molecula -Dsonar.sources=. -Dsonar.host.url=https://sonarcloud.io -Dsonar.go.coverage.reportPaths=coverage.out,results/coverage*.out -Dsonar.go.tests.reportPaths=test-report.out,results/report* -Dsonar.javascript.lcov.reportPaths=lattice/coverage/lcov.info + needs: + - job: run go tests future + - job: run jest tests + - job: clustertests build for linux amd64: stage: build @@ -292,7 +292,6 @@ build arm container fb: # 3. make sure docker/docker-compose is installed # 4. make sure the git config is done `git config --global --add url."ssh://git@github.com/".insteadOf "https://github.com/"` # 5. Add deploy key github.com/molecula/featurebase/settings/keys and add public key in .ssh folder of gitlab-runner user -# TODO: (I think) get clustertests coverage added to coverage report clustertests: variables: PROJECT: clustertests_${CI_CONCURRENT_ID} @@ -500,8 +499,12 @@ perf_able: variables: PARENT_PIPELINE_ID: $CI_PIPELINE_ID +# only run on merge to master, the whole process takes about 10 minutes in total +# (which is a long time) perf_delete: - stage: integration + stage: performance + rules: + - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $CI_PIPELINE_SOURCE == "push"' timeout: 2h image: registry.gitlab.com/gitlab-org/cloud-deploy/aws-base:latest variables: From fc41870dfe3f1310ad9e58e9ca434eb9cad1e514 Mon Sep 17 00:00:00 2001 From: reesporte <45641995+reesporte@users.noreply.github.com> Date: Tue, 15 Mar 2022 14:40:42 -0500 Subject: [PATCH 3/3] remove linux amd64 dependency Co-authored-by: souhailanoor <90720110+souhailanoor@users.noreply.github.com> --- .gitlab/.gitlab-ci.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 65c3d7254..9cde8e07a 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -559,7 +559,6 @@ perf_delete: - aws autoscaling set-instance-protection --instance-ids "$INSTANCE_ID" --auto-scaling-group-name $ASG_NAME --no-protected-from-scale-in --profile $INFRA_PROFILE needs: - job: build for linux arm64 - - job: build for linux amd64 s3 dump tag: stage: post build