diff --git a/.gitignore b/.gitignore index d7356ece0..6758e781b 100644 --- a/.gitignore +++ b/.gitignore @@ -18,3 +18,5 @@ pilosa launch.json .terraform.lock.hcl __pycache__/ +report.xml +outputs.json diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 55bbde528..c32893e8c 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -13,7 +13,6 @@ include: variables: GOVERSION: "1.16.10" - stages: - lint - test @@ -27,7 +26,7 @@ golangci-lint: extends: .go-cache allow_failure: false rules: - - if: '$CI_PIPELINE_SOURCE == "push"' + - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"' script: - echo "Checking for issues in new code" - golangci-lint run -v @@ -38,7 +37,7 @@ build lattice: variables: CI: "false" rules: - - if: '$CI_PIPELINE_SOURCE == "push"' + - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"' script: - cd lattice - yarn install @@ -58,7 +57,7 @@ run jest tests: variables: CI: "true" rules: - - if: '$CI_PIPELINE_SOURCE == "push"' + - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"' script: - echo "Testing lattice..." - cd lattice @@ -73,7 +72,7 @@ run go tests: image: golang:$GOVERSION extends: .go-cache rules: - - if: '$CI_PIPELINE_SOURCE == "push"' + - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"' script: - echo "Running featurebase unit tests..." - PKG_LIST=$(go list ./... | grep -v internal/clustertests | paste -s -d, -) @@ -87,7 +86,7 @@ run go tests future: image: golang:1.17.3 extends: .go-cache rules: - - if: '$CI_PIPELINE_SOURCE == "push"' + - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"' script: - echo "Running featurebase unit tests..." - PKG_LIST=$(go list ./... | grep -v internal/clustertests | paste -s -d, -) @@ -96,12 +95,11 @@ run go tests future: paths: - coverage.out - run go tests with output: stage: test image: golang:$GOVERSION rules: - - if: '$CI_PIPELINE_SOURCE == "push"' + - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"' script: - echo "Running featurebase unit tests to capture JSON output..." - go test -json > test-report.out @@ -115,7 +113,7 @@ upload to sonarcloud: variables: SONAR_TOKEN: $SONAR_TOKEN rules: - - if: '$CI_PIPELINE_SOURCE == "push"' + - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"' script: - sonar-scanner -Dsonar.projectKey=molecula_featurebase -Dsonar.organization=molecula -Dsonar.sources=. -Dsonar.host.url=https://sonarcloud.io -Dsonar.go.coverage.reportPaths=coverage.out -Dsonar.go.tests.reportPaths=test-report.out -Dsonar.javascript.lcov.reportPaths=lattice/coverage/lcov.info needs: @@ -127,7 +125,7 @@ build for linux amd64: stage: build image: golang:$GOVERSION rules: - - if: '$CI_PIPELINE_SOURCE == "push"' + - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"' script: - rm -r lattice - tar -xvf lattice.tar.gz @@ -142,7 +140,7 @@ build for linux arm64: stage: build image: golang:$GOVERSION rules: - - if: '$CI_PIPELINE_SOURCE == "push"' + - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"' script: - rm -r lattice - tar -xvf lattice.tar.gz @@ -157,7 +155,7 @@ build for darwin amd64: stage: build image: golang:$GOVERSION rules: - - if: '$CI_PIPELINE_SOURCE == "push"' + - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"' script: - rm -r lattice - tar -xvf lattice.tar.gz @@ -172,7 +170,7 @@ build for darwin arm64: stage: build image: golang:$GOVERSION rules: - - if: '$CI_PIPELINE_SOURCE == "push"' + - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"' script: - rm -r lattice - tar -xvf lattice.tar.gz @@ -187,7 +185,7 @@ package for linux amd64: stage: build image: golang:$GOVERSION rules: - - if: '$CI_PIPELINE_SOURCE == "push"' + - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"' variables: GOOS: "linux" GOARCH: "amd64" @@ -209,7 +207,7 @@ build container fb: tags: - shell rules: - - if: '$CI_PIPELINE_SOURCE == "push"' + - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"' before_script: - echo "${DOCKER_DEPLOY_TOKEN}" | docker login -u ${DOCKER_DEPLOY_USER} --password-stdin ${CI_REGISTRY} script: @@ -218,46 +216,18 @@ build container fb: - docker push $tag - echo Created docker featurebase image with tag "$tag" -# deploy EC2 instance, configure and run featurebase -# diabling for now - will come back and refactor POK -# deploy node for linux amd64: -# stage: integration -# image: registry.gitlab.com/gitlab-org/cloud-deploy/aws-base:latest -# variables: -# PROFILE: "default" -# AWS_SSH_PRIVATE_KEY: $AWS_FBCI_SSH_KEY -# rules: -# - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH' -# before_script: -# - aws configure set aws_access_key_id $AWS_FBCI_ACCESS_KEY_ID -# - aws configure set aws_secret_access_key $AWS_FBCI_SECRET_ACCESS_KEY -# - aws configure set region "us-east-2" -# - aws configure set aws_profile $PROFILE -# - echo $AWS_FBCI_SSH_KEY > gitlab-featurebase-ci.pem -# - chmod 400 gitlab-featurebase-ci.pem -# - 'which ssh-agent || ( apt-get update -y && apt-get install openssh-client -y )' -# - eval `ssh-agent -s` -# - mkdir -p ~/.ssh -# - echo "$AWS_FBCI_SSH_KEY" | ssh-add - -# - chmod 700 /root/.ssh -# - '[[ -f /.dockerenv ]] && echo -e "Host *\n\tStrictHostKeyChecking no\n\n" > ~/.ssh/config' -# - apt update && apt -y install jq -# script: -# - ./qa/scripts/deployNode.sh $PROFILE -# needs: -# - job: build for linux amd64 - -gauntlet: - stage: gauntlet - timeout: 4h +smoke test: + stage: integration image: registry.gitlab.com/gitlab-org/cloud-deploy/aws-base:latest variables: - PROFILE: "default" + PROFILE: "service-terraform" AWS_SSH_PRIVATE_KEY: $AWS_FBCI_SSH_KEY AWS_ACCESS_KEY_ID: $AWS_FBCI_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY: $AWS_FBCI_SECRET_ACCESS_KEY + TF_VAR_cluster_prefix: "" + TF_VAR_branch: "" rules: - - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && ($CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web")' + - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"' before_script: - apt-get update && apt-get install -y gnupg software-properties-common curl git - curl -fsSL https://apt.releases.hashicorp.com/gpg | apt-key add - @@ -281,6 +251,65 @@ gauntlet: - wget https://go.dev/dl/go1.17.5.linux-amd64.tar.gz - tar -C /usr/local -xzf go1.17.5.linux-amd64.tar.gz - export PATH=$PATH:/usr/local/go/bin + - TF_VAR_cluster_prefix="smoke-$(openssl rand -base64 12 | tr -d /=+ | cut -c -16)" + - echo "Cluster Prefix --> $TF_VAR_cluster_prefix" + - TF_VAR_branch=$CI_COMMIT_BRANCH + - echo "Branch --> $TF_VAR_branch" + script: + - ./qa/scripts/setupSmokeTest.sh + - ./qa/scripts/testSmokeTest.sh + after_script: + - ./qa/scripts/teardownSmokeTest.sh + needs: + - job: build for linux arm64 + artifacts: + when: always + paths: + - report.xml + reports: + junit: report.xml + +gauntlet: + stage: gauntlet + timeout: 4h + image: registry.gitlab.com/gitlab-org/cloud-deploy/aws-base:latest + variables: + PROFILE: "default" + AWS_SSH_PRIVATE_KEY: $AWS_FBCI_SSH_KEY + AWS_ACCESS_KEY_ID: $AWS_FBCI_ACCESS_KEY_ID + AWS_SECRET_ACCESS_KEY: $AWS_FBCI_SECRET_ACCESS_KEY + TF_VAR_cluster_prefix: "" + TF_VAR_branch: "" + rules: + - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH' + - if: '$CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"' + before_script: + - apt-get update && apt-get install -y gnupg software-properties-common curl git + - curl -fsSL https://apt.releases.hashicorp.com/gpg | apt-key add - + - apt-add-repository "deb [arch=amd64] https://apt.releases.hashicorp.com $(lsb_release -cs) main" + - apt-get update && apt-get install terraform + - aws configure set aws_access_key_id $AWS_FBCI_ACCESS_KEY_ID + - aws configure set aws_secret_access_key $AWS_FBCI_SECRET_ACCESS_KEY + - aws configure set region "us-east-2" + - aws configure set aws_profile $PROFILE + - echo $AWS_FBCI_SSH_KEY > gitlab-featurebase-ci.pem + - chmod 400 gitlab-featurebase-ci.pem + - 'which ssh-agent || ( apt-get update -y && apt-get install openssh-client -y )' + - eval $(ssh-agent -s) + - mkdir -p ~/.ssh + - echo $AWS_FBCI_SSH_KEY > /root/.ssh/gitlab-featurebase-ci.pem + - chmod 400 /root/.ssh/gitlab-featurebase-ci.pem + - echo "$AWS_FBCI_SSH_KEY" | ssh-add - + - chmod 700 /root/.ssh + - '[[ -f /.dockerenv ]] && echo -e "Host *\n\tStrictHostKeyChecking no\n\n" > ~/.ssh/config' + - apt update && apt -y install jq wget + - wget https://go.dev/dl/go1.17.5.linux-amd64.tar.gz + - tar -C /usr/local -xzf go1.17.5.linux-amd64.tar.gz + - export PATH=$PATH:/usr/local/go/bin + - TF_VAR_cluster_prefix="smoke-$(openssl rand -base64 12 | tr -d /=+ | cut -c -16)" + - echo "Cluster Prefix --> $TF_VAR_cluster_prefix" + - TF_VAR_branch=$CI_COMMIT_BRANCH + - echo "Branch --> $TF_VAR_branch" script: - ./qa/scripts/setupSamsungGauntlet.sh - ./qa/scripts/testSamsungGauntlet.sh diff --git a/qa/scripts/cloud-init.sh b/qa/scripts/cloud-init.sh deleted file mode 100755 index 8d710d0b2..000000000 --- a/qa/scripts/cloud-init.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/bash -ex -# generate log -exec > >(tee /var/log/user-data.log|logger -t user-data -s 2>/dev/console) 2>&1 - -# Install packages -yum update -y -yum install postgresql -y - -# Configure host system -echo 'cat /proc/sys/fs/file-max' -sysctl -w fs.file-max=262144 -sysctl -p -echo 'cat /proc/sys/fs/file-max' - -# yum install golang -y # latest verion in ec2 is 1.15.14 -# install go 1.16.9 manually -curl -O https://dl.google.com/go/go1.16.10.linux-amd64.tar.gz -tar xvf go1.16.10.linux-amd64.tar.gz -chown -R root:root ./go -mv go /usr/local -echo "export PATH=/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/home/ec2-user/.local/bin:/home/ec2-user/bin:/usr/local/go/bin" | tee -a /etc/profile > /dev/null -source /etc/profile - -# install aws session manager pluggin -curl "https://s3.amazonaws.com/session-manager-downloads/plugin/latest/linux_64bit/session-manager-plugin.rpm" -o "session-manager-plugin.rpm" -yum install -y session-manager-plugin.rpm diff --git a/qa/scripts/config.json b/qa/scripts/config.json deleted file mode 100644 index e69de29bb..000000000 diff --git a/qa/scripts/configureFeatureBase.json b/qa/scripts/configureFeatureBase.json deleted file mode 100644 index 6afd135d9..000000000 --- a/qa/scripts/configureFeatureBase.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "Parameters": { - "commands": [ - "#!/bin/bash", - "mv /home/ec2-user/featurebase_linux_amd64 /usr/local/bin/featurebase", - "mv /home/ec2-user/featurebase.conf /etc/", - "mv /home/ec2-user/featurebase.service /etc/systemd/system/", - "adduser molecula", - "sudo mkdir /var/log/molecula", - "sudo chown molecula /var/log/molecula", - "sudo mkdir -p /opt/molecula/featurebase", - "sudo chown molecula /opt/molecula/featurebase", - "systemctl daemon-reload", - "sudo systemctl start featurebase", - "sudo systemctl enable featurebase", - "sudo systemctl status featurebase", - "curl localhost:10101" - ] - } -} \ No newline at end of file diff --git a/qa/scripts/deployNode.sh b/qa/scripts/deployNode.sh deleted file mode 100755 index 7b5f53796..000000000 --- a/qa/scripts/deployNode.sh +++ /dev/null @@ -1,98 +0,0 @@ -#!/bin/bash - -# To run script: ./deployNode.sh $PROFILE - -# default to the VPC initially created -VPC=${VPC:-vpc-0582f594d7d2ca2d4} - -function deploy_node() { - # get AMI, security group and subnet ID - AMI=$(aws ssm get-parameters --names /aws/service/ami-amazon-linux-latest/amzn2-ami-hvm-x86_64-ebs --query 'Parameters[0].[Value]' --output text --profile $PROFILE) - if [[ $? > 0 ]]; then - echo "aws session manager failed to find AMI" - exit 1 - fi - - SECURITY_GROUP=$(aws ec2 describe-security-groups --filters "Name=vpc-id,Values=$VPC" Name=group-name,Values=default --query 'SecurityGroups[*].[GroupId]' --output text --profile $PROFILE) - if [[ $? > 0 ]]; then - echo "aws session manager failed to find security group" - exit 1 - fi - - SUBNET_ID=$(aws ec2 describe-subnets --filters "Name=vpc-id,Values=$VPC" 'Name=availability-zone,Values=us-east-2a' --query 'Subnets[0].SubnetId' --output text --profile $PROFILE) - if [[ $? > 0 ]]; then - echo "aws session manager failed to find subnet ID" - exit 1 - fi - - # launch EC2 instance and get instance ID - aws ec2 run-instances --image-id $AMI --instance-type $INSTANCE --security-group-ids $SECURITY_GROUP --subnet-id $SUBNET_ID --key-name gitlab-featurebase-dev --tag-specifications 'ResourceType=instance,Tags=[{Key=Name,Value=linux-amd64-node}]' --profile $PROFILE --user-data file://./qa/scripts/cloud-init.sh --iam-instance-profile Name=featurebase-dev-ssm > config.json - if [[ $? > 0 ]]; then - echo "aws run-instances failed to launch a new EC2 instance" - exit 1 - fi - - INSTANCE_ID=$(jq '.Instances | .[0] |.InstanceId' config.json | tr -d '"') - echo "aws run-instances succeeded in launching a new EC2 instance with instance ID: " $INSTANCE_ID -} - -function initialize_featurebase() { - # get IP for node - for i in {0..24} - do - IP=$(aws ec2 describe-instances --instance-ids $INSTANCE_ID --filters 'Name=instance-state-name, Values=running' --query 'Reservations[*].Instances[*].PublicIpAddress' --output text --profile $PROFILE) - if [ -n "$IP" ]; then - echo "Public IP for EC2 instance found: " $IP - break - fi - - if [[ $? > 0 ]]; then - echo "aws cli describe-instances command failed to find public IP" - terminate_node - exit 1 - fi - - sleep 5 - done - - sleep 60 # to allow enough time for node to be ready for use - - # copy featurebase binary and files to ec2 instance - scp -o StrictHostKeyChecking=no -i gitlab-featurebase-dev.pem featurebase_linux_amd64 ./qa/scripts/featurebase.conf ./qa/scripts/featurebase.service ec2-user@$IP:. - if [[ $? > 0 ]]; then - echo "scp of featurebase binary, service and config files to EC2 instance failed" - terminate_node - exit 1 - fi - - # execute script to configure featurebase on the EC2 node - aws ssm send-command --document-name "AWS-RunShellScript" --instance-ids $INSTANCE_ID --cli-input-json file://./qa/scripts/configureFeatureBase.json --profile $PROFILE --region $REGION - if [[ $? > 0 ]]; then - echo "aws cli session manager send-command failed" - terminate_node - exit 1 - fi -} - -function terminate_node() { - aws ec2 terminate-instances --instance-ids $INSTANCE_ID --profile $PROFILE -} - -# Pass variables to shell script -PROFILE=$1 -shift - -# set some variables -INSTANCE="t3a.large" -REGION="us-east-2" - -# get AMI, security group and subnet for EC2 instance, -# launch instance, save instance Id and run cloud-init to set up node env -deploy_node - -# Get IP for instance, scp featurebase binary, config and service files; -# set up featurebase config in node -initialize_featurebase - -# terminate node -terminate_node diff --git a/qa/scripts/deploySingleNodeCluster.sh b/qa/scripts/deploySingleNodeCluster.sh deleted file mode 100644 index d4d1240ec..000000000 --- a/qa/scripts/deploySingleNodeCluster.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/bash - -# To run script: ./deploySingleNodeCluster.sh -# requires TF_VAR_gitlab_token env var to be set - -echo “$(pwd)” - -pushd ./qa/tf/ci/singlenode -export TF_IN_AUTOMATION=1 -terraform init -input=false -terraform apply -input=false -auto-approve -popd - -# configure Featurebase - -# step 1a: get IPs of the cluster - - - -# step 1b: get IPs of the ingest nodes - -# step 2: write a featurebase.conf file - -# step 3: write featurebase.service - -# step 4: start featurebase - -# step 5: verify featurebase running \ No newline at end of file diff --git a/qa/scripts/featurebase.conf b/qa/scripts/featurebase.conf deleted file mode 100644 index 7716d0f6b..000000000 --- a/qa/scripts/featurebase.conf +++ /dev/null @@ -1,30 +0,0 @@ -name = "pilosa1" -bind = "0.0.0.0:10101" -bind-grpc = "0.0.0.0:20101" - -data-dir = "/opt/molecula/featurebase" -log-path = "/var/log/molecula/featurebase.log" - -max-file-count=900000 -max-map-count=900000 - -long-query-time = "10s" - -[postgres] - - bind = "localhost:55432" - -[cluster] - - name = "cluster1" - replicas = 1 - -[etcd] - - listen-client-address = "http://localhost:10401" - listen-peer-address = "http://localhost:10301" - initial-cluster = "pilosa1=http://localhost:10301" - -[metric] - - service = "prometheus" \ No newline at end of file diff --git a/qa/scripts/featurebase.service b/qa/scripts/featurebase.service deleted file mode 100644 index a543b92af..000000000 --- a/qa/scripts/featurebase.service +++ /dev/null @@ -1,13 +0,0 @@ -# Not Ansible managed - -[Unit] -Description="Service for FeatureBase" - -[Service] -RestartSec=30 -Restart=on-failure -EnvironmentFile= -User=molecula -ExecStart=/usr/local/bin/featurebase server -c /etc/featurebase.conf - -[Install] \ No newline at end of file diff --git a/qa/scripts/ingestWorkload.sh b/qa/scripts/ingestWorkload.sh deleted file mode 100755 index 6b87fb762..000000000 --- a/qa/scripts/ingestWorkload.sh +++ /dev/null @@ -1,84 +0,0 @@ -#!/usr/bin/env bash - -# path for featurebase binary -FEATUREBASE_PATH=/usr/local/bin - -# path for directory with csv directory files for all fields to be ingested -CSV_DIR_PATH=/data - - -# To run: -# ./ingestWorkload.sh {Local host & port for featurebase} {initialize flag} - -function delete_field { - if (($INITIALIZE == 0)); - then - curl -XDELETE $HOST/index/$INDEX/field/$FIELD - fi -} - -# Script to replicate samsung workload of deleting and re-ingesting fields every night -# outline delete and re-ingest workload -function ingest_int_field { - delete_field - curl -XPOST $HOST/index/$INDEX/field/$FIELD -d '{"options": {"type": "int", "min": 0, "max":'$MAX'}}' - $FEATUREBASE_PATH/featurebase import --host $HOST -i $INDEX -f $FIELD $CSV_FILE -} - -function ingest_time_field { - delete_field - curl -XPOST $HOST/index/$INDEX/field/$FIELD -d '{"options": {"keys": true, "type": "time", "timeQuantum": "YMD"}}' - $FEATUREBASE_PATH/featurebase import --host $HOST -i $INDEX -f $FIELD $CSV_FILE -} - -function ingest_set_field { - delete_field - curl -XPOST $HOST/index/$INDEX/field/$FIELD -d '{"options": {"keys": true}}' - $FEATUREBASE_PATH/featurebase import --host $HOST -i $INDEX -f $FIELD $CSV_FILE -} - -# featurebase host & port -HOST=$1 -shift - -# intialize flag - 0:disabled, 1:enabled - creates the index and fields for testing -INITIALIZE=$1 -shift - -# get a list of csv files in the directory -CSV_FILES=`ls $CSV_DIR_PATH/*.csv` - -# assign index name -INDEX="samsung" -if (($INITIALIZE == 1)); -then - curl -XPOST $HOST/index/$INDEX -fi - -# perform delete and re-ingest for all fields -for CSV_FILE in ${CSV_FILES[@]} - do - # get field name from csv file path - FIELD="$(basename $CSV_FILE .csv)" - if [[ "$FIELD" == *"age"* ]]; - then - MAX=100 - ingest_int_field - elif [[ "$FIELD" == *"identifier"* ]]; - then - MAX=$((2**63 - 1)) # compute max value for 64bit - ingest_int_field - elif [[ "$FIELD" == *"ip"* ]]; - then - MAX=$((2**31 - 1)) # compute max value for 32bit - ingest_int_field - elif [[ "$FIELD" == *"time"* ]]; - then - ingest_time_field - else - ingest_set_field - fi - done - - - diff --git a/qa/scripts/perf.sh b/qa/scripts/perf.sh deleted file mode 100644 index 41a734d59..000000000 --- a/qa/scripts/perf.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash -echo >&2 "performance testing" -time ./simulacraData diff --git a/qa/scripts/runSamsungGauntlet.sh b/qa/scripts/runSamsungGauntlet.sh index d99a03baa..85fd22c67 100644 --- a/qa/scripts/runSamsungGauntlet.sh +++ b/qa/scripts/runSamsungGauntlet.sh @@ -1,8 +1,5 @@ #!/bin/bash - -#openssl rand -base64 32 | tr -d /=+ | cut -c -16 - ./setupSamsungGauntlet.sh ./testSamsungGauntlet.sh ./teardownSamsungGauntlet.sh diff --git a/qa/scripts/runSmokeTest.sh b/qa/scripts/runSmokeTest.sh new file mode 100644 index 000000000..554a234cc --- /dev/null +++ b/qa/scripts/runSmokeTest.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +./setupSmokeTest.sh +./testSmokeTest.sh +./teardownSmokeTest.sh diff --git a/qa/scripts/setupSamsungGauntlet.sh b/qa/scripts/setupSamsungGauntlet.sh index 6a867c18d..1c741be78 100755 --- a/qa/scripts/setupSamsungGauntlet.sh +++ b/qa/scripts/setupSamsungGauntlet.sh @@ -9,14 +9,14 @@ echo "Running terraform init..." terraform init -input=false echo "Running terraform apply..." terraform apply -input=false -auto-approve -terraform output -json > samsung-gauntlet.json +terraform output -json > outputs.json popd # get the bastion host -BASTION=$(cat ./qa/tf/gauntlet/samsung/samsung-gauntlet.json | jq -r '[.ingest_ips][0]["value"][0]') +BASTION=$(cat ./qa/tf/gauntlet/samsung/outputs.json | jq -r '[.ingest_ips][0]["value"][0]') echo "using bastion ${BASTION}" -NODE=$(cat ./qa/tf/gauntlet/samsung/samsung-gauntlet.json | jq -r '[.data_node_ips][0]["value"][0]') +NODE=$(cat ./qa/tf/gauntlet/samsung/outputs.json | jq -r '[.data_node_ips][0]["value"][0]') echo "using node ${NODE}" # remember that the nodes will take at least 2 mins to be up and going and finish cloud-init diff --git a/qa/scripts/setupSmokeTest.sh b/qa/scripts/setupSmokeTest.sh new file mode 100755 index 000000000..766d9a82b --- /dev/null +++ b/qa/scripts/setupSmokeTest.sh @@ -0,0 +1,38 @@ +#!/bin/bash + +# To run script: ./setupSmokeTest.sh +# requires TF_VAR_gitlab_token env var to be set + +pushd ./qa/tf/ci/smoketest +export TF_IN_AUTOMATION=1 +echo "Running terraform init..." +terraform init -input=false +echo "Running terraform apply..." +terraform apply -input=false -auto-approve +terraform output -json > outputs.json +popd + +# get the bastion host +BASTION=$(cat ./qa/tf/ci/smoketest/outputs.json | jq -r '[.ingest_ips][0]["value"][0]') +echo "using bastion ${BASTION}" + +NODE=$(cat ./qa/tf/ci/smoketest/outputs.json | jq -r '[.data_node_ips][0]["value"][0]') +echo "using node ${NODE}" + +# remember that the nodes will take at least 2 mins to be up and going and finish cloud-init +echo "Waiting for cluster to become available..." +# jaffee - I do wanna do a loop here, but I give up, and am running home to sleep... -POK +sleep 150 + +# verify featurebase running +echo "Verifying featurebase cluster running..." +ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${BASTION} "curl -s http://${NODE}:10101/status" +if (( $? != 0 )) +then + echo "Featurebase cluster not running" + exit 1 +fi + +echo "Cluster running." + + diff --git a/qa/scripts/teardownSmokeTest.sh b/qa/scripts/teardownSmokeTest.sh new file mode 100755 index 000000000..1e67d2895 --- /dev/null +++ b/qa/scripts/teardownSmokeTest.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +# To run script: ./teardownSmokeTest.sh +# requires TF_VAR_gitlab_token env var to be set + +cd qa/tf/ci/smoketest +export TF_IN_AUTOMATION=1 +terraform destroy -auto-approve diff --git a/qa/scripts/testSamsungGauntlet.sh b/qa/scripts/testSamsungGauntlet.sh index ff8d13cb7..efe0b8951 100755 --- a/qa/scripts/testSamsungGauntlet.sh +++ b/qa/scripts/testSamsungGauntlet.sh @@ -1,10 +1,10 @@ #!/bin/bash # get the bastion host -BASTION=$(cat ./qa/tf/gauntlet/samsung/samsung-gauntlet.json | jq -r '[.ingest_ips][0]["value"][0]') +BASTION=$(cat ./qa/tf/gauntlet/samsung/outputs.json | jq -r '[.ingest_ips][0]["value"][0]') echo "using bastion ${BASTION}" -NODE=$(cat ./qa/tf/gauntlet/samsung/samsung-gauntlet.json | jq -r '[.data_node_ips][0]["value"][0]') +NODE=$(cat ./qa/tf/gauntlet/samsung/outputs.json | jq -r '[.data_node_ips][0]["value"][0]') echo "using node ${NODE}" # generate csv files diff --git a/qa/scripts/testSmokeTest.sh b/qa/scripts/testSmokeTest.sh new file mode 100755 index 000000000..ed748cf97 --- /dev/null +++ b/qa/scripts/testSmokeTest.sh @@ -0,0 +1,35 @@ +#!/bin/bash + +# get the bastion host +BASTION=$(cat ./qa/tf/ci/smoketest/outputs.json | jq -r '[.ingest_ips][0]["value"][0]') +echo "using bastion ${BASTION}" + +NODE=$(cat ./qa/tf/ci/smoketest/outputs.json | jq -r '[.data_node_ips][0]["value"][0]') +echo "using node ${NODE}" + +echo "Copying tests to remote" +scp -r -i ~/.ssh/gitlab-featurebase-ci.pem ./qa/testcases/smoketest/*.py ec2-user@${BASTION}:/data +if (( $? != 0 )) +then + echo "Copy failed" + exit 1 +fi + +# run smoke test +echo "Running smoke test..." +ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o "StrictHostKeyChecking no" ec2-user@${BASTION} " pushd /data; pytest --junitxml=report.xml; popd" +if (( $? != 0 )) +then + echo "Unable to run smoke test" + exit 1 +fi + +echo "Copying test report to local" +scp -r -i ~/.ssh/gitlab-featurebase-ci.pem ec2-user@${BASTION}:/data/report.xml report.xml +if (( $? != 0 )) +then + echo "Copy failed" + exit 1 +fi + +echo "Smoke test complete" \ No newline at end of file diff --git a/qa/testcases/smoketest/test_smoke.py b/qa/testcases/smoketest/test_smoke.py new file mode 100644 index 000000000..ba45124ff --- /dev/null +++ b/qa/testcases/smoketest/test_smoke.py @@ -0,0 +1,7 @@ +# content of test_smoke.py +def inc(x): + return x + 1 + + +def test_answer(): + assert inc(3) == 5 \ No newline at end of file diff --git a/qa/tf/.modules/featurebase-cluster/main.tf b/qa/tf/.modules/featurebase-cluster/main.tf index f8c612e49..133226e75 100644 --- a/qa/tf/.modules/featurebase-cluster/main.tf +++ b/qa/tf/.modules/featurebase-cluster/main.tf @@ -46,7 +46,7 @@ resource "aws_instance" "fb_cluster_nodes" { Role = "cluster_node" } - user_data = base64encode(templatefile("${path.module}/setup_cluster_node.sh.tpl", { gitlab_token = var.gitlab_token, cluster_prefix = var.cluster_prefix, node_count = var.fb_data_node_count, fb_cluster_replica_count = var.fb_cluster_replica_count, region = var.region })) + user_data = base64encode(templatefile("${path.module}/setup_cluster_node.sh.tpl", { gitlab_token = var.gitlab_token, branch = var.branch, cluster_prefix = var.cluster_prefix, node_count = var.fb_data_node_count, fb_cluster_replica_count = var.fb_cluster_replica_count, region = var.region })) } resource "aws_instance" "fb_ingest" { @@ -79,7 +79,7 @@ resource "aws_instance" "fb_ingest" { Role = "ingest_node" } - user_data = base64encode(templatefile("${path.module}/setup_ingest_node.sh.tpl", { gitlab_token = var.gitlab_token, cluster_prefix = var.cluster_prefix, node_count = var.fb_ingest_node_count, this_node = count.index, region = var.region })) + user_data = base64encode(templatefile("${path.module}/setup_ingest_node.sh.tpl", { gitlab_token = var.gitlab_token, branch = var.branch, cluster_prefix = var.cluster_prefix, node_count = var.fb_ingest_node_count, this_node = count.index, region = var.region })) } resource "aws_key_pair" "gitlab-featurebase-ci" { diff --git a/qa/tf/.modules/featurebase-cluster/setup_cluster_node.sh.tpl b/qa/tf/.modules/featurebase-cluster/setup_cluster_node.sh.tpl index 05d55d973..9a7e71a93 100644 --- a/qa/tf/.modules/featurebase-cluster/setup_cluster_node.sh.tpl +++ b/qa/tf/.modules/featurebase-cluster/setup_cluster_node.sh.tpl @@ -164,7 +164,7 @@ write_featurebase_service_file #get the featurebase binary and put in in the right spot echo "Getting featurebase binary..." -curl --header "PRIVATE-TOKEN: ${gitlab_token}" -o "/home/ec2-user/featurebase_linux_arm64" https://gitlab.com/api/v4/projects/molecula%2Ffeaturebase/jobs/artifacts/master/raw/featurebase_linux_arm64?job=build%20for%20linux%20arm64 +curl --fail --header "PRIVATE-TOKEN: ${gitlab_token}" -o "/home/ec2-user/featurebase_linux_arm64" https://gitlab.com/api/v4/projects/molecula%2Ffeaturebase/jobs/artifacts/${branch}/raw/featurebase_linux_arm64?job=build%20for%20linux%20arm64 chown ec2-user:ec2-user "/home/ec2-user/featurebase_linux_arm64" chmod ugo+x "/home/ec2-user/featurebase_linux_arm64" diff --git a/qa/tf/.modules/featurebase-cluster/setup_ingest_node.sh.tpl b/qa/tf/.modules/featurebase-cluster/setup_ingest_node.sh.tpl index 1e93149b1..8c032f0e5 100644 --- a/qa/tf/.modules/featurebase-cluster/setup_ingest_node.sh.tpl +++ b/qa/tf/.modules/featurebase-cluster/setup_ingest_node.sh.tpl @@ -48,7 +48,7 @@ get_aws_instance_id wait_on_all_ingest_ips echo "Getting featurebase binary..." -curl --header "PRIVATE-TOKEN: ${gitlab_token}" -o "/home/ec2-user/featurebase_linux_arm64" https://gitlab.com/api/v4/projects/molecula%2Ffeaturebase/jobs/artifacts/master/raw/featurebase_linux_arm64?job=build%20for%20linux%20arm64 +curl --fail --header "PRIVATE-TOKEN: ${gitlab_token}" -o "/home/ec2-user/featurebase_linux_arm64" https://gitlab.com/api/v4/projects/molecula%2Ffeaturebase/jobs/artifacts/${branch}/raw/featurebase_linux_arm64?job=build%20for%20linux%20arm64 chown ec2-user:ec2-user "/home/ec2-user/featurebase_linux_arm64" chmod ugo+x "/home/ec2-user/featurebase_linux_arm64" @@ -62,5 +62,11 @@ sudo mount /dev/nvme1n1 /data sudo chown -R ec2-user /data +echo "Installing pytest" +pip3 install -U pytest +pip3 install -U requests + +echo "Done." + diff --git a/qa/tf/.modules/featurebase-cluster/variables.tf b/qa/tf/.modules/featurebase-cluster/variables.tf index dd0da90c7..dcb159820 100644 --- a/qa/tf/.modules/featurebase-cluster/variables.tf +++ b/qa/tf/.modules/featurebase-cluster/variables.tf @@ -91,3 +91,8 @@ variable "gitlab_token" { description = "Gitlab API token" type = string } + +variable "branch" { + description = "The branch we are on" + type = string +} diff --git a/qa/tf/ci/singlenode/main.tf b/qa/tf/ci/smoketest/main.tf similarity index 77% rename from qa/tf/ci/singlenode/main.tf rename to qa/tf/ci/smoketest/main.tf index d50f41182..80753a1f9 100644 --- a/qa/tf/ci/singlenode/main.tf +++ b/qa/tf/ci/smoketest/main.tf @@ -1,10 +1,11 @@ module "ci-cluster" { source = "../../.modules/featurebase-cluster" - cluster_prefix = "ci-single-node" + cluster_prefix = var.cluster_prefix region = var.region profile = var.profile fb_data_node_type = "m6g.large" fb_data_node_count = 1 gitlab_token = var.gitlab_token + branch = var.branch } diff --git a/qa/tf/ci/singlenode/outputs.tf b/qa/tf/ci/smoketest/outputs.tf similarity index 100% rename from qa/tf/ci/singlenode/outputs.tf rename to qa/tf/ci/smoketest/outputs.tf diff --git a/qa/tf/ci/singlenode/provider.tf b/qa/tf/ci/smoketest/provider.tf similarity index 100% rename from qa/tf/ci/singlenode/provider.tf rename to qa/tf/ci/smoketest/provider.tf diff --git a/qa/tf/ci/smoketest/terraform.tfstate.backup b/qa/tf/ci/smoketest/terraform.tfstate.backup new file mode 100644 index 000000000..f2f74f7a8 --- /dev/null +++ b/qa/tf/ci/smoketest/terraform.tfstate.backup @@ -0,0 +1,1596 @@ +{ + "version": 4, + "terraform_version": "1.1.2", + "serial": 255, + "lineage": "bf91272c-d504-5c98-ce46-2ced1888bf74", + "outputs": { + "data_node_ips": { + "value": [ + "10.0.1.152" + ], + "type": [ + "tuple", + [ + "string" + ] + ] + }, + "ingest_ips": { + "value": [ + "3.128.203.136" + ], + "type": [ + "tuple", + [ + "string" + ] + ] + } + }, + "resources": [ + { + "module": "module.ci-cluster", + "mode": "data", + "type": "aws_ami", + "name": "amazon_linux_2", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "architecture": "arm64", + "arn": "arn:aws:ec2:us-east-2::image/ami-0b09f36be67d32fff", + "block_device_mappings": [ + { + "device_name": "/dev/xvda", + "ebs": { + "delete_on_termination": "true", + "encrypted": "false", + "iops": "0", + "snapshot_id": "snap-0617b00e90bae012b", + "throughput": "0", + "volume_size": "8", + "volume_type": "gp2" + }, + "no_device": "", + "virtual_name": "" + } + ], + "creation_date": "2021-12-01T19:36:11.000Z", + "description": "Amazon Linux 2 LTS Arm64 AMI 2.0.20211201.0 arm64 HVM gp2", + "ena_support": true, + "executable_users": null, + "filter": [ + { + "name": "architecture", + "values": [ + "arm64" + ] + }, + { + "name": "name", + "values": [ + "amzn2-ami-hvm-*" + ] + }, + { + "name": "virtualization-type", + "values": [ + "hvm" + ] + } + ], + "hypervisor": "xen", + "id": "ami-0b09f36be67d32fff", + "image_id": "ami-0b09f36be67d32fff", + "image_location": "amazon/amzn2-ami-hvm-2.0.20211201.0-arm64-gp2", + "image_owner_alias": "amazon", + "image_type": "machine", + "kernel_id": null, + "most_recent": true, + "name": "amzn2-ami-hvm-2.0.20211201.0-arm64-gp2", + "name_regex": null, + "owner_id": "137112412989", + "owners": [ + "amazon" + ], + "platform": null, + "platform_details": "Linux/UNIX", + "product_codes": [], + "public": true, + "ramdisk_id": null, + "root_device_name": "/dev/xvda", + "root_device_type": "ebs", + "root_snapshot_id": "snap-0617b00e90bae012b", + "sriov_net_support": "simple", + "state": "available", + "state_reason": { + "code": "UNSET", + "message": "UNSET" + }, + "tags": {}, + "usage_operation": "RunInstances", + "virtualization_type": "hvm" + }, + "sensitive_attributes": [] + } + ] + }, + { + "module": "module.ci-cluster", + "mode": "managed", + "type": "aws_iam_instance_profile", + "name": "fb_cluster_node_profile", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "arn": "arn:aws:iam::941206295814:instance-profile/fb_cluster_node_profile", + "create_date": "2022-01-05T19:35:38Z", + "id": "fb_cluster_node_profile", + "name": "fb_cluster_node_profile", + "name_prefix": null, + "path": "/", + "role": "fb_cluster_node", + "tags": null, + "tags_all": {}, + "unique_id": "AIPA5WJCEKUDB6OWYZPTW" + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "module.ci-cluster.aws_iam_role.fb_cluster_node_role" + ] + } + ] + }, + { + "module": "module.ci-cluster", + "mode": "managed", + "type": "aws_iam_role", + "name": "fb_cluster_node_role", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "arn": "arn:aws:iam::941206295814:role/fb_cluster_node", + "assume_role_policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Sid\":\"\",\"Effect\":\"Allow\",\"Principal\":{\"Service\":\"ec2.amazonaws.com\"},\"Action\":\"sts:AssumeRole\"}]}", + "create_date": "2022-01-05T19:35:35Z", + "description": "", + "force_detach_policies": false, + "id": "fb_cluster_node", + "inline_policy": [ + { + "name": "ec2_read_all", + "policy": "{\"Statement\":[{\"Action\":[\"ec2:Describe*\"],\"Effect\":\"Allow\",\"Resource\":\"*\"}],\"Version\":\"2012-10-17\"}" + } + ], + "managed_policy_arns": [], + "max_session_duration": 3600, + "name": "fb_cluster_node", + "name_prefix": "", + "path": "/", + "permissions_boundary": null, + "tags": null, + "tags_all": {}, + "unique_id": "AROA5WJCEKUDN4ZISIR3N" + }, + "sensitive_attributes": [], + "private": "bnVsbA==" + } + ] + }, + { + "module": "module.ci-cluster", + "mode": "managed", + "type": "aws_instance", + "name": "fb_cluster_nodes", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "index_key": 0, + "schema_version": 1, + "attributes": { + "ami": "ami-0b09f36be67d32fff", + "arn": "arn:aws:ec2:us-east-2:941206295814:instance/i-08ac53ed9d7ac2ac1", + "associate_public_ip_address": false, + "availability_zone": "us-east-2a", + "capacity_reservation_specification": [ + { + "capacity_reservation_preference": "open", + "capacity_reservation_target": [] + } + ], + "cpu_core_count": 2, + "cpu_threads_per_core": 1, + "credit_specification": [], + "disable_api_termination": false, + "ebs_block_device": [ + { + "delete_on_termination": true, + "device_name": "/dev/sdb", + "encrypted": false, + "iops": 3000, + "kms_key_id": "", + "snapshot_id": "", + "tags": {}, + "throughput": 125, + "volume_id": "vol-0041b5d58bde3da13", + "volume_size": 100, + "volume_type": "gp3" + } + ], + "ebs_optimized": false, + "enclave_options": [ + { + "enabled": false + } + ], + "ephemeral_block_device": [], + "get_password_data": false, + "hibernation": false, + "host_id": null, + "iam_instance_profile": "fb_cluster_node_profile", + "id": "i-08ac53ed9d7ac2ac1", + "instance_initiated_shutdown_behavior": "stop", + "instance_state": "running", + "instance_type": "m6g.large", + "ipv6_address_count": 0, + "ipv6_addresses": [], + "key_name": "gitlab-featurebase-ci", + "launch_template": [], + "metadata_options": [ + { + "http_endpoint": "enabled", + "http_put_response_hop_limit": 1, + "http_tokens": "optional" + } + ], + "monitoring": true, + "network_interface": [], + "outpost_arn": "", + "password_data": "", + "placement_group": "", + "placement_partition_number": null, + "primary_network_interface_id": "eni-0b00c3636cad95ae5", + "private_dns": "ip-10-0-1-152.us-east-2.compute.internal", + "private_ip": "10.0.1.152", + "public_dns": "", + "public_ip": "", + "root_block_device": [ + { + "delete_on_termination": true, + "device_name": "/dev/xvda", + "encrypted": false, + "iops": 3000, + "kms_key_id": "", + "tags": null, + "throughput": 125, + "volume_id": "vol-0f12ff15c510db547", + "volume_size": 20, + "volume_type": "gp3" + } + ], + "secondary_private_ips": [], + "security_groups": [], + "source_dest_check": true, + "subnet_id": "subnet-0dcb2339122bafc95", + "tags": { + "Name": "smoke-X8A0attf2zz6EhnV-featurebase-cluster-0", + "Prefix": "smoke-X8A0attf2zz6EhnV", + "Role": "cluster_node" + }, + "tags_all": { + "Name": "smoke-X8A0attf2zz6EhnV-featurebase-cluster-0", + "Prefix": "smoke-X8A0attf2zz6EhnV", + "Role": "cluster_node" + }, + "tenancy": "default", + "timeouts": null, + "user_data": "efedaaed014dc69321f5e633ebc5dc436baf7b1e", + "user_data_base64": null, + "volume_tags": null, + "vpc_security_group_ids": [ + "sg-0580fa9cbc0493d77" + ] + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjo2MDAwMDAwMDAwMDAsImRlbGV0ZSI6MTIwMDAwMDAwMDAwMCwidXBkYXRlIjo2MDAwMDAwMDAwMDB9LCJzY2hlbWFfdmVyc2lvbiI6IjEifQ==", + "dependencies": [ + "module.ci-cluster.aws_iam_instance_profile.fb_cluster_node_profile", + "module.ci-cluster.aws_iam_role.fb_cluster_node_role", + "module.ci-cluster.aws_key_pair.gitlab-featurebase-ci", + "module.ci-cluster.aws_security_group.featurebase", + "module.ci-cluster.data.aws_ami.amazon_linux_2", + "module.ci-cluster.module.vpc.aws_subnet.private", + "module.ci-cluster.module.vpc.aws_vpc.this", + "module.ci-cluster.module.vpc.aws_vpc_ipv4_cidr_block_association.this" + ] + } + ] + }, + { + "module": "module.ci-cluster", + "mode": "managed", + "type": "aws_instance", + "name": "fb_ingest", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "index_key": 0, + "schema_version": 1, + "attributes": { + "ami": "ami-0b09f36be67d32fff", + "arn": "arn:aws:ec2:us-east-2:941206295814:instance/i-07f30af63526e9a6b", + "associate_public_ip_address": true, + "availability_zone": "us-east-2a", + "capacity_reservation_specification": [ + { + "capacity_reservation_preference": "open", + "capacity_reservation_target": [] + } + ], + "cpu_core_count": 8, + "cpu_threads_per_core": 1, + "credit_specification": [], + "disable_api_termination": false, + "ebs_block_device": [ + { + "delete_on_termination": true, + "device_name": "/dev/sdb", + "encrypted": false, + "iops": 3000, + "kms_key_id": "", + "snapshot_id": "", + "tags": {}, + "throughput": 125, + "volume_id": "vol-0a7270e5a68789fbb", + "volume_size": 100, + "volume_type": "gp3" + } + ], + "ebs_optimized": false, + "enclave_options": [ + { + "enabled": false + } + ], + "ephemeral_block_device": [], + "get_password_data": false, + "hibernation": false, + "host_id": null, + "iam_instance_profile": "fb_cluster_node_profile", + "id": "i-07f30af63526e9a6b", + "instance_initiated_shutdown_behavior": "stop", + "instance_state": "running", + "instance_type": "c6g.2xlarge", + "ipv6_address_count": 0, + "ipv6_addresses": [], + "key_name": "gitlab-featurebase-ci", + "launch_template": [], + "metadata_options": [ + { + "http_endpoint": "enabled", + "http_put_response_hop_limit": 1, + "http_tokens": "optional" + } + ], + "monitoring": true, + "network_interface": [], + "outpost_arn": "", + "password_data": "", + "placement_group": "", + "placement_partition_number": null, + "primary_network_interface_id": "eni-0c855451ce29250c9", + "private_dns": "ip-10-0-101-214.us-east-2.compute.internal", + "private_ip": "10.0.101.214", + "public_dns": "", + "public_ip": "3.128.203.136", + "root_block_device": [ + { + "delete_on_termination": true, + "device_name": "/dev/xvda", + "encrypted": false, + "iops": 3000, + "kms_key_id": "", + "tags": null, + "throughput": 125, + "volume_id": "vol-021c14b5593bdd999", + "volume_size": 20, + "volume_type": "gp3" + } + ], + "secondary_private_ips": [], + "security_groups": [], + "source_dest_check": true, + "subnet_id": "subnet-09f6ef1380254341b", + "tags": { + "Name": "smoke-X8A0attf2zz6EhnV-featurebase-ingest-0", + "Prefix": "smoke-X8A0attf2zz6EhnV", + "Role": "ingest_node" + }, + "tags_all": { + "Name": "smoke-X8A0attf2zz6EhnV-featurebase-ingest-0", + "Prefix": "smoke-X8A0attf2zz6EhnV", + "Role": "ingest_node" + }, + "tenancy": "default", + "timeouts": null, + "user_data": "38da76e780fedb19ef551fe9d3c84540d1823453", + "user_data_base64": null, + "volume_tags": null, + "vpc_security_group_ids": [ + "sg-08089f99e02a87452" + ] + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjo2MDAwMDAwMDAwMDAsImRlbGV0ZSI6MTIwMDAwMDAwMDAwMCwidXBkYXRlIjo2MDAwMDAwMDAwMDB9LCJzY2hlbWFfdmVyc2lvbiI6IjEifQ==", + "dependencies": [ + "module.ci-cluster.aws_iam_instance_profile.fb_cluster_node_profile", + "module.ci-cluster.aws_iam_role.fb_cluster_node_role", + "module.ci-cluster.aws_key_pair.gitlab-featurebase-ci", + "module.ci-cluster.aws_security_group.ingest", + "module.ci-cluster.data.aws_ami.amazon_linux_2", + "module.ci-cluster.module.vpc.aws_subnet.public", + "module.ci-cluster.module.vpc.aws_vpc.this", + "module.ci-cluster.module.vpc.aws_vpc_ipv4_cidr_block_association.this" + ] + } + ] + }, + { + "module": "module.ci-cluster", + "mode": "managed", + "type": "aws_key_pair", + "name": "gitlab-featurebase-ci", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 1, + "attributes": { + "arn": "arn:aws:ec2:us-east-2:941206295814:key-pair/gitlab-featurebase-ci", + "fingerprint": "69:e5:4b:15:8d:1f:22:61:de:08:a9:ee:f3:29:5c:69", + "id": "gitlab-featurebase-ci", + "key_name": "gitlab-featurebase-ci", + "key_name_prefix": "", + "key_pair_id": "key-0a36382ce2a87c44e", + "public_key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC91hhpVHNonAG7ku2ugpxEskf9KHeyHJPQJT26OHrMUw7R+T5A8TjqSzTau07sXQ/E9SO3ebV8SJ5PqeaQOnQB8VEvVNK0DjQH7ppvNg1Rfs42FZT9ttzTMvOjsSbK3vZTHXdoKQEdC9NxBwSkFIRGQojK1HUOq9xGrw31fA1OjSwlpLcbx7yyg18lcqW6UOptnVR8U9Yy9qQ5jZF1HtkQ6L9J+gv4o1UyNAUK2bopeGiXpBc3PQ/CFaFT2h/aqLBP66qAHsHVyAFD3PIRtplC5EHa8jXDgLacEls0uF7Q3kRPxvzcuo4g4VkOn1rDy9qH3vd2hT3aKVnM73FIDUiL", + "tags": null, + "tags_all": {} + }, + "sensitive_attributes": [], + "private": "eyJzY2hlbWFfdmVyc2lvbiI6IjEifQ==" + } + ] + }, + { + "module": "module.ci-cluster", + "mode": "managed", + "type": "aws_security_group", + "name": "featurebase", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 1, + "attributes": { + "arn": "arn:aws:ec2:us-east-2:941206295814:security-group/sg-0580fa9cbc0493d77", + "description": "Allow featurebase inbound traffic", + "egress": [ + { + "cidr_blocks": [ + "0.0.0.0/0" + ], + "description": "", + "from_port": 0, + "ipv6_cidr_blocks": [ + "::/0" + ], + "prefix_list_ids": [], + "protocol": "-1", + "security_groups": [], + "self": false, + "to_port": 0 + } + ], + "id": "sg-0580fa9cbc0493d77", + "ingress": [ + { + "cidr_blocks": [ + "0.0.0.0/0" + ], + "description": "SSH", + "from_port": 22, + "ipv6_cidr_blocks": [ + "::/0" + ], + "prefix_list_ids": [], + "protocol": "tcp", + "security_groups": [], + "self": false, + "to_port": 22 + }, + { + "cidr_blocks": [ + "10.0.0.0/16" + ], + "description": "GRPC from Internal", + "from_port": 20101, + "ipv6_cidr_blocks": [], + "prefix_list_ids": [], + "protocol": "tcp", + "security_groups": [], + "self": false, + "to_port": 20101 + }, + { + "cidr_blocks": [ + "10.0.0.0/16" + ], + "description": "PostgreSQL from Internal", + "from_port": 55432, + "ipv6_cidr_blocks": [], + "prefix_list_ids": [], + "protocol": "tcp", + "security_groups": [], + "self": false, + "to_port": 55432 + }, + { + "cidr_blocks": [ + "10.0.0.0/16" + ], + "description": "TLS from Internal", + "from_port": 10101, + "ipv6_cidr_blocks": [], + "prefix_list_ids": [], + "protocol": "tcp", + "security_groups": [], + "self": false, + "to_port": 10101 + }, + { + "cidr_blocks": [ + "10.0.0.0/16" + ], + "description": "etcd from internal 2", + "from_port": 10401, + "ipv6_cidr_blocks": [], + "prefix_list_ids": [], + "protocol": "tcp", + "security_groups": [], + "self": false, + "to_port": 10401 + }, + { + "cidr_blocks": [ + "10.0.0.0/16" + ], + "description": "etcd from internal", + "from_port": 10301, + "ipv6_cidr_blocks": [], + "prefix_list_ids": [], + "protocol": "tcp", + "security_groups": [], + "self": false, + "to_port": 10301 + } + ], + "name": "allow_featurebase", + "name_prefix": "", + "owner_id": "941206295814", + "revoke_rules_on_delete": false, + "tags": { + "Name": "allow_featurebase" + }, + "tags_all": { + "Name": "allow_featurebase" + }, + "timeouts": null, + "vpc_id": "vpc-035fcf75548026b23" + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjo2MDAwMDAwMDAwMDAsImRlbGV0ZSI6OTAwMDAwMDAwMDAwfSwic2NoZW1hX3ZlcnNpb24iOiIxIn0=", + "dependencies": [ + "module.ci-cluster.module.vpc.aws_vpc.this" + ] + } + ] + }, + { + "module": "module.ci-cluster", + "mode": "managed", + "type": "aws_security_group", + "name": "ingest", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 1, + "attributes": { + "arn": "arn:aws:ec2:us-east-2:941206295814:security-group/sg-08089f99e02a87452", + "description": "Allow ingest inbound traffic", + "egress": [ + { + "cidr_blocks": [ + "0.0.0.0/0" + ], + "description": "", + "from_port": 0, + "ipv6_cidr_blocks": [ + "::/0" + ], + "prefix_list_ids": [], + "protocol": "-1", + "security_groups": [], + "self": false, + "to_port": 0 + } + ], + "id": "sg-08089f99e02a87452", + "ingress": [ + { + "cidr_blocks": [ + "0.0.0.0/0" + ], + "description": "", + "from_port": 10101, + "ipv6_cidr_blocks": [ + "::/0" + ], + "prefix_list_ids": [], + "protocol": "tcp", + "security_groups": [], + "self": false, + "to_port": 10101 + }, + { + "cidr_blocks": [ + "0.0.0.0/0" + ], + "description": "SSH", + "from_port": 22, + "ipv6_cidr_blocks": [ + "::/0" + ], + "prefix_list_ids": [], + "protocol": "tcp", + "security_groups": [], + "self": false, + "to_port": 22 + } + ], + "name": "allow_ingest", + "name_prefix": "", + "owner_id": "941206295814", + "revoke_rules_on_delete": false, + "tags": { + "Name": "allow_ingest" + }, + "tags_all": { + "Name": "allow_ingest" + }, + "timeouts": null, + "vpc_id": "vpc-035fcf75548026b23" + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjo2MDAwMDAwMDAwMDAsImRlbGV0ZSI6OTAwMDAwMDAwMDAwfSwic2NoZW1hX3ZlcnNpb24iOiIxIn0=", + "dependencies": [ + "module.ci-cluster.module.vpc.aws_vpc.this" + ] + } + ] + }, + { + "module": "module.ci-cluster.module.vpc", + "mode": "managed", + "type": "aws_eip", + "name": "nat", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "index_key": 0, + "schema_version": 0, + "attributes": { + "address": null, + "allocation_id": "eipalloc-099cedbe3c5820bbe", + "associate_with_private_ip": null, + "association_id": "", + "carrier_ip": "", + "customer_owned_ip": "", + "customer_owned_ipv4_pool": "", + "domain": "vpc", + "id": "eipalloc-099cedbe3c5820bbe", + "instance": "", + "network_border_group": "us-east-2", + "network_interface": "", + "private_dns": null, + "private_ip": "", + "public_dns": "ec2-3-17-85-138.us-east-2.compute.amazonaws.com", + "public_ip": "3.17.85.138", + "public_ipv4_pool": "amazon", + "tags": { + "Name": "smoke-X8A0attf2zz6EhnV" + }, + "tags_all": { + "Name": "smoke-X8A0attf2zz6EhnV" + }, + "timeouts": null, + "vpc": true + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiZGVsZXRlIjoxODAwMDAwMDAwMDAsInJlYWQiOjkwMDAwMDAwMDAwMCwidXBkYXRlIjozMDAwMDAwMDAwMDB9fQ==" + }, + { + "index_key": 1, + "schema_version": 0, + "attributes": { + "address": null, + "allocation_id": "eipalloc-0fca65223cdfd313a", + "associate_with_private_ip": null, + "association_id": "", + "carrier_ip": "", + "customer_owned_ip": "", + "customer_owned_ipv4_pool": "", + "domain": "vpc", + "id": "eipalloc-0fca65223cdfd313a", + "instance": "", + "network_border_group": "us-east-2", + "network_interface": "", + "private_dns": null, + "private_ip": "", + "public_dns": "ec2-3-135-112-2.us-east-2.compute.amazonaws.com", + "public_ip": "3.135.112.2", + "public_ipv4_pool": "amazon", + "tags": { + "Name": "smoke-X8A0attf2zz6EhnV" + }, + "tags_all": { + "Name": "smoke-X8A0attf2zz6EhnV" + }, + "timeouts": null, + "vpc": true + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiZGVsZXRlIjoxODAwMDAwMDAwMDAsInJlYWQiOjkwMDAwMDAwMDAwMCwidXBkYXRlIjozMDAwMDAwMDAwMDB9fQ==" + }, + { + "index_key": 2, + "schema_version": 0, + "attributes": { + "address": null, + "allocation_id": "eipalloc-09323e2a47e394df7", + "associate_with_private_ip": null, + "association_id": "", + "carrier_ip": "", + "customer_owned_ip": "", + "customer_owned_ipv4_pool": "", + "domain": "vpc", + "id": "eipalloc-09323e2a47e394df7", + "instance": "", + "network_border_group": "us-east-2", + "network_interface": "", + "private_dns": null, + "private_ip": "", + "public_dns": "ec2-3-128-36-233.us-east-2.compute.amazonaws.com", + "public_ip": "3.128.36.233", + "public_ipv4_pool": "amazon", + "tags": { + "Name": "smoke-X8A0attf2zz6EhnV" + }, + "tags_all": { + "Name": "smoke-X8A0attf2zz6EhnV" + }, + "timeouts": null, + "vpc": true + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiZGVsZXRlIjoxODAwMDAwMDAwMDAsInJlYWQiOjkwMDAwMDAwMDAwMCwidXBkYXRlIjozMDAwMDAwMDAwMDB9fQ==" + } + ] + }, + { + "module": "module.ci-cluster.module.vpc", + "mode": "managed", + "type": "aws_internet_gateway", + "name": "this", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "index_key": 0, + "schema_version": 0, + "attributes": { + "arn": "arn:aws:ec2:us-east-2:941206295814:internet-gateway/igw-069e52ef204598fdd", + "id": "igw-069e52ef204598fdd", + "owner_id": "941206295814", + "tags": { + "Name": "smoke-X8A0attf2zz6EhnV" + }, + "tags_all": { + "Name": "smoke-X8A0attf2zz6EhnV" + }, + "vpc_id": "vpc-035fcf75548026b23" + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "module.ci-cluster.module.vpc.aws_vpc.this", + "module.ci-cluster.module.vpc.aws_vpc_ipv4_cidr_block_association.this" + ] + } + ] + }, + { + "module": "module.ci-cluster.module.vpc", + "mode": "managed", + "type": "aws_nat_gateway", + "name": "this", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "index_key": 0, + "schema_version": 0, + "attributes": { + "allocation_id": "eipalloc-099cedbe3c5820bbe", + "connectivity_type": "public", + "id": "nat-01960cd93ff94dc13", + "network_interface_id": "eni-0da9e0562c5e62a35", + "private_ip": "10.0.101.138", + "public_ip": "3.17.85.138", + "subnet_id": "subnet-09f6ef1380254341b", + "tags": { + "Name": "smoke-X8A0attf2zz6EhnV" + }, + "tags_all": { + "Name": "smoke-X8A0attf2zz6EhnV" + } + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "module.ci-cluster.module.vpc.aws_eip.nat", + "module.ci-cluster.module.vpc.aws_internet_gateway.this", + "module.ci-cluster.module.vpc.aws_subnet.public", + "module.ci-cluster.module.vpc.aws_vpc.this", + "module.ci-cluster.module.vpc.aws_vpc_ipv4_cidr_block_association.this" + ] + }, + { + "index_key": 1, + "schema_version": 0, + "attributes": { + "allocation_id": "eipalloc-0fca65223cdfd313a", + "connectivity_type": "public", + "id": "nat-03883494dfdfcc234", + "network_interface_id": "eni-0f234f97b35663c64", + "private_ip": "10.0.102.238", + "public_ip": "3.135.112.2", + "subnet_id": "subnet-063459780d5b84c86", + "tags": { + "Name": "smoke-X8A0attf2zz6EhnV" + }, + "tags_all": { + "Name": "smoke-X8A0attf2zz6EhnV" + } + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "module.ci-cluster.module.vpc.aws_eip.nat", + "module.ci-cluster.module.vpc.aws_internet_gateway.this", + "module.ci-cluster.module.vpc.aws_subnet.public", + "module.ci-cluster.module.vpc.aws_vpc.this", + "module.ci-cluster.module.vpc.aws_vpc_ipv4_cidr_block_association.this" + ] + }, + { + "index_key": 2, + "schema_version": 0, + "attributes": { + "allocation_id": "eipalloc-09323e2a47e394df7", + "connectivity_type": "public", + "id": "nat-05bbd0bcd20938896", + "network_interface_id": "eni-0a36a85d8383cac8e", + "private_ip": "10.0.103.228", + "public_ip": "3.128.36.233", + "subnet_id": "subnet-0eae785053155c08b", + "tags": { + "Name": "smoke-X8A0attf2zz6EhnV" + }, + "tags_all": { + "Name": "smoke-X8A0attf2zz6EhnV" + } + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "module.ci-cluster.module.vpc.aws_eip.nat", + "module.ci-cluster.module.vpc.aws_internet_gateway.this", + "module.ci-cluster.module.vpc.aws_subnet.public", + "module.ci-cluster.module.vpc.aws_vpc.this", + "module.ci-cluster.module.vpc.aws_vpc_ipv4_cidr_block_association.this" + ] + } + ] + }, + { + "module": "module.ci-cluster.module.vpc", + "mode": "managed", + "type": "aws_route", + "name": "private_nat_gateway", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "index_key": 0, + "schema_version": 0, + "attributes": { + "carrier_gateway_id": "", + "destination_cidr_block": "0.0.0.0/0", + "destination_ipv6_cidr_block": "", + "destination_prefix_list_id": "", + "egress_only_gateway_id": "", + "gateway_id": "", + "id": "r-rtb-06122e232dba3e71d1080289494", + "instance_id": "", + "instance_owner_id": "", + "local_gateway_id": "", + "nat_gateway_id": "nat-01960cd93ff94dc13", + "network_interface_id": "", + "origin": "CreateRoute", + "route_table_id": "rtb-06122e232dba3e71d", + "state": "active", + "timeouts": { + "create": "5m", + "delete": null, + "update": null + }, + "transit_gateway_id": "", + "vpc_endpoint_id": "", + "vpc_peering_connection_id": "" + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjozMDAwMDAwMDAwMDAsImRlbGV0ZSI6MzAwMDAwMDAwMDAwLCJ1cGRhdGUiOjEyMDAwMDAwMDAwMH19", + "dependencies": [ + "module.ci-cluster.module.vpc.aws_eip.nat", + "module.ci-cluster.module.vpc.aws_internet_gateway.this", + "module.ci-cluster.module.vpc.aws_nat_gateway.this", + "module.ci-cluster.module.vpc.aws_route_table.private", + "module.ci-cluster.module.vpc.aws_subnet.public", + "module.ci-cluster.module.vpc.aws_vpc.this", + "module.ci-cluster.module.vpc.aws_vpc_ipv4_cidr_block_association.this" + ] + }, + { + "index_key": 1, + "schema_version": 0, + "attributes": { + "carrier_gateway_id": "", + "destination_cidr_block": "0.0.0.0/0", + "destination_ipv6_cidr_block": "", + "destination_prefix_list_id": "", + "egress_only_gateway_id": "", + "gateway_id": "", + "id": "r-rtb-0757c96161330927f1080289494", + "instance_id": "", + "instance_owner_id": "", + "local_gateway_id": "", + "nat_gateway_id": "nat-03883494dfdfcc234", + "network_interface_id": "", + "origin": "CreateRoute", + "route_table_id": "rtb-0757c96161330927f", + "state": "active", + "timeouts": { + "create": "5m", + "delete": null, + "update": null + }, + "transit_gateway_id": "", + "vpc_endpoint_id": "", + "vpc_peering_connection_id": "" + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjozMDAwMDAwMDAwMDAsImRlbGV0ZSI6MzAwMDAwMDAwMDAwLCJ1cGRhdGUiOjEyMDAwMDAwMDAwMH19", + "dependencies": [ + "module.ci-cluster.module.vpc.aws_eip.nat", + "module.ci-cluster.module.vpc.aws_internet_gateway.this", + "module.ci-cluster.module.vpc.aws_nat_gateway.this", + "module.ci-cluster.module.vpc.aws_route_table.private", + "module.ci-cluster.module.vpc.aws_subnet.public", + "module.ci-cluster.module.vpc.aws_vpc.this", + "module.ci-cluster.module.vpc.aws_vpc_ipv4_cidr_block_association.this" + ] + }, + { + "index_key": 2, + "schema_version": 0, + "attributes": { + "carrier_gateway_id": "", + "destination_cidr_block": "0.0.0.0/0", + "destination_ipv6_cidr_block": "", + "destination_prefix_list_id": "", + "egress_only_gateway_id": "", + "gateway_id": "", + "id": "r-rtb-0584788728d6485171080289494", + "instance_id": "", + "instance_owner_id": "", + "local_gateway_id": "", + "nat_gateway_id": "nat-05bbd0bcd20938896", + "network_interface_id": "", + "origin": "CreateRoute", + "route_table_id": "rtb-0584788728d648517", + "state": "active", + "timeouts": { + "create": "5m", + "delete": null, + "update": null + }, + "transit_gateway_id": "", + "vpc_endpoint_id": "", + "vpc_peering_connection_id": "" + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjozMDAwMDAwMDAwMDAsImRlbGV0ZSI6MzAwMDAwMDAwMDAwLCJ1cGRhdGUiOjEyMDAwMDAwMDAwMH19", + "dependencies": [ + "module.ci-cluster.module.vpc.aws_eip.nat", + "module.ci-cluster.module.vpc.aws_internet_gateway.this", + "module.ci-cluster.module.vpc.aws_nat_gateway.this", + "module.ci-cluster.module.vpc.aws_route_table.private", + "module.ci-cluster.module.vpc.aws_subnet.public", + "module.ci-cluster.module.vpc.aws_vpc.this", + "module.ci-cluster.module.vpc.aws_vpc_ipv4_cidr_block_association.this" + ] + } + ] + }, + { + "module": "module.ci-cluster.module.vpc", + "mode": "managed", + "type": "aws_route", + "name": "public_internet_gateway", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "index_key": 0, + "schema_version": 0, + "attributes": { + "carrier_gateway_id": "", + "destination_cidr_block": "0.0.0.0/0", + "destination_ipv6_cidr_block": "", + "destination_prefix_list_id": "", + "egress_only_gateway_id": "", + "gateway_id": "igw-069e52ef204598fdd", + "id": "r-rtb-08768a10ff241df961080289494", + "instance_id": "", + "instance_owner_id": "", + "local_gateway_id": "", + "nat_gateway_id": "", + "network_interface_id": "", + "origin": "CreateRoute", + "route_table_id": "rtb-08768a10ff241df96", + "state": "active", + "timeouts": { + "create": "5m", + "delete": null, + "update": null + }, + "transit_gateway_id": "", + "vpc_endpoint_id": "", + "vpc_peering_connection_id": "" + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjozMDAwMDAwMDAwMDAsImRlbGV0ZSI6MzAwMDAwMDAwMDAwLCJ1cGRhdGUiOjEyMDAwMDAwMDAwMH19", + "dependencies": [ + "module.ci-cluster.module.vpc.aws_internet_gateway.this", + "module.ci-cluster.module.vpc.aws_route_table.public", + "module.ci-cluster.module.vpc.aws_vpc.this", + "module.ci-cluster.module.vpc.aws_vpc_ipv4_cidr_block_association.this" + ] + } + ] + }, + { + "module": "module.ci-cluster.module.vpc", + "mode": "managed", + "type": "aws_route_table", + "name": "private", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "index_key": 0, + "schema_version": 0, + "attributes": { + "arn": "arn:aws:ec2:us-east-2:941206295814:route-table/rtb-06122e232dba3e71d", + "id": "rtb-06122e232dba3e71d", + "owner_id": "941206295814", + "propagating_vgws": [], + "route": [], + "tags": { + "Name": "smoke-X8A0attf2zz6EhnV" + }, + "tags_all": { + "Name": "smoke-X8A0attf2zz6EhnV" + }, + "timeouts": null, + "vpc_id": "vpc-035fcf75548026b23" + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjozMDAwMDAwMDAwMDAsImRlbGV0ZSI6MzAwMDAwMDAwMDAwLCJ1cGRhdGUiOjEyMDAwMDAwMDAwMH19", + "dependencies": [ + "module.ci-cluster.module.vpc.aws_vpc.this", + "module.ci-cluster.module.vpc.aws_vpc_ipv4_cidr_block_association.this" + ] + }, + { + "index_key": 1, + "schema_version": 0, + "attributes": { + "arn": "arn:aws:ec2:us-east-2:941206295814:route-table/rtb-0757c96161330927f", + "id": "rtb-0757c96161330927f", + "owner_id": "941206295814", + "propagating_vgws": [], + "route": [], + "tags": { + "Name": "smoke-X8A0attf2zz6EhnV" + }, + "tags_all": { + "Name": "smoke-X8A0attf2zz6EhnV" + }, + "timeouts": null, + "vpc_id": "vpc-035fcf75548026b23" + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjozMDAwMDAwMDAwMDAsImRlbGV0ZSI6MzAwMDAwMDAwMDAwLCJ1cGRhdGUiOjEyMDAwMDAwMDAwMH19", + "dependencies": [ + "module.ci-cluster.module.vpc.aws_vpc.this", + "module.ci-cluster.module.vpc.aws_vpc_ipv4_cidr_block_association.this" + ] + }, + { + "index_key": 2, + "schema_version": 0, + "attributes": { + "arn": "arn:aws:ec2:us-east-2:941206295814:route-table/rtb-0584788728d648517", + "id": "rtb-0584788728d648517", + "owner_id": "941206295814", + "propagating_vgws": [], + "route": [], + "tags": { + "Name": "smoke-X8A0attf2zz6EhnV" + }, + "tags_all": { + "Name": "smoke-X8A0attf2zz6EhnV" + }, + "timeouts": null, + "vpc_id": "vpc-035fcf75548026b23" + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjozMDAwMDAwMDAwMDAsImRlbGV0ZSI6MzAwMDAwMDAwMDAwLCJ1cGRhdGUiOjEyMDAwMDAwMDAwMH19", + "dependencies": [ + "module.ci-cluster.module.vpc.aws_vpc.this", + "module.ci-cluster.module.vpc.aws_vpc_ipv4_cidr_block_association.this" + ] + } + ] + }, + { + "module": "module.ci-cluster.module.vpc", + "mode": "managed", + "type": "aws_route_table", + "name": "public", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "index_key": 0, + "schema_version": 0, + "attributes": { + "arn": "arn:aws:ec2:us-east-2:941206295814:route-table/rtb-08768a10ff241df96", + "id": "rtb-08768a10ff241df96", + "owner_id": "941206295814", + "propagating_vgws": [], + "route": [], + "tags": { + "Name": "smoke-X8A0attf2zz6EhnV" + }, + "tags_all": { + "Name": "smoke-X8A0attf2zz6EhnV" + }, + "timeouts": null, + "vpc_id": "vpc-035fcf75548026b23" + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjozMDAwMDAwMDAwMDAsImRlbGV0ZSI6MzAwMDAwMDAwMDAwLCJ1cGRhdGUiOjEyMDAwMDAwMDAwMH19", + "dependencies": [ + "module.ci-cluster.module.vpc.aws_vpc.this", + "module.ci-cluster.module.vpc.aws_vpc_ipv4_cidr_block_association.this" + ] + } + ] + }, + { + "module": "module.ci-cluster.module.vpc", + "mode": "managed", + "type": "aws_route_table_association", + "name": "private", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "index_key": 0, + "schema_version": 0, + "attributes": { + "gateway_id": "", + "id": "rtbassoc-0e02368a84fbc483d", + "route_table_id": "rtb-06122e232dba3e71d", + "subnet_id": "subnet-0dcb2339122bafc95" + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "module.ci-cluster.module.vpc.aws_route_table.private", + "module.ci-cluster.module.vpc.aws_subnet.private", + "module.ci-cluster.module.vpc.aws_vpc.this", + "module.ci-cluster.module.vpc.aws_vpc_ipv4_cidr_block_association.this" + ] + }, + { + "index_key": 1, + "schema_version": 0, + "attributes": { + "gateway_id": "", + "id": "rtbassoc-03a94dc33d8ce77ee", + "route_table_id": "rtb-0757c96161330927f", + "subnet_id": "subnet-0981f7f434ca09e8e" + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "module.ci-cluster.module.vpc.aws_route_table.private", + "module.ci-cluster.module.vpc.aws_subnet.private", + "module.ci-cluster.module.vpc.aws_vpc.this", + "module.ci-cluster.module.vpc.aws_vpc_ipv4_cidr_block_association.this" + ] + }, + { + "index_key": 2, + "schema_version": 0, + "attributes": { + "gateway_id": "", + "id": "rtbassoc-0bc6944b6e56191de", + "route_table_id": "rtb-0584788728d648517", + "subnet_id": "subnet-0353db6d8c1c7ad43" + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "module.ci-cluster.module.vpc.aws_route_table.private", + "module.ci-cluster.module.vpc.aws_subnet.private", + "module.ci-cluster.module.vpc.aws_vpc.this", + "module.ci-cluster.module.vpc.aws_vpc_ipv4_cidr_block_association.this" + ] + } + ] + }, + { + "module": "module.ci-cluster.module.vpc", + "mode": "managed", + "type": "aws_route_table_association", + "name": "public", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "index_key": 0, + "schema_version": 0, + "attributes": { + "gateway_id": "", + "id": "rtbassoc-03dfd5b74f16f8cd8", + "route_table_id": "rtb-08768a10ff241df96", + "subnet_id": "subnet-09f6ef1380254341b" + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "module.ci-cluster.module.vpc.aws_route_table.public", + "module.ci-cluster.module.vpc.aws_subnet.public", + "module.ci-cluster.module.vpc.aws_vpc.this", + "module.ci-cluster.module.vpc.aws_vpc_ipv4_cidr_block_association.this" + ] + }, + { + "index_key": 1, + "schema_version": 0, + "attributes": { + "gateway_id": "", + "id": "rtbassoc-0628c1dc5f82384f6", + "route_table_id": "rtb-08768a10ff241df96", + "subnet_id": "subnet-063459780d5b84c86" + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "module.ci-cluster.module.vpc.aws_route_table.public", + "module.ci-cluster.module.vpc.aws_subnet.public", + "module.ci-cluster.module.vpc.aws_vpc.this", + "module.ci-cluster.module.vpc.aws_vpc_ipv4_cidr_block_association.this" + ] + }, + { + "index_key": 2, + "schema_version": 0, + "attributes": { + "gateway_id": "", + "id": "rtbassoc-0b730b5a81f9e2de4", + "route_table_id": "rtb-08768a10ff241df96", + "subnet_id": "subnet-0eae785053155c08b" + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "module.ci-cluster.module.vpc.aws_route_table.public", + "module.ci-cluster.module.vpc.aws_subnet.public", + "module.ci-cluster.module.vpc.aws_vpc.this", + "module.ci-cluster.module.vpc.aws_vpc_ipv4_cidr_block_association.this" + ] + } + ] + }, + { + "module": "module.ci-cluster.module.vpc", + "mode": "managed", + "type": "aws_subnet", + "name": "private", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "index_key": 0, + "schema_version": 1, + "attributes": { + "arn": "arn:aws:ec2:us-east-2:941206295814:subnet/subnet-0dcb2339122bafc95", + "assign_ipv6_address_on_creation": false, + "availability_zone": "us-east-2a", + "availability_zone_id": "use2-az1", + "cidr_block": "10.0.1.0/24", + "customer_owned_ipv4_pool": "", + "id": "subnet-0dcb2339122bafc95", + "ipv6_cidr_block": "", + "ipv6_cidr_block_association_id": "", + "map_customer_owned_ip_on_launch": false, + "map_public_ip_on_launch": false, + "outpost_arn": "", + "owner_id": "941206295814", + "tags": { + "Name": "smoke-X8A0attf2zz6EhnV" + }, + "tags_all": { + "Name": "smoke-X8A0attf2zz6EhnV" + }, + "timeouts": null, + "vpc_id": "vpc-035fcf75548026b23" + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjo2MDAwMDAwMDAwMDAsImRlbGV0ZSI6MTIwMDAwMDAwMDAwMH0sInNjaGVtYV92ZXJzaW9uIjoiMSJ9", + "dependencies": [ + "module.ci-cluster.module.vpc.aws_vpc.this", + "module.ci-cluster.module.vpc.aws_vpc_ipv4_cidr_block_association.this" + ] + }, + { + "index_key": 1, + "schema_version": 1, + "attributes": { + "arn": "arn:aws:ec2:us-east-2:941206295814:subnet/subnet-0981f7f434ca09e8e", + "assign_ipv6_address_on_creation": false, + "availability_zone": "us-east-2b", + "availability_zone_id": "use2-az2", + "cidr_block": "10.0.2.0/24", + "customer_owned_ipv4_pool": "", + "id": "subnet-0981f7f434ca09e8e", + "ipv6_cidr_block": "", + "ipv6_cidr_block_association_id": "", + "map_customer_owned_ip_on_launch": false, + "map_public_ip_on_launch": false, + "outpost_arn": "", + "owner_id": "941206295814", + "tags": { + "Name": "smoke-X8A0attf2zz6EhnV" + }, + "tags_all": { + "Name": "smoke-X8A0attf2zz6EhnV" + }, + "timeouts": null, + "vpc_id": "vpc-035fcf75548026b23" + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjo2MDAwMDAwMDAwMDAsImRlbGV0ZSI6MTIwMDAwMDAwMDAwMH0sInNjaGVtYV92ZXJzaW9uIjoiMSJ9", + "dependencies": [ + "module.ci-cluster.module.vpc.aws_vpc.this", + "module.ci-cluster.module.vpc.aws_vpc_ipv4_cidr_block_association.this" + ] + }, + { + "index_key": 2, + "schema_version": 1, + "attributes": { + "arn": "arn:aws:ec2:us-east-2:941206295814:subnet/subnet-0353db6d8c1c7ad43", + "assign_ipv6_address_on_creation": false, + "availability_zone": "us-east-2c", + "availability_zone_id": "use2-az3", + "cidr_block": "10.0.3.0/24", + "customer_owned_ipv4_pool": "", + "id": "subnet-0353db6d8c1c7ad43", + "ipv6_cidr_block": "", + "ipv6_cidr_block_association_id": "", + "map_customer_owned_ip_on_launch": false, + "map_public_ip_on_launch": false, + "outpost_arn": "", + "owner_id": "941206295814", + "tags": { + "Name": "smoke-X8A0attf2zz6EhnV" + }, + "tags_all": { + "Name": "smoke-X8A0attf2zz6EhnV" + }, + "timeouts": null, + "vpc_id": "vpc-035fcf75548026b23" + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjo2MDAwMDAwMDAwMDAsImRlbGV0ZSI6MTIwMDAwMDAwMDAwMH0sInNjaGVtYV92ZXJzaW9uIjoiMSJ9", + "dependencies": [ + "module.ci-cluster.module.vpc.aws_vpc.this", + "module.ci-cluster.module.vpc.aws_vpc_ipv4_cidr_block_association.this" + ] + } + ] + }, + { + "module": "module.ci-cluster.module.vpc", + "mode": "managed", + "type": "aws_subnet", + "name": "public", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "index_key": 0, + "schema_version": 1, + "attributes": { + "arn": "arn:aws:ec2:us-east-2:941206295814:subnet/subnet-09f6ef1380254341b", + "assign_ipv6_address_on_creation": false, + "availability_zone": "us-east-2a", + "availability_zone_id": "use2-az1", + "cidr_block": "10.0.101.0/24", + "customer_owned_ipv4_pool": "", + "id": "subnet-09f6ef1380254341b", + "ipv6_cidr_block": "", + "ipv6_cidr_block_association_id": "", + "map_customer_owned_ip_on_launch": false, + "map_public_ip_on_launch": true, + "outpost_arn": "", + "owner_id": "941206295814", + "tags": { + "Name": "smoke-X8A0attf2zz6EhnV" + }, + "tags_all": { + "Name": "smoke-X8A0attf2zz6EhnV" + }, + "timeouts": null, + "vpc_id": "vpc-035fcf75548026b23" + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjo2MDAwMDAwMDAwMDAsImRlbGV0ZSI6MTIwMDAwMDAwMDAwMH0sInNjaGVtYV92ZXJzaW9uIjoiMSJ9", + "dependencies": [ + "module.ci-cluster.module.vpc.aws_vpc.this", + "module.ci-cluster.module.vpc.aws_vpc_ipv4_cidr_block_association.this" + ] + }, + { + "index_key": 1, + "schema_version": 1, + "attributes": { + "arn": "arn:aws:ec2:us-east-2:941206295814:subnet/subnet-063459780d5b84c86", + "assign_ipv6_address_on_creation": false, + "availability_zone": "us-east-2b", + "availability_zone_id": "use2-az2", + "cidr_block": "10.0.102.0/24", + "customer_owned_ipv4_pool": "", + "id": "subnet-063459780d5b84c86", + "ipv6_cidr_block": "", + "ipv6_cidr_block_association_id": "", + "map_customer_owned_ip_on_launch": false, + "map_public_ip_on_launch": true, + "outpost_arn": "", + "owner_id": "941206295814", + "tags": { + "Name": "smoke-X8A0attf2zz6EhnV" + }, + "tags_all": { + "Name": "smoke-X8A0attf2zz6EhnV" + }, + "timeouts": null, + "vpc_id": "vpc-035fcf75548026b23" + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjo2MDAwMDAwMDAwMDAsImRlbGV0ZSI6MTIwMDAwMDAwMDAwMH0sInNjaGVtYV92ZXJzaW9uIjoiMSJ9", + "dependencies": [ + "module.ci-cluster.module.vpc.aws_vpc.this", + "module.ci-cluster.module.vpc.aws_vpc_ipv4_cidr_block_association.this" + ] + }, + { + "index_key": 2, + "schema_version": 1, + "attributes": { + "arn": "arn:aws:ec2:us-east-2:941206295814:subnet/subnet-0eae785053155c08b", + "assign_ipv6_address_on_creation": false, + "availability_zone": "us-east-2c", + "availability_zone_id": "use2-az3", + "cidr_block": "10.0.103.0/24", + "customer_owned_ipv4_pool": "", + "id": "subnet-0eae785053155c08b", + "ipv6_cidr_block": "", + "ipv6_cidr_block_association_id": "", + "map_customer_owned_ip_on_launch": false, + "map_public_ip_on_launch": true, + "outpost_arn": "", + "owner_id": "941206295814", + "tags": { + "Name": "smoke-X8A0attf2zz6EhnV" + }, + "tags_all": { + "Name": "smoke-X8A0attf2zz6EhnV" + }, + "timeouts": null, + "vpc_id": "vpc-035fcf75548026b23" + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjo2MDAwMDAwMDAwMDAsImRlbGV0ZSI6MTIwMDAwMDAwMDAwMH0sInNjaGVtYV92ZXJzaW9uIjoiMSJ9", + "dependencies": [ + "module.ci-cluster.module.vpc.aws_vpc.this", + "module.ci-cluster.module.vpc.aws_vpc_ipv4_cidr_block_association.this" + ] + } + ] + }, + { + "module": "module.ci-cluster.module.vpc", + "mode": "managed", + "type": "aws_vpc", + "name": "this", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "index_key": 0, + "schema_version": 1, + "attributes": { + "arn": "arn:aws:ec2:us-east-2:941206295814:vpc/vpc-035fcf75548026b23", + "assign_generated_ipv6_cidr_block": false, + "cidr_block": "10.0.0.0/16", + "default_network_acl_id": "acl-0db479d99cf0759f8", + "default_route_table_id": "rtb-0a264d3195bef2a21", + "default_security_group_id": "sg-025b70c95323fd7e6", + "dhcp_options_id": "dopt-0398725faf4f782c8", + "enable_classiclink": null, + "enable_classiclink_dns_support": null, + "enable_dns_hostnames": false, + "enable_dns_support": true, + "id": "vpc-035fcf75548026b23", + "instance_tenancy": "default", + "ipv4_ipam_pool_id": null, + "ipv4_netmask_length": null, + "ipv6_association_id": "", + "ipv6_cidr_block": "", + "ipv6_ipam_pool_id": null, + "ipv6_netmask_length": null, + "main_route_table_id": "rtb-0a264d3195bef2a21", + "owner_id": "941206295814", + "tags": { + "Name": "smoke-X8A0attf2zz6EhnV" + }, + "tags_all": { + "Name": "smoke-X8A0attf2zz6EhnV" + } + }, + "sensitive_attributes": [], + "private": "eyJzY2hlbWFfdmVyc2lvbiI6IjEifQ==" + } + ] + } + ] +} diff --git a/qa/tf/ci/singlenode/tf.auto.tfvars b/qa/tf/ci/smoketest/tf.auto.tfvars similarity index 100% rename from qa/tf/ci/singlenode/tf.auto.tfvars rename to qa/tf/ci/smoketest/tf.auto.tfvars diff --git a/qa/tf/ci/singlenode/variables.tf b/qa/tf/ci/smoketest/variables.tf similarity index 63% rename from qa/tf/ci/singlenode/variables.tf rename to qa/tf/ci/smoketest/variables.tf index e87a8f517..bab877497 100644 --- a/qa/tf/ci/singlenode/variables.tf +++ b/qa/tf/ci/smoketest/variables.tf @@ -11,4 +11,14 @@ variable "profile" { variable "gitlab_token" { description = "The API token for taking to Gitlab API - expected to come from an env variable." type = string +} + +variable "cluster_prefix" { + type = string + description = "This is a identifier that will be prefixed to created resources" +} + +variable "branch" { + type = string + description = "The branch we are on" } \ No newline at end of file diff --git a/qa/tf/gauntlet/samsung/main.tf b/qa/tf/gauntlet/samsung/main.tf index 926e54cc3..8a5101d7c 100644 --- a/qa/tf/gauntlet/samsung/main.tf +++ b/qa/tf/gauntlet/samsung/main.tf @@ -10,4 +10,5 @@ module "samsung-cluster" { fb_ingest_disk_iops = 10000 fb_ingest_node_count = 1 gitlab_token = var.gitlab_token + branch = var.branch } diff --git a/qa/tf/gauntlet/samsung/samsung-gauntlet.json b/qa/tf/gauntlet/samsung/samsung-gauntlet.json deleted file mode 100644 index 8760c7322..000000000 --- a/qa/tf/gauntlet/samsung/samsung-gauntlet.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - "data_node_ips": { - "sensitive": false, - "type": [ - "tuple", - [ - "string", - "string", - "string" - ] - ], - "value": [ - "10.0.1.144", - "10.0.2.108", - "10.0.3.178" - ] - }, - "ingest_ips": { - "sensitive": false, - "type": [ - "tuple", - [ - "string" - ] - ], - "value": [ - "3.145.104.76" - ] - } -} diff --git a/qa/tf/gauntlet/samsung/variables.tf b/qa/tf/gauntlet/samsung/variables.tf index e87a8f517..bab877497 100644 --- a/qa/tf/gauntlet/samsung/variables.tf +++ b/qa/tf/gauntlet/samsung/variables.tf @@ -11,4 +11,14 @@ variable "profile" { variable "gitlab_token" { description = "The API token for taking to Gitlab API - expected to come from an env variable." type = string +} + +variable "cluster_prefix" { + type = string + description = "This is a identifier that will be prefixed to created resources" +} + +variable "branch" { + type = string + description = "The branch we are on" } \ No newline at end of file