diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index c0e279b47..3f1b4eb0a 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -219,7 +219,7 @@ smoke test: - docker - fbsmoke rules: - - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"' + - if: '$CI_PIPELINE_SOURCE == "push"' before_script: - apt-get update && apt-get install -y gnupg software-properties-common curl git - curl -fsSL https://apt.releases.hashicorp.com/gpg | apt-key add - @@ -262,49 +262,55 @@ smoke test: reports: junit: report.xml -# gauntlet: -# stage: gauntlet -# timeout: 4h -# image: registry.gitlab.com/gitlab-org/cloud-deploy/aws-base:latest -# variables: -# PROFILE: "default" -# AWS_SSH_PRIVATE_KEY: $AWS_FBCI_SSH_KEY -# AWS_ACCESS_KEY_ID: $AWS_FBCI_ACCESS_KEY_ID -# AWS_SECRET_ACCESS_KEY: $AWS_FBCI_SECRET_ACCESS_KEY -# TF_VAR_cluster_prefix: "" -# TF_VAR_branch: "" -# rules: -# - if: '$CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"' -# - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH' -# before_script: -# - apt-get update && apt-get install -y gnupg software-properties-common curl git -# - curl -fsSL https://apt.releases.hashicorp.com/gpg | apt-key add - -# - apt-add-repository "deb [arch=amd64] https://apt.releases.hashicorp.com $(lsb_release -cs) main" -# - apt-get update && apt-get install terraform -# - aws configure set aws_access_key_id $AWS_FBCI_ACCESS_KEY_ID -# - aws configure set aws_secret_access_key $AWS_FBCI_SECRET_ACCESS_KEY -# - aws configure set region "us-east-2" -# - aws configure set aws_profile $PROFILE -# - echo $AWS_FBCI_SSH_KEY > gitlab-featurebase-ci.pem -# - chmod 400 gitlab-featurebase-ci.pem -# - 'which ssh-agent || ( apt-get update -y && apt-get install openssh-client -y )' -# - eval $(ssh-agent -s) -# - mkdir -p ~/.ssh -# - echo $AWS_FBCI_SSH_KEY > /root/.ssh/gitlab-featurebase-ci.pem -# - chmod 400 /root/.ssh/gitlab-featurebase-ci.pem -# - echo "$AWS_FBCI_SSH_KEY" | ssh-add - -# - chmod 700 /root/.ssh -# - '[[ -f /.dockerenv ]] && echo -e "Host *\n\tStrictHostKeyChecking no\n\n" > ~/.ssh/config' -# - apt update && apt -y install jq wget -# - wget https://go.dev/dl/go1.17.5.linux-amd64.tar.gz -# - tar -C /usr/local -xzf go1.17.5.linux-amd64.tar.gz -# - export PATH=$PATH:/usr/local/go/bin -# - TF_VAR_cluster_prefix="smoke-$(openssl rand -base64 12 | tr -d /=+ | cut -c -16)" -# - echo "Cluster Prefix --> $TF_VAR_cluster_prefix" -# - TF_VAR_branch=$CI_COMMIT_BRANCH -# - echo "Branch --> $TF_VAR_branch" -# script: -# - ./qa/scripts/setupSamsungGauntlet.sh -# - ./qa/scripts/testSamsungGauntlet.sh -# after_script: -# - ./qa/scripts/teardownSamsungGauntlet.sh + +gauntlet: + stage: gauntlet + timeout: 4h + image: registry.gitlab.com/gitlab-org/cloud-deploy/aws-base:latest + variables: + PROFILE: "service-terraform" + AWS_SSH_PRIVATE_KEY: $AWS_FBCI_SSH_KEY + AWS_ACCESS_KEY_ID: $AWS_FBCI_ACCESS_KEY_ID + AWS_SECRET_ACCESS_KEY: $AWS_FBCI_SECRET_ACCESS_KEY + TF_VAR_cluster_prefix: "" + TF_VAR_branch: "" + tags: + - aws + - docker + - fbsmoke + rules: + - if: '$CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"' + - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH' + before_script: + - apt-get update && apt-get install -y gnupg software-properties-common curl git + - curl -fsSL https://apt.releases.hashicorp.com/gpg | apt-key add - + - apt-add-repository "deb [arch=amd64] https://apt.releases.hashicorp.com $(lsb_release -cs) main" + - apt-get update && apt-get install terraform + - aws configure set aws_access_key_id $AWS_FBCI_ACCESS_KEY_ID + - aws configure set aws_secret_access_key $AWS_FBCI_SECRET_ACCESS_KEY + - aws configure set region "us-east-2" + - aws configure set aws_profile $PROFILE + - echo $AWS_FBCI_SSH_KEY > gitlab-featurebase-ci.pem + - chmod 400 gitlab-featurebase-ci.pem + - 'which ssh-agent || ( apt-get update -y && apt-get install openssh-client -y )' + - eval $(ssh-agent -s) + - mkdir -p ~/.ssh + - echo $AWS_FBCI_SSH_KEY > /root/.ssh/gitlab-featurebase-ci.pem + - chmod 400 /root/.ssh/gitlab-featurebase-ci.pem + - echo "$AWS_FBCI_SSH_KEY" | ssh-add - + - chmod 700 /root/.ssh + - '[[ -f /.dockerenv ]] && echo -e "Host *\n\tStrictHostKeyChecking no\n\n" > ~/.ssh/config' + - apt update && apt -y install jq wget + - wget https://go.dev/dl/go1.17.5.linux-amd64.tar.gz + - tar -C /usr/local -xzf go1.17.5.linux-amd64.tar.gz + - export PATH=$PATH:/usr/local/go/bin + - TF_VAR_cluster_prefix="gauntlet-$(openssl rand -base64 12 | tr -d /=+ | cut -c -16)" + - echo "Cluster Prefix --> $TF_VAR_cluster_prefix" + - TF_VAR_branch=$CI_COMMIT_BRANCH + - echo "Branch --> $TF_VAR_branch" + script: + - ./qa/scripts/setupSamsungGauntlet.sh + - ./qa/scripts/testSamsungGauntlet.sh + after_script: + - ./qa/scripts/teardownSamsungGauntlet.sh + diff --git a/qa/scripts/runSamsungGauntlet.sh b/qa/scripts/runSamsungGauntlet.sh index 85fd22c67..4b10e0d7c 100644 --- a/qa/scripts/runSamsungGauntlet.sh +++ b/qa/scripts/runSamsungGauntlet.sh @@ -1,5 +1,22 @@ #!/bin/bash -./setupSamsungGauntlet.sh -./testSamsungGauntlet.sh -./teardownSamsungGauntlet.sh +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) + +# requires TF_VAR_gitlab_token env var to be set +if [ -z ${TF_VAR_gitlab_token+x} ]; then echo "TF_VAR_gitlab_token is unset"; else echo "TF_VAR_gitlab_token is set to '$TF_VAR_gitlab_token'"; fi + +# requires TF_VAR_branch env var to be set +if [ -z ${TF_VAR_branch+x} ]; then echo "TF_VAR_branch is unset"; else echo "TF_VAR_branch is set to '$TF_VAR_branch'"; fi + +# requires TF_VAR_cluster_prefix env var to be set +if [ -z ${TF_VAR_cluster_prefix+x} ]; then + echo "setting TF_VAR_cluster_prefix"; + export TF_VAR_cluster_prefix="gauntlet-$(openssl rand -base64 12 | tr -d /=+ | cut -c -16)" + echo "TF_VAR_cluster_prefix is set to '$TF_VAR_cluster_prefix'"; +else + echo "TF_VAR_cluster_prefix is set to '$TF_VAR_cluster_prefix'"; +fi + +$SCRIPT_DIR/setupSamsungGauntlet.sh +#$SCRIPT_DIR/testSamsungGauntlet.sh +#$SCRIPT_DIR/teardownSamsungGauntlet.sh diff --git a/qa/scripts/runSmokeTest.sh b/qa/scripts/runSmokeTest.sh old mode 100644 new mode 100755 index 554a234cc..96866fd6d --- a/qa/scripts/runSmokeTest.sh +++ b/qa/scripts/runSmokeTest.sh @@ -1,5 +1,22 @@ #!/bin/bash -./setupSmokeTest.sh -./testSmokeTest.sh -./teardownSmokeTest.sh +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) + +# requires TF_VAR_gitlab_token env var to be set +if [ -z ${TF_VAR_gitlab_token+x} ]; then echo "TF_VAR_gitlab_token is unset"; else echo "TF_VAR_gitlab_token is set to '$TF_VAR_gitlab_token'"; fi + +# requires TF_VAR_branch env var to be set +if [ -z ${TF_VAR_branch+x} ]; then echo "TF_VAR_branch is unset"; else echo "TF_VAR_branch is set to '$TF_VAR_branch'"; fi + +# requires TF_VAR_cluster_prefix env var to be set +if [ -z ${TF_VAR_cluster_prefix+x} ]; then + echo "setting TF_VAR_cluster_prefix"; + export TF_VAR_cluster_prefix="smoke-$(openssl rand -base64 12 | tr -d /=+ | cut -c -16)" + echo "TF_VAR_cluster_prefix is set to '$TF_VAR_cluster_prefix'"; +else + echo "TF_VAR_cluster_prefix is set to '$TF_VAR_cluster_prefix'"; +fi + +$SCRIPT_DIR/setupSmokeTest.sh +$SCRIPT_DIR/testSmokeTest.sh +$SCRIPT_DIR/teardownSmokeTest.sh diff --git a/qa/scripts/setupSamsungGauntlet.sh b/qa/scripts/setupSamsungGauntlet.sh index d14da6729..1229c0044 100755 --- a/qa/scripts/setupSamsungGauntlet.sh +++ b/qa/scripts/setupSamsungGauntlet.sh @@ -12,7 +12,6 @@ if [ -z ${TF_VAR_branch+x} ]; then echo "TF_VAR_branch is unset"; else echo "TF_ SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) source $SCRIPT_DIR/utilCluster.sh - pushd ./qa/tf/gauntlet/samsung echo "Running terraform init..." terraform init -input=false @@ -22,23 +21,44 @@ terraform output -json > outputs.json popd # get the first ingest host -INGESTNODE0=$(cat ./qa/tf/ci/smoketest/outputs.json | jq -r '[.ingest_ips][0]["value"][0]') +INGESTNODE0=$(cat ./qa/tf/gauntlet/samsung/outputs.json | jq -r '[.ingest_ips][0]["value"][0]') echo "using INGESTNODE0 ${INGESTNODE0}" # get the first data host -DATANODE0=$(cat ./qa/tf/ci/smoketest/outputs.json | jq -r '[.data_node_ips][0]["value"][0]') +DATANODE0=$(cat ./qa/tf/gauntlet/samsung/outputs.json | jq -r '[.data_node_ips][0]["value"][0]') echo "using DATANODE0 ${DATANODE0}" + +DEPLOYED_CLUSTER_PREFIX=$(cat ./qa/tf/gauntlet/samsung/outputs.json | jq -r '[.cluster_prefix][0]["value"]') +echo "Using DEPLOYED_CLUSTER_PREFIX: ${DEPLOYED_CLUSTER_PREFIX}" + +DEPLOYED_CLUSTER_REPLICA_COUNT=$(cat ./qa/tf/gauntlet/samsung/outputs.json | jq -r '[.fb_cluster_replica_count][0]["value"]') +echo "Using DEPLOYED_CLUSTER_REPLICA_COUNT: ${DEPLOYED_CLUSTDEPLOYED_CLUSTER_REPLICA_COUNTER_PREFIX}" + +DEPLOYED_DATA_IPS=$(cat ./qa/tf/gauntlet/samsung/outputs.json | jq -r '[.data_node_ips][0]["value"][]') +echo "DEPLOYED_DATA_IPS: {" +echo "${DEPLOYED_DATA_IPS}" +echo "}" + +DEPLOYED_DATA_IPS_LEN=`echo "$DEPLOYED_DATA_IPS" | wc -l` + +DEPLOYED_INGEST_IPS=$(cat ./qa/tf/gauntlet/samsung/outputs.json | jq -r '[.ingest_ips][0]["value"][]') +echo "DEPLOYED_INGEST_IPS: {" +echo "${DEPLOYED_INGEST_IPS}" +echo "}" + +DEPLOYED_INGEST_IPS_LEN=`echo "$DEPLOYED_INGEST_IPS" | wc -l` + #wait until we can connect to one of the hosts for i in {0..24} do ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no -o ConnectTimeout=10 ec2-user@${DATANODE0} "pwd" if [ $? -eq 0 ] then - echo "Cluster is up after $${i} tries." + echo "Cluster is up after ${i} tries." break fi - sleep 10s + sleep 10 done ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no -o ConnectTimeout=10 ec2-user@${DATANODE0} "pwd" diff --git a/qa/scripts/setupSmokeTest.sh b/qa/scripts/setupSmokeTest.sh index 37eba6879..fac0d7111 100755 --- a/qa/scripts/setupSmokeTest.sh +++ b/qa/scripts/setupSmokeTest.sh @@ -28,13 +28,34 @@ echo "using INGESTNODE0 ${INGESTNODE0}" DATANODE0=$(cat ./qa/tf/ci/smoketest/outputs.json | jq -r '[.data_node_ips][0]["value"][0]') echo "using DATANODE0 ${DATANODE0}" +DEPLOYED_CLUSTER_PREFIX=$(cat ./qa/tf/ci/smoketest/outputs.json | jq -r '[.cluster_prefix][0]["value"]') +echo "Using DEPLOYED_CLUSTER_PREFIX: ${DEPLOYED_CLUSTER_PREFIX}" + +DEPLOYED_CLUSTER_REPLICA_COUNT=$(cat ./qa/tf/ci/smoketest/outputs.json | jq -r '[.fb_cluster_replica_count][0]["value"]') +echo "Using DEPLOYED_CLUSTER_REPLICA_COUNT: ${DEPLOYED_CLUSTDEPLOYED_CLUSTER_REPLICA_COUNTER_PREFIX}" + +DEPLOYED_DATA_IPS=$(cat ./qa/tf/ci/smoketest/outputs.json | jq -r '[.data_node_ips][0]["value"][]') +echo "DEPLOYED_DATA_IPS: {" +echo "${DEPLOYED_DATA_IPS}" +echo "}" + +DEPLOYED_DATA_IPS_LEN=`echo "$DEPLOYED_DATA_IPS" | wc -l` + +DEPLOYED_INGEST_IPS=$(cat ./qa/tf/ci/smoketest/outputs.json | jq -r '[.ingest_ips][0]["value"][]') +echo "DEPLOYED_INGEST_IPS: {" +echo "${DEPLOYED_INGEST_IPS}" +echo "}" + +DEPLOYED_INGEST_IPS_LEN=`echo "$DEPLOYED_INGEST_IPS" | wc -l` + + #wait until we can connect to one of the hosts for i in {0..24} do ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no -o ConnectTimeout=10 ec2-user@${DATANODE0} "pwd" if [ $? -eq 0 ] then - echo "Cluster is up after $${i} tries." + echo "Cluster is up after ${i} tries." break fi sleep 10 diff --git a/qa/scripts/testSamsungGauntlet.sh b/qa/scripts/testSamsungGauntlet.sh index efe0b8951..8240588ba 100755 --- a/qa/scripts/testSamsungGauntlet.sh +++ b/qa/scripts/testSamsungGauntlet.sh @@ -1,48 +1,64 @@ #!/bin/bash -# get the bastion host -BASTION=$(cat ./qa/tf/gauntlet/samsung/outputs.json | jq -r '[.ingest_ips][0]["value"][0]') -echo "using bastion ${BASTION}" +# get the first ingest host +INGESTNODE0=$(cat ./qa/tf/gauntlet/samsung/outputs.json | jq -r '[.ingest_ips][0]["value"][0]') +echo "using INGESTNODE0 ${INGESTNODE0}" -NODE=$(cat ./qa/tf/gauntlet/samsung/outputs.json | jq -r '[.data_node_ips][0]["value"][0]') -echo "using node ${NODE}" +# get the first data host +DATANODE0=$(cat ./qa/tf/gauntlet/samsung/outputs.json | jq -r '[.data_node_ips][0]["value"][0]') +echo "using DATANODE0 ${DATANODE0}" # generate csv files +echo "Building simulacraData..." GOOS=linux GOARCH=arm64 go build ./qa/simulacraData/... -scp -i ~/.ssh/gitlab-featurebase-ci.pem simulacraData ec2-user@${BASTION}:/data +if (( $? != 0 )) +then + echo "Build failed" + exit 1 +fi +echo "Copying simulacraData..." +scp -i ~/.ssh/gitlab-featurebase-ci.pem simulacraData ec2-user@${INGESTNODE0}:/data if (( $? != 0 )) then echo "Copy failed" exit 1 fi -ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem ec2-user@${BASTION} "cd /data && /data/simulacraData" +echo "Running simulacraData..." +ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem ec2-user@${INGESTNODE0} "cd /data && /data/simulacraData" if (( $? != 0 )) then echo "Making big files failed" exit 1 fi +echo "Running simulacraData done." # ingest these files the way that samsung does it -scp -i ~/.ssh/gitlab-featurebase-ci.pem ./qa/scripts/testSamsungPayload.sh ec2-user@${BASTION}: +echo "Copying testSamsungPayload.sh..." +scp -i ~/.ssh/gitlab-featurebase-ci.pem ./qa/scripts/testSamsungPayload.sh ec2-user@${INGESTNODE0}: if (( $? != 0 )) then - echo "Copy ingest script failed" + echo "Copying testSamsungPayload.sh failed" exit 1 fi -ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem ec2-user@${BASTION} "./testSamsungPayload.sh http://${NODE}:10101 1" +echo "Running (1) testSamsungPayload.sh..." +ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem ec2-user@${INGESTNODE0} "./testSamsungPayload.sh http://${DATANODE0}:10101 1" if (( $? != 0 )) then echo "Running 1 testSamsungPayload.sh failed" exit 1 fi -ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem ec2-user@${BASTION} "./testSamsungPayload.sh http://${NODE}:10101 0" +echo "Running (0) testSamsungPayload.sh..." +ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem ec2-user@${INGESTNODE0} "./testSamsungPayload.sh http://${DATANODE0}:10101 0" if (( $? != 0 )) then echo "Running 0 testSamsungPayload.sh failed" exit 1 fi -# query workload that runs \ No newline at end of file +# query workload that runs + + +echo "Done." \ No newline at end of file diff --git a/qa/scripts/testSamsungPayload.sh b/qa/scripts/testSamsungPayload.sh index 8c36c63dc..85752b409 100755 --- a/qa/scripts/testSamsungPayload.sh +++ b/qa/scripts/testSamsungPayload.sh @@ -6,7 +6,6 @@ FEATUREBASE_PATH=/usr/local/bin # path for directory with csv directory files for all fields to be ingested CSV_DIR_PATH=/data - # To run: # ./testSamsungPayload.sh {Local host & port for featurebase} {initialize flag} diff --git a/qa/scripts/testSmokeTest.sh b/qa/scripts/testSmokeTest.sh index 19d84e265..24576b132 100755 --- a/qa/scripts/testSmokeTest.sh +++ b/qa/scripts/testSmokeTest.sh @@ -28,12 +28,8 @@ fi # run smoke test echo "Running smoke test..." -ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o "StrictHostKeyChecking no" ec2-user@${INGESTNODE0} "pushd /data; ~/.local/bin/pytest --junitxml=report.xml; popd" -if (( $? != 0 )) -then - echo "Unable to run smoke test" - exit 1 -fi +ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o "StrictHostKeyChecking no" ec2-user@${INGESTNODE0} "cd /data; ~/.local/bin/pytest --junitxml=report.xml" +SMOKETESTRESULT=$? echo "Copying test report to local" scp -r -i ~/.ssh/gitlab-featurebase-ci.pem ec2-user@${INGESTNODE0}:/data/report.xml report.xml @@ -43,4 +39,11 @@ then exit 1 fi -echo "Smoke test complete" \ No newline at end of file +if (( $SMOKETESTRESULT != 0 )) +then + echo "Smoke test complete with test failures" +else + echo "Smoke test complete" +fi + +exit $SMOKETESTRESULT \ No newline at end of file diff --git a/qa/scripts/utilCluster.sh b/qa/scripts/utilCluster.sh index d0bc49184..f62e07208 100644 --- a/qa/scripts/utilCluster.sh +++ b/qa/scripts/utilCluster.sh @@ -218,26 +218,6 @@ generateInitialClusterString() { } setupClusterNodes() { - DEPLOYED_CLUSTER_PREFIX=$(cat ./qa/tf/ci/smoketest/outputs.json | jq -r '[.cluster_prefix][0]["value"]') - echo "Using DEPLOYED_CLUSTER_PREFIX: ${DEPLOYED_CLUSTER_PREFIX}" - - DEPLOYED_CLUSTER_REPLICA_COUNT=$(cat ./qa/tf/ci/smoketest/outputs.json | jq -r '[.fb_cluster_replica_count][0]["value"]') - echo "Using DEPLOYED_CLUSTER_REPLICA_COUNT: ${DEPLOYED_CLUSTDEPLOYED_CLUSTER_REPLICA_COUNTER_PREFIX}" - - DEPLOYED_DATA_IPS=$(cat ./qa/tf/ci/smoketest/outputs.json | jq -r '[.data_node_ips][0]["value"][]') - echo "DEPLOYED_DATA_IPS: {" - echo "${DEPLOYED_DATA_IPS}" - echo "}" - - DEPLOYED_DATA_IPS_LEN=`echo "$DEPLOYED_DATA_IPS" | wc -l` - - DEPLOYED_INGEST_IPS=$(cat ./qa/tf/ci/smoketest/outputs.json | jq -r '[.ingest_ips][0]["value"][]') - echo "DEPLOYED_INGEST_IPS: {" - echo "${DEPLOYED_INGEST_IPS}" - echo "}" - - DEPLOYED_INGEST_IPS_LEN=`echo "$DEPLOYED_INGEST_IPS" | wc -l` - #data nodes generateInitialClusterString diff --git a/qa/simulacraData/simulacra_data.go b/qa/simulacraData/simulacra_data.go index 5ec269780..a6af37b67 100644 --- a/qa/simulacraData/simulacra_data.go +++ b/qa/simulacraData/simulacra_data.go @@ -38,29 +38,41 @@ var countryList [246]string = [...]string{"ABW", "AFG", "AGO", "AIA", "ALA", "AL const totalRecords int = 200000000 func main() { + log.Println("generating age field...") if err := GenerateAgeField(totalRecords); err != nil { log.Fatalf("unable to generate age field: %v", err) } + log.Println("generating age field done.") + log.Println("generating ip field...") if err := GenerateIPField(totalRecords); err != nil { log.Fatalf("unable to generate IP field: %v", err) } + log.Println("generating ip field done.") + log.Println("generating identifier field...") if err := GenerateArbIdField(totalRecords); err != nil { log.Fatalf("unable to generate indentifier field: %v", err) } + log.Println("generating identifier field done.") + log.Println("generating opt in field...") if err := GenerateOptInField(totalRecords); err != nil { log.Fatalf("unable to generate opt in field: %v", err) } + log.Println("generating opt in field done.") + log.Println("generating country field...") if err := GenerateCountryField(totalRecords); err != nil { log.Fatalf("unable to generate country field: %v", err) } + log.Println("generating country field done.") + log.Println("generating time field...") if err := GenerateTimeField(totalRecords); err != nil { log.Fatalf("unable to generate time field: %v", err) } + log.Println("generating time field done.") } func GenerateAgeField(requestedRecords int) error { @@ -85,7 +97,9 @@ func GenerateAgeField(requestedRecords int) error { } } - + } + if i%2000000 == 0 { + log.Printf("generating age field (%d)", i) } } err1 := writer.Flush() @@ -120,6 +134,9 @@ func GenerateIPField(requestedRecords int) error { return errors.Wrap(err, "unable to write to ip.csv") } } + if i%2000000 == 0 { + log.Printf("generating ip field (%d)", i) + } } err1 := writer.Flush() @@ -155,6 +172,9 @@ func GenerateArbIdField(requestedRecords int) error { return errors.Wrap(err, "unable to write to identifier.csv") } } + if i%2000000 == 0 { + log.Printf("generating identifier field (%d)", i) + } } err1 := writer.Flush() @@ -213,7 +233,9 @@ func GenerateTimeField(requestedRecords int) error { return errors.Wrap(err, "unable to write to time.csv") } } - + if i%2000000 == 0 { + log.Printf("generating time field (%d)", i) + } } err1 := writer.Flush() if err1 != nil { @@ -266,6 +288,9 @@ func GenerateOptInField(requestedRecords int) error { } } } + if i%2000000 == 0 { + log.Printf("generating opt in field (%d)", i) + } } err1 := writer.Flush() @@ -315,7 +340,9 @@ func GenerateCountryField(requestedRecords int) error { } } } - + if i%2000000 == 0 { + log.Printf("generating country field (%d)", i) + } } err1 := writer.Flush() diff --git a/qa/tf/ci/smoketest/main.tf b/qa/tf/ci/smoketest/main.tf index 3ebd5eb08..40f51a89a 100644 --- a/qa/tf/ci/smoketest/main.tf +++ b/qa/tf/ci/smoketest/main.tf @@ -6,6 +6,7 @@ module "ci-cluster" { profile = var.profile fb_data_node_type = "m6g.large" fb_data_node_count = 1 + fb_ingest_type = "m6g.large" vpc_id = "vpc-05a26a122f961dc2b" vpc_cidr_block = "10.0.0.0/16" vpc_public_subnets = ["subnet-066b4b922b54e51a2","subnet-037b8884269a69025","subnet-08482631514426210",] diff --git a/qa/tf/ci/smoketest/terraform.tfstate.backup b/qa/tf/ci/smoketest/terraform.tfstate.backup index 5bb4c8639..b4998f14f 100644 --- a/qa/tf/ci/smoketest/terraform.tfstate.backup +++ b/qa/tf/ci/smoketest/terraform.tfstate.backup @@ -1,8 +1,710 @@ { "version": 4, "terraform_version": "1.1.2", - "serial": 178, + "serial": 203, "lineage": "0f5e8a05-0e94-e86f-f384-26086bd40585", - "outputs": {}, - "resources": [] + "outputs": { + "cluster_prefix": { + "value": "smoke-BzP3aSw62HwEPFS", + "type": "string" + }, + "data_node_ips": { + "value": [ + "10.0.1.185" + ], + "type": [ + "tuple", + [ + "string" + ] + ] + }, + "fb_cluster_replica_count": { + "value": 1, + "type": "number" + }, + "ingest_ips": { + "value": [ + "3.144.237.6" + ], + "type": [ + "tuple", + [ + "string" + ] + ] + } + }, + "resources": [ + { + "module": "module.ci-cluster", + "mode": "data", + "type": "aws_ami", + "name": "amazon_linux_2", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "architecture": "arm64", + "arn": "arn:aws:ec2:us-east-2::image/ami-0b09f36be67d32fff", + "block_device_mappings": [ + { + "device_name": "/dev/xvda", + "ebs": { + "delete_on_termination": "true", + "encrypted": "false", + "iops": "0", + "snapshot_id": "snap-0617b00e90bae012b", + "throughput": "0", + "volume_size": "8", + "volume_type": "gp2" + }, + "no_device": "", + "virtual_name": "" + } + ], + "creation_date": "2021-12-01T19:36:11.000Z", + "description": "Amazon Linux 2 LTS Arm64 AMI 2.0.20211201.0 arm64 HVM gp2", + "ena_support": true, + "executable_users": null, + "filter": [ + { + "name": "architecture", + "values": [ + "arm64" + ] + }, + { + "name": "name", + "values": [ + "amzn2-ami-hvm-*" + ] + }, + { + "name": "virtualization-type", + "values": [ + "hvm" + ] + } + ], + "hypervisor": "xen", + "id": "ami-0b09f36be67d32fff", + "image_id": "ami-0b09f36be67d32fff", + "image_location": "amazon/amzn2-ami-hvm-2.0.20211201.0-arm64-gp2", + "image_owner_alias": "amazon", + "image_type": "machine", + "kernel_id": null, + "most_recent": true, + "name": "amzn2-ami-hvm-2.0.20211201.0-arm64-gp2", + "name_regex": null, + "owner_id": "137112412989", + "owners": [ + "amazon" + ], + "platform": null, + "platform_details": "Linux/UNIX", + "product_codes": [], + "public": true, + "ramdisk_id": null, + "root_device_name": "/dev/xvda", + "root_device_type": "ebs", + "root_snapshot_id": "snap-0617b00e90bae012b", + "sriov_net_support": "simple", + "state": "available", + "state_reason": { + "code": "UNSET", + "message": "UNSET" + }, + "tags": {}, + "usage_operation": "RunInstances", + "virtualization_type": "hvm" + }, + "sensitive_attributes": [] + } + ] + }, + { + "module": "module.ci-cluster", + "mode": "managed", + "type": "aws_iam_instance_profile", + "name": "fb_cluster_node_profile", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "arn": "arn:aws:iam::977373308795:instance-profile/smoke-BzP3aSw62HwEPFS-fb_cluster_node_profile", + "create_date": "2022-01-09T17:47:01Z", + "id": "smoke-BzP3aSw62HwEPFS-fb_cluster_node_profile", + "name": "smoke-BzP3aSw62HwEPFS-fb_cluster_node_profile", + "name_prefix": null, + "path": "/", + "role": "smoke-BzP3aSw62HwEPFS-fb_cluster_node", + "tags": null, + "tags_all": {}, + "unique_id": "AIPA6HD75E55XTIU7KRKL" + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "module.ci-cluster.aws_iam_role.fb_cluster_node_role" + ] + } + ] + }, + { + "module": "module.ci-cluster", + "mode": "managed", + "type": "aws_iam_role", + "name": "fb_cluster_node_role", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "arn": "arn:aws:iam::977373308795:role/smoke-BzP3aSw62HwEPFS-fb_cluster_node", + "assume_role_policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Sid\":\"\",\"Effect\":\"Allow\",\"Principal\":{\"Service\":\"ec2.amazonaws.com\"},\"Action\":\"sts:AssumeRole\"}]}", + "create_date": "2022-01-09T17:46:58Z", + "description": "", + "force_detach_policies": false, + "id": "smoke-BzP3aSw62HwEPFS-fb_cluster_node", + "inline_policy": [ + { + "name": "ec2_read_all", + "policy": "{\"Statement\":[{\"Action\":[\"ec2:Describe*\"],\"Effect\":\"Allow\",\"Resource\":\"*\"}],\"Version\":\"2012-10-17\"}" + } + ], + "managed_policy_arns": [], + "max_session_duration": 3600, + "name": "smoke-BzP3aSw62HwEPFS-fb_cluster_node", + "name_prefix": "", + "path": "/", + "permissions_boundary": null, + "tags": null, + "tags_all": {}, + "unique_id": "AROA6HD75E55YXA4Z4XK2" + }, + "sensitive_attributes": [], + "private": "bnVsbA==" + } + ] + }, + { + "module": "module.ci-cluster", + "mode": "managed", + "type": "aws_instance", + "name": "fb_cluster_nodes", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "index_key": 0, + "schema_version": 1, + "attributes": { + "ami": "ami-0b09f36be67d32fff", + "arn": "arn:aws:ec2:us-east-2:977373308795:instance/i-077d0596e47712614", + "associate_public_ip_address": false, + "availability_zone": "us-east-2a", + "capacity_reservation_specification": [ + { + "capacity_reservation_preference": "open", + "capacity_reservation_target": [] + } + ], + "cpu_core_count": 2, + "cpu_threads_per_core": 1, + "credit_specification": [], + "disable_api_termination": false, + "ebs_block_device": [ + { + "delete_on_termination": true, + "device_name": "/dev/sdb", + "encrypted": false, + "iops": 3000, + "kms_key_id": "", + "snapshot_id": "", + "tags": {}, + "throughput": 125, + "volume_id": "vol-05eb623ed8991e3ec", + "volume_size": 100, + "volume_type": "gp3" + } + ], + "ebs_optimized": false, + "enclave_options": [ + { + "enabled": false + } + ], + "ephemeral_block_device": [], + "get_password_data": false, + "hibernation": false, + "host_id": null, + "iam_instance_profile": "smoke-BzP3aSw62HwEPFS-fb_cluster_node_profile", + "id": "i-077d0596e47712614", + "instance_initiated_shutdown_behavior": "stop", + "instance_state": "running", + "instance_type": "m6g.large", + "ipv6_address_count": 0, + "ipv6_addresses": [], + "key_name": "smoke-BzP3aSw62HwEPFS-gitlab-ci", + "launch_template": [], + "metadata_options": [ + { + "http_endpoint": "enabled", + "http_put_response_hop_limit": 1, + "http_tokens": "optional" + } + ], + "monitoring": true, + "network_interface": [], + "outpost_arn": "", + "password_data": "", + "placement_group": "", + "placement_partition_number": null, + "primary_network_interface_id": "eni-076ed20f66a9f3552", + "private_dns": "ip-10-0-1-185.us-east-2.compute.internal", + "private_ip": "10.0.1.185", + "public_dns": "", + "public_ip": "", + "root_block_device": [ + { + "delete_on_termination": true, + "device_name": "/dev/xvda", + "encrypted": false, + "iops": 3000, + "kms_key_id": "", + "tags": null, + "throughput": 125, + "volume_id": "vol-03c02dd3fdafd0aa0", + "volume_size": 20, + "volume_type": "gp3" + } + ], + "secondary_private_ips": [], + "security_groups": [], + "source_dest_check": true, + "subnet_id": "subnet-050b1219d78f2db1b", + "tags": { + "Name": "smoke-BzP3aSw62HwEPFS-featurebase-cluster-0", + "Prefix": "smoke-BzP3aSw62HwEPFS", + "Role": "cluster_node" + }, + "tags_all": { + "Name": "smoke-BzP3aSw62HwEPFS-featurebase-cluster-0", + "Prefix": "smoke-BzP3aSw62HwEPFS", + "Role": "cluster_node" + }, + "tenancy": "default", + "timeouts": null, + "user_data": null, + "user_data_base64": null, + "volume_tags": null, + "vpc_security_group_ids": [ + "sg-06b14c4162509fb5f" + ] + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjo2MDAwMDAwMDAwMDAsImRlbGV0ZSI6MTIwMDAwMDAwMDAwMCwidXBkYXRlIjo2MDAwMDAwMDAwMDB9LCJzY2hlbWFfdmVyc2lvbiI6IjEifQ==", + "dependencies": [ + "module.ci-cluster.aws_iam_instance_profile.fb_cluster_node_profile", + "module.ci-cluster.aws_iam_role.fb_cluster_node_role", + "module.ci-cluster.aws_key_pair.gitlab-featurebase-ci", + "module.ci-cluster.aws_security_group.featurebase", + "module.ci-cluster.data.aws_ami.amazon_linux_2" + ] + } + ] + }, + { + "module": "module.ci-cluster", + "mode": "managed", + "type": "aws_instance", + "name": "fb_ingest", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "index_key": 0, + "schema_version": 1, + "attributes": { + "ami": "ami-0b09f36be67d32fff", + "arn": "arn:aws:ec2:us-east-2:977373308795:instance/i-0ca325881e8a1dd8a", + "associate_public_ip_address": true, + "availability_zone": "us-east-2a", + "capacity_reservation_specification": [ + { + "capacity_reservation_preference": "open", + "capacity_reservation_target": [] + } + ], + "cpu_core_count": 8, + "cpu_threads_per_core": 1, + "credit_specification": [], + "disable_api_termination": false, + "ebs_block_device": [ + { + "delete_on_termination": true, + "device_name": "/dev/sdb", + "encrypted": false, + "iops": 3000, + "kms_key_id": "", + "snapshot_id": "", + "tags": {}, + "throughput": 125, + "volume_id": "vol-0803614106f4d6033", + "volume_size": 100, + "volume_type": "gp3" + } + ], + "ebs_optimized": false, + "enclave_options": [ + { + "enabled": false + } + ], + "ephemeral_block_device": [], + "get_password_data": false, + "hibernation": false, + "host_id": null, + "iam_instance_profile": "smoke-BzP3aSw62HwEPFS-fb_cluster_node_profile", + "id": "i-0ca325881e8a1dd8a", + "instance_initiated_shutdown_behavior": "stop", + "instance_state": "running", + "instance_type": "c6g.2xlarge", + "ipv6_address_count": 0, + "ipv6_addresses": [], + "key_name": "smoke-BzP3aSw62HwEPFS-gitlab-ci", + "launch_template": [], + "metadata_options": [ + { + "http_endpoint": "enabled", + "http_put_response_hop_limit": 1, + "http_tokens": "optional" + } + ], + "monitoring": true, + "network_interface": [], + "outpost_arn": "", + "password_data": "", + "placement_group": "", + "placement_partition_number": null, + "primary_network_interface_id": "eni-06216f96b73bcf92d", + "private_dns": "ip-10-0-101-105.us-east-2.compute.internal", + "private_ip": "10.0.101.105", + "public_dns": "", + "public_ip": "3.144.237.6", + "root_block_device": [ + { + "delete_on_termination": true, + "device_name": "/dev/xvda", + "encrypted": false, + "iops": 3000, + "kms_key_id": "", + "tags": null, + "throughput": 125, + "volume_id": "vol-070f8a9022e93f9ee", + "volume_size": 20, + "volume_type": "gp3" + } + ], + "secondary_private_ips": [], + "security_groups": [], + "source_dest_check": true, + "subnet_id": "subnet-066b4b922b54e51a2", + "tags": { + "Name": "smoke-BzP3aSw62HwEPFS-featurebase-ingest-0", + "Prefix": "smoke-BzP3aSw62HwEPFS", + "Role": "ingest_node" + }, + "tags_all": { + "Name": "smoke-BzP3aSw62HwEPFS-featurebase-ingest-0", + "Prefix": "smoke-BzP3aSw62HwEPFS", + "Role": "ingest_node" + }, + "tenancy": "default", + "timeouts": null, + "user_data": null, + "user_data_base64": null, + "volume_tags": null, + "vpc_security_group_ids": [ + "sg-062ddd221720047f4" + ] + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjo2MDAwMDAwMDAwMDAsImRlbGV0ZSI6MTIwMDAwMDAwMDAwMCwidXBkYXRlIjo2MDAwMDAwMDAwMDB9LCJzY2hlbWFfdmVyc2lvbiI6IjEifQ==", + "dependencies": [ + "module.ci-cluster.aws_iam_instance_profile.fb_cluster_node_profile", + "module.ci-cluster.aws_iam_role.fb_cluster_node_role", + "module.ci-cluster.aws_key_pair.gitlab-featurebase-ci", + "module.ci-cluster.aws_security_group.ingest", + "module.ci-cluster.data.aws_ami.amazon_linux_2" + ] + } + ] + }, + { + "module": "module.ci-cluster", + "mode": "managed", + "type": "aws_key_pair", + "name": "gitlab-featurebase-ci", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 1, + "attributes": { + "arn": "arn:aws:ec2:us-east-2:977373308795:key-pair/smoke-BzP3aSw62HwEPFS-gitlab-ci", + "fingerprint": "69:e5:4b:15:8d:1f:22:61:de:08:a9:ee:f3:29:5c:69", + "id": "smoke-BzP3aSw62HwEPFS-gitlab-ci", + "key_name": "smoke-BzP3aSw62HwEPFS-gitlab-ci", + "key_name_prefix": "", + "key_pair_id": "key-01ca26b2795043890", + "public_key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC91hhpVHNonAG7ku2ugpxEskf9KHeyHJPQJT26OHrMUw7R+T5A8TjqSzTau07sXQ/E9SO3ebV8SJ5PqeaQOnQB8VEvVNK0DjQH7ppvNg1Rfs42FZT9ttzTMvOjsSbK3vZTHXdoKQEdC9NxBwSkFIRGQojK1HUOq9xGrw31fA1OjSwlpLcbx7yyg18lcqW6UOptnVR8U9Yy9qQ5jZF1HtkQ6L9J+gv4o1UyNAUK2bopeGiXpBc3PQ/CFaFT2h/aqLBP66qAHsHVyAFD3PIRtplC5EHa8jXDgLacEls0uF7Q3kRPxvzcuo4g4VkOn1rDy9qH3vd2hT3aKVnM73FIDUiL", + "tags": null, + "tags_all": {} + }, + "sensitive_attributes": [], + "private": "eyJzY2hlbWFfdmVyc2lvbiI6IjEifQ==" + } + ] + }, + { + "module": "module.ci-cluster", + "mode": "managed", + "type": "aws_security_group", + "name": "featurebase", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 1, + "attributes": { + "arn": "arn:aws:ec2:us-east-2:977373308795:security-group/sg-06b14c4162509fb5f", + "description": "Allow featurebase inbound traffic", + "egress": [ + { + "cidr_blocks": [ + "0.0.0.0/0" + ], + "description": "", + "from_port": 0, + "ipv6_cidr_blocks": [ + "::/0" + ], + "prefix_list_ids": [], + "protocol": "-1", + "security_groups": [], + "self": false, + "to_port": 0 + } + ], + "id": "sg-06b14c4162509fb5f", + "ingress": [ + { + "cidr_blocks": [ + "0.0.0.0/0" + ], + "description": "SSH", + "from_port": 22, + "ipv6_cidr_blocks": [ + "::/0" + ], + "prefix_list_ids": [], + "protocol": "tcp", + "security_groups": [], + "self": false, + "to_port": 22 + }, + { + "cidr_blocks": [ + "0.0.0.0/0" + ], + "description": "icmp from Anywhere", + "from_port": -1, + "ipv6_cidr_blocks": [], + "prefix_list_ids": [], + "protocol": "icmp", + "security_groups": [], + "self": false, + "to_port": -1 + }, + { + "cidr_blocks": [ + "10.0.0.0/16" + ], + "description": "etcd from internal 2", + "from_port": 10401, + "ipv6_cidr_blocks": [], + "prefix_list_ids": [], + "protocol": "tcp", + "security_groups": [], + "self": false, + "to_port": 10401 + }, + { + "cidr_blocks": [ + "10.0.0.0/16" + ], + "description": "etcd from internal", + "from_port": 10301, + "ipv6_cidr_blocks": [], + "prefix_list_ids": [], + "protocol": "tcp", + "security_groups": [], + "self": false, + "to_port": 10301 + }, + { + "cidr_blocks": [ + "10.0.0.0/8", + "172.31.0.0/16" + ], + "description": "GRPC from Internal", + "from_port": 20101, + "ipv6_cidr_blocks": [], + "prefix_list_ids": [], + "protocol": "tcp", + "security_groups": [], + "self": false, + "to_port": 20101 + }, + { + "cidr_blocks": [ + "10.0.0.0/8", + "172.31.0.0/16" + ], + "description": "HTTP from Internal", + "from_port": 10101, + "ipv6_cidr_blocks": [], + "prefix_list_ids": [], + "protocol": "tcp", + "security_groups": [], + "self": false, + "to_port": 10101 + }, + { + "cidr_blocks": [ + "10.0.0.0/8", + "172.31.0.0/16" + ], + "description": "PostgreSQL from Internal", + "from_port": 55432, + "ipv6_cidr_blocks": [], + "prefix_list_ids": [], + "protocol": "tcp", + "security_groups": [], + "self": false, + "to_port": 55432 + } + ], + "name": "smoke-BzP3aSw62HwEPFS-allow_featurebase", + "name_prefix": "", + "owner_id": "977373308795", + "revoke_rules_on_delete": false, + "tags": { + "Name": "allow_featurebase" + }, + "tags_all": { + "Name": "allow_featurebase" + }, + "timeouts": null, + "vpc_id": "vpc-05a26a122f961dc2b" + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjo2MDAwMDAwMDAwMDAsImRlbGV0ZSI6OTAwMDAwMDAwMDAwfSwic2NoZW1hX3ZlcnNpb24iOiIxIn0=" + } + ] + }, + { + "module": "module.ci-cluster", + "mode": "managed", + "type": "aws_security_group", + "name": "ingest", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 1, + "attributes": { + "arn": "arn:aws:ec2:us-east-2:977373308795:security-group/sg-062ddd221720047f4", + "description": "Allow ingest inbound traffic", + "egress": [ + { + "cidr_blocks": [ + "0.0.0.0/0" + ], + "description": "", + "from_port": 0, + "ipv6_cidr_blocks": [ + "::/0" + ], + "prefix_list_ids": [], + "protocol": "-1", + "security_groups": [], + "self": false, + "to_port": 0 + } + ], + "id": "sg-062ddd221720047f4", + "ingress": [ + { + "cidr_blocks": [ + "0.0.0.0/0" + ], + "description": "", + "from_port": 10101, + "ipv6_cidr_blocks": [ + "::/0" + ], + "prefix_list_ids": [], + "protocol": "tcp", + "security_groups": [], + "self": false, + "to_port": 10101 + }, + { + "cidr_blocks": [ + "0.0.0.0/0" + ], + "description": "SSH", + "from_port": 22, + "ipv6_cidr_blocks": [ + "::/0" + ], + "prefix_list_ids": [], + "protocol": "tcp", + "security_groups": [], + "self": false, + "to_port": 22 + }, + { + "cidr_blocks": [ + "0.0.0.0/0" + ], + "description": "icmp from Anywhere", + "from_port": -1, + "ipv6_cidr_blocks": [], + "prefix_list_ids": [], + "protocol": "icmp", + "security_groups": [], + "self": false, + "to_port": -1 + } + ], + "name": "smoke-BzP3aSw62HwEPFS-allow_ingest", + "name_prefix": "", + "owner_id": "977373308795", + "revoke_rules_on_delete": false, + "tags": { + "Name": "allow_ingest" + }, + "tags_all": { + "Name": "allow_ingest" + }, + "timeouts": null, + "vpc_id": "vpc-05a26a122f961dc2b" + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjo2MDAwMDAwMDAwMDAsImRlbGV0ZSI6OTAwMDAwMDAwMDAwfSwic2NoZW1hX3ZlcnNpb24iOiIxIn0=" + } + ] + } + ] } diff --git a/qa/tf/gauntlet/samsung/main.tf b/qa/tf/gauntlet/samsung/main.tf index a4bdd2321..f86b7db70 100644 --- a/qa/tf/gauntlet/samsung/main.tf +++ b/qa/tf/gauntlet/samsung/main.tf @@ -1,6 +1,6 @@ module "samsung-cluster" { source = "../../.modules/featurebase-cluster" - cluster_prefix = "samsung-gauntlet" + cluster_prefix = var.cluster_prefix region = var.region profile = var.profile fb_data_node_type = "m6g.xlarge" diff --git a/qa/tf/gauntlet/samsung/terraform.tfstate.backup b/qa/tf/gauntlet/samsung/terraform.tfstate.backup deleted file mode 100644 index 2622f5631..000000000 --- a/qa/tf/gauntlet/samsung/terraform.tfstate.backup +++ /dev/null @@ -1,938 +0,0 @@ -{ - "version": 4, - "terraform_version": "1.1.2", - "serial": 20, - "lineage": "febac4ac-400a-d207-d2d1-64c55f14767b", - "outputs": { - "data_node_ips": { - "value": [ - "10.0.1.197", - "10.0.2.133", - "10.0.3.244" - ], - "type": [ - "tuple", - [ - "string", - "string", - "string" - ] - ] - }, - "ingest_ips": { - "value": [ - "3.145.96.245" - ], - "type": [ - "tuple", - [ - "string" - ] - ] - } - }, - "resources": [ - { - "module": "module.samsung-cluster", - "mode": "data", - "type": "aws_ami", - "name": "amazon_linux_2", - "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", - "instances": [ - { - "schema_version": 0, - "attributes": { - "architecture": "arm64", - "arn": "arn:aws:ec2:us-east-2::image/ami-0b09f36be67d32fff", - "block_device_mappings": [ - { - "device_name": "/dev/xvda", - "ebs": { - "delete_on_termination": "true", - "encrypted": "false", - "iops": "0", - "snapshot_id": "snap-0617b00e90bae012b", - "throughput": "0", - "volume_size": "8", - "volume_type": "gp2" - }, - "no_device": "", - "virtual_name": "" - } - ], - "creation_date": "2021-12-01T19:36:11.000Z", - "description": "Amazon Linux 2 LTS Arm64 AMI 2.0.20211201.0 arm64 HVM gp2", - "ena_support": true, - "executable_users": null, - "filter": [ - { - "name": "architecture", - "values": [ - "arm64" - ] - }, - { - "name": "name", - "values": [ - "amzn2-ami-hvm-*" - ] - }, - { - "name": "virtualization-type", - "values": [ - "hvm" - ] - } - ], - "hypervisor": "xen", - "id": "ami-0b09f36be67d32fff", - "image_id": "ami-0b09f36be67d32fff", - "image_location": "amazon/amzn2-ami-hvm-2.0.20211201.0-arm64-gp2", - "image_owner_alias": "amazon", - "image_type": "machine", - "kernel_id": null, - "most_recent": true, - "name": "amzn2-ami-hvm-2.0.20211201.0-arm64-gp2", - "name_regex": null, - "owner_id": "137112412989", - "owners": [ - "amazon" - ], - "platform": null, - "platform_details": "Linux/UNIX", - "product_codes": [], - "public": true, - "ramdisk_id": null, - "root_device_name": "/dev/xvda", - "root_device_type": "ebs", - "root_snapshot_id": "snap-0617b00e90bae012b", - "sriov_net_support": "simple", - "state": "available", - "state_reason": { - "code": "UNSET", - "message": "UNSET" - }, - "tags": {}, - "usage_operation": "RunInstances", - "virtualization_type": "hvm" - }, - "sensitive_attributes": [] - } - ] - }, - { - "module": "module.samsung-cluster", - "mode": "managed", - "type": "aws_iam_instance_profile", - "name": "fb_cluster_node_profile", - "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", - "instances": [ - { - "schema_version": 0, - "attributes": { - "arn": "arn:aws:iam::977373308795:instance-profile/samsung-gauntlet-fb_cluster_node_profile", - "create_date": "2022-01-07T23:16:27Z", - "id": "samsung-gauntlet-fb_cluster_node_profile", - "name": "samsung-gauntlet-fb_cluster_node_profile", - "name_prefix": null, - "path": "/", - "role": "samsung-gauntlet-fb_cluster_node", - "tags": {}, - "tags_all": {}, - "unique_id": "AIPA6HD75E55ZM7XC2IBD" - }, - "sensitive_attributes": [], - "private": "bnVsbA==", - "dependencies": [ - "module.samsung-cluster.aws_iam_role.fb_cluster_node_role" - ] - } - ] - }, - { - "module": "module.samsung-cluster", - "mode": "managed", - "type": "aws_iam_role", - "name": "fb_cluster_node_role", - "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", - "instances": [ - { - "schema_version": 0, - "attributes": { - "arn": "arn:aws:iam::977373308795:role/samsung-gauntlet-fb_cluster_node", - "assume_role_policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Sid\":\"\",\"Effect\":\"Allow\",\"Principal\":{\"Service\":\"ec2.amazonaws.com\"},\"Action\":\"sts:AssumeRole\"}]}", - "create_date": "2022-01-07T23:16:05Z", - "description": "", - "force_detach_policies": false, - "id": "samsung-gauntlet-fb_cluster_node", - "inline_policy": [ - { - "name": "ec2_read_all", - "policy": "{\"Statement\":[{\"Action\":[\"ec2:Describe*\"],\"Effect\":\"Allow\",\"Resource\":\"*\"}],\"Version\":\"2012-10-17\"}" - } - ], - "managed_policy_arns": [], - "max_session_duration": 3600, - "name": "samsung-gauntlet-fb_cluster_node", - "name_prefix": "", - "path": "/", - "permissions_boundary": null, - "tags": {}, - "tags_all": {}, - "unique_id": "AROA6HD75E55VM3GADWTD" - }, - "sensitive_attributes": [], - "private": "bnVsbA==" - } - ] - }, - { - "module": "module.samsung-cluster", - "mode": "managed", - "type": "aws_instance", - "name": "fb_cluster_nodes", - "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", - "instances": [ - { - "index_key": 0, - "schema_version": 1, - "attributes": { - "ami": "ami-0b09f36be67d32fff", - "arn": "arn:aws:ec2:us-east-2:977373308795:instance/i-09664bb4472386327", - "associate_public_ip_address": false, - "availability_zone": "us-east-2a", - "capacity_reservation_specification": [ - { - "capacity_reservation_preference": "open", - "capacity_reservation_target": [] - } - ], - "cpu_core_count": 4, - "cpu_threads_per_core": 1, - "credit_specification": [], - "disable_api_termination": false, - "ebs_block_device": [ - { - "delete_on_termination": true, - "device_name": "/dev/sdb", - "encrypted": false, - "iops": 10000, - "kms_key_id": "", - "snapshot_id": "", - "tags": {}, - "throughput": 125, - "volume_id": "vol-04c28c1cb5143ddf7", - "volume_size": 100, - "volume_type": "gp3" - } - ], - "ebs_optimized": false, - "enclave_options": [ - { - "enabled": false - } - ], - "ephemeral_block_device": [], - "get_password_data": false, - "hibernation": false, - "host_id": null, - "iam_instance_profile": "samsung-gauntlet-fb_cluster_node_profile", - "id": "i-09664bb4472386327", - "instance_initiated_shutdown_behavior": "stop", - "instance_state": "running", - "instance_type": "m6g.xlarge", - "ipv6_address_count": 0, - "ipv6_addresses": [], - "key_name": "samsung-gauntlet-gitlab-ci", - "launch_template": [], - "metadata_options": [ - { - "http_endpoint": "enabled", - "http_put_response_hop_limit": 1, - "http_tokens": "optional" - } - ], - "monitoring": true, - "network_interface": [], - "outpost_arn": "", - "password_data": "", - "placement_group": "", - "placement_partition_number": null, - "primary_network_interface_id": "eni-0809b96866ccfa203", - "private_dns": "ip-10-0-1-197.us-east-2.compute.internal", - "private_ip": "10.0.1.197", - "public_dns": "", - "public_ip": "", - "root_block_device": [ - { - "delete_on_termination": true, - "device_name": "/dev/xvda", - "encrypted": false, - "iops": 3000, - "kms_key_id": "", - "tags": {}, - "throughput": 125, - "volume_id": "vol-031f177e3f36fe051", - "volume_size": 20, - "volume_type": "gp3" - } - ], - "secondary_private_ips": [], - "security_groups": [], - "source_dest_check": true, - "subnet_id": "subnet-050b1219d78f2db1b", - "tags": { - "Name": "samsung-gauntlet-featurebase-cluster-0", - "Prefix": "samsung-gauntlet", - "Role": "cluster_node" - }, - "tags_all": { - "Name": "samsung-gauntlet-featurebase-cluster-0", - "Prefix": "samsung-gauntlet", - "Role": "cluster_node" - }, - "tenancy": "default", - "timeouts": null, - "user_data": "ed849ffb2caa5f32ccaf0571e91c3f6d9a54faac", - "user_data_base64": null, - "volume_tags": null, - "vpc_security_group_ids": [ - "sg-04d60067fbf393f98" - ] - }, - "sensitive_attributes": [], - "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjo2MDAwMDAwMDAwMDAsImRlbGV0ZSI6MTIwMDAwMDAwMDAwMCwidXBkYXRlIjo2MDAwMDAwMDAwMDB9LCJzY2hlbWFfdmVyc2lvbiI6IjEifQ==", - "dependencies": [ - "module.samsung-cluster.aws_iam_role.fb_cluster_node_role", - "module.samsung-cluster.aws_key_pair.gitlab-featurebase-ci", - "module.samsung-cluster.aws_security_group.featurebase", - "module.samsung-cluster.data.aws_ami.amazon_linux_2", - "module.samsung-cluster.aws_iam_instance_profile.fb_cluster_node_profile" - ] - }, - { - "index_key": 1, - "schema_version": 1, - "attributes": { - "ami": "ami-0b09f36be67d32fff", - "arn": "arn:aws:ec2:us-east-2:977373308795:instance/i-031b4e15ea7645578", - "associate_public_ip_address": false, - "availability_zone": "us-east-2b", - "capacity_reservation_specification": [ - { - "capacity_reservation_preference": "open", - "capacity_reservation_target": [] - } - ], - "cpu_core_count": 4, - "cpu_threads_per_core": 1, - "credit_specification": [], - "disable_api_termination": false, - "ebs_block_device": [ - { - "delete_on_termination": true, - "device_name": "/dev/sdb", - "encrypted": false, - "iops": 10000, - "kms_key_id": "", - "snapshot_id": "", - "tags": {}, - "throughput": 125, - "volume_id": "vol-0fd362b5516d0c017", - "volume_size": 100, - "volume_type": "gp3" - } - ], - "ebs_optimized": false, - "enclave_options": [ - { - "enabled": false - } - ], - "ephemeral_block_device": [], - "get_password_data": false, - "hibernation": false, - "host_id": null, - "iam_instance_profile": "samsung-gauntlet-fb_cluster_node_profile", - "id": "i-031b4e15ea7645578", - "instance_initiated_shutdown_behavior": "stop", - "instance_state": "running", - "instance_type": "m6g.xlarge", - "ipv6_address_count": 0, - "ipv6_addresses": [], - "key_name": "samsung-gauntlet-gitlab-ci", - "launch_template": [], - "metadata_options": [ - { - "http_endpoint": "enabled", - "http_put_response_hop_limit": 1, - "http_tokens": "optional" - } - ], - "monitoring": true, - "network_interface": [], - "outpost_arn": "", - "password_data": "", - "placement_group": "", - "placement_partition_number": null, - "primary_network_interface_id": "eni-0b844cf311584941e", - "private_dns": "ip-10-0-2-133.us-east-2.compute.internal", - "private_ip": "10.0.2.133", - "public_dns": "", - "public_ip": "", - "root_block_device": [ - { - "delete_on_termination": true, - "device_name": "/dev/xvda", - "encrypted": false, - "iops": 3000, - "kms_key_id": "", - "tags": null, - "throughput": 125, - "volume_id": "vol-0c3fe8fb90d0542c0", - "volume_size": 20, - "volume_type": "gp3" - } - ], - "secondary_private_ips": [], - "security_groups": [], - "source_dest_check": true, - "subnet_id": "subnet-0d623c769e086e46e", - "tags": { - "Name": "samsung-gauntlet-featurebase-cluster-1", - "Prefix": "samsung-gauntlet", - "Role": "cluster_node" - }, - "tags_all": { - "Name": "samsung-gauntlet-featurebase-cluster-1", - "Prefix": "samsung-gauntlet", - "Role": "cluster_node" - }, - "tenancy": "default", - "timeouts": null, - "user_data": "ed849ffb2caa5f32ccaf0571e91c3f6d9a54faac", - "user_data_base64": null, - "volume_tags": null, - "vpc_security_group_ids": [ - "sg-04d60067fbf393f98" - ] - }, - "sensitive_attributes": [], - "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjo2MDAwMDAwMDAwMDAsImRlbGV0ZSI6MTIwMDAwMDAwMDAwMCwidXBkYXRlIjo2MDAwMDAwMDAwMDB9LCJzY2hlbWFfdmVyc2lvbiI6IjEifQ==", - "dependencies": [ - "module.samsung-cluster.aws_iam_instance_profile.fb_cluster_node_profile", - "module.samsung-cluster.aws_key_pair.gitlab-featurebase-ci", - "module.samsung-cluster.aws_security_group.featurebase", - "module.samsung-cluster.data.aws_ami.amazon_linux_2" - ] - }, - { - "index_key": 2, - "schema_version": 1, - "attributes": { - "ami": "ami-0b09f36be67d32fff", - "arn": "arn:aws:ec2:us-east-2:977373308795:instance/i-0a37c057bee7ba28d", - "associate_public_ip_address": false, - "availability_zone": "us-east-2c", - "capacity_reservation_specification": [ - { - "capacity_reservation_preference": "open", - "capacity_reservation_target": [] - } - ], - "cpu_core_count": 4, - "cpu_threads_per_core": 1, - "credit_specification": [], - "disable_api_termination": false, - "ebs_block_device": [ - { - "delete_on_termination": true, - "device_name": "/dev/sdb", - "encrypted": false, - "iops": 10000, - "kms_key_id": "", - "snapshot_id": "", - "tags": {}, - "throughput": 125, - "volume_id": "vol-0203d9c0080bccc2a", - "volume_size": 100, - "volume_type": "gp3" - } - ], - "ebs_optimized": false, - "enclave_options": [ - { - "enabled": false - } - ], - "ephemeral_block_device": [], - "get_password_data": false, - "hibernation": false, - "host_id": null, - "iam_instance_profile": "samsung-gauntlet-fb_cluster_node_profile", - "id": "i-0a37c057bee7ba28d", - "instance_initiated_shutdown_behavior": "stop", - "instance_state": "running", - "instance_type": "m6g.xlarge", - "ipv6_address_count": 0, - "ipv6_addresses": [], - "key_name": "samsung-gauntlet-gitlab-ci", - "launch_template": [], - "metadata_options": [ - { - "http_endpoint": "enabled", - "http_put_response_hop_limit": 1, - "http_tokens": "optional" - } - ], - "monitoring": true, - "network_interface": [], - "outpost_arn": "", - "password_data": "", - "placement_group": "", - "placement_partition_number": null, - "primary_network_interface_id": "eni-03d10f8c14edc12b2", - "private_dns": "ip-10-0-3-244.us-east-2.compute.internal", - "private_ip": "10.0.3.244", - "public_dns": "", - "public_ip": "", - "root_block_device": [ - { - "delete_on_termination": true, - "device_name": "/dev/xvda", - "encrypted": false, - "iops": 3000, - "kms_key_id": "", - "tags": null, - "throughput": 125, - "volume_id": "vol-06b96553464e68c08", - "volume_size": 20, - "volume_type": "gp3" - } - ], - "secondary_private_ips": [], - "security_groups": [], - "source_dest_check": true, - "subnet_id": "subnet-07155281789c6d33b", - "tags": { - "Name": "samsung-gauntlet-featurebase-cluster-2", - "Prefix": "samsung-gauntlet", - "Role": "cluster_node" - }, - "tags_all": { - "Name": "samsung-gauntlet-featurebase-cluster-2", - "Prefix": "samsung-gauntlet", - "Role": "cluster_node" - }, - "tenancy": "default", - "timeouts": null, - "user_data": "ed849ffb2caa5f32ccaf0571e91c3f6d9a54faac", - "user_data_base64": null, - "volume_tags": null, - "vpc_security_group_ids": [ - "sg-04d60067fbf393f98" - ] - }, - "sensitive_attributes": [], - "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjo2MDAwMDAwMDAwMDAsImRlbGV0ZSI6MTIwMDAwMDAwMDAwMCwidXBkYXRlIjo2MDAwMDAwMDAwMDB9LCJzY2hlbWFfdmVyc2lvbiI6IjEifQ==", - "dependencies": [ - "module.samsung-cluster.aws_iam_instance_profile.fb_cluster_node_profile", - "module.samsung-cluster.aws_key_pair.gitlab-featurebase-ci", - "module.samsung-cluster.aws_security_group.featurebase", - "module.samsung-cluster.data.aws_ami.amazon_linux_2" - ] - } - ] - }, - { - "module": "module.samsung-cluster", - "mode": "managed", - "type": "aws_instance", - "name": "fb_ingest", - "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", - "instances": [ - { - "index_key": 0, - "schema_version": 1, - "attributes": { - "ami": "ami-0b09f36be67d32fff", - "arn": "arn:aws:ec2:us-east-2:977373308795:instance/i-05fe29bb7763ef362", - "associate_public_ip_address": true, - "availability_zone": "us-east-2a", - "capacity_reservation_specification": [ - { - "capacity_reservation_preference": "open", - "capacity_reservation_target": [] - } - ], - "cpu_core_count": 2, - "cpu_threads_per_core": 1, - "credit_specification": [], - "disable_api_termination": false, - "ebs_block_device": [ - { - "delete_on_termination": true, - "device_name": "/dev/sdb", - "encrypted": false, - "iops": 10000, - "kms_key_id": "", - "snapshot_id": "", - "tags": {}, - "throughput": 125, - "volume_id": "vol-08bc1374e4a0ef2d8", - "volume_size": 100, - "volume_type": "gp3" - } - ], - "ebs_optimized": false, - "enclave_options": [ - { - "enabled": false - } - ], - "ephemeral_block_device": [], - "get_password_data": false, - "hibernation": false, - "host_id": null, - "iam_instance_profile": "samsung-gauntlet-fb_cluster_node_profile", - "id": "i-05fe29bb7763ef362", - "instance_initiated_shutdown_behavior": "stop", - "instance_state": "running", - "instance_type": "m6g.large", - "ipv6_address_count": 0, - "ipv6_addresses": [], - "key_name": "samsung-gauntlet-gitlab-ci", - "launch_template": [], - "metadata_options": [ - { - "http_endpoint": "enabled", - "http_put_response_hop_limit": 1, - "http_tokens": "optional" - } - ], - "monitoring": true, - "network_interface": [], - "outpost_arn": "", - "password_data": "", - "placement_group": "", - "placement_partition_number": null, - "primary_network_interface_id": "eni-0dac7645cbd9504c0", - "private_dns": "ip-10-0-101-110.us-east-2.compute.internal", - "private_ip": "10.0.101.110", - "public_dns": "", - "public_ip": "3.145.96.245", - "root_block_device": [ - { - "delete_on_termination": true, - "device_name": "/dev/xvda", - "encrypted": false, - "iops": 3000, - "kms_key_id": "", - "tags": {}, - "throughput": 125, - "volume_id": "vol-054d4f37ba9a40786", - "volume_size": 20, - "volume_type": "gp3" - } - ], - "secondary_private_ips": [], - "security_groups": [], - "source_dest_check": true, - "subnet_id": "subnet-066b4b922b54e51a2", - "tags": { - "Name": "samsung-gauntlet-featurebase-ingest-0", - "Prefix": "samsung-gauntlet", - "Role": "ingest_node" - }, - "tags_all": { - "Name": "samsung-gauntlet-featurebase-ingest-0", - "Prefix": "samsung-gauntlet", - "Role": "ingest_node" - }, - "tenancy": "default", - "timeouts": null, - "user_data": "a511760e647134f82a8c6862bace75462b4be450", - "user_data_base64": null, - "volume_tags": null, - "vpc_security_group_ids": [ - "sg-04819516ceb6d6a6d" - ] - }, - "sensitive_attributes": [], - "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjo2MDAwMDAwMDAwMDAsImRlbGV0ZSI6MTIwMDAwMDAwMDAwMCwidXBkYXRlIjo2MDAwMDAwMDAwMDB9LCJzY2hlbWFfdmVyc2lvbiI6IjEifQ==", - "dependencies": [ - "module.samsung-cluster.aws_iam_instance_profile.fb_cluster_node_profile", - "module.samsung-cluster.aws_iam_role.fb_cluster_node_role", - "module.samsung-cluster.aws_key_pair.gitlab-featurebase-ci", - "module.samsung-cluster.aws_security_group.ingest", - "module.samsung-cluster.data.aws_ami.amazon_linux_2" - ] - } - ] - }, - { - "module": "module.samsung-cluster", - "mode": "managed", - "type": "aws_key_pair", - "name": "gitlab-featurebase-ci", - "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", - "instances": [ - { - "schema_version": 1, - "attributes": { - "arn": "arn:aws:ec2:us-east-2:977373308795:key-pair/samsung-gauntlet-gitlab-ci", - "fingerprint": "69:e5:4b:15:8d:1f:22:61:de:08:a9:ee:f3:29:5c:69", - "id": "samsung-gauntlet-gitlab-ci", - "key_name": "samsung-gauntlet-gitlab-ci", - "key_name_prefix": "", - "key_pair_id": "key-0f23e421e2db9bc94", - "public_key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC91hhpVHNonAG7ku2ugpxEskf9KHeyHJPQJT26OHrMUw7R+T5A8TjqSzTau07sXQ/E9SO3ebV8SJ5PqeaQOnQB8VEvVNK0DjQH7ppvNg1Rfs42FZT9ttzTMvOjsSbK3vZTHXdoKQEdC9NxBwSkFIRGQojK1HUOq9xGrw31fA1OjSwlpLcbx7yyg18lcqW6UOptnVR8U9Yy9qQ5jZF1HtkQ6L9J+gv4o1UyNAUK2bopeGiXpBc3PQ/CFaFT2h/aqLBP66qAHsHVyAFD3PIRtplC5EHa8jXDgLacEls0uF7Q3kRPxvzcuo4g4VkOn1rDy9qH3vd2hT3aKVnM73FIDUiL", - "tags": {}, - "tags_all": {} - }, - "sensitive_attributes": [], - "private": "eyJzY2hlbWFfdmVyc2lvbiI6IjEifQ==" - } - ] - }, - { - "module": "module.samsung-cluster", - "mode": "managed", - "type": "aws_security_group", - "name": "featurebase", - "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", - "instances": [ - { - "schema_version": 1, - "attributes": { - "arn": "arn:aws:ec2:us-east-2:977373308795:security-group/sg-04d60067fbf393f98", - "description": "Allow featurebase inbound traffic", - "egress": [ - { - "cidr_blocks": [ - "0.0.0.0/0" - ], - "description": "", - "from_port": 0, - "ipv6_cidr_blocks": [ - "::/0" - ], - "prefix_list_ids": [], - "protocol": "-1", - "security_groups": [], - "self": false, - "to_port": 0 - } - ], - "id": "sg-04d60067fbf393f98", - "ingress": [ - { - "cidr_blocks": [ - "0.0.0.0/0" - ], - "description": "SSH", - "from_port": 22, - "ipv6_cidr_blocks": [ - "::/0" - ], - "prefix_list_ids": [], - "protocol": "tcp", - "security_groups": [], - "self": false, - "to_port": 22 - }, - { - "cidr_blocks": [ - "0.0.0.0/0" - ], - "description": "icmp from Anywhere", - "from_port": -1, - "ipv6_cidr_blocks": [], - "prefix_list_ids": [], - "protocol": "icmp", - "security_groups": [], - "self": false, - "to_port": -1 - }, - { - "cidr_blocks": [ - "10.0.0.0/16" - ], - "description": "etcd from internal 2", - "from_port": 10401, - "ipv6_cidr_blocks": [], - "prefix_list_ids": [], - "protocol": "tcp", - "security_groups": [], - "self": false, - "to_port": 10401 - }, - { - "cidr_blocks": [ - "10.0.0.0/16" - ], - "description": "etcd from internal", - "from_port": 10301, - "ipv6_cidr_blocks": [], - "prefix_list_ids": [], - "protocol": "tcp", - "security_groups": [], - "self": false, - "to_port": 10301 - }, - { - "cidr_blocks": [ - "10.0.0.0/8", - "172.31.0.0/16" - ], - "description": "GRPC from Internal", - "from_port": 20101, - "ipv6_cidr_blocks": [], - "prefix_list_ids": [], - "protocol": "tcp", - "security_groups": [], - "self": false, - "to_port": 20101 - }, - { - "cidr_blocks": [ - "10.0.0.0/8", - "172.31.0.0/16" - ], - "description": "HTTP from Internal", - "from_port": 10101, - "ipv6_cidr_blocks": [], - "prefix_list_ids": [], - "protocol": "tcp", - "security_groups": [], - "self": false, - "to_port": 10101 - }, - { - "cidr_blocks": [ - "10.0.0.0/8", - "172.31.0.0/16" - ], - "description": "PostgreSQL from Internal", - "from_port": 55432, - "ipv6_cidr_blocks": [], - "prefix_list_ids": [], - "protocol": "tcp", - "security_groups": [], - "self": false, - "to_port": 55432 - } - ], - "name": "samsung-gauntlet-allow_featurebase", - "name_prefix": "", - "owner_id": "977373308795", - "revoke_rules_on_delete": false, - "tags": { - "Name": "allow_featurebase" - }, - "tags_all": { - "Name": "allow_featurebase" - }, - "timeouts": null, - "vpc_id": "vpc-05a26a122f961dc2b" - }, - "sensitive_attributes": [], - "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjo2MDAwMDAwMDAwMDAsImRlbGV0ZSI6OTAwMDAwMDAwMDAwfSwic2NoZW1hX3ZlcnNpb24iOiIxIn0=" - } - ] - }, - { - "module": "module.samsung-cluster", - "mode": "managed", - "type": "aws_security_group", - "name": "ingest", - "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", - "instances": [ - { - "schema_version": 1, - "attributes": { - "arn": "arn:aws:ec2:us-east-2:977373308795:security-group/sg-04819516ceb6d6a6d", - "description": "Allow ingest inbound traffic", - "egress": [ - { - "cidr_blocks": [ - "0.0.0.0/0" - ], - "description": "", - "from_port": 0, - "ipv6_cidr_blocks": [ - "::/0" - ], - "prefix_list_ids": [], - "protocol": "-1", - "security_groups": [], - "self": false, - "to_port": 0 - } - ], - "id": "sg-04819516ceb6d6a6d", - "ingress": [ - { - "cidr_blocks": [ - "0.0.0.0/0" - ], - "description": "", - "from_port": 10101, - "ipv6_cidr_blocks": [ - "::/0" - ], - "prefix_list_ids": [], - "protocol": "tcp", - "security_groups": [], - "self": false, - "to_port": 10101 - }, - { - "cidr_blocks": [ - "0.0.0.0/0" - ], - "description": "SSH", - "from_port": 22, - "ipv6_cidr_blocks": [ - "::/0" - ], - "prefix_list_ids": [], - "protocol": "tcp", - "security_groups": [], - "self": false, - "to_port": 22 - }, - { - "cidr_blocks": [ - "0.0.0.0/0" - ], - "description": "icmp from Anywhere", - "from_port": -1, - "ipv6_cidr_blocks": [], - "prefix_list_ids": [], - "protocol": "icmp", - "security_groups": [], - "self": false, - "to_port": -1 - } - ], - "name": "samsung-gauntlet-allow_ingest", - "name_prefix": "", - "owner_id": "977373308795", - "revoke_rules_on_delete": false, - "tags": { - "Name": "allow_ingest" - }, - "tags_all": { - "Name": "allow_ingest" - }, - "timeouts": null, - "vpc_id": "vpc-05a26a122f961dc2b" - }, - "sensitive_attributes": [], - "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjo2MDAwMDAwMDAwMDAsImRlbGV0ZSI6OTAwMDAwMDAwMDAwfSwic2NoZW1hX3ZlcnNpb24iOiIxIn0=" - } - ] - } - ] -}