From 286953c5fc1fd6c0f7b8240c11a1f79126739bd6 Mon Sep 17 00:00:00 2001 From: pokeeffe-molecula Date: Sun, 9 Jan 2022 11:53:14 -0600 Subject: [PATCH 01/13] getting gauntlet working --- qa/scripts/runSamsungGauntlet.sh | 23 +- qa/scripts/runSmokeTest.sh | 23 +- qa/scripts/setupSamsungGauntlet.sh | 27 +- qa/scripts/setupSmokeTest.sh | 23 +- qa/scripts/testSmokeTest.sh | 11 +- qa/scripts/utilCluster.sh | 20 - qa/tf/ci/smoketest/main.tf | 1 + qa/tf/ci/smoketest/terraform.tfstate.backup | 708 +++++++++++++++++- qa/tf/gauntlet/samsung/main.tf | 2 +- .../gauntlet/samsung/terraform.tfstate.backup | 128 ++-- 10 files changed, 866 insertions(+), 100 deletions(-) mode change 100644 => 100755 qa/scripts/runSmokeTest.sh diff --git a/qa/scripts/runSamsungGauntlet.sh b/qa/scripts/runSamsungGauntlet.sh index 85fd22c67..4b10e0d7c 100644 --- a/qa/scripts/runSamsungGauntlet.sh +++ b/qa/scripts/runSamsungGauntlet.sh @@ -1,5 +1,22 @@ #!/bin/bash -./setupSamsungGauntlet.sh -./testSamsungGauntlet.sh -./teardownSamsungGauntlet.sh +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) + +# requires TF_VAR_gitlab_token env var to be set +if [ -z ${TF_VAR_gitlab_token+x} ]; then echo "TF_VAR_gitlab_token is unset"; else echo "TF_VAR_gitlab_token is set to '$TF_VAR_gitlab_token'"; fi + +# requires TF_VAR_branch env var to be set +if [ -z ${TF_VAR_branch+x} ]; then echo "TF_VAR_branch is unset"; else echo "TF_VAR_branch is set to '$TF_VAR_branch'"; fi + +# requires TF_VAR_cluster_prefix env var to be set +if [ -z ${TF_VAR_cluster_prefix+x} ]; then + echo "setting TF_VAR_cluster_prefix"; + export TF_VAR_cluster_prefix="gauntlet-$(openssl rand -base64 12 | tr -d /=+ | cut -c -16)" + echo "TF_VAR_cluster_prefix is set to '$TF_VAR_cluster_prefix'"; +else + echo "TF_VAR_cluster_prefix is set to '$TF_VAR_cluster_prefix'"; +fi + +$SCRIPT_DIR/setupSamsungGauntlet.sh +#$SCRIPT_DIR/testSamsungGauntlet.sh +#$SCRIPT_DIR/teardownSamsungGauntlet.sh diff --git a/qa/scripts/runSmokeTest.sh b/qa/scripts/runSmokeTest.sh old mode 100644 new mode 100755 index 554a234cc..96866fd6d --- a/qa/scripts/runSmokeTest.sh +++ b/qa/scripts/runSmokeTest.sh @@ -1,5 +1,22 @@ #!/bin/bash -./setupSmokeTest.sh -./testSmokeTest.sh -./teardownSmokeTest.sh +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) + +# requires TF_VAR_gitlab_token env var to be set +if [ -z ${TF_VAR_gitlab_token+x} ]; then echo "TF_VAR_gitlab_token is unset"; else echo "TF_VAR_gitlab_token is set to '$TF_VAR_gitlab_token'"; fi + +# requires TF_VAR_branch env var to be set +if [ -z ${TF_VAR_branch+x} ]; then echo "TF_VAR_branch is unset"; else echo "TF_VAR_branch is set to '$TF_VAR_branch'"; fi + +# requires TF_VAR_cluster_prefix env var to be set +if [ -z ${TF_VAR_cluster_prefix+x} ]; then + echo "setting TF_VAR_cluster_prefix"; + export TF_VAR_cluster_prefix="smoke-$(openssl rand -base64 12 | tr -d /=+ | cut -c -16)" + echo "TF_VAR_cluster_prefix is set to '$TF_VAR_cluster_prefix'"; +else + echo "TF_VAR_cluster_prefix is set to '$TF_VAR_cluster_prefix'"; +fi + +$SCRIPT_DIR/setupSmokeTest.sh +$SCRIPT_DIR/testSmokeTest.sh +$SCRIPT_DIR/teardownSmokeTest.sh diff --git a/qa/scripts/setupSamsungGauntlet.sh b/qa/scripts/setupSamsungGauntlet.sh index d14da6729..7e8b684b3 100755 --- a/qa/scripts/setupSamsungGauntlet.sh +++ b/qa/scripts/setupSamsungGauntlet.sh @@ -22,20 +22,41 @@ terraform output -json > outputs.json popd # get the first ingest host -INGESTNODE0=$(cat ./qa/tf/ci/smoketest/outputs.json | jq -r '[.ingest_ips][0]["value"][0]') +INGESTNODE0=$(cat ./qa/tf/gauntlet/samsung/outputs.json | jq -r '[.ingest_ips][0]["value"][0]') echo "using INGESTNODE0 ${INGESTNODE0}" # get the first data host -DATANODE0=$(cat ./qa/tf/ci/smoketest/outputs.json | jq -r '[.data_node_ips][0]["value"][0]') +DATANODE0=$(cat ./qa/tf/gauntlet/samsung/outputs.json | jq -r '[.data_node_ips][0]["value"][0]') echo "using DATANODE0 ${DATANODE0}" + +DEPLOYED_CLUSTER_PREFIX=$(cat ./qa/tf/gauntlet/samsung/outputs.json | jq -r '[.cluster_prefix][0]["value"]') +echo "Using DEPLOYED_CLUSTER_PREFIX: ${DEPLOYED_CLUSTER_PREFIX}" + +DEPLOYED_CLUSTER_REPLICA_COUNT=$(cat ./qa/tf/gauntlet/samsung/outputs.json | jq -r '[.fb_cluster_replica_count][0]["value"]') +echo "Using DEPLOYED_CLUSTER_REPLICA_COUNT: ${DEPLOYED_CLUSTDEPLOYED_CLUSTER_REPLICA_COUNTER_PREFIX}" + +DEPLOYED_DATA_IPS=$(cat ./qa/tf/gauntlet/samsung/outputs.json | jq -r '[.data_node_ips][0]["value"][]') +echo "DEPLOYED_DATA_IPS: {" +echo "${DEPLOYED_DATA_IPS}" +echo "}" + +DEPLOYED_DATA_IPS_LEN=`echo "$DEPLOYED_DATA_IPS" | wc -l` + +DEPLOYED_INGEST_IPS=$(cat ./qa/tf/gauntlet/samsung/outputs.json | jq -r '[.ingest_ips][0]["value"][]') +echo "DEPLOYED_INGEST_IPS: {" +echo "${DEPLOYED_INGEST_IPS}" +echo "}" + +DEPLOYED_INGEST_IPS_LEN=`echo "$DEPLOYED_INGEST_IPS" | wc -l` + #wait until we can connect to one of the hosts for i in {0..24} do ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no -o ConnectTimeout=10 ec2-user@${DATANODE0} "pwd" if [ $? -eq 0 ] then - echo "Cluster is up after $${i} tries." + echo "Cluster is up after ${i} tries." break fi sleep 10s diff --git a/qa/scripts/setupSmokeTest.sh b/qa/scripts/setupSmokeTest.sh index 37eba6879..fac0d7111 100755 --- a/qa/scripts/setupSmokeTest.sh +++ b/qa/scripts/setupSmokeTest.sh @@ -28,13 +28,34 @@ echo "using INGESTNODE0 ${INGESTNODE0}" DATANODE0=$(cat ./qa/tf/ci/smoketest/outputs.json | jq -r '[.data_node_ips][0]["value"][0]') echo "using DATANODE0 ${DATANODE0}" +DEPLOYED_CLUSTER_PREFIX=$(cat ./qa/tf/ci/smoketest/outputs.json | jq -r '[.cluster_prefix][0]["value"]') +echo "Using DEPLOYED_CLUSTER_PREFIX: ${DEPLOYED_CLUSTER_PREFIX}" + +DEPLOYED_CLUSTER_REPLICA_COUNT=$(cat ./qa/tf/ci/smoketest/outputs.json | jq -r '[.fb_cluster_replica_count][0]["value"]') +echo "Using DEPLOYED_CLUSTER_REPLICA_COUNT: ${DEPLOYED_CLUSTDEPLOYED_CLUSTER_REPLICA_COUNTER_PREFIX}" + +DEPLOYED_DATA_IPS=$(cat ./qa/tf/ci/smoketest/outputs.json | jq -r '[.data_node_ips][0]["value"][]') +echo "DEPLOYED_DATA_IPS: {" +echo "${DEPLOYED_DATA_IPS}" +echo "}" + +DEPLOYED_DATA_IPS_LEN=`echo "$DEPLOYED_DATA_IPS" | wc -l` + +DEPLOYED_INGEST_IPS=$(cat ./qa/tf/ci/smoketest/outputs.json | jq -r '[.ingest_ips][0]["value"][]') +echo "DEPLOYED_INGEST_IPS: {" +echo "${DEPLOYED_INGEST_IPS}" +echo "}" + +DEPLOYED_INGEST_IPS_LEN=`echo "$DEPLOYED_INGEST_IPS" | wc -l` + + #wait until we can connect to one of the hosts for i in {0..24} do ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no -o ConnectTimeout=10 ec2-user@${DATANODE0} "pwd" if [ $? -eq 0 ] then - echo "Cluster is up after $${i} tries." + echo "Cluster is up after ${i} tries." break fi sleep 10 diff --git a/qa/scripts/testSmokeTest.sh b/qa/scripts/testSmokeTest.sh index 19d84e265..7b4f134a0 100755 --- a/qa/scripts/testSmokeTest.sh +++ b/qa/scripts/testSmokeTest.sh @@ -28,12 +28,8 @@ fi # run smoke test echo "Running smoke test..." -ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o "StrictHostKeyChecking no" ec2-user@${INGESTNODE0} "pushd /data; ~/.local/bin/pytest --junitxml=report.xml; popd" -if (( $? != 0 )) -then - echo "Unable to run smoke test" - exit 1 -fi +ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o "StrictHostKeyChecking no" ec2-user@${INGESTNODE0} "cd /data; ~/.local/bin/pytest --junitxml=report.xml" +SMOKETESTRESULT=$? echo "Copying test report to local" scp -r -i ~/.ssh/gitlab-featurebase-ci.pem ec2-user@${INGESTNODE0}:/data/report.xml report.xml @@ -43,4 +39,5 @@ then exit 1 fi -echo "Smoke test complete" \ No newline at end of file +echo "Smoke test complete" +exit $SMOKETESTRESULT \ No newline at end of file diff --git a/qa/scripts/utilCluster.sh b/qa/scripts/utilCluster.sh index d0bc49184..f62e07208 100644 --- a/qa/scripts/utilCluster.sh +++ b/qa/scripts/utilCluster.sh @@ -218,26 +218,6 @@ generateInitialClusterString() { } setupClusterNodes() { - DEPLOYED_CLUSTER_PREFIX=$(cat ./qa/tf/ci/smoketest/outputs.json | jq -r '[.cluster_prefix][0]["value"]') - echo "Using DEPLOYED_CLUSTER_PREFIX: ${DEPLOYED_CLUSTER_PREFIX}" - - DEPLOYED_CLUSTER_REPLICA_COUNT=$(cat ./qa/tf/ci/smoketest/outputs.json | jq -r '[.fb_cluster_replica_count][0]["value"]') - echo "Using DEPLOYED_CLUSTER_REPLICA_COUNT: ${DEPLOYED_CLUSTDEPLOYED_CLUSTER_REPLICA_COUNTER_PREFIX}" - - DEPLOYED_DATA_IPS=$(cat ./qa/tf/ci/smoketest/outputs.json | jq -r '[.data_node_ips][0]["value"][]') - echo "DEPLOYED_DATA_IPS: {" - echo "${DEPLOYED_DATA_IPS}" - echo "}" - - DEPLOYED_DATA_IPS_LEN=`echo "$DEPLOYED_DATA_IPS" | wc -l` - - DEPLOYED_INGEST_IPS=$(cat ./qa/tf/ci/smoketest/outputs.json | jq -r '[.ingest_ips][0]["value"][]') - echo "DEPLOYED_INGEST_IPS: {" - echo "${DEPLOYED_INGEST_IPS}" - echo "}" - - DEPLOYED_INGEST_IPS_LEN=`echo "$DEPLOYED_INGEST_IPS" | wc -l` - #data nodes generateInitialClusterString diff --git a/qa/tf/ci/smoketest/main.tf b/qa/tf/ci/smoketest/main.tf index 3ebd5eb08..40f51a89a 100644 --- a/qa/tf/ci/smoketest/main.tf +++ b/qa/tf/ci/smoketest/main.tf @@ -6,6 +6,7 @@ module "ci-cluster" { profile = var.profile fb_data_node_type = "m6g.large" fb_data_node_count = 1 + fb_ingest_type = "m6g.large" vpc_id = "vpc-05a26a122f961dc2b" vpc_cidr_block = "10.0.0.0/16" vpc_public_subnets = ["subnet-066b4b922b54e51a2","subnet-037b8884269a69025","subnet-08482631514426210",] diff --git a/qa/tf/ci/smoketest/terraform.tfstate.backup b/qa/tf/ci/smoketest/terraform.tfstate.backup index 5bb4c8639..b4998f14f 100644 --- a/qa/tf/ci/smoketest/terraform.tfstate.backup +++ b/qa/tf/ci/smoketest/terraform.tfstate.backup @@ -1,8 +1,710 @@ { "version": 4, "terraform_version": "1.1.2", - "serial": 178, + "serial": 203, "lineage": "0f5e8a05-0e94-e86f-f384-26086bd40585", - "outputs": {}, - "resources": [] + "outputs": { + "cluster_prefix": { + "value": "smoke-BzP3aSw62HwEPFS", + "type": "string" + }, + "data_node_ips": { + "value": [ + "10.0.1.185" + ], + "type": [ + "tuple", + [ + "string" + ] + ] + }, + "fb_cluster_replica_count": { + "value": 1, + "type": "number" + }, + "ingest_ips": { + "value": [ + "3.144.237.6" + ], + "type": [ + "tuple", + [ + "string" + ] + ] + } + }, + "resources": [ + { + "module": "module.ci-cluster", + "mode": "data", + "type": "aws_ami", + "name": "amazon_linux_2", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "architecture": "arm64", + "arn": "arn:aws:ec2:us-east-2::image/ami-0b09f36be67d32fff", + "block_device_mappings": [ + { + "device_name": "/dev/xvda", + "ebs": { + "delete_on_termination": "true", + "encrypted": "false", + "iops": "0", + "snapshot_id": "snap-0617b00e90bae012b", + "throughput": "0", + "volume_size": "8", + "volume_type": "gp2" + }, + "no_device": "", + "virtual_name": "" + } + ], + "creation_date": "2021-12-01T19:36:11.000Z", + "description": "Amazon Linux 2 LTS Arm64 AMI 2.0.20211201.0 arm64 HVM gp2", + "ena_support": true, + "executable_users": null, + "filter": [ + { + "name": "architecture", + "values": [ + "arm64" + ] + }, + { + "name": "name", + "values": [ + "amzn2-ami-hvm-*" + ] + }, + { + "name": "virtualization-type", + "values": [ + "hvm" + ] + } + ], + "hypervisor": "xen", + "id": "ami-0b09f36be67d32fff", + "image_id": "ami-0b09f36be67d32fff", + "image_location": "amazon/amzn2-ami-hvm-2.0.20211201.0-arm64-gp2", + "image_owner_alias": "amazon", + "image_type": "machine", + "kernel_id": null, + "most_recent": true, + "name": "amzn2-ami-hvm-2.0.20211201.0-arm64-gp2", + "name_regex": null, + "owner_id": "137112412989", + "owners": [ + "amazon" + ], + "platform": null, + "platform_details": "Linux/UNIX", + "product_codes": [], + "public": true, + "ramdisk_id": null, + "root_device_name": "/dev/xvda", + "root_device_type": "ebs", + "root_snapshot_id": "snap-0617b00e90bae012b", + "sriov_net_support": "simple", + "state": "available", + "state_reason": { + "code": "UNSET", + "message": "UNSET" + }, + "tags": {}, + "usage_operation": "RunInstances", + "virtualization_type": "hvm" + }, + "sensitive_attributes": [] + } + ] + }, + { + "module": "module.ci-cluster", + "mode": "managed", + "type": "aws_iam_instance_profile", + "name": "fb_cluster_node_profile", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "arn": "arn:aws:iam::977373308795:instance-profile/smoke-BzP3aSw62HwEPFS-fb_cluster_node_profile", + "create_date": "2022-01-09T17:47:01Z", + "id": "smoke-BzP3aSw62HwEPFS-fb_cluster_node_profile", + "name": "smoke-BzP3aSw62HwEPFS-fb_cluster_node_profile", + "name_prefix": null, + "path": "/", + "role": "smoke-BzP3aSw62HwEPFS-fb_cluster_node", + "tags": null, + "tags_all": {}, + "unique_id": "AIPA6HD75E55XTIU7KRKL" + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "module.ci-cluster.aws_iam_role.fb_cluster_node_role" + ] + } + ] + }, + { + "module": "module.ci-cluster", + "mode": "managed", + "type": "aws_iam_role", + "name": "fb_cluster_node_role", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "arn": "arn:aws:iam::977373308795:role/smoke-BzP3aSw62HwEPFS-fb_cluster_node", + "assume_role_policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Sid\":\"\",\"Effect\":\"Allow\",\"Principal\":{\"Service\":\"ec2.amazonaws.com\"},\"Action\":\"sts:AssumeRole\"}]}", + "create_date": "2022-01-09T17:46:58Z", + "description": "", + "force_detach_policies": false, + "id": "smoke-BzP3aSw62HwEPFS-fb_cluster_node", + "inline_policy": [ + { + "name": "ec2_read_all", + "policy": "{\"Statement\":[{\"Action\":[\"ec2:Describe*\"],\"Effect\":\"Allow\",\"Resource\":\"*\"}],\"Version\":\"2012-10-17\"}" + } + ], + "managed_policy_arns": [], + "max_session_duration": 3600, + "name": "smoke-BzP3aSw62HwEPFS-fb_cluster_node", + "name_prefix": "", + "path": "/", + "permissions_boundary": null, + "tags": null, + "tags_all": {}, + "unique_id": "AROA6HD75E55YXA4Z4XK2" + }, + "sensitive_attributes": [], + "private": "bnVsbA==" + } + ] + }, + { + "module": "module.ci-cluster", + "mode": "managed", + "type": "aws_instance", + "name": "fb_cluster_nodes", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "index_key": 0, + "schema_version": 1, + "attributes": { + "ami": "ami-0b09f36be67d32fff", + "arn": "arn:aws:ec2:us-east-2:977373308795:instance/i-077d0596e47712614", + "associate_public_ip_address": false, + "availability_zone": "us-east-2a", + "capacity_reservation_specification": [ + { + "capacity_reservation_preference": "open", + "capacity_reservation_target": [] + } + ], + "cpu_core_count": 2, + "cpu_threads_per_core": 1, + "credit_specification": [], + "disable_api_termination": false, + "ebs_block_device": [ + { + "delete_on_termination": true, + "device_name": "/dev/sdb", + "encrypted": false, + "iops": 3000, + "kms_key_id": "", + "snapshot_id": "", + "tags": {}, + "throughput": 125, + "volume_id": "vol-05eb623ed8991e3ec", + "volume_size": 100, + "volume_type": "gp3" + } + ], + "ebs_optimized": false, + "enclave_options": [ + { + "enabled": false + } + ], + "ephemeral_block_device": [], + "get_password_data": false, + "hibernation": false, + "host_id": null, + "iam_instance_profile": "smoke-BzP3aSw62HwEPFS-fb_cluster_node_profile", + "id": "i-077d0596e47712614", + "instance_initiated_shutdown_behavior": "stop", + "instance_state": "running", + "instance_type": "m6g.large", + "ipv6_address_count": 0, + "ipv6_addresses": [], + "key_name": "smoke-BzP3aSw62HwEPFS-gitlab-ci", + "launch_template": [], + "metadata_options": [ + { + "http_endpoint": "enabled", + "http_put_response_hop_limit": 1, + "http_tokens": "optional" + } + ], + "monitoring": true, + "network_interface": [], + "outpost_arn": "", + "password_data": "", + "placement_group": "", + "placement_partition_number": null, + "primary_network_interface_id": "eni-076ed20f66a9f3552", + "private_dns": "ip-10-0-1-185.us-east-2.compute.internal", + "private_ip": "10.0.1.185", + "public_dns": "", + "public_ip": "", + "root_block_device": [ + { + "delete_on_termination": true, + "device_name": "/dev/xvda", + "encrypted": false, + "iops": 3000, + "kms_key_id": "", + "tags": null, + "throughput": 125, + "volume_id": "vol-03c02dd3fdafd0aa0", + "volume_size": 20, + "volume_type": "gp3" + } + ], + "secondary_private_ips": [], + "security_groups": [], + "source_dest_check": true, + "subnet_id": "subnet-050b1219d78f2db1b", + "tags": { + "Name": "smoke-BzP3aSw62HwEPFS-featurebase-cluster-0", + "Prefix": "smoke-BzP3aSw62HwEPFS", + "Role": "cluster_node" + }, + "tags_all": { + "Name": "smoke-BzP3aSw62HwEPFS-featurebase-cluster-0", + "Prefix": "smoke-BzP3aSw62HwEPFS", + "Role": "cluster_node" + }, + "tenancy": "default", + "timeouts": null, + "user_data": null, + "user_data_base64": null, + "volume_tags": null, + "vpc_security_group_ids": [ + "sg-06b14c4162509fb5f" + ] + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjo2MDAwMDAwMDAwMDAsImRlbGV0ZSI6MTIwMDAwMDAwMDAwMCwidXBkYXRlIjo2MDAwMDAwMDAwMDB9LCJzY2hlbWFfdmVyc2lvbiI6IjEifQ==", + "dependencies": [ + "module.ci-cluster.aws_iam_instance_profile.fb_cluster_node_profile", + "module.ci-cluster.aws_iam_role.fb_cluster_node_role", + "module.ci-cluster.aws_key_pair.gitlab-featurebase-ci", + "module.ci-cluster.aws_security_group.featurebase", + "module.ci-cluster.data.aws_ami.amazon_linux_2" + ] + } + ] + }, + { + "module": "module.ci-cluster", + "mode": "managed", + "type": "aws_instance", + "name": "fb_ingest", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "index_key": 0, + "schema_version": 1, + "attributes": { + "ami": "ami-0b09f36be67d32fff", + "arn": "arn:aws:ec2:us-east-2:977373308795:instance/i-0ca325881e8a1dd8a", + "associate_public_ip_address": true, + "availability_zone": "us-east-2a", + "capacity_reservation_specification": [ + { + "capacity_reservation_preference": "open", + "capacity_reservation_target": [] + } + ], + "cpu_core_count": 8, + "cpu_threads_per_core": 1, + "credit_specification": [], + "disable_api_termination": false, + "ebs_block_device": [ + { + "delete_on_termination": true, + "device_name": "/dev/sdb", + "encrypted": false, + "iops": 3000, + "kms_key_id": "", + "snapshot_id": "", + "tags": {}, + "throughput": 125, + "volume_id": "vol-0803614106f4d6033", + "volume_size": 100, + "volume_type": "gp3" + } + ], + "ebs_optimized": false, + "enclave_options": [ + { + "enabled": false + } + ], + "ephemeral_block_device": [], + "get_password_data": false, + "hibernation": false, + "host_id": null, + "iam_instance_profile": "smoke-BzP3aSw62HwEPFS-fb_cluster_node_profile", + "id": "i-0ca325881e8a1dd8a", + "instance_initiated_shutdown_behavior": "stop", + "instance_state": "running", + "instance_type": "c6g.2xlarge", + "ipv6_address_count": 0, + "ipv6_addresses": [], + "key_name": "smoke-BzP3aSw62HwEPFS-gitlab-ci", + "launch_template": [], + "metadata_options": [ + { + "http_endpoint": "enabled", + "http_put_response_hop_limit": 1, + "http_tokens": "optional" + } + ], + "monitoring": true, + "network_interface": [], + "outpost_arn": "", + "password_data": "", + "placement_group": "", + "placement_partition_number": null, + "primary_network_interface_id": "eni-06216f96b73bcf92d", + "private_dns": "ip-10-0-101-105.us-east-2.compute.internal", + "private_ip": "10.0.101.105", + "public_dns": "", + "public_ip": "3.144.237.6", + "root_block_device": [ + { + "delete_on_termination": true, + "device_name": "/dev/xvda", + "encrypted": false, + "iops": 3000, + "kms_key_id": "", + "tags": null, + "throughput": 125, + "volume_id": "vol-070f8a9022e93f9ee", + "volume_size": 20, + "volume_type": "gp3" + } + ], + "secondary_private_ips": [], + "security_groups": [], + "source_dest_check": true, + "subnet_id": "subnet-066b4b922b54e51a2", + "tags": { + "Name": "smoke-BzP3aSw62HwEPFS-featurebase-ingest-0", + "Prefix": "smoke-BzP3aSw62HwEPFS", + "Role": "ingest_node" + }, + "tags_all": { + "Name": "smoke-BzP3aSw62HwEPFS-featurebase-ingest-0", + "Prefix": "smoke-BzP3aSw62HwEPFS", + "Role": "ingest_node" + }, + "tenancy": "default", + "timeouts": null, + "user_data": null, + "user_data_base64": null, + "volume_tags": null, + "vpc_security_group_ids": [ + "sg-062ddd221720047f4" + ] + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjo2MDAwMDAwMDAwMDAsImRlbGV0ZSI6MTIwMDAwMDAwMDAwMCwidXBkYXRlIjo2MDAwMDAwMDAwMDB9LCJzY2hlbWFfdmVyc2lvbiI6IjEifQ==", + "dependencies": [ + "module.ci-cluster.aws_iam_instance_profile.fb_cluster_node_profile", + "module.ci-cluster.aws_iam_role.fb_cluster_node_role", + "module.ci-cluster.aws_key_pair.gitlab-featurebase-ci", + "module.ci-cluster.aws_security_group.ingest", + "module.ci-cluster.data.aws_ami.amazon_linux_2" + ] + } + ] + }, + { + "module": "module.ci-cluster", + "mode": "managed", + "type": "aws_key_pair", + "name": "gitlab-featurebase-ci", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 1, + "attributes": { + "arn": "arn:aws:ec2:us-east-2:977373308795:key-pair/smoke-BzP3aSw62HwEPFS-gitlab-ci", + "fingerprint": "69:e5:4b:15:8d:1f:22:61:de:08:a9:ee:f3:29:5c:69", + "id": "smoke-BzP3aSw62HwEPFS-gitlab-ci", + "key_name": "smoke-BzP3aSw62HwEPFS-gitlab-ci", + "key_name_prefix": "", + "key_pair_id": "key-01ca26b2795043890", + "public_key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC91hhpVHNonAG7ku2ugpxEskf9KHeyHJPQJT26OHrMUw7R+T5A8TjqSzTau07sXQ/E9SO3ebV8SJ5PqeaQOnQB8VEvVNK0DjQH7ppvNg1Rfs42FZT9ttzTMvOjsSbK3vZTHXdoKQEdC9NxBwSkFIRGQojK1HUOq9xGrw31fA1OjSwlpLcbx7yyg18lcqW6UOptnVR8U9Yy9qQ5jZF1HtkQ6L9J+gv4o1UyNAUK2bopeGiXpBc3PQ/CFaFT2h/aqLBP66qAHsHVyAFD3PIRtplC5EHa8jXDgLacEls0uF7Q3kRPxvzcuo4g4VkOn1rDy9qH3vd2hT3aKVnM73FIDUiL", + "tags": null, + "tags_all": {} + }, + "sensitive_attributes": [], + "private": "eyJzY2hlbWFfdmVyc2lvbiI6IjEifQ==" + } + ] + }, + { + "module": "module.ci-cluster", + "mode": "managed", + "type": "aws_security_group", + "name": "featurebase", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 1, + "attributes": { + "arn": "arn:aws:ec2:us-east-2:977373308795:security-group/sg-06b14c4162509fb5f", + "description": "Allow featurebase inbound traffic", + "egress": [ + { + "cidr_blocks": [ + "0.0.0.0/0" + ], + "description": "", + "from_port": 0, + "ipv6_cidr_blocks": [ + "::/0" + ], + "prefix_list_ids": [], + "protocol": "-1", + "security_groups": [], + "self": false, + "to_port": 0 + } + ], + "id": "sg-06b14c4162509fb5f", + "ingress": [ + { + "cidr_blocks": [ + "0.0.0.0/0" + ], + "description": "SSH", + "from_port": 22, + "ipv6_cidr_blocks": [ + "::/0" + ], + "prefix_list_ids": [], + "protocol": "tcp", + "security_groups": [], + "self": false, + "to_port": 22 + }, + { + "cidr_blocks": [ + "0.0.0.0/0" + ], + "description": "icmp from Anywhere", + "from_port": -1, + "ipv6_cidr_blocks": [], + "prefix_list_ids": [], + "protocol": "icmp", + "security_groups": [], + "self": false, + "to_port": -1 + }, + { + "cidr_blocks": [ + "10.0.0.0/16" + ], + "description": "etcd from internal 2", + "from_port": 10401, + "ipv6_cidr_blocks": [], + "prefix_list_ids": [], + "protocol": "tcp", + "security_groups": [], + "self": false, + "to_port": 10401 + }, + { + "cidr_blocks": [ + "10.0.0.0/16" + ], + "description": "etcd from internal", + "from_port": 10301, + "ipv6_cidr_blocks": [], + "prefix_list_ids": [], + "protocol": "tcp", + "security_groups": [], + "self": false, + "to_port": 10301 + }, + { + "cidr_blocks": [ + "10.0.0.0/8", + "172.31.0.0/16" + ], + "description": "GRPC from Internal", + "from_port": 20101, + "ipv6_cidr_blocks": [], + "prefix_list_ids": [], + "protocol": "tcp", + "security_groups": [], + "self": false, + "to_port": 20101 + }, + { + "cidr_blocks": [ + "10.0.0.0/8", + "172.31.0.0/16" + ], + "description": "HTTP from Internal", + "from_port": 10101, + "ipv6_cidr_blocks": [], + "prefix_list_ids": [], + "protocol": "tcp", + "security_groups": [], + "self": false, + "to_port": 10101 + }, + { + "cidr_blocks": [ + "10.0.0.0/8", + "172.31.0.0/16" + ], + "description": "PostgreSQL from Internal", + "from_port": 55432, + "ipv6_cidr_blocks": [], + "prefix_list_ids": [], + "protocol": "tcp", + "security_groups": [], + "self": false, + "to_port": 55432 + } + ], + "name": "smoke-BzP3aSw62HwEPFS-allow_featurebase", + "name_prefix": "", + "owner_id": "977373308795", + "revoke_rules_on_delete": false, + "tags": { + "Name": "allow_featurebase" + }, + "tags_all": { + "Name": "allow_featurebase" + }, + "timeouts": null, + "vpc_id": "vpc-05a26a122f961dc2b" + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjo2MDAwMDAwMDAwMDAsImRlbGV0ZSI6OTAwMDAwMDAwMDAwfSwic2NoZW1hX3ZlcnNpb24iOiIxIn0=" + } + ] + }, + { + "module": "module.ci-cluster", + "mode": "managed", + "type": "aws_security_group", + "name": "ingest", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 1, + "attributes": { + "arn": "arn:aws:ec2:us-east-2:977373308795:security-group/sg-062ddd221720047f4", + "description": "Allow ingest inbound traffic", + "egress": [ + { + "cidr_blocks": [ + "0.0.0.0/0" + ], + "description": "", + "from_port": 0, + "ipv6_cidr_blocks": [ + "::/0" + ], + "prefix_list_ids": [], + "protocol": "-1", + "security_groups": [], + "self": false, + "to_port": 0 + } + ], + "id": "sg-062ddd221720047f4", + "ingress": [ + { + "cidr_blocks": [ + "0.0.0.0/0" + ], + "description": "", + "from_port": 10101, + "ipv6_cidr_blocks": [ + "::/0" + ], + "prefix_list_ids": [], + "protocol": "tcp", + "security_groups": [], + "self": false, + "to_port": 10101 + }, + { + "cidr_blocks": [ + "0.0.0.0/0" + ], + "description": "SSH", + "from_port": 22, + "ipv6_cidr_blocks": [ + "::/0" + ], + "prefix_list_ids": [], + "protocol": "tcp", + "security_groups": [], + "self": false, + "to_port": 22 + }, + { + "cidr_blocks": [ + "0.0.0.0/0" + ], + "description": "icmp from Anywhere", + "from_port": -1, + "ipv6_cidr_blocks": [], + "prefix_list_ids": [], + "protocol": "icmp", + "security_groups": [], + "self": false, + "to_port": -1 + } + ], + "name": "smoke-BzP3aSw62HwEPFS-allow_ingest", + "name_prefix": "", + "owner_id": "977373308795", + "revoke_rules_on_delete": false, + "tags": { + "Name": "allow_ingest" + }, + "tags_all": { + "Name": "allow_ingest" + }, + "timeouts": null, + "vpc_id": "vpc-05a26a122f961dc2b" + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjo2MDAwMDAwMDAwMDAsImRlbGV0ZSI6OTAwMDAwMDAwMDAwfSwic2NoZW1hX3ZlcnNpb24iOiIxIn0=" + } + ] + } + ] } diff --git a/qa/tf/gauntlet/samsung/main.tf b/qa/tf/gauntlet/samsung/main.tf index a4bdd2321..f86b7db70 100644 --- a/qa/tf/gauntlet/samsung/main.tf +++ b/qa/tf/gauntlet/samsung/main.tf @@ -1,6 +1,6 @@ module "samsung-cluster" { source = "../../.modules/featurebase-cluster" - cluster_prefix = "samsung-gauntlet" + cluster_prefix = var.cluster_prefix region = var.region profile = var.profile fb_data_node_type = "m6g.xlarge" diff --git a/qa/tf/gauntlet/samsung/terraform.tfstate.backup b/qa/tf/gauntlet/samsung/terraform.tfstate.backup index 2622f5631..264cb8522 100644 --- a/qa/tf/gauntlet/samsung/terraform.tfstate.backup +++ b/qa/tf/gauntlet/samsung/terraform.tfstate.backup @@ -1,14 +1,18 @@ { "version": 4, "terraform_version": "1.1.2", - "serial": 20, + "serial": 42, "lineage": "febac4ac-400a-d207-d2d1-64c55f14767b", "outputs": { + "cluster_prefix": { + "value": "samsung-gauntlet", + "type": "string" + }, "data_node_ips": { "value": [ - "10.0.1.197", - "10.0.2.133", - "10.0.3.244" + "10.0.1.153", + "10.0.2.15", + "10.0.3.89" ], "type": [ "tuple", @@ -19,9 +23,13 @@ ] ] }, + "fb_cluster_replica_count": { + "value": 1, + "type": "number" + }, "ingest_ips": { "value": [ - "3.145.96.245" + "18.217.165.151" ], "type": [ "tuple", @@ -131,7 +139,7 @@ "schema_version": 0, "attributes": { "arn": "arn:aws:iam::977373308795:instance-profile/samsung-gauntlet-fb_cluster_node_profile", - "create_date": "2022-01-07T23:16:27Z", + "create_date": "2022-01-09T17:25:36Z", "id": "samsung-gauntlet-fb_cluster_node_profile", "name": "samsung-gauntlet-fb_cluster_node_profile", "name_prefix": null, @@ -139,7 +147,7 @@ "role": "samsung-gauntlet-fb_cluster_node", "tags": {}, "tags_all": {}, - "unique_id": "AIPA6HD75E55ZM7XC2IBD" + "unique_id": "AIPA6HD75E55STNZ2F4C3" }, "sensitive_attributes": [], "private": "bnVsbA==", @@ -161,7 +169,7 @@ "attributes": { "arn": "arn:aws:iam::977373308795:role/samsung-gauntlet-fb_cluster_node", "assume_role_policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Sid\":\"\",\"Effect\":\"Allow\",\"Principal\":{\"Service\":\"ec2.amazonaws.com\"},\"Action\":\"sts:AssumeRole\"}]}", - "create_date": "2022-01-07T23:16:05Z", + "create_date": "2022-01-09T17:25:34Z", "description": "", "force_detach_policies": false, "id": "samsung-gauntlet-fb_cluster_node", @@ -179,7 +187,7 @@ "permissions_boundary": null, "tags": {}, "tags_all": {}, - "unique_id": "AROA6HD75E55VM3GADWTD" + "unique_id": "AROA6HD75E552T4UVUF5M" }, "sensitive_attributes": [], "private": "bnVsbA==" @@ -198,7 +206,7 @@ "schema_version": 1, "attributes": { "ami": "ami-0b09f36be67d32fff", - "arn": "arn:aws:ec2:us-east-2:977373308795:instance/i-09664bb4472386327", + "arn": "arn:aws:ec2:us-east-2:977373308795:instance/i-0480cda8df43c65c2", "associate_public_ip_address": false, "availability_zone": "us-east-2a", "capacity_reservation_specification": [ @@ -221,7 +229,7 @@ "snapshot_id": "", "tags": {}, "throughput": 125, - "volume_id": "vol-04c28c1cb5143ddf7", + "volume_id": "vol-048ea5053dd0f6c4d", "volume_size": 100, "volume_type": "gp3" } @@ -237,7 +245,7 @@ "hibernation": false, "host_id": null, "iam_instance_profile": "samsung-gauntlet-fb_cluster_node_profile", - "id": "i-09664bb4472386327", + "id": "i-0480cda8df43c65c2", "instance_initiated_shutdown_behavior": "stop", "instance_state": "running", "instance_type": "m6g.xlarge", @@ -258,9 +266,9 @@ "password_data": "", "placement_group": "", "placement_partition_number": null, - "primary_network_interface_id": "eni-0809b96866ccfa203", - "private_dns": "ip-10-0-1-197.us-east-2.compute.internal", - "private_ip": "10.0.1.197", + "primary_network_interface_id": "eni-0dc00e2676f102cbb", + "private_dns": "ip-10-0-1-153.us-east-2.compute.internal", + "private_ip": "10.0.1.153", "public_dns": "", "public_ip": "", "root_block_device": [ @@ -272,7 +280,7 @@ "kms_key_id": "", "tags": {}, "throughput": 125, - "volume_id": "vol-031f177e3f36fe051", + "volume_id": "vol-0d4c17942d878d0c9", "volume_size": 20, "volume_type": "gp3" } @@ -293,21 +301,21 @@ }, "tenancy": "default", "timeouts": null, - "user_data": "ed849ffb2caa5f32ccaf0571e91c3f6d9a54faac", + "user_data": null, "user_data_base64": null, "volume_tags": null, "vpc_security_group_ids": [ - "sg-04d60067fbf393f98" + "sg-026cd014f6dd154b1" ] }, "sensitive_attributes": [], "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjo2MDAwMDAwMDAwMDAsImRlbGV0ZSI6MTIwMDAwMDAwMDAwMCwidXBkYXRlIjo2MDAwMDAwMDAwMDB9LCJzY2hlbWFfdmVyc2lvbiI6IjEifQ==", "dependencies": [ + "module.samsung-cluster.aws_iam_instance_profile.fb_cluster_node_profile", "module.samsung-cluster.aws_iam_role.fb_cluster_node_role", "module.samsung-cluster.aws_key_pair.gitlab-featurebase-ci", "module.samsung-cluster.aws_security_group.featurebase", - "module.samsung-cluster.data.aws_ami.amazon_linux_2", - "module.samsung-cluster.aws_iam_instance_profile.fb_cluster_node_profile" + "module.samsung-cluster.data.aws_ami.amazon_linux_2" ] }, { @@ -315,7 +323,7 @@ "schema_version": 1, "attributes": { "ami": "ami-0b09f36be67d32fff", - "arn": "arn:aws:ec2:us-east-2:977373308795:instance/i-031b4e15ea7645578", + "arn": "arn:aws:ec2:us-east-2:977373308795:instance/i-0bbf310c24c1c46d5", "associate_public_ip_address": false, "availability_zone": "us-east-2b", "capacity_reservation_specification": [ @@ -338,7 +346,7 @@ "snapshot_id": "", "tags": {}, "throughput": 125, - "volume_id": "vol-0fd362b5516d0c017", + "volume_id": "vol-07325e290952e5a6b", "volume_size": 100, "volume_type": "gp3" } @@ -354,7 +362,7 @@ "hibernation": false, "host_id": null, "iam_instance_profile": "samsung-gauntlet-fb_cluster_node_profile", - "id": "i-031b4e15ea7645578", + "id": "i-0bbf310c24c1c46d5", "instance_initiated_shutdown_behavior": "stop", "instance_state": "running", "instance_type": "m6g.xlarge", @@ -375,9 +383,9 @@ "password_data": "", "placement_group": "", "placement_partition_number": null, - "primary_network_interface_id": "eni-0b844cf311584941e", - "private_dns": "ip-10-0-2-133.us-east-2.compute.internal", - "private_ip": "10.0.2.133", + "primary_network_interface_id": "eni-0745bbcd1bae80fd4", + "private_dns": "ip-10-0-2-15.us-east-2.compute.internal", + "private_ip": "10.0.2.15", "public_dns": "", "public_ip": "", "root_block_device": [ @@ -387,9 +395,9 @@ "encrypted": false, "iops": 3000, "kms_key_id": "", - "tags": null, + "tags": {}, "throughput": 125, - "volume_id": "vol-0c3fe8fb90d0542c0", + "volume_id": "vol-0b13e3dc72826a74a", "volume_size": 20, "volume_type": "gp3" } @@ -410,20 +418,21 @@ }, "tenancy": "default", "timeouts": null, - "user_data": "ed849ffb2caa5f32ccaf0571e91c3f6d9a54faac", + "user_data": null, "user_data_base64": null, "volume_tags": null, "vpc_security_group_ids": [ - "sg-04d60067fbf393f98" + "sg-026cd014f6dd154b1" ] }, "sensitive_attributes": [], "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjo2MDAwMDAwMDAwMDAsImRlbGV0ZSI6MTIwMDAwMDAwMDAwMCwidXBkYXRlIjo2MDAwMDAwMDAwMDB9LCJzY2hlbWFfdmVyc2lvbiI6IjEifQ==", "dependencies": [ + "module.samsung-cluster.data.aws_ami.amazon_linux_2", "module.samsung-cluster.aws_iam_instance_profile.fb_cluster_node_profile", + "module.samsung-cluster.aws_iam_role.fb_cluster_node_role", "module.samsung-cluster.aws_key_pair.gitlab-featurebase-ci", - "module.samsung-cluster.aws_security_group.featurebase", - "module.samsung-cluster.data.aws_ami.amazon_linux_2" + "module.samsung-cluster.aws_security_group.featurebase" ] }, { @@ -431,7 +440,7 @@ "schema_version": 1, "attributes": { "ami": "ami-0b09f36be67d32fff", - "arn": "arn:aws:ec2:us-east-2:977373308795:instance/i-0a37c057bee7ba28d", + "arn": "arn:aws:ec2:us-east-2:977373308795:instance/i-0b9def9ba1e299921", "associate_public_ip_address": false, "availability_zone": "us-east-2c", "capacity_reservation_specification": [ @@ -454,7 +463,7 @@ "snapshot_id": "", "tags": {}, "throughput": 125, - "volume_id": "vol-0203d9c0080bccc2a", + "volume_id": "vol-099fe812da7384809", "volume_size": 100, "volume_type": "gp3" } @@ -470,7 +479,7 @@ "hibernation": false, "host_id": null, "iam_instance_profile": "samsung-gauntlet-fb_cluster_node_profile", - "id": "i-0a37c057bee7ba28d", + "id": "i-0b9def9ba1e299921", "instance_initiated_shutdown_behavior": "stop", "instance_state": "running", "instance_type": "m6g.xlarge", @@ -491,9 +500,9 @@ "password_data": "", "placement_group": "", "placement_partition_number": null, - "primary_network_interface_id": "eni-03d10f8c14edc12b2", - "private_dns": "ip-10-0-3-244.us-east-2.compute.internal", - "private_ip": "10.0.3.244", + "primary_network_interface_id": "eni-011390db576e1ef11", + "private_dns": "ip-10-0-3-89.us-east-2.compute.internal", + "private_ip": "10.0.3.89", "public_dns": "", "public_ip": "", "root_block_device": [ @@ -503,9 +512,9 @@ "encrypted": false, "iops": 3000, "kms_key_id": "", - "tags": null, + "tags": {}, "throughput": 125, - "volume_id": "vol-06b96553464e68c08", + "volume_id": "vol-0705e352c214b476b", "volume_size": 20, "volume_type": "gp3" } @@ -526,17 +535,18 @@ }, "tenancy": "default", "timeouts": null, - "user_data": "ed849ffb2caa5f32ccaf0571e91c3f6d9a54faac", + "user_data": null, "user_data_base64": null, "volume_tags": null, "vpc_security_group_ids": [ - "sg-04d60067fbf393f98" + "sg-026cd014f6dd154b1" ] }, "sensitive_attributes": [], "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjo2MDAwMDAwMDAwMDAsImRlbGV0ZSI6MTIwMDAwMDAwMDAwMCwidXBkYXRlIjo2MDAwMDAwMDAwMDB9LCJzY2hlbWFfdmVyc2lvbiI6IjEifQ==", "dependencies": [ "module.samsung-cluster.aws_iam_instance_profile.fb_cluster_node_profile", + "module.samsung-cluster.aws_iam_role.fb_cluster_node_role", "module.samsung-cluster.aws_key_pair.gitlab-featurebase-ci", "module.samsung-cluster.aws_security_group.featurebase", "module.samsung-cluster.data.aws_ami.amazon_linux_2" @@ -556,7 +566,7 @@ "schema_version": 1, "attributes": { "ami": "ami-0b09f36be67d32fff", - "arn": "arn:aws:ec2:us-east-2:977373308795:instance/i-05fe29bb7763ef362", + "arn": "arn:aws:ec2:us-east-2:977373308795:instance/i-0205a6a18916b76d4", "associate_public_ip_address": true, "availability_zone": "us-east-2a", "capacity_reservation_specification": [ @@ -579,7 +589,7 @@ "snapshot_id": "", "tags": {}, "throughput": 125, - "volume_id": "vol-08bc1374e4a0ef2d8", + "volume_id": "vol-0f2d419dec323106a", "volume_size": 100, "volume_type": "gp3" } @@ -595,7 +605,7 @@ "hibernation": false, "host_id": null, "iam_instance_profile": "samsung-gauntlet-fb_cluster_node_profile", - "id": "i-05fe29bb7763ef362", + "id": "i-0205a6a18916b76d4", "instance_initiated_shutdown_behavior": "stop", "instance_state": "running", "instance_type": "m6g.large", @@ -616,11 +626,11 @@ "password_data": "", "placement_group": "", "placement_partition_number": null, - "primary_network_interface_id": "eni-0dac7645cbd9504c0", - "private_dns": "ip-10-0-101-110.us-east-2.compute.internal", - "private_ip": "10.0.101.110", + "primary_network_interface_id": "eni-0353d32a9a9cb44e1", + "private_dns": "ip-10-0-101-14.us-east-2.compute.internal", + "private_ip": "10.0.101.14", "public_dns": "", - "public_ip": "3.145.96.245", + "public_ip": "18.217.165.151", "root_block_device": [ { "delete_on_termination": true, @@ -630,7 +640,7 @@ "kms_key_id": "", "tags": {}, "throughput": 125, - "volume_id": "vol-054d4f37ba9a40786", + "volume_id": "vol-00ffed175f7e1b82f", "volume_size": 20, "volume_type": "gp3" } @@ -651,21 +661,21 @@ }, "tenancy": "default", "timeouts": null, - "user_data": "a511760e647134f82a8c6862bace75462b4be450", + "user_data": null, "user_data_base64": null, "volume_tags": null, "vpc_security_group_ids": [ - "sg-04819516ceb6d6a6d" + "sg-020719cac2e5f2055" ] }, "sensitive_attributes": [], "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjo2MDAwMDAwMDAwMDAsImRlbGV0ZSI6MTIwMDAwMDAwMDAwMCwidXBkYXRlIjo2MDAwMDAwMDAwMDB9LCJzY2hlbWFfdmVyc2lvbiI6IjEifQ==", "dependencies": [ + "module.samsung-cluster.data.aws_ami.amazon_linux_2", "module.samsung-cluster.aws_iam_instance_profile.fb_cluster_node_profile", "module.samsung-cluster.aws_iam_role.fb_cluster_node_role", "module.samsung-cluster.aws_key_pair.gitlab-featurebase-ci", - "module.samsung-cluster.aws_security_group.ingest", - "module.samsung-cluster.data.aws_ami.amazon_linux_2" + "module.samsung-cluster.aws_security_group.ingest" ] } ] @@ -685,7 +695,7 @@ "id": "samsung-gauntlet-gitlab-ci", "key_name": "samsung-gauntlet-gitlab-ci", "key_name_prefix": "", - "key_pair_id": "key-0f23e421e2db9bc94", + "key_pair_id": "key-0c3cbdc0e74874b3d", "public_key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC91hhpVHNonAG7ku2ugpxEskf9KHeyHJPQJT26OHrMUw7R+T5A8TjqSzTau07sXQ/E9SO3ebV8SJ5PqeaQOnQB8VEvVNK0DjQH7ppvNg1Rfs42FZT9ttzTMvOjsSbK3vZTHXdoKQEdC9NxBwSkFIRGQojK1HUOq9xGrw31fA1OjSwlpLcbx7yyg18lcqW6UOptnVR8U9Yy9qQ5jZF1HtkQ6L9J+gv4o1UyNAUK2bopeGiXpBc3PQ/CFaFT2h/aqLBP66qAHsHVyAFD3PIRtplC5EHa8jXDgLacEls0uF7Q3kRPxvzcuo4g4VkOn1rDy9qH3vd2hT3aKVnM73FIDUiL", "tags": {}, "tags_all": {} @@ -705,7 +715,7 @@ { "schema_version": 1, "attributes": { - "arn": "arn:aws:ec2:us-east-2:977373308795:security-group/sg-04d60067fbf393f98", + "arn": "arn:aws:ec2:us-east-2:977373308795:security-group/sg-026cd014f6dd154b1", "description": "Allow featurebase inbound traffic", "egress": [ { @@ -724,7 +734,7 @@ "to_port": 0 } ], - "id": "sg-04d60067fbf393f98", + "id": "sg-026cd014f6dd154b1", "ingress": [ { "cidr_blocks": [ @@ -851,7 +861,7 @@ { "schema_version": 1, "attributes": { - "arn": "arn:aws:ec2:us-east-2:977373308795:security-group/sg-04819516ceb6d6a6d", + "arn": "arn:aws:ec2:us-east-2:977373308795:security-group/sg-020719cac2e5f2055", "description": "Allow ingest inbound traffic", "egress": [ { @@ -870,7 +880,7 @@ "to_port": 0 } ], - "id": "sg-04819516ceb6d6a6d", + "id": "sg-020719cac2e5f2055", "ingress": [ { "cidr_blocks": [ From 320ae1a8b28eb2d332a6ef615d1fd656def78ea5 Mon Sep 17 00:00:00 2001 From: pokeeffe-molecula Date: Mon, 10 Jan 2022 08:24:11 -0600 Subject: [PATCH 02/13] make test failures clearer --- qa/scripts/testSmokeTest.sh | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/qa/scripts/testSmokeTest.sh b/qa/scripts/testSmokeTest.sh index 7b4f134a0..24576b132 100755 --- a/qa/scripts/testSmokeTest.sh +++ b/qa/scripts/testSmokeTest.sh @@ -39,5 +39,11 @@ then exit 1 fi -echo "Smoke test complete" -exit $SMOKETESTRESULT \ No newline at end of file +if (( $SMOKETESTRESULT != 0 )) +then + echo "Smoke test complete with test failures" +else + echo "Smoke test complete" +fi + +exit $SMOKETESTRESULT \ No newline at end of file From d4e9852b09101203983bf89302b9a182c4c70c68 Mon Sep 17 00:00:00 2001 From: pokeeffe-molecula Date: Mon, 10 Jan 2022 10:28:51 -0600 Subject: [PATCH 03/13] try to get gauntlet to run e2e on GitLab --- .gitlab/.gitlab-ci.yml | 93 +++++---- .../gauntlet/samsung/terraform.tfstate.backup | 190 +++++++++--------- 2 files changed, 144 insertions(+), 139 deletions(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 5f65fc704..f325bf552 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -274,49 +274,54 @@ smoke test: reports: junit: report.xml -# gauntlet: -# stage: gauntlet -# timeout: 4h -# image: registry.gitlab.com/gitlab-org/cloud-deploy/aws-base:latest -# variables: -# PROFILE: "default" -# AWS_SSH_PRIVATE_KEY: $AWS_FBCI_SSH_KEY -# AWS_ACCESS_KEY_ID: $AWS_FBCI_ACCESS_KEY_ID -# AWS_SECRET_ACCESS_KEY: $AWS_FBCI_SECRET_ACCESS_KEY -# TF_VAR_cluster_prefix: "" -# TF_VAR_branch: "" -# rules: + gauntlet: + stage: gauntlet + timeout: 4h + image: registry.gitlab.com/gitlab-org/cloud-deploy/aws-base:latest + variables: + PROFILE: "service-terraform" + AWS_SSH_PRIVATE_KEY: $AWS_FBCI_SSH_KEY + AWS_ACCESS_KEY_ID: $AWS_FBCI_ACCESS_KEY_ID + AWS_SECRET_ACCESS_KEY: $AWS_FBCI_SECRET_ACCESS_KEY + TF_VAR_cluster_prefix: "" + TF_VAR_branch: "" + tags: + - aws + - docker + - fbsmoke + rules: # - if: '$CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"' -# - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH' -# before_script: -# - apt-get update && apt-get install -y gnupg software-properties-common curl git -# - curl -fsSL https://apt.releases.hashicorp.com/gpg | apt-key add - -# - apt-add-repository "deb [arch=amd64] https://apt.releases.hashicorp.com $(lsb_release -cs) main" -# - apt-get update && apt-get install terraform -# - aws configure set aws_access_key_id $AWS_FBCI_ACCESS_KEY_ID -# - aws configure set aws_secret_access_key $AWS_FBCI_SECRET_ACCESS_KEY -# - aws configure set region "us-east-2" -# - aws configure set aws_profile $PROFILE -# - echo $AWS_FBCI_SSH_KEY > gitlab-featurebase-ci.pem -# - chmod 400 gitlab-featurebase-ci.pem -# - 'which ssh-agent || ( apt-get update -y && apt-get install openssh-client -y )' -# - eval $(ssh-agent -s) -# - mkdir -p ~/.ssh -# - echo $AWS_FBCI_SSH_KEY > /root/.ssh/gitlab-featurebase-ci.pem -# - chmod 400 /root/.ssh/gitlab-featurebase-ci.pem -# - echo "$AWS_FBCI_SSH_KEY" | ssh-add - -# - chmod 700 /root/.ssh -# - '[[ -f /.dockerenv ]] && echo -e "Host *\n\tStrictHostKeyChecking no\n\n" > ~/.ssh/config' -# - apt update && apt -y install jq wget -# - wget https://go.dev/dl/go1.17.5.linux-amd64.tar.gz -# - tar -C /usr/local -xzf go1.17.5.linux-amd64.tar.gz -# - export PATH=$PATH:/usr/local/go/bin -# - TF_VAR_cluster_prefix="smoke-$(openssl rand -base64 12 | tr -d /=+ | cut -c -16)" -# - echo "Cluster Prefix --> $TF_VAR_cluster_prefix" -# - TF_VAR_branch=$CI_COMMIT_BRANCH -# - echo "Branch --> $TF_VAR_branch" -# script: -# - ./qa/scripts/setupSamsungGauntlet.sh + - if: '$CI_PIPELINE_SOURCE == "push"' + - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH' + before_script: + - apt-get update && apt-get install -y gnupg software-properties-common curl git + - curl -fsSL https://apt.releases.hashicorp.com/gpg | apt-key add - + - apt-add-repository "deb [arch=amd64] https://apt.releases.hashicorp.com $(lsb_release -cs) main" + - apt-get update && apt-get install terraform + - aws configure set aws_access_key_id $AWS_FBCI_ACCESS_KEY_ID + - aws configure set aws_secret_access_key $AWS_FBCI_SECRET_ACCESS_KEY + - aws configure set region "us-east-2" + - aws configure set aws_profile $PROFILE + - echo $AWS_FBCI_SSH_KEY > gitlab-featurebase-ci.pem + - chmod 400 gitlab-featurebase-ci.pem + - 'which ssh-agent || ( apt-get update -y && apt-get install openssh-client -y )' + - eval $(ssh-agent -s) + - mkdir -p ~/.ssh + - echo $AWS_FBCI_SSH_KEY > /root/.ssh/gitlab-featurebase-ci.pem + - chmod 400 /root/.ssh/gitlab-featurebase-ci.pem + - echo "$AWS_FBCI_SSH_KEY" | ssh-add - + - chmod 700 /root/.ssh + - '[[ -f /.dockerenv ]] && echo -e "Host *\n\tStrictHostKeyChecking no\n\n" > ~/.ssh/config' + - apt update && apt -y install jq wget + - wget https://go.dev/dl/go1.17.5.linux-amd64.tar.gz + - tar -C /usr/local -xzf go1.17.5.linux-amd64.tar.gz + - export PATH=$PATH:/usr/local/go/bin + - TF_VAR_cluster_prefix="gauntlet-$(openssl rand -base64 12 | tr -d /=+ | cut -c -16)" + - echo "Cluster Prefix --> $TF_VAR_cluster_prefix" + - TF_VAR_branch=$CI_COMMIT_BRANCH + - echo "Branch --> $TF_VAR_branch" + script: + - ./qa/scripts/setupSamsungGauntlet.sh # - ./qa/scripts/testSamsungGauntlet.sh -# after_script: -# - ./qa/scripts/teardownSamsungGauntlet.sh + after_script: + - ./qa/scripts/teardownSamsungGauntlet.sh diff --git a/qa/tf/gauntlet/samsung/terraform.tfstate.backup b/qa/tf/gauntlet/samsung/terraform.tfstate.backup index 264cb8522..bc74976b2 100644 --- a/qa/tf/gauntlet/samsung/terraform.tfstate.backup +++ b/qa/tf/gauntlet/samsung/terraform.tfstate.backup @@ -1,18 +1,18 @@ { "version": 4, "terraform_version": "1.1.2", - "serial": 42, + "serial": 63, "lineage": "febac4ac-400a-d207-d2d1-64c55f14767b", "outputs": { "cluster_prefix": { - "value": "samsung-gauntlet", + "value": "gauntlet-wFQOOzXB51R3ebr", "type": "string" }, "data_node_ips": { "value": [ - "10.0.1.153", - "10.0.2.15", - "10.0.3.89" + "10.0.1.141", + "10.0.2.81", + "10.0.3.251" ], "type": [ "tuple", @@ -29,7 +29,7 @@ }, "ingest_ips": { "value": [ - "18.217.165.151" + "3.144.13.48" ], "type": [ "tuple", @@ -138,16 +138,16 @@ { "schema_version": 0, "attributes": { - "arn": "arn:aws:iam::977373308795:instance-profile/samsung-gauntlet-fb_cluster_node_profile", - "create_date": "2022-01-09T17:25:36Z", - "id": "samsung-gauntlet-fb_cluster_node_profile", - "name": "samsung-gauntlet-fb_cluster_node_profile", + "arn": "arn:aws:iam::977373308795:instance-profile/gauntlet-wFQOOzXB51R3ebr-fb_cluster_node_profile", + "create_date": "2022-01-10T16:18:40Z", + "id": "gauntlet-wFQOOzXB51R3ebr-fb_cluster_node_profile", + "name": "gauntlet-wFQOOzXB51R3ebr-fb_cluster_node_profile", "name_prefix": null, "path": "/", - "role": "samsung-gauntlet-fb_cluster_node", - "tags": {}, + "role": "gauntlet-wFQOOzXB51R3ebr-fb_cluster_node", + "tags": null, "tags_all": {}, - "unique_id": "AIPA6HD75E55STNZ2F4C3" + "unique_id": "AIPA6HD75E55UMCWYCTQW" }, "sensitive_attributes": [], "private": "bnVsbA==", @@ -167,12 +167,12 @@ { "schema_version": 0, "attributes": { - "arn": "arn:aws:iam::977373308795:role/samsung-gauntlet-fb_cluster_node", + "arn": "arn:aws:iam::977373308795:role/gauntlet-wFQOOzXB51R3ebr-fb_cluster_node", "assume_role_policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Sid\":\"\",\"Effect\":\"Allow\",\"Principal\":{\"Service\":\"ec2.amazonaws.com\"},\"Action\":\"sts:AssumeRole\"}]}", - "create_date": "2022-01-09T17:25:34Z", + "create_date": "2022-01-10T16:18:38Z", "description": "", "force_detach_policies": false, - "id": "samsung-gauntlet-fb_cluster_node", + "id": "gauntlet-wFQOOzXB51R3ebr-fb_cluster_node", "inline_policy": [ { "name": "ec2_read_all", @@ -181,13 +181,13 @@ ], "managed_policy_arns": [], "max_session_duration": 3600, - "name": "samsung-gauntlet-fb_cluster_node", + "name": "gauntlet-wFQOOzXB51R3ebr-fb_cluster_node", "name_prefix": "", "path": "/", "permissions_boundary": null, - "tags": {}, + "tags": null, "tags_all": {}, - "unique_id": "AROA6HD75E552T4UVUF5M" + "unique_id": "AROA6HD75E55XTJYYBTLB" }, "sensitive_attributes": [], "private": "bnVsbA==" @@ -206,7 +206,7 @@ "schema_version": 1, "attributes": { "ami": "ami-0b09f36be67d32fff", - "arn": "arn:aws:ec2:us-east-2:977373308795:instance/i-0480cda8df43c65c2", + "arn": "arn:aws:ec2:us-east-2:977373308795:instance/i-0d8c02e5d3e02b13c", "associate_public_ip_address": false, "availability_zone": "us-east-2a", "capacity_reservation_specification": [ @@ -229,7 +229,7 @@ "snapshot_id": "", "tags": {}, "throughput": 125, - "volume_id": "vol-048ea5053dd0f6c4d", + "volume_id": "vol-0aec666a06a62cf56", "volume_size": 100, "volume_type": "gp3" } @@ -244,14 +244,14 @@ "get_password_data": false, "hibernation": false, "host_id": null, - "iam_instance_profile": "samsung-gauntlet-fb_cluster_node_profile", - "id": "i-0480cda8df43c65c2", + "iam_instance_profile": "gauntlet-wFQOOzXB51R3ebr-fb_cluster_node_profile", + "id": "i-0d8c02e5d3e02b13c", "instance_initiated_shutdown_behavior": "stop", "instance_state": "running", "instance_type": "m6g.xlarge", "ipv6_address_count": 0, "ipv6_addresses": [], - "key_name": "samsung-gauntlet-gitlab-ci", + "key_name": "gauntlet-wFQOOzXB51R3ebr-gitlab-ci", "launch_template": [], "metadata_options": [ { @@ -266,9 +266,9 @@ "password_data": "", "placement_group": "", "placement_partition_number": null, - "primary_network_interface_id": "eni-0dc00e2676f102cbb", - "private_dns": "ip-10-0-1-153.us-east-2.compute.internal", - "private_ip": "10.0.1.153", + "primary_network_interface_id": "eni-0c80f27ddcedba46a", + "private_dns": "ip-10-0-1-141.us-east-2.compute.internal", + "private_ip": "10.0.1.141", "public_dns": "", "public_ip": "", "root_block_device": [ @@ -278,9 +278,9 @@ "encrypted": false, "iops": 3000, "kms_key_id": "", - "tags": {}, + "tags": null, "throughput": 125, - "volume_id": "vol-0d4c17942d878d0c9", + "volume_id": "vol-0d9377bbf488e54ef", "volume_size": 20, "volume_type": "gp3" } @@ -290,13 +290,13 @@ "source_dest_check": true, "subnet_id": "subnet-050b1219d78f2db1b", "tags": { - "Name": "samsung-gauntlet-featurebase-cluster-0", - "Prefix": "samsung-gauntlet", + "Name": "gauntlet-wFQOOzXB51R3ebr-featurebase-cluster-0", + "Prefix": "gauntlet-wFQOOzXB51R3ebr", "Role": "cluster_node" }, "tags_all": { - "Name": "samsung-gauntlet-featurebase-cluster-0", - "Prefix": "samsung-gauntlet", + "Name": "gauntlet-wFQOOzXB51R3ebr-featurebase-cluster-0", + "Prefix": "gauntlet-wFQOOzXB51R3ebr", "Role": "cluster_node" }, "tenancy": "default", @@ -305,7 +305,7 @@ "user_data_base64": null, "volume_tags": null, "vpc_security_group_ids": [ - "sg-026cd014f6dd154b1" + "sg-09c4419bed8bf5876" ] }, "sensitive_attributes": [], @@ -323,7 +323,7 @@ "schema_version": 1, "attributes": { "ami": "ami-0b09f36be67d32fff", - "arn": "arn:aws:ec2:us-east-2:977373308795:instance/i-0bbf310c24c1c46d5", + "arn": "arn:aws:ec2:us-east-2:977373308795:instance/i-0e50a8965846142ca", "associate_public_ip_address": false, "availability_zone": "us-east-2b", "capacity_reservation_specification": [ @@ -346,7 +346,7 @@ "snapshot_id": "", "tags": {}, "throughput": 125, - "volume_id": "vol-07325e290952e5a6b", + "volume_id": "vol-0b02da40377abf65d", "volume_size": 100, "volume_type": "gp3" } @@ -361,14 +361,14 @@ "get_password_data": false, "hibernation": false, "host_id": null, - "iam_instance_profile": "samsung-gauntlet-fb_cluster_node_profile", - "id": "i-0bbf310c24c1c46d5", + "iam_instance_profile": "gauntlet-wFQOOzXB51R3ebr-fb_cluster_node_profile", + "id": "i-0e50a8965846142ca", "instance_initiated_shutdown_behavior": "stop", "instance_state": "running", "instance_type": "m6g.xlarge", "ipv6_address_count": 0, "ipv6_addresses": [], - "key_name": "samsung-gauntlet-gitlab-ci", + "key_name": "gauntlet-wFQOOzXB51R3ebr-gitlab-ci", "launch_template": [], "metadata_options": [ { @@ -383,9 +383,9 @@ "password_data": "", "placement_group": "", "placement_partition_number": null, - "primary_network_interface_id": "eni-0745bbcd1bae80fd4", - "private_dns": "ip-10-0-2-15.us-east-2.compute.internal", - "private_ip": "10.0.2.15", + "primary_network_interface_id": "eni-0bdf01f21c7c83aa7", + "private_dns": "ip-10-0-2-81.us-east-2.compute.internal", + "private_ip": "10.0.2.81", "public_dns": "", "public_ip": "", "root_block_device": [ @@ -395,9 +395,9 @@ "encrypted": false, "iops": 3000, "kms_key_id": "", - "tags": {}, + "tags": null, "throughput": 125, - "volume_id": "vol-0b13e3dc72826a74a", + "volume_id": "vol-0598ebf8df0eaf12f", "volume_size": 20, "volume_type": "gp3" } @@ -407,13 +407,13 @@ "source_dest_check": true, "subnet_id": "subnet-0d623c769e086e46e", "tags": { - "Name": "samsung-gauntlet-featurebase-cluster-1", - "Prefix": "samsung-gauntlet", + "Name": "gauntlet-wFQOOzXB51R3ebr-featurebase-cluster-1", + "Prefix": "gauntlet-wFQOOzXB51R3ebr", "Role": "cluster_node" }, "tags_all": { - "Name": "samsung-gauntlet-featurebase-cluster-1", - "Prefix": "samsung-gauntlet", + "Name": "gauntlet-wFQOOzXB51R3ebr-featurebase-cluster-1", + "Prefix": "gauntlet-wFQOOzXB51R3ebr", "Role": "cluster_node" }, "tenancy": "default", @@ -422,17 +422,17 @@ "user_data_base64": null, "volume_tags": null, "vpc_security_group_ids": [ - "sg-026cd014f6dd154b1" + "sg-09c4419bed8bf5876" ] }, "sensitive_attributes": [], "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjo2MDAwMDAwMDAwMDAsImRlbGV0ZSI6MTIwMDAwMDAwMDAwMCwidXBkYXRlIjo2MDAwMDAwMDAwMDB9LCJzY2hlbWFfdmVyc2lvbiI6IjEifQ==", "dependencies": [ - "module.samsung-cluster.data.aws_ami.amazon_linux_2", "module.samsung-cluster.aws_iam_instance_profile.fb_cluster_node_profile", "module.samsung-cluster.aws_iam_role.fb_cluster_node_role", "module.samsung-cluster.aws_key_pair.gitlab-featurebase-ci", - "module.samsung-cluster.aws_security_group.featurebase" + "module.samsung-cluster.aws_security_group.featurebase", + "module.samsung-cluster.data.aws_ami.amazon_linux_2" ] }, { @@ -440,7 +440,7 @@ "schema_version": 1, "attributes": { "ami": "ami-0b09f36be67d32fff", - "arn": "arn:aws:ec2:us-east-2:977373308795:instance/i-0b9def9ba1e299921", + "arn": "arn:aws:ec2:us-east-2:977373308795:instance/i-0db45f32178ab7fa7", "associate_public_ip_address": false, "availability_zone": "us-east-2c", "capacity_reservation_specification": [ @@ -463,7 +463,7 @@ "snapshot_id": "", "tags": {}, "throughput": 125, - "volume_id": "vol-099fe812da7384809", + "volume_id": "vol-090a2fe1dc5701335", "volume_size": 100, "volume_type": "gp3" } @@ -478,14 +478,14 @@ "get_password_data": false, "hibernation": false, "host_id": null, - "iam_instance_profile": "samsung-gauntlet-fb_cluster_node_profile", - "id": "i-0b9def9ba1e299921", + "iam_instance_profile": "gauntlet-wFQOOzXB51R3ebr-fb_cluster_node_profile", + "id": "i-0db45f32178ab7fa7", "instance_initiated_shutdown_behavior": "stop", "instance_state": "running", "instance_type": "m6g.xlarge", "ipv6_address_count": 0, "ipv6_addresses": [], - "key_name": "samsung-gauntlet-gitlab-ci", + "key_name": "gauntlet-wFQOOzXB51R3ebr-gitlab-ci", "launch_template": [], "metadata_options": [ { @@ -500,9 +500,9 @@ "password_data": "", "placement_group": "", "placement_partition_number": null, - "primary_network_interface_id": "eni-011390db576e1ef11", - "private_dns": "ip-10-0-3-89.us-east-2.compute.internal", - "private_ip": "10.0.3.89", + "primary_network_interface_id": "eni-0a39a93f26db70f1b", + "private_dns": "ip-10-0-3-251.us-east-2.compute.internal", + "private_ip": "10.0.3.251", "public_dns": "", "public_ip": "", "root_block_device": [ @@ -512,9 +512,9 @@ "encrypted": false, "iops": 3000, "kms_key_id": "", - "tags": {}, + "tags": null, "throughput": 125, - "volume_id": "vol-0705e352c214b476b", + "volume_id": "vol-04ca311d0f009fbd4", "volume_size": 20, "volume_type": "gp3" } @@ -524,13 +524,13 @@ "source_dest_check": true, "subnet_id": "subnet-07155281789c6d33b", "tags": { - "Name": "samsung-gauntlet-featurebase-cluster-2", - "Prefix": "samsung-gauntlet", + "Name": "gauntlet-wFQOOzXB51R3ebr-featurebase-cluster-2", + "Prefix": "gauntlet-wFQOOzXB51R3ebr", "Role": "cluster_node" }, "tags_all": { - "Name": "samsung-gauntlet-featurebase-cluster-2", - "Prefix": "samsung-gauntlet", + "Name": "gauntlet-wFQOOzXB51R3ebr-featurebase-cluster-2", + "Prefix": "gauntlet-wFQOOzXB51R3ebr", "Role": "cluster_node" }, "tenancy": "default", @@ -539,7 +539,7 @@ "user_data_base64": null, "volume_tags": null, "vpc_security_group_ids": [ - "sg-026cd014f6dd154b1" + "sg-09c4419bed8bf5876" ] }, "sensitive_attributes": [], @@ -566,7 +566,7 @@ "schema_version": 1, "attributes": { "ami": "ami-0b09f36be67d32fff", - "arn": "arn:aws:ec2:us-east-2:977373308795:instance/i-0205a6a18916b76d4", + "arn": "arn:aws:ec2:us-east-2:977373308795:instance/i-02d2e54ebcc03d50f", "associate_public_ip_address": true, "availability_zone": "us-east-2a", "capacity_reservation_specification": [ @@ -589,7 +589,7 @@ "snapshot_id": "", "tags": {}, "throughput": 125, - "volume_id": "vol-0f2d419dec323106a", + "volume_id": "vol-0f3f70d88829a6b81", "volume_size": 100, "volume_type": "gp3" } @@ -604,14 +604,14 @@ "get_password_data": false, "hibernation": false, "host_id": null, - "iam_instance_profile": "samsung-gauntlet-fb_cluster_node_profile", - "id": "i-0205a6a18916b76d4", + "iam_instance_profile": "gauntlet-wFQOOzXB51R3ebr-fb_cluster_node_profile", + "id": "i-02d2e54ebcc03d50f", "instance_initiated_shutdown_behavior": "stop", "instance_state": "running", "instance_type": "m6g.large", "ipv6_address_count": 0, "ipv6_addresses": [], - "key_name": "samsung-gauntlet-gitlab-ci", + "key_name": "gauntlet-wFQOOzXB51R3ebr-gitlab-ci", "launch_template": [], "metadata_options": [ { @@ -626,11 +626,11 @@ "password_data": "", "placement_group": "", "placement_partition_number": null, - "primary_network_interface_id": "eni-0353d32a9a9cb44e1", - "private_dns": "ip-10-0-101-14.us-east-2.compute.internal", - "private_ip": "10.0.101.14", + "primary_network_interface_id": "eni-087162265d04f2649", + "private_dns": "ip-10-0-101-116.us-east-2.compute.internal", + "private_ip": "10.0.101.116", "public_dns": "", - "public_ip": "18.217.165.151", + "public_ip": "3.144.13.48", "root_block_device": [ { "delete_on_termination": true, @@ -638,9 +638,9 @@ "encrypted": false, "iops": 3000, "kms_key_id": "", - "tags": {}, + "tags": null, "throughput": 125, - "volume_id": "vol-00ffed175f7e1b82f", + "volume_id": "vol-0c996c15a2fbed5da", "volume_size": 20, "volume_type": "gp3" } @@ -650,13 +650,13 @@ "source_dest_check": true, "subnet_id": "subnet-066b4b922b54e51a2", "tags": { - "Name": "samsung-gauntlet-featurebase-ingest-0", - "Prefix": "samsung-gauntlet", + "Name": "gauntlet-wFQOOzXB51R3ebr-featurebase-ingest-0", + "Prefix": "gauntlet-wFQOOzXB51R3ebr", "Role": "ingest_node" }, "tags_all": { - "Name": "samsung-gauntlet-featurebase-ingest-0", - "Prefix": "samsung-gauntlet", + "Name": "gauntlet-wFQOOzXB51R3ebr-featurebase-ingest-0", + "Prefix": "gauntlet-wFQOOzXB51R3ebr", "Role": "ingest_node" }, "tenancy": "default", @@ -665,17 +665,17 @@ "user_data_base64": null, "volume_tags": null, "vpc_security_group_ids": [ - "sg-020719cac2e5f2055" + "sg-0426fd89db464e42d" ] }, "sensitive_attributes": [], "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjo2MDAwMDAwMDAwMDAsImRlbGV0ZSI6MTIwMDAwMDAwMDAwMCwidXBkYXRlIjo2MDAwMDAwMDAwMDB9LCJzY2hlbWFfdmVyc2lvbiI6IjEifQ==", "dependencies": [ - "module.samsung-cluster.data.aws_ami.amazon_linux_2", "module.samsung-cluster.aws_iam_instance_profile.fb_cluster_node_profile", "module.samsung-cluster.aws_iam_role.fb_cluster_node_role", "module.samsung-cluster.aws_key_pair.gitlab-featurebase-ci", - "module.samsung-cluster.aws_security_group.ingest" + "module.samsung-cluster.aws_security_group.ingest", + "module.samsung-cluster.data.aws_ami.amazon_linux_2" ] } ] @@ -690,14 +690,14 @@ { "schema_version": 1, "attributes": { - "arn": "arn:aws:ec2:us-east-2:977373308795:key-pair/samsung-gauntlet-gitlab-ci", + "arn": "arn:aws:ec2:us-east-2:977373308795:key-pair/gauntlet-wFQOOzXB51R3ebr-gitlab-ci", "fingerprint": "69:e5:4b:15:8d:1f:22:61:de:08:a9:ee:f3:29:5c:69", - "id": "samsung-gauntlet-gitlab-ci", - "key_name": "samsung-gauntlet-gitlab-ci", + "id": "gauntlet-wFQOOzXB51R3ebr-gitlab-ci", + "key_name": "gauntlet-wFQOOzXB51R3ebr-gitlab-ci", "key_name_prefix": "", - "key_pair_id": "key-0c3cbdc0e74874b3d", + "key_pair_id": "key-00da6c5ab0ac3a651", "public_key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC91hhpVHNonAG7ku2ugpxEskf9KHeyHJPQJT26OHrMUw7R+T5A8TjqSzTau07sXQ/E9SO3ebV8SJ5PqeaQOnQB8VEvVNK0DjQH7ppvNg1Rfs42FZT9ttzTMvOjsSbK3vZTHXdoKQEdC9NxBwSkFIRGQojK1HUOq9xGrw31fA1OjSwlpLcbx7yyg18lcqW6UOptnVR8U9Yy9qQ5jZF1HtkQ6L9J+gv4o1UyNAUK2bopeGiXpBc3PQ/CFaFT2h/aqLBP66qAHsHVyAFD3PIRtplC5EHa8jXDgLacEls0uF7Q3kRPxvzcuo4g4VkOn1rDy9qH3vd2hT3aKVnM73FIDUiL", - "tags": {}, + "tags": null, "tags_all": {} }, "sensitive_attributes": [], @@ -715,7 +715,7 @@ { "schema_version": 1, "attributes": { - "arn": "arn:aws:ec2:us-east-2:977373308795:security-group/sg-026cd014f6dd154b1", + "arn": "arn:aws:ec2:us-east-2:977373308795:security-group/sg-09c4419bed8bf5876", "description": "Allow featurebase inbound traffic", "egress": [ { @@ -734,7 +734,7 @@ "to_port": 0 } ], - "id": "sg-026cd014f6dd154b1", + "id": "sg-09c4419bed8bf5876", "ingress": [ { "cidr_blocks": [ @@ -833,7 +833,7 @@ "to_port": 55432 } ], - "name": "samsung-gauntlet-allow_featurebase", + "name": "gauntlet-wFQOOzXB51R3ebr-allow_featurebase", "name_prefix": "", "owner_id": "977373308795", "revoke_rules_on_delete": false, @@ -861,7 +861,7 @@ { "schema_version": 1, "attributes": { - "arn": "arn:aws:ec2:us-east-2:977373308795:security-group/sg-020719cac2e5f2055", + "arn": "arn:aws:ec2:us-east-2:977373308795:security-group/sg-0426fd89db464e42d", "description": "Allow ingest inbound traffic", "egress": [ { @@ -880,7 +880,7 @@ "to_port": 0 } ], - "id": "sg-020719cac2e5f2055", + "id": "sg-0426fd89db464e42d", "ingress": [ { "cidr_blocks": [ @@ -926,7 +926,7 @@ "to_port": -1 } ], - "name": "samsung-gauntlet-allow_ingest", + "name": "gauntlet-wFQOOzXB51R3ebr-allow_ingest", "name_prefix": "", "owner_id": "977373308795", "revoke_rules_on_delete": false, From 63bf6dee2e2eabb773d22db444a77ff0d5733c49 Mon Sep 17 00:00:00 2001 From: pokeeffe-molecula Date: Mon, 10 Jan 2022 10:38:11 -0600 Subject: [PATCH 04/13] fix yaml --- .gitlab/.gitlab-ci.yml | 3 +- .../gauntlet/samsung/terraform.tfstate.backup | 36 +++++++++---------- 2 files changed, 19 insertions(+), 20 deletions(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index f325bf552..6e45523c5 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -290,8 +290,7 @@ smoke test: - docker - fbsmoke rules: -# - if: '$CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"' - - if: '$CI_PIPELINE_SOURCE == "push"' + - if: '$CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"' - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH' before_script: - apt-get update && apt-get install -y gnupg software-properties-common curl git diff --git a/qa/tf/gauntlet/samsung/terraform.tfstate.backup b/qa/tf/gauntlet/samsung/terraform.tfstate.backup index bc74976b2..5a8a4917a 100644 --- a/qa/tf/gauntlet/samsung/terraform.tfstate.backup +++ b/qa/tf/gauntlet/samsung/terraform.tfstate.backup @@ -1,7 +1,7 @@ { "version": 4, "terraform_version": "1.1.2", - "serial": 63, + "serial": 64, "lineage": "febac4ac-400a-d207-d2d1-64c55f14767b", "outputs": { "cluster_prefix": { @@ -145,7 +145,7 @@ "name_prefix": null, "path": "/", "role": "gauntlet-wFQOOzXB51R3ebr-fb_cluster_node", - "tags": null, + "tags": {}, "tags_all": {}, "unique_id": "AIPA6HD75E55UMCWYCTQW" }, @@ -185,7 +185,7 @@ "name_prefix": "", "path": "/", "permissions_boundary": null, - "tags": null, + "tags": {}, "tags_all": {}, "unique_id": "AROA6HD75E55XTJYYBTLB" }, @@ -278,7 +278,7 @@ "encrypted": false, "iops": 3000, "kms_key_id": "", - "tags": null, + "tags": {}, "throughput": 125, "volume_id": "vol-0d9377bbf488e54ef", "volume_size": 20, @@ -311,11 +311,11 @@ "sensitive_attributes": [], "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjo2MDAwMDAwMDAwMDAsImRlbGV0ZSI6MTIwMDAwMDAwMDAwMCwidXBkYXRlIjo2MDAwMDAwMDAwMDB9LCJzY2hlbWFfdmVyc2lvbiI6IjEifQ==", "dependencies": [ - "module.samsung-cluster.aws_iam_instance_profile.fb_cluster_node_profile", - "module.samsung-cluster.aws_iam_role.fb_cluster_node_role", "module.samsung-cluster.aws_key_pair.gitlab-featurebase-ci", "module.samsung-cluster.aws_security_group.featurebase", - "module.samsung-cluster.data.aws_ami.amazon_linux_2" + "module.samsung-cluster.data.aws_ami.amazon_linux_2", + "module.samsung-cluster.aws_iam_instance_profile.fb_cluster_node_profile", + "module.samsung-cluster.aws_iam_role.fb_cluster_node_role" ] }, { @@ -395,7 +395,7 @@ "encrypted": false, "iops": 3000, "kms_key_id": "", - "tags": null, + "tags": {}, "throughput": 125, "volume_id": "vol-0598ebf8df0eaf12f", "volume_size": 20, @@ -428,11 +428,11 @@ "sensitive_attributes": [], "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjo2MDAwMDAwMDAwMDAsImRlbGV0ZSI6MTIwMDAwMDAwMDAwMCwidXBkYXRlIjo2MDAwMDAwMDAwMDB9LCJzY2hlbWFfdmVyc2lvbiI6IjEifQ==", "dependencies": [ + "module.samsung-cluster.aws_security_group.featurebase", + "module.samsung-cluster.data.aws_ami.amazon_linux_2", "module.samsung-cluster.aws_iam_instance_profile.fb_cluster_node_profile", "module.samsung-cluster.aws_iam_role.fb_cluster_node_role", - "module.samsung-cluster.aws_key_pair.gitlab-featurebase-ci", - "module.samsung-cluster.aws_security_group.featurebase", - "module.samsung-cluster.data.aws_ami.amazon_linux_2" + "module.samsung-cluster.aws_key_pair.gitlab-featurebase-ci" ] }, { @@ -512,7 +512,7 @@ "encrypted": false, "iops": 3000, "kms_key_id": "", - "tags": null, + "tags": {}, "throughput": 125, "volume_id": "vol-04ca311d0f009fbd4", "volume_size": 20, @@ -545,11 +545,11 @@ "sensitive_attributes": [], "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjo2MDAwMDAwMDAwMDAsImRlbGV0ZSI6MTIwMDAwMDAwMDAwMCwidXBkYXRlIjo2MDAwMDAwMDAwMDB9LCJzY2hlbWFfdmVyc2lvbiI6IjEifQ==", "dependencies": [ - "module.samsung-cluster.aws_iam_instance_profile.fb_cluster_node_profile", "module.samsung-cluster.aws_iam_role.fb_cluster_node_role", "module.samsung-cluster.aws_key_pair.gitlab-featurebase-ci", "module.samsung-cluster.aws_security_group.featurebase", - "module.samsung-cluster.data.aws_ami.amazon_linux_2" + "module.samsung-cluster.data.aws_ami.amazon_linux_2", + "module.samsung-cluster.aws_iam_instance_profile.fb_cluster_node_profile" ] } ] @@ -638,7 +638,7 @@ "encrypted": false, "iops": 3000, "kms_key_id": "", - "tags": null, + "tags": {}, "throughput": 125, "volume_id": "vol-0c996c15a2fbed5da", "volume_size": 20, @@ -671,11 +671,11 @@ "sensitive_attributes": [], "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjo2MDAwMDAwMDAwMDAsImRlbGV0ZSI6MTIwMDAwMDAwMDAwMCwidXBkYXRlIjo2MDAwMDAwMDAwMDB9LCJzY2hlbWFfdmVyc2lvbiI6IjEifQ==", "dependencies": [ + "module.samsung-cluster.data.aws_ami.amazon_linux_2", "module.samsung-cluster.aws_iam_instance_profile.fb_cluster_node_profile", "module.samsung-cluster.aws_iam_role.fb_cluster_node_role", "module.samsung-cluster.aws_key_pair.gitlab-featurebase-ci", - "module.samsung-cluster.aws_security_group.ingest", - "module.samsung-cluster.data.aws_ami.amazon_linux_2" + "module.samsung-cluster.aws_security_group.ingest" ] } ] @@ -697,7 +697,7 @@ "key_name_prefix": "", "key_pair_id": "key-00da6c5ab0ac3a651", "public_key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC91hhpVHNonAG7ku2ugpxEskf9KHeyHJPQJT26OHrMUw7R+T5A8TjqSzTau07sXQ/E9SO3ebV8SJ5PqeaQOnQB8VEvVNK0DjQH7ppvNg1Rfs42FZT9ttzTMvOjsSbK3vZTHXdoKQEdC9NxBwSkFIRGQojK1HUOq9xGrw31fA1OjSwlpLcbx7yyg18lcqW6UOptnVR8U9Yy9qQ5jZF1HtkQ6L9J+gv4o1UyNAUK2bopeGiXpBc3PQ/CFaFT2h/aqLBP66qAHsHVyAFD3PIRtplC5EHa8jXDgLacEls0uF7Q3kRPxvzcuo4g4VkOn1rDy9qH3vd2hT3aKVnM73FIDUiL", - "tags": null, + "tags": {}, "tags_all": {} }, "sensitive_attributes": [], From d3b64941a5385068126d14e745459fd45d50b452 Mon Sep 17 00:00:00 2001 From: pokeeffe-molecula Date: Mon, 10 Jan 2022 10:42:48 -0600 Subject: [PATCH 05/13] fix yaml (again) --- .gitlab/.gitlab-ci.yml | 100 ++++++++++++++++++++--------------------- 1 file changed, 50 insertions(+), 50 deletions(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 6e45523c5..1678b3b1e 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -274,53 +274,53 @@ smoke test: reports: junit: report.xml - gauntlet: - stage: gauntlet - timeout: 4h - image: registry.gitlab.com/gitlab-org/cloud-deploy/aws-base:latest - variables: - PROFILE: "service-terraform" - AWS_SSH_PRIVATE_KEY: $AWS_FBCI_SSH_KEY - AWS_ACCESS_KEY_ID: $AWS_FBCI_ACCESS_KEY_ID - AWS_SECRET_ACCESS_KEY: $AWS_FBCI_SECRET_ACCESS_KEY - TF_VAR_cluster_prefix: "" - TF_VAR_branch: "" - tags: - - aws - - docker - - fbsmoke - rules: - - if: '$CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"' - - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH' - before_script: - - apt-get update && apt-get install -y gnupg software-properties-common curl git - - curl -fsSL https://apt.releases.hashicorp.com/gpg | apt-key add - - - apt-add-repository "deb [arch=amd64] https://apt.releases.hashicorp.com $(lsb_release -cs) main" - - apt-get update && apt-get install terraform - - aws configure set aws_access_key_id $AWS_FBCI_ACCESS_KEY_ID - - aws configure set aws_secret_access_key $AWS_FBCI_SECRET_ACCESS_KEY - - aws configure set region "us-east-2" - - aws configure set aws_profile $PROFILE - - echo $AWS_FBCI_SSH_KEY > gitlab-featurebase-ci.pem - - chmod 400 gitlab-featurebase-ci.pem - - 'which ssh-agent || ( apt-get update -y && apt-get install openssh-client -y )' - - eval $(ssh-agent -s) - - mkdir -p ~/.ssh - - echo $AWS_FBCI_SSH_KEY > /root/.ssh/gitlab-featurebase-ci.pem - - chmod 400 /root/.ssh/gitlab-featurebase-ci.pem - - echo "$AWS_FBCI_SSH_KEY" | ssh-add - - - chmod 700 /root/.ssh - - '[[ -f /.dockerenv ]] && echo -e "Host *\n\tStrictHostKeyChecking no\n\n" > ~/.ssh/config' - - apt update && apt -y install jq wget - - wget https://go.dev/dl/go1.17.5.linux-amd64.tar.gz - - tar -C /usr/local -xzf go1.17.5.linux-amd64.tar.gz - - export PATH=$PATH:/usr/local/go/bin - - TF_VAR_cluster_prefix="gauntlet-$(openssl rand -base64 12 | tr -d /=+ | cut -c -16)" - - echo "Cluster Prefix --> $TF_VAR_cluster_prefix" - - TF_VAR_branch=$CI_COMMIT_BRANCH - - echo "Branch --> $TF_VAR_branch" - script: - - ./qa/scripts/setupSamsungGauntlet.sh -# - ./qa/scripts/testSamsungGauntlet.sh - after_script: - - ./qa/scripts/teardownSamsungGauntlet.sh +# gauntlet: +# stage: gauntlet +# timeout: 4h +# image: registry.gitlab.com/gitlab-org/cloud-deploy/aws-base:latest +# variables: +# PROFILE: "service-terraform" +# AWS_SSH_PRIVATE_KEY: $AWS_FBCI_SSH_KEY +# AWS_ACCESS_KEY_ID: $AWS_FBCI_ACCESS_KEY_ID +# AWS_SECRET_ACCESS_KEY: $AWS_FBCI_SECRET_ACCESS_KEY +# TF_VAR_cluster_prefix: "" +# TF_VAR_branch: "" +# tags: +# - aws +# - docker +# - fbsmoke +# rules: +# - if: '$CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"' +# - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH' +# before_script: +# - apt-get update && apt-get install -y gnupg software-properties-common curl git +# - curl -fsSL https://apt.releases.hashicorp.com/gpg | apt-key add - +# - apt-add-repository "deb [arch=amd64] https://apt.releases.hashicorp.com $(lsb_release -cs) main" +# - apt-get update && apt-get install terraform +# - aws configure set aws_access_key_id $AWS_FBCI_ACCESS_KEY_ID +# - aws configure set aws_secret_access_key $AWS_FBCI_SECRET_ACCESS_KEY +# - aws configure set region "us-east-2" +# - aws configure set aws_profile $PROFILE +# - echo $AWS_FBCI_SSH_KEY > gitlab-featurebase-ci.pem +# - chmod 400 gitlab-featurebase-ci.pem +# - 'which ssh-agent || ( apt-get update -y && apt-get install openssh-client -y )' +# - eval $(ssh-agent -s) +# - mkdir -p ~/.ssh +# - echo $AWS_FBCI_SSH_KEY > /root/.ssh/gitlab-featurebase-ci.pem +# - chmod 400 /root/.ssh/gitlab-featurebase-ci.pem +# - echo "$AWS_FBCI_SSH_KEY" | ssh-add - +# - chmod 700 /root/.ssh +# - '[[ -f /.dockerenv ]] && echo -e "Host *\n\tStrictHostKeyChecking no\n\n" > ~/.ssh/config' +# - apt update && apt -y install jq wget +# - wget https://go.dev/dl/go1.17.5.linux-amd64.tar.gz +# - tar -C /usr/local -xzf go1.17.5.linux-amd64.tar.gz +# - export PATH=$PATH:/usr/local/go/bin +# - TF_VAR_cluster_prefix="gauntlet-$(openssl rand -base64 12 | tr -d /=+ | cut -c -16)" +# - echo "Cluster Prefix --> $TF_VAR_cluster_prefix" +# - TF_VAR_branch=$CI_COMMIT_BRANCH +# - echo "Branch --> $TF_VAR_branch" +# script: +# - ./qa/scripts/setupSamsungGauntlet.sh +# # - ./qa/scripts/testSamsungGauntlet.sh +# after_script: +# - ./qa/scripts/teardownSamsungGauntlet.sh From d35273c3b59773442ec5175452266a9578c54fdd Mon Sep 17 00:00:00 2001 From: pokeeffe-molecula Date: Mon, 10 Jan 2022 11:59:32 -0600 Subject: [PATCH 06/13] Update .gitlab-ci.yml --- .gitlab/.gitlab-ci.yml | 51 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 1678b3b1e..53d7ad4da 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -274,6 +274,57 @@ smoke test: reports: junit: report.xml + +gauntlet: + stage: gauntlet + image: registry.gitlab.com/gitlab-org/cloud-deploy/aws-base:latest + variables: + PROFILE: "service-terraform" + AWS_SSH_PRIVATE_KEY: $AWS_FBCI_SSH_KEY + AWS_ACCESS_KEY_ID: $AWS_FBCI_ACCESS_KEY_ID + AWS_SECRET_ACCESS_KEY: $AWS_FBCI_SECRET_ACCESS_KEY + TF_VAR_cluster_prefix: "" + TF_VAR_branch: "" + tags: + - aws + - docker + - fbsmoke + rules: + - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"' +# - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH' + before_script: + - apt-get update && apt-get install -y gnupg software-properties-common curl git + - curl -fsSL https://apt.releases.hashicorp.com/gpg | apt-key add - + - apt-add-repository "deb [arch=amd64] https://apt.releases.hashicorp.com $(lsb_release -cs) main" + - apt-get update && apt-get install terraform + - aws configure set aws_access_key_id $AWS_FBCI_ACCESS_KEY_ID + - aws configure set aws_secret_access_key $AWS_FBCI_SECRET_ACCESS_KEY + - aws configure set region "us-east-2" + - aws configure set aws_profile $PROFILE + - echo $AWS_FBCI_SSH_KEY > gitlab-featurebase-ci.pem + - chmod 400 gitlab-featurebase-ci.pem + - 'which ssh-agent || ( apt-get update -y && apt-get install openssh-client -y )' + - eval $(ssh-agent -s) + - mkdir -p ~/.ssh + - echo $AWS_FBCI_SSH_KEY > /root/.ssh/gitlab-featurebase-ci.pem + - chmod 400 /root/.ssh/gitlab-featurebase-ci.pem + - echo "$AWS_FBCI_SSH_KEY" | ssh-add - + - chmod 700 /root/.ssh + - '[[ -f /.dockerenv ]] && echo -e "Host *\n\tStrictHostKeyChecking no\n\n" > ~/.ssh/config' + - apt update && apt -y install jq wget + - wget https://go.dev/dl/go1.17.5.linux-amd64.tar.gz + - tar -C /usr/local -xzf go1.17.5.linux-amd64.tar.gz + - export PATH=$PATH:/usr/local/go/bin + - TF_VAR_cluster_prefix="gauntlet-$(openssl rand -base64 12 | tr -d /=+ | cut -c -16)" + - echo "Cluster Prefix --> $TF_VAR_cluster_prefix" + - TF_VAR_branch=$CI_COMMIT_BRANCH + - echo "Branch --> $TF_VAR_branch" + script: + - ./qa/scripts/setupSamsungGauntlet.sh +# - ./qa/scripts/testSamsungGauntlet.sh + after_script: + - ./qa/scripts/teardownSamsungGauntlet.sh + # gauntlet: # stage: gauntlet # timeout: 4h From e4e667215f541e8040401f05b29491c59ce459e0 Mon Sep 17 00:00:00 2001 From: pokeeffe-molecula Date: Mon, 10 Jan 2022 12:19:06 -0600 Subject: [PATCH 07/13] try to run full gauntlet --- .gitlab/.gitlab-ci.yml | 52 +----------------------------------------- 1 file changed, 1 insertion(+), 51 deletions(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 53d7ad4da..80960fcfb 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -321,57 +321,7 @@ gauntlet: - echo "Branch --> $TF_VAR_branch" script: - ./qa/scripts/setupSamsungGauntlet.sh -# - ./qa/scripts/testSamsungGauntlet.sh + - ./qa/scripts/testSamsungGauntlet.sh after_script: - ./qa/scripts/teardownSamsungGauntlet.sh -# gauntlet: -# stage: gauntlet -# timeout: 4h -# image: registry.gitlab.com/gitlab-org/cloud-deploy/aws-base:latest -# variables: -# PROFILE: "service-terraform" -# AWS_SSH_PRIVATE_KEY: $AWS_FBCI_SSH_KEY -# AWS_ACCESS_KEY_ID: $AWS_FBCI_ACCESS_KEY_ID -# AWS_SECRET_ACCESS_KEY: $AWS_FBCI_SECRET_ACCESS_KEY -# TF_VAR_cluster_prefix: "" -# TF_VAR_branch: "" -# tags: -# - aws -# - docker -# - fbsmoke -# rules: -# - if: '$CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"' -# - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH' -# before_script: -# - apt-get update && apt-get install -y gnupg software-properties-common curl git -# - curl -fsSL https://apt.releases.hashicorp.com/gpg | apt-key add - -# - apt-add-repository "deb [arch=amd64] https://apt.releases.hashicorp.com $(lsb_release -cs) main" -# - apt-get update && apt-get install terraform -# - aws configure set aws_access_key_id $AWS_FBCI_ACCESS_KEY_ID -# - aws configure set aws_secret_access_key $AWS_FBCI_SECRET_ACCESS_KEY -# - aws configure set region "us-east-2" -# - aws configure set aws_profile $PROFILE -# - echo $AWS_FBCI_SSH_KEY > gitlab-featurebase-ci.pem -# - chmod 400 gitlab-featurebase-ci.pem -# - 'which ssh-agent || ( apt-get update -y && apt-get install openssh-client -y )' -# - eval $(ssh-agent -s) -# - mkdir -p ~/.ssh -# - echo $AWS_FBCI_SSH_KEY > /root/.ssh/gitlab-featurebase-ci.pem -# - chmod 400 /root/.ssh/gitlab-featurebase-ci.pem -# - echo "$AWS_FBCI_SSH_KEY" | ssh-add - -# - chmod 700 /root/.ssh -# - '[[ -f /.dockerenv ]] && echo -e "Host *\n\tStrictHostKeyChecking no\n\n" > ~/.ssh/config' -# - apt update && apt -y install jq wget -# - wget https://go.dev/dl/go1.17.5.linux-amd64.tar.gz -# - tar -C /usr/local -xzf go1.17.5.linux-amd64.tar.gz -# - export PATH=$PATH:/usr/local/go/bin -# - TF_VAR_cluster_prefix="gauntlet-$(openssl rand -base64 12 | tr -d /=+ | cut -c -16)" -# - echo "Cluster Prefix --> $TF_VAR_cluster_prefix" -# - TF_VAR_branch=$CI_COMMIT_BRANCH -# - echo "Branch --> $TF_VAR_branch" -# script: -# - ./qa/scripts/setupSamsungGauntlet.sh -# # - ./qa/scripts/testSamsungGauntlet.sh -# after_script: -# - ./qa/scripts/teardownSamsungGauntlet.sh From 5ec9d7cf9611e66042c67de7253267e92f897a91 Mon Sep 17 00:00:00 2001 From: pokeeffe-molecula Date: Mon, 10 Jan 2022 16:11:03 -0600 Subject: [PATCH 08/13] Now with more gauntlet --- .gitlab/.gitlab-ci.yml | 1 + qa/scripts/setupSamsungGauntlet.sh | 3 +- qa/scripts/testSamsungGauntlet.sh | 40 +++-- qa/scripts/testSamsungPayload.sh | 1 - .../gauntlet/samsung/terraform.tfstate.backup | 154 +++++++++--------- 5 files changed, 107 insertions(+), 92 deletions(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 80960fcfb..c957f245b 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -277,6 +277,7 @@ smoke test: gauntlet: stage: gauntlet + timeout: 4h image: registry.gitlab.com/gitlab-org/cloud-deploy/aws-base:latest variables: PROFILE: "service-terraform" diff --git a/qa/scripts/setupSamsungGauntlet.sh b/qa/scripts/setupSamsungGauntlet.sh index 7e8b684b3..1229c0044 100755 --- a/qa/scripts/setupSamsungGauntlet.sh +++ b/qa/scripts/setupSamsungGauntlet.sh @@ -12,7 +12,6 @@ if [ -z ${TF_VAR_branch+x} ]; then echo "TF_VAR_branch is unset"; else echo "TF_ SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) source $SCRIPT_DIR/utilCluster.sh - pushd ./qa/tf/gauntlet/samsung echo "Running terraform init..." terraform init -input=false @@ -59,7 +58,7 @@ do echo "Cluster is up after ${i} tries." break fi - sleep 10s + sleep 10 done ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no -o ConnectTimeout=10 ec2-user@${DATANODE0} "pwd" diff --git a/qa/scripts/testSamsungGauntlet.sh b/qa/scripts/testSamsungGauntlet.sh index efe0b8951..8240588ba 100755 --- a/qa/scripts/testSamsungGauntlet.sh +++ b/qa/scripts/testSamsungGauntlet.sh @@ -1,48 +1,64 @@ #!/bin/bash -# get the bastion host -BASTION=$(cat ./qa/tf/gauntlet/samsung/outputs.json | jq -r '[.ingest_ips][0]["value"][0]') -echo "using bastion ${BASTION}" +# get the first ingest host +INGESTNODE0=$(cat ./qa/tf/gauntlet/samsung/outputs.json | jq -r '[.ingest_ips][0]["value"][0]') +echo "using INGESTNODE0 ${INGESTNODE0}" -NODE=$(cat ./qa/tf/gauntlet/samsung/outputs.json | jq -r '[.data_node_ips][0]["value"][0]') -echo "using node ${NODE}" +# get the first data host +DATANODE0=$(cat ./qa/tf/gauntlet/samsung/outputs.json | jq -r '[.data_node_ips][0]["value"][0]') +echo "using DATANODE0 ${DATANODE0}" # generate csv files +echo "Building simulacraData..." GOOS=linux GOARCH=arm64 go build ./qa/simulacraData/... -scp -i ~/.ssh/gitlab-featurebase-ci.pem simulacraData ec2-user@${BASTION}:/data +if (( $? != 0 )) +then + echo "Build failed" + exit 1 +fi +echo "Copying simulacraData..." +scp -i ~/.ssh/gitlab-featurebase-ci.pem simulacraData ec2-user@${INGESTNODE0}:/data if (( $? != 0 )) then echo "Copy failed" exit 1 fi -ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem ec2-user@${BASTION} "cd /data && /data/simulacraData" +echo "Running simulacraData..." +ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem ec2-user@${INGESTNODE0} "cd /data && /data/simulacraData" if (( $? != 0 )) then echo "Making big files failed" exit 1 fi +echo "Running simulacraData done." # ingest these files the way that samsung does it -scp -i ~/.ssh/gitlab-featurebase-ci.pem ./qa/scripts/testSamsungPayload.sh ec2-user@${BASTION}: +echo "Copying testSamsungPayload.sh..." +scp -i ~/.ssh/gitlab-featurebase-ci.pem ./qa/scripts/testSamsungPayload.sh ec2-user@${INGESTNODE0}: if (( $? != 0 )) then - echo "Copy ingest script failed" + echo "Copying testSamsungPayload.sh failed" exit 1 fi -ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem ec2-user@${BASTION} "./testSamsungPayload.sh http://${NODE}:10101 1" +echo "Running (1) testSamsungPayload.sh..." +ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem ec2-user@${INGESTNODE0} "./testSamsungPayload.sh http://${DATANODE0}:10101 1" if (( $? != 0 )) then echo "Running 1 testSamsungPayload.sh failed" exit 1 fi -ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem ec2-user@${BASTION} "./testSamsungPayload.sh http://${NODE}:10101 0" +echo "Running (0) testSamsungPayload.sh..." +ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem ec2-user@${INGESTNODE0} "./testSamsungPayload.sh http://${DATANODE0}:10101 0" if (( $? != 0 )) then echo "Running 0 testSamsungPayload.sh failed" exit 1 fi -# query workload that runs \ No newline at end of file +# query workload that runs + + +echo "Done." \ No newline at end of file diff --git a/qa/scripts/testSamsungPayload.sh b/qa/scripts/testSamsungPayload.sh index 8c36c63dc..85752b409 100755 --- a/qa/scripts/testSamsungPayload.sh +++ b/qa/scripts/testSamsungPayload.sh @@ -6,7 +6,6 @@ FEATUREBASE_PATH=/usr/local/bin # path for directory with csv directory files for all fields to be ingested CSV_DIR_PATH=/data - # To run: # ./testSamsungPayload.sh {Local host & port for featurebase} {initialize flag} diff --git a/qa/tf/gauntlet/samsung/terraform.tfstate.backup b/qa/tf/gauntlet/samsung/terraform.tfstate.backup index 5a8a4917a..dedd3874f 100644 --- a/qa/tf/gauntlet/samsung/terraform.tfstate.backup +++ b/qa/tf/gauntlet/samsung/terraform.tfstate.backup @@ -1,7 +1,7 @@ { "version": 4, "terraform_version": "1.1.2", - "serial": 64, + "serial": 85, "lineage": "febac4ac-400a-d207-d2d1-64c55f14767b", "outputs": { "cluster_prefix": { @@ -10,9 +10,9 @@ }, "data_node_ips": { "value": [ - "10.0.1.141", - "10.0.2.81", - "10.0.3.251" + "10.0.1.126", + "10.0.2.53", + "10.0.3.15" ], "type": [ "tuple", @@ -29,7 +29,7 @@ }, "ingest_ips": { "value": [ - "3.144.13.48" + "13.59.231.222" ], "type": [ "tuple", @@ -51,7 +51,7 @@ "schema_version": 0, "attributes": { "architecture": "arm64", - "arn": "arn:aws:ec2:us-east-2::image/ami-0b09f36be67d32fff", + "arn": "arn:aws:ec2:us-east-2::image/ami-088e1f338c3b87d1a", "block_device_mappings": [ { "device_name": "/dev/xvda", @@ -59,7 +59,7 @@ "delete_on_termination": "true", "encrypted": "false", "iops": "0", - "snapshot_id": "snap-0617b00e90bae012b", + "snapshot_id": "snap-0f9ae89577e61b172", "throughput": "0", "volume_size": "8", "volume_type": "gp2" @@ -68,8 +68,8 @@ "virtual_name": "" } ], - "creation_date": "2021-12-01T19:36:11.000Z", - "description": "Amazon Linux 2 LTS Arm64 AMI 2.0.20211201.0 arm64 HVM gp2", + "creation_date": "2022-01-05T21:55:03.000Z", + "description": "Amazon Linux 2 LTS Arm64 AMI 2.0.20211223.0 arm64 HVM gp2", "ena_support": true, "executable_users": null, "filter": [ @@ -93,14 +93,14 @@ } ], "hypervisor": "xen", - "id": "ami-0b09f36be67d32fff", - "image_id": "ami-0b09f36be67d32fff", - "image_location": "amazon/amzn2-ami-hvm-2.0.20211201.0-arm64-gp2", + "id": "ami-088e1f338c3b87d1a", + "image_id": "ami-088e1f338c3b87d1a", + "image_location": "amazon/amzn2-ami-hvm-2.0.20211223.0-arm64-gp2", "image_owner_alias": "amazon", "image_type": "machine", "kernel_id": null, "most_recent": true, - "name": "amzn2-ami-hvm-2.0.20211201.0-arm64-gp2", + "name": "amzn2-ami-hvm-2.0.20211223.0-arm64-gp2", "name_regex": null, "owner_id": "137112412989", "owners": [ @@ -113,7 +113,7 @@ "ramdisk_id": null, "root_device_name": "/dev/xvda", "root_device_type": "ebs", - "root_snapshot_id": "snap-0617b00e90bae012b", + "root_snapshot_id": "snap-0f9ae89577e61b172", "sriov_net_support": "simple", "state": "available", "state_reason": { @@ -139,15 +139,15 @@ "schema_version": 0, "attributes": { "arn": "arn:aws:iam::977373308795:instance-profile/gauntlet-wFQOOzXB51R3ebr-fb_cluster_node_profile", - "create_date": "2022-01-10T16:18:40Z", + "create_date": "2022-01-10T19:49:14Z", "id": "gauntlet-wFQOOzXB51R3ebr-fb_cluster_node_profile", "name": "gauntlet-wFQOOzXB51R3ebr-fb_cluster_node_profile", "name_prefix": null, "path": "/", "role": "gauntlet-wFQOOzXB51R3ebr-fb_cluster_node", - "tags": {}, + "tags": null, "tags_all": {}, - "unique_id": "AIPA6HD75E55UMCWYCTQW" + "unique_id": "AIPA6HD75E55QU3MSOJ6U" }, "sensitive_attributes": [], "private": "bnVsbA==", @@ -169,7 +169,7 @@ "attributes": { "arn": "arn:aws:iam::977373308795:role/gauntlet-wFQOOzXB51R3ebr-fb_cluster_node", "assume_role_policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Sid\":\"\",\"Effect\":\"Allow\",\"Principal\":{\"Service\":\"ec2.amazonaws.com\"},\"Action\":\"sts:AssumeRole\"}]}", - "create_date": "2022-01-10T16:18:38Z", + "create_date": "2022-01-10T19:49:11Z", "description": "", "force_detach_policies": false, "id": "gauntlet-wFQOOzXB51R3ebr-fb_cluster_node", @@ -185,9 +185,9 @@ "name_prefix": "", "path": "/", "permissions_boundary": null, - "tags": {}, + "tags": null, "tags_all": {}, - "unique_id": "AROA6HD75E55XTJYYBTLB" + "unique_id": "AROA6HD75E55VSTOHIPD7" }, "sensitive_attributes": [], "private": "bnVsbA==" @@ -205,8 +205,8 @@ "index_key": 0, "schema_version": 1, "attributes": { - "ami": "ami-0b09f36be67d32fff", - "arn": "arn:aws:ec2:us-east-2:977373308795:instance/i-0d8c02e5d3e02b13c", + "ami": "ami-088e1f338c3b87d1a", + "arn": "arn:aws:ec2:us-east-2:977373308795:instance/i-02dd4026474cd7bd2", "associate_public_ip_address": false, "availability_zone": "us-east-2a", "capacity_reservation_specification": [ @@ -229,7 +229,7 @@ "snapshot_id": "", "tags": {}, "throughput": 125, - "volume_id": "vol-0aec666a06a62cf56", + "volume_id": "vol-03c5743e3f610783c", "volume_size": 100, "volume_type": "gp3" } @@ -245,7 +245,7 @@ "hibernation": false, "host_id": null, "iam_instance_profile": "gauntlet-wFQOOzXB51R3ebr-fb_cluster_node_profile", - "id": "i-0d8c02e5d3e02b13c", + "id": "i-02dd4026474cd7bd2", "instance_initiated_shutdown_behavior": "stop", "instance_state": "running", "instance_type": "m6g.xlarge", @@ -266,9 +266,9 @@ "password_data": "", "placement_group": "", "placement_partition_number": null, - "primary_network_interface_id": "eni-0c80f27ddcedba46a", - "private_dns": "ip-10-0-1-141.us-east-2.compute.internal", - "private_ip": "10.0.1.141", + "primary_network_interface_id": "eni-0120997558c61093e", + "private_dns": "ip-10-0-1-126.us-east-2.compute.internal", + "private_ip": "10.0.1.126", "public_dns": "", "public_ip": "", "root_block_device": [ @@ -278,9 +278,9 @@ "encrypted": false, "iops": 3000, "kms_key_id": "", - "tags": {}, + "tags": null, "throughput": 125, - "volume_id": "vol-0d9377bbf488e54ef", + "volume_id": "vol-06b2a1b78f6264c4d", "volume_size": 20, "volume_type": "gp3" } @@ -305,25 +305,25 @@ "user_data_base64": null, "volume_tags": null, "vpc_security_group_ids": [ - "sg-09c4419bed8bf5876" + "sg-029350ea6d628a8b7" ] }, "sensitive_attributes": [], "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjo2MDAwMDAwMDAwMDAsImRlbGV0ZSI6MTIwMDAwMDAwMDAwMCwidXBkYXRlIjo2MDAwMDAwMDAwMDB9LCJzY2hlbWFfdmVyc2lvbiI6IjEifQ==", "dependencies": [ + "module.samsung-cluster.aws_iam_instance_profile.fb_cluster_node_profile", + "module.samsung-cluster.aws_iam_role.fb_cluster_node_role", "module.samsung-cluster.aws_key_pair.gitlab-featurebase-ci", "module.samsung-cluster.aws_security_group.featurebase", - "module.samsung-cluster.data.aws_ami.amazon_linux_2", - "module.samsung-cluster.aws_iam_instance_profile.fb_cluster_node_profile", - "module.samsung-cluster.aws_iam_role.fb_cluster_node_role" + "module.samsung-cluster.data.aws_ami.amazon_linux_2" ] }, { "index_key": 1, "schema_version": 1, "attributes": { - "ami": "ami-0b09f36be67d32fff", - "arn": "arn:aws:ec2:us-east-2:977373308795:instance/i-0e50a8965846142ca", + "ami": "ami-088e1f338c3b87d1a", + "arn": "arn:aws:ec2:us-east-2:977373308795:instance/i-05512fdf5b06be3cd", "associate_public_ip_address": false, "availability_zone": "us-east-2b", "capacity_reservation_specification": [ @@ -346,7 +346,7 @@ "snapshot_id": "", "tags": {}, "throughput": 125, - "volume_id": "vol-0b02da40377abf65d", + "volume_id": "vol-03c4fbc66f16dd900", "volume_size": 100, "volume_type": "gp3" } @@ -362,7 +362,7 @@ "hibernation": false, "host_id": null, "iam_instance_profile": "gauntlet-wFQOOzXB51R3ebr-fb_cluster_node_profile", - "id": "i-0e50a8965846142ca", + "id": "i-05512fdf5b06be3cd", "instance_initiated_shutdown_behavior": "stop", "instance_state": "running", "instance_type": "m6g.xlarge", @@ -383,9 +383,9 @@ "password_data": "", "placement_group": "", "placement_partition_number": null, - "primary_network_interface_id": "eni-0bdf01f21c7c83aa7", - "private_dns": "ip-10-0-2-81.us-east-2.compute.internal", - "private_ip": "10.0.2.81", + "primary_network_interface_id": "eni-058dc3caa2e9f1468", + "private_dns": "ip-10-0-2-53.us-east-2.compute.internal", + "private_ip": "10.0.2.53", "public_dns": "", "public_ip": "", "root_block_device": [ @@ -395,9 +395,9 @@ "encrypted": false, "iops": 3000, "kms_key_id": "", - "tags": {}, + "tags": null, "throughput": 125, - "volume_id": "vol-0598ebf8df0eaf12f", + "volume_id": "vol-031baade3831d550d", "volume_size": 20, "volume_type": "gp3" } @@ -422,25 +422,25 @@ "user_data_base64": null, "volume_tags": null, "vpc_security_group_ids": [ - "sg-09c4419bed8bf5876" + "sg-029350ea6d628a8b7" ] }, "sensitive_attributes": [], "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjo2MDAwMDAwMDAwMDAsImRlbGV0ZSI6MTIwMDAwMDAwMDAwMCwidXBkYXRlIjo2MDAwMDAwMDAwMDB9LCJzY2hlbWFfdmVyc2lvbiI6IjEifQ==", "dependencies": [ - "module.samsung-cluster.aws_security_group.featurebase", - "module.samsung-cluster.data.aws_ami.amazon_linux_2", "module.samsung-cluster.aws_iam_instance_profile.fb_cluster_node_profile", "module.samsung-cluster.aws_iam_role.fb_cluster_node_role", - "module.samsung-cluster.aws_key_pair.gitlab-featurebase-ci" + "module.samsung-cluster.aws_key_pair.gitlab-featurebase-ci", + "module.samsung-cluster.aws_security_group.featurebase", + "module.samsung-cluster.data.aws_ami.amazon_linux_2" ] }, { "index_key": 2, "schema_version": 1, "attributes": { - "ami": "ami-0b09f36be67d32fff", - "arn": "arn:aws:ec2:us-east-2:977373308795:instance/i-0db45f32178ab7fa7", + "ami": "ami-088e1f338c3b87d1a", + "arn": "arn:aws:ec2:us-east-2:977373308795:instance/i-0419c1e88127c9f10", "associate_public_ip_address": false, "availability_zone": "us-east-2c", "capacity_reservation_specification": [ @@ -463,7 +463,7 @@ "snapshot_id": "", "tags": {}, "throughput": 125, - "volume_id": "vol-090a2fe1dc5701335", + "volume_id": "vol-0bb728053ae788784", "volume_size": 100, "volume_type": "gp3" } @@ -479,7 +479,7 @@ "hibernation": false, "host_id": null, "iam_instance_profile": "gauntlet-wFQOOzXB51R3ebr-fb_cluster_node_profile", - "id": "i-0db45f32178ab7fa7", + "id": "i-0419c1e88127c9f10", "instance_initiated_shutdown_behavior": "stop", "instance_state": "running", "instance_type": "m6g.xlarge", @@ -500,9 +500,9 @@ "password_data": "", "placement_group": "", "placement_partition_number": null, - "primary_network_interface_id": "eni-0a39a93f26db70f1b", - "private_dns": "ip-10-0-3-251.us-east-2.compute.internal", - "private_ip": "10.0.3.251", + "primary_network_interface_id": "eni-01e566cbb9b7c3900", + "private_dns": "ip-10-0-3-15.us-east-2.compute.internal", + "private_ip": "10.0.3.15", "public_dns": "", "public_ip": "", "root_block_device": [ @@ -512,9 +512,9 @@ "encrypted": false, "iops": 3000, "kms_key_id": "", - "tags": {}, + "tags": null, "throughput": 125, - "volume_id": "vol-04ca311d0f009fbd4", + "volume_id": "vol-07b5aa90375034d8c", "volume_size": 20, "volume_type": "gp3" } @@ -539,17 +539,17 @@ "user_data_base64": null, "volume_tags": null, "vpc_security_group_ids": [ - "sg-09c4419bed8bf5876" + "sg-029350ea6d628a8b7" ] }, "sensitive_attributes": [], "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjo2MDAwMDAwMDAwMDAsImRlbGV0ZSI6MTIwMDAwMDAwMDAwMCwidXBkYXRlIjo2MDAwMDAwMDAwMDB9LCJzY2hlbWFfdmVyc2lvbiI6IjEifQ==", "dependencies": [ + "module.samsung-cluster.aws_iam_instance_profile.fb_cluster_node_profile", "module.samsung-cluster.aws_iam_role.fb_cluster_node_role", "module.samsung-cluster.aws_key_pair.gitlab-featurebase-ci", "module.samsung-cluster.aws_security_group.featurebase", - "module.samsung-cluster.data.aws_ami.amazon_linux_2", - "module.samsung-cluster.aws_iam_instance_profile.fb_cluster_node_profile" + "module.samsung-cluster.data.aws_ami.amazon_linux_2" ] } ] @@ -565,8 +565,8 @@ "index_key": 0, "schema_version": 1, "attributes": { - "ami": "ami-0b09f36be67d32fff", - "arn": "arn:aws:ec2:us-east-2:977373308795:instance/i-02d2e54ebcc03d50f", + "ami": "ami-088e1f338c3b87d1a", + "arn": "arn:aws:ec2:us-east-2:977373308795:instance/i-021d8144e6c7c519f", "associate_public_ip_address": true, "availability_zone": "us-east-2a", "capacity_reservation_specification": [ @@ -589,7 +589,7 @@ "snapshot_id": "", "tags": {}, "throughput": 125, - "volume_id": "vol-0f3f70d88829a6b81", + "volume_id": "vol-0a3a332ca01d6b706", "volume_size": 100, "volume_type": "gp3" } @@ -605,7 +605,7 @@ "hibernation": false, "host_id": null, "iam_instance_profile": "gauntlet-wFQOOzXB51R3ebr-fb_cluster_node_profile", - "id": "i-02d2e54ebcc03d50f", + "id": "i-021d8144e6c7c519f", "instance_initiated_shutdown_behavior": "stop", "instance_state": "running", "instance_type": "m6g.large", @@ -626,11 +626,11 @@ "password_data": "", "placement_group": "", "placement_partition_number": null, - "primary_network_interface_id": "eni-087162265d04f2649", - "private_dns": "ip-10-0-101-116.us-east-2.compute.internal", - "private_ip": "10.0.101.116", + "primary_network_interface_id": "eni-05417a5a74984b658", + "private_dns": "ip-10-0-101-34.us-east-2.compute.internal", + "private_ip": "10.0.101.34", "public_dns": "", - "public_ip": "3.144.13.48", + "public_ip": "13.59.231.222", "root_block_device": [ { "delete_on_termination": true, @@ -638,9 +638,9 @@ "encrypted": false, "iops": 3000, "kms_key_id": "", - "tags": {}, + "tags": null, "throughput": 125, - "volume_id": "vol-0c996c15a2fbed5da", + "volume_id": "vol-055425b9087c74984", "volume_size": 20, "volume_type": "gp3" } @@ -665,17 +665,17 @@ "user_data_base64": null, "volume_tags": null, "vpc_security_group_ids": [ - "sg-0426fd89db464e42d" + "sg-0a66d187ba05935ce" ] }, "sensitive_attributes": [], "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjo2MDAwMDAwMDAwMDAsImRlbGV0ZSI6MTIwMDAwMDAwMDAwMCwidXBkYXRlIjo2MDAwMDAwMDAwMDB9LCJzY2hlbWFfdmVyc2lvbiI6IjEifQ==", "dependencies": [ - "module.samsung-cluster.data.aws_ami.amazon_linux_2", "module.samsung-cluster.aws_iam_instance_profile.fb_cluster_node_profile", "module.samsung-cluster.aws_iam_role.fb_cluster_node_role", "module.samsung-cluster.aws_key_pair.gitlab-featurebase-ci", - "module.samsung-cluster.aws_security_group.ingest" + "module.samsung-cluster.aws_security_group.ingest", + "module.samsung-cluster.data.aws_ami.amazon_linux_2" ] } ] @@ -695,9 +695,9 @@ "id": "gauntlet-wFQOOzXB51R3ebr-gitlab-ci", "key_name": "gauntlet-wFQOOzXB51R3ebr-gitlab-ci", "key_name_prefix": "", - "key_pair_id": "key-00da6c5ab0ac3a651", + "key_pair_id": "key-03d84801e3ffaeec7", "public_key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC91hhpVHNonAG7ku2ugpxEskf9KHeyHJPQJT26OHrMUw7R+T5A8TjqSzTau07sXQ/E9SO3ebV8SJ5PqeaQOnQB8VEvVNK0DjQH7ppvNg1Rfs42FZT9ttzTMvOjsSbK3vZTHXdoKQEdC9NxBwSkFIRGQojK1HUOq9xGrw31fA1OjSwlpLcbx7yyg18lcqW6UOptnVR8U9Yy9qQ5jZF1HtkQ6L9J+gv4o1UyNAUK2bopeGiXpBc3PQ/CFaFT2h/aqLBP66qAHsHVyAFD3PIRtplC5EHa8jXDgLacEls0uF7Q3kRPxvzcuo4g4VkOn1rDy9qH3vd2hT3aKVnM73FIDUiL", - "tags": {}, + "tags": null, "tags_all": {} }, "sensitive_attributes": [], @@ -715,7 +715,7 @@ { "schema_version": 1, "attributes": { - "arn": "arn:aws:ec2:us-east-2:977373308795:security-group/sg-09c4419bed8bf5876", + "arn": "arn:aws:ec2:us-east-2:977373308795:security-group/sg-029350ea6d628a8b7", "description": "Allow featurebase inbound traffic", "egress": [ { @@ -734,7 +734,7 @@ "to_port": 0 } ], - "id": "sg-09c4419bed8bf5876", + "id": "sg-029350ea6d628a8b7", "ingress": [ { "cidr_blocks": [ @@ -861,7 +861,7 @@ { "schema_version": 1, "attributes": { - "arn": "arn:aws:ec2:us-east-2:977373308795:security-group/sg-0426fd89db464e42d", + "arn": "arn:aws:ec2:us-east-2:977373308795:security-group/sg-0a66d187ba05935ce", "description": "Allow ingest inbound traffic", "egress": [ { @@ -880,7 +880,7 @@ "to_port": 0 } ], - "id": "sg-0426fd89db464e42d", + "id": "sg-0a66d187ba05935ce", "ingress": [ { "cidr_blocks": [ From 4b192ee8bf3c104567368307bf76ccf727e40a73 Mon Sep 17 00:00:00 2001 From: pokeeffe-molecula Date: Mon, 10 Jan 2022 18:02:45 -0600 Subject: [PATCH 09/13] added progress reporting --- qa/simulacraData/simulacra_data.go | 33 +++++++++++++++++++++++++++--- 1 file changed, 30 insertions(+), 3 deletions(-) diff --git a/qa/simulacraData/simulacra_data.go b/qa/simulacraData/simulacra_data.go index 5ec269780..a6af37b67 100644 --- a/qa/simulacraData/simulacra_data.go +++ b/qa/simulacraData/simulacra_data.go @@ -38,29 +38,41 @@ var countryList [246]string = [...]string{"ABW", "AFG", "AGO", "AIA", "ALA", "AL const totalRecords int = 200000000 func main() { + log.Println("generating age field...") if err := GenerateAgeField(totalRecords); err != nil { log.Fatalf("unable to generate age field: %v", err) } + log.Println("generating age field done.") + log.Println("generating ip field...") if err := GenerateIPField(totalRecords); err != nil { log.Fatalf("unable to generate IP field: %v", err) } + log.Println("generating ip field done.") + log.Println("generating identifier field...") if err := GenerateArbIdField(totalRecords); err != nil { log.Fatalf("unable to generate indentifier field: %v", err) } + log.Println("generating identifier field done.") + log.Println("generating opt in field...") if err := GenerateOptInField(totalRecords); err != nil { log.Fatalf("unable to generate opt in field: %v", err) } + log.Println("generating opt in field done.") + log.Println("generating country field...") if err := GenerateCountryField(totalRecords); err != nil { log.Fatalf("unable to generate country field: %v", err) } + log.Println("generating country field done.") + log.Println("generating time field...") if err := GenerateTimeField(totalRecords); err != nil { log.Fatalf("unable to generate time field: %v", err) } + log.Println("generating time field done.") } func GenerateAgeField(requestedRecords int) error { @@ -85,7 +97,9 @@ func GenerateAgeField(requestedRecords int) error { } } - + } + if i%2000000 == 0 { + log.Printf("generating age field (%d)", i) } } err1 := writer.Flush() @@ -120,6 +134,9 @@ func GenerateIPField(requestedRecords int) error { return errors.Wrap(err, "unable to write to ip.csv") } } + if i%2000000 == 0 { + log.Printf("generating ip field (%d)", i) + } } err1 := writer.Flush() @@ -155,6 +172,9 @@ func GenerateArbIdField(requestedRecords int) error { return errors.Wrap(err, "unable to write to identifier.csv") } } + if i%2000000 == 0 { + log.Printf("generating identifier field (%d)", i) + } } err1 := writer.Flush() @@ -213,7 +233,9 @@ func GenerateTimeField(requestedRecords int) error { return errors.Wrap(err, "unable to write to time.csv") } } - + if i%2000000 == 0 { + log.Printf("generating time field (%d)", i) + } } err1 := writer.Flush() if err1 != nil { @@ -266,6 +288,9 @@ func GenerateOptInField(requestedRecords int) error { } } } + if i%2000000 == 0 { + log.Printf("generating opt in field (%d)", i) + } } err1 := writer.Flush() @@ -315,7 +340,9 @@ func GenerateCountryField(requestedRecords int) error { } } } - + if i%2000000 == 0 { + log.Printf("generating country field (%d)", i) + } } err1 := writer.Flush() From 394a6e98542a4904c65b672564e48a7ae2563651 Mon Sep 17 00:00:00 2001 From: pokeeffe-molecula Date: Tue, 11 Jan 2022 08:47:02 -0600 Subject: [PATCH 10/13] switching gauntlet to scheduled --- .gitlab/.gitlab-ci.yml | 4 +- .../gauntlet/samsung/terraform.tfstate.backup | 948 ------------------ 2 files changed, 2 insertions(+), 950 deletions(-) delete mode 100644 qa/tf/gauntlet/samsung/terraform.tfstate.backup diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index c957f245b..fdcafb813 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -291,8 +291,8 @@ gauntlet: - docker - fbsmoke rules: - - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"' -# - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH' + - if: '$CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"' + - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH' before_script: - apt-get update && apt-get install -y gnupg software-properties-common curl git - curl -fsSL https://apt.releases.hashicorp.com/gpg | apt-key add - diff --git a/qa/tf/gauntlet/samsung/terraform.tfstate.backup b/qa/tf/gauntlet/samsung/terraform.tfstate.backup deleted file mode 100644 index dedd3874f..000000000 --- a/qa/tf/gauntlet/samsung/terraform.tfstate.backup +++ /dev/null @@ -1,948 +0,0 @@ -{ - "version": 4, - "terraform_version": "1.1.2", - "serial": 85, - "lineage": "febac4ac-400a-d207-d2d1-64c55f14767b", - "outputs": { - "cluster_prefix": { - "value": "gauntlet-wFQOOzXB51R3ebr", - "type": "string" - }, - "data_node_ips": { - "value": [ - "10.0.1.126", - "10.0.2.53", - "10.0.3.15" - ], - "type": [ - "tuple", - [ - "string", - "string", - "string" - ] - ] - }, - "fb_cluster_replica_count": { - "value": 1, - "type": "number" - }, - "ingest_ips": { - "value": [ - "13.59.231.222" - ], - "type": [ - "tuple", - [ - "string" - ] - ] - } - }, - "resources": [ - { - "module": "module.samsung-cluster", - "mode": "data", - "type": "aws_ami", - "name": "amazon_linux_2", - "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", - "instances": [ - { - "schema_version": 0, - "attributes": { - "architecture": "arm64", - "arn": "arn:aws:ec2:us-east-2::image/ami-088e1f338c3b87d1a", - "block_device_mappings": [ - { - "device_name": "/dev/xvda", - "ebs": { - "delete_on_termination": "true", - "encrypted": "false", - "iops": "0", - "snapshot_id": "snap-0f9ae89577e61b172", - "throughput": "0", - "volume_size": "8", - "volume_type": "gp2" - }, - "no_device": "", - "virtual_name": "" - } - ], - "creation_date": "2022-01-05T21:55:03.000Z", - "description": "Amazon Linux 2 LTS Arm64 AMI 2.0.20211223.0 arm64 HVM gp2", - "ena_support": true, - "executable_users": null, - "filter": [ - { - "name": "architecture", - "values": [ - "arm64" - ] - }, - { - "name": "name", - "values": [ - "amzn2-ami-hvm-*" - ] - }, - { - "name": "virtualization-type", - "values": [ - "hvm" - ] - } - ], - "hypervisor": "xen", - "id": "ami-088e1f338c3b87d1a", - "image_id": "ami-088e1f338c3b87d1a", - "image_location": "amazon/amzn2-ami-hvm-2.0.20211223.0-arm64-gp2", - "image_owner_alias": "amazon", - "image_type": "machine", - "kernel_id": null, - "most_recent": true, - "name": "amzn2-ami-hvm-2.0.20211223.0-arm64-gp2", - "name_regex": null, - "owner_id": "137112412989", - "owners": [ - "amazon" - ], - "platform": null, - "platform_details": "Linux/UNIX", - "product_codes": [], - "public": true, - "ramdisk_id": null, - "root_device_name": "/dev/xvda", - "root_device_type": "ebs", - "root_snapshot_id": "snap-0f9ae89577e61b172", - "sriov_net_support": "simple", - "state": "available", - "state_reason": { - "code": "UNSET", - "message": "UNSET" - }, - "tags": {}, - "usage_operation": "RunInstances", - "virtualization_type": "hvm" - }, - "sensitive_attributes": [] - } - ] - }, - { - "module": "module.samsung-cluster", - "mode": "managed", - "type": "aws_iam_instance_profile", - "name": "fb_cluster_node_profile", - "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", - "instances": [ - { - "schema_version": 0, - "attributes": { - "arn": "arn:aws:iam::977373308795:instance-profile/gauntlet-wFQOOzXB51R3ebr-fb_cluster_node_profile", - "create_date": "2022-01-10T19:49:14Z", - "id": "gauntlet-wFQOOzXB51R3ebr-fb_cluster_node_profile", - "name": "gauntlet-wFQOOzXB51R3ebr-fb_cluster_node_profile", - "name_prefix": null, - "path": "/", - "role": "gauntlet-wFQOOzXB51R3ebr-fb_cluster_node", - "tags": null, - "tags_all": {}, - "unique_id": "AIPA6HD75E55QU3MSOJ6U" - }, - "sensitive_attributes": [], - "private": "bnVsbA==", - "dependencies": [ - "module.samsung-cluster.aws_iam_role.fb_cluster_node_role" - ] - } - ] - }, - { - "module": "module.samsung-cluster", - "mode": "managed", - "type": "aws_iam_role", - "name": "fb_cluster_node_role", - "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", - "instances": [ - { - "schema_version": 0, - "attributes": { - "arn": "arn:aws:iam::977373308795:role/gauntlet-wFQOOzXB51R3ebr-fb_cluster_node", - "assume_role_policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Sid\":\"\",\"Effect\":\"Allow\",\"Principal\":{\"Service\":\"ec2.amazonaws.com\"},\"Action\":\"sts:AssumeRole\"}]}", - "create_date": "2022-01-10T19:49:11Z", - "description": "", - "force_detach_policies": false, - "id": "gauntlet-wFQOOzXB51R3ebr-fb_cluster_node", - "inline_policy": [ - { - "name": "ec2_read_all", - "policy": "{\"Statement\":[{\"Action\":[\"ec2:Describe*\"],\"Effect\":\"Allow\",\"Resource\":\"*\"}],\"Version\":\"2012-10-17\"}" - } - ], - "managed_policy_arns": [], - "max_session_duration": 3600, - "name": "gauntlet-wFQOOzXB51R3ebr-fb_cluster_node", - "name_prefix": "", - "path": "/", - "permissions_boundary": null, - "tags": null, - "tags_all": {}, - "unique_id": "AROA6HD75E55VSTOHIPD7" - }, - "sensitive_attributes": [], - "private": "bnVsbA==" - } - ] - }, - { - "module": "module.samsung-cluster", - "mode": "managed", - "type": "aws_instance", - "name": "fb_cluster_nodes", - "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", - "instances": [ - { - "index_key": 0, - "schema_version": 1, - "attributes": { - "ami": "ami-088e1f338c3b87d1a", - "arn": "arn:aws:ec2:us-east-2:977373308795:instance/i-02dd4026474cd7bd2", - "associate_public_ip_address": false, - "availability_zone": "us-east-2a", - "capacity_reservation_specification": [ - { - "capacity_reservation_preference": "open", - "capacity_reservation_target": [] - } - ], - "cpu_core_count": 4, - "cpu_threads_per_core": 1, - "credit_specification": [], - "disable_api_termination": false, - "ebs_block_device": [ - { - "delete_on_termination": true, - "device_name": "/dev/sdb", - "encrypted": false, - "iops": 10000, - "kms_key_id": "", - "snapshot_id": "", - "tags": {}, - "throughput": 125, - "volume_id": "vol-03c5743e3f610783c", - "volume_size": 100, - "volume_type": "gp3" - } - ], - "ebs_optimized": false, - "enclave_options": [ - { - "enabled": false - } - ], - "ephemeral_block_device": [], - "get_password_data": false, - "hibernation": false, - "host_id": null, - "iam_instance_profile": "gauntlet-wFQOOzXB51R3ebr-fb_cluster_node_profile", - "id": "i-02dd4026474cd7bd2", - "instance_initiated_shutdown_behavior": "stop", - "instance_state": "running", - "instance_type": "m6g.xlarge", - "ipv6_address_count": 0, - "ipv6_addresses": [], - "key_name": "gauntlet-wFQOOzXB51R3ebr-gitlab-ci", - "launch_template": [], - "metadata_options": [ - { - "http_endpoint": "enabled", - "http_put_response_hop_limit": 1, - "http_tokens": "optional" - } - ], - "monitoring": true, - "network_interface": [], - "outpost_arn": "", - "password_data": "", - "placement_group": "", - "placement_partition_number": null, - "primary_network_interface_id": "eni-0120997558c61093e", - "private_dns": "ip-10-0-1-126.us-east-2.compute.internal", - "private_ip": "10.0.1.126", - "public_dns": "", - "public_ip": "", - "root_block_device": [ - { - "delete_on_termination": true, - "device_name": "/dev/xvda", - "encrypted": false, - "iops": 3000, - "kms_key_id": "", - "tags": null, - "throughput": 125, - "volume_id": "vol-06b2a1b78f6264c4d", - "volume_size": 20, - "volume_type": "gp3" - } - ], - "secondary_private_ips": [], - "security_groups": [], - "source_dest_check": true, - "subnet_id": "subnet-050b1219d78f2db1b", - "tags": { - "Name": "gauntlet-wFQOOzXB51R3ebr-featurebase-cluster-0", - "Prefix": "gauntlet-wFQOOzXB51R3ebr", - "Role": "cluster_node" - }, - "tags_all": { - "Name": "gauntlet-wFQOOzXB51R3ebr-featurebase-cluster-0", - "Prefix": "gauntlet-wFQOOzXB51R3ebr", - "Role": "cluster_node" - }, - "tenancy": "default", - "timeouts": null, - "user_data": null, - "user_data_base64": null, - "volume_tags": null, - "vpc_security_group_ids": [ - "sg-029350ea6d628a8b7" - ] - }, - "sensitive_attributes": [], - "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjo2MDAwMDAwMDAwMDAsImRlbGV0ZSI6MTIwMDAwMDAwMDAwMCwidXBkYXRlIjo2MDAwMDAwMDAwMDB9LCJzY2hlbWFfdmVyc2lvbiI6IjEifQ==", - "dependencies": [ - "module.samsung-cluster.aws_iam_instance_profile.fb_cluster_node_profile", - "module.samsung-cluster.aws_iam_role.fb_cluster_node_role", - "module.samsung-cluster.aws_key_pair.gitlab-featurebase-ci", - "module.samsung-cluster.aws_security_group.featurebase", - "module.samsung-cluster.data.aws_ami.amazon_linux_2" - ] - }, - { - "index_key": 1, - "schema_version": 1, - "attributes": { - "ami": "ami-088e1f338c3b87d1a", - "arn": "arn:aws:ec2:us-east-2:977373308795:instance/i-05512fdf5b06be3cd", - "associate_public_ip_address": false, - "availability_zone": "us-east-2b", - "capacity_reservation_specification": [ - { - "capacity_reservation_preference": "open", - "capacity_reservation_target": [] - } - ], - "cpu_core_count": 4, - "cpu_threads_per_core": 1, - "credit_specification": [], - "disable_api_termination": false, - "ebs_block_device": [ - { - "delete_on_termination": true, - "device_name": "/dev/sdb", - "encrypted": false, - "iops": 10000, - "kms_key_id": "", - "snapshot_id": "", - "tags": {}, - "throughput": 125, - "volume_id": "vol-03c4fbc66f16dd900", - "volume_size": 100, - "volume_type": "gp3" - } - ], - "ebs_optimized": false, - "enclave_options": [ - { - "enabled": false - } - ], - "ephemeral_block_device": [], - "get_password_data": false, - "hibernation": false, - "host_id": null, - "iam_instance_profile": "gauntlet-wFQOOzXB51R3ebr-fb_cluster_node_profile", - "id": "i-05512fdf5b06be3cd", - "instance_initiated_shutdown_behavior": "stop", - "instance_state": "running", - "instance_type": "m6g.xlarge", - "ipv6_address_count": 0, - "ipv6_addresses": [], - "key_name": "gauntlet-wFQOOzXB51R3ebr-gitlab-ci", - "launch_template": [], - "metadata_options": [ - { - "http_endpoint": "enabled", - "http_put_response_hop_limit": 1, - "http_tokens": "optional" - } - ], - "monitoring": true, - "network_interface": [], - "outpost_arn": "", - "password_data": "", - "placement_group": "", - "placement_partition_number": null, - "primary_network_interface_id": "eni-058dc3caa2e9f1468", - "private_dns": "ip-10-0-2-53.us-east-2.compute.internal", - "private_ip": "10.0.2.53", - "public_dns": "", - "public_ip": "", - "root_block_device": [ - { - "delete_on_termination": true, - "device_name": "/dev/xvda", - "encrypted": false, - "iops": 3000, - "kms_key_id": "", - "tags": null, - "throughput": 125, - "volume_id": "vol-031baade3831d550d", - "volume_size": 20, - "volume_type": "gp3" - } - ], - "secondary_private_ips": [], - "security_groups": [], - "source_dest_check": true, - "subnet_id": "subnet-0d623c769e086e46e", - "tags": { - "Name": "gauntlet-wFQOOzXB51R3ebr-featurebase-cluster-1", - "Prefix": "gauntlet-wFQOOzXB51R3ebr", - "Role": "cluster_node" - }, - "tags_all": { - "Name": "gauntlet-wFQOOzXB51R3ebr-featurebase-cluster-1", - "Prefix": "gauntlet-wFQOOzXB51R3ebr", - "Role": "cluster_node" - }, - "tenancy": "default", - "timeouts": null, - "user_data": null, - "user_data_base64": null, - "volume_tags": null, - "vpc_security_group_ids": [ - "sg-029350ea6d628a8b7" - ] - }, - "sensitive_attributes": [], - "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjo2MDAwMDAwMDAwMDAsImRlbGV0ZSI6MTIwMDAwMDAwMDAwMCwidXBkYXRlIjo2MDAwMDAwMDAwMDB9LCJzY2hlbWFfdmVyc2lvbiI6IjEifQ==", - "dependencies": [ - "module.samsung-cluster.aws_iam_instance_profile.fb_cluster_node_profile", - "module.samsung-cluster.aws_iam_role.fb_cluster_node_role", - "module.samsung-cluster.aws_key_pair.gitlab-featurebase-ci", - "module.samsung-cluster.aws_security_group.featurebase", - "module.samsung-cluster.data.aws_ami.amazon_linux_2" - ] - }, - { - "index_key": 2, - "schema_version": 1, - "attributes": { - "ami": "ami-088e1f338c3b87d1a", - "arn": "arn:aws:ec2:us-east-2:977373308795:instance/i-0419c1e88127c9f10", - "associate_public_ip_address": false, - "availability_zone": "us-east-2c", - "capacity_reservation_specification": [ - { - "capacity_reservation_preference": "open", - "capacity_reservation_target": [] - } - ], - "cpu_core_count": 4, - "cpu_threads_per_core": 1, - "credit_specification": [], - "disable_api_termination": false, - "ebs_block_device": [ - { - "delete_on_termination": true, - "device_name": "/dev/sdb", - "encrypted": false, - "iops": 10000, - "kms_key_id": "", - "snapshot_id": "", - "tags": {}, - "throughput": 125, - "volume_id": "vol-0bb728053ae788784", - "volume_size": 100, - "volume_type": "gp3" - } - ], - "ebs_optimized": false, - "enclave_options": [ - { - "enabled": false - } - ], - "ephemeral_block_device": [], - "get_password_data": false, - "hibernation": false, - "host_id": null, - "iam_instance_profile": "gauntlet-wFQOOzXB51R3ebr-fb_cluster_node_profile", - "id": "i-0419c1e88127c9f10", - "instance_initiated_shutdown_behavior": "stop", - "instance_state": "running", - "instance_type": "m6g.xlarge", - "ipv6_address_count": 0, - "ipv6_addresses": [], - "key_name": "gauntlet-wFQOOzXB51R3ebr-gitlab-ci", - "launch_template": [], - "metadata_options": [ - { - "http_endpoint": "enabled", - "http_put_response_hop_limit": 1, - "http_tokens": "optional" - } - ], - "monitoring": true, - "network_interface": [], - "outpost_arn": "", - "password_data": "", - "placement_group": "", - "placement_partition_number": null, - "primary_network_interface_id": "eni-01e566cbb9b7c3900", - "private_dns": "ip-10-0-3-15.us-east-2.compute.internal", - "private_ip": "10.0.3.15", - "public_dns": "", - "public_ip": "", - "root_block_device": [ - { - "delete_on_termination": true, - "device_name": "/dev/xvda", - "encrypted": false, - "iops": 3000, - "kms_key_id": "", - "tags": null, - "throughput": 125, - "volume_id": "vol-07b5aa90375034d8c", - "volume_size": 20, - "volume_type": "gp3" - } - ], - "secondary_private_ips": [], - "security_groups": [], - "source_dest_check": true, - "subnet_id": "subnet-07155281789c6d33b", - "tags": { - "Name": "gauntlet-wFQOOzXB51R3ebr-featurebase-cluster-2", - "Prefix": "gauntlet-wFQOOzXB51R3ebr", - "Role": "cluster_node" - }, - "tags_all": { - "Name": "gauntlet-wFQOOzXB51R3ebr-featurebase-cluster-2", - "Prefix": "gauntlet-wFQOOzXB51R3ebr", - "Role": "cluster_node" - }, - "tenancy": "default", - "timeouts": null, - "user_data": null, - "user_data_base64": null, - "volume_tags": null, - "vpc_security_group_ids": [ - "sg-029350ea6d628a8b7" - ] - }, - "sensitive_attributes": [], - "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjo2MDAwMDAwMDAwMDAsImRlbGV0ZSI6MTIwMDAwMDAwMDAwMCwidXBkYXRlIjo2MDAwMDAwMDAwMDB9LCJzY2hlbWFfdmVyc2lvbiI6IjEifQ==", - "dependencies": [ - "module.samsung-cluster.aws_iam_instance_profile.fb_cluster_node_profile", - "module.samsung-cluster.aws_iam_role.fb_cluster_node_role", - "module.samsung-cluster.aws_key_pair.gitlab-featurebase-ci", - "module.samsung-cluster.aws_security_group.featurebase", - "module.samsung-cluster.data.aws_ami.amazon_linux_2" - ] - } - ] - }, - { - "module": "module.samsung-cluster", - "mode": "managed", - "type": "aws_instance", - "name": "fb_ingest", - "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", - "instances": [ - { - "index_key": 0, - "schema_version": 1, - "attributes": { - "ami": "ami-088e1f338c3b87d1a", - "arn": "arn:aws:ec2:us-east-2:977373308795:instance/i-021d8144e6c7c519f", - "associate_public_ip_address": true, - "availability_zone": "us-east-2a", - "capacity_reservation_specification": [ - { - "capacity_reservation_preference": "open", - "capacity_reservation_target": [] - } - ], - "cpu_core_count": 2, - "cpu_threads_per_core": 1, - "credit_specification": [], - "disable_api_termination": false, - "ebs_block_device": [ - { - "delete_on_termination": true, - "device_name": "/dev/sdb", - "encrypted": false, - "iops": 10000, - "kms_key_id": "", - "snapshot_id": "", - "tags": {}, - "throughput": 125, - "volume_id": "vol-0a3a332ca01d6b706", - "volume_size": 100, - "volume_type": "gp3" - } - ], - "ebs_optimized": false, - "enclave_options": [ - { - "enabled": false - } - ], - "ephemeral_block_device": [], - "get_password_data": false, - "hibernation": false, - "host_id": null, - "iam_instance_profile": "gauntlet-wFQOOzXB51R3ebr-fb_cluster_node_profile", - "id": "i-021d8144e6c7c519f", - "instance_initiated_shutdown_behavior": "stop", - "instance_state": "running", - "instance_type": "m6g.large", - "ipv6_address_count": 0, - "ipv6_addresses": [], - "key_name": "gauntlet-wFQOOzXB51R3ebr-gitlab-ci", - "launch_template": [], - "metadata_options": [ - { - "http_endpoint": "enabled", - "http_put_response_hop_limit": 1, - "http_tokens": "optional" - } - ], - "monitoring": true, - "network_interface": [], - "outpost_arn": "", - "password_data": "", - "placement_group": "", - "placement_partition_number": null, - "primary_network_interface_id": "eni-05417a5a74984b658", - "private_dns": "ip-10-0-101-34.us-east-2.compute.internal", - "private_ip": "10.0.101.34", - "public_dns": "", - "public_ip": "13.59.231.222", - "root_block_device": [ - { - "delete_on_termination": true, - "device_name": "/dev/xvda", - "encrypted": false, - "iops": 3000, - "kms_key_id": "", - "tags": null, - "throughput": 125, - "volume_id": "vol-055425b9087c74984", - "volume_size": 20, - "volume_type": "gp3" - } - ], - "secondary_private_ips": [], - "security_groups": [], - "source_dest_check": true, - "subnet_id": "subnet-066b4b922b54e51a2", - "tags": { - "Name": "gauntlet-wFQOOzXB51R3ebr-featurebase-ingest-0", - "Prefix": "gauntlet-wFQOOzXB51R3ebr", - "Role": "ingest_node" - }, - "tags_all": { - "Name": "gauntlet-wFQOOzXB51R3ebr-featurebase-ingest-0", - "Prefix": "gauntlet-wFQOOzXB51R3ebr", - "Role": "ingest_node" - }, - "tenancy": "default", - "timeouts": null, - "user_data": null, - "user_data_base64": null, - "volume_tags": null, - "vpc_security_group_ids": [ - "sg-0a66d187ba05935ce" - ] - }, - "sensitive_attributes": [], - "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjo2MDAwMDAwMDAwMDAsImRlbGV0ZSI6MTIwMDAwMDAwMDAwMCwidXBkYXRlIjo2MDAwMDAwMDAwMDB9LCJzY2hlbWFfdmVyc2lvbiI6IjEifQ==", - "dependencies": [ - "module.samsung-cluster.aws_iam_instance_profile.fb_cluster_node_profile", - "module.samsung-cluster.aws_iam_role.fb_cluster_node_role", - "module.samsung-cluster.aws_key_pair.gitlab-featurebase-ci", - "module.samsung-cluster.aws_security_group.ingest", - "module.samsung-cluster.data.aws_ami.amazon_linux_2" - ] - } - ] - }, - { - "module": "module.samsung-cluster", - "mode": "managed", - "type": "aws_key_pair", - "name": "gitlab-featurebase-ci", - "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", - "instances": [ - { - "schema_version": 1, - "attributes": { - "arn": "arn:aws:ec2:us-east-2:977373308795:key-pair/gauntlet-wFQOOzXB51R3ebr-gitlab-ci", - "fingerprint": "69:e5:4b:15:8d:1f:22:61:de:08:a9:ee:f3:29:5c:69", - "id": "gauntlet-wFQOOzXB51R3ebr-gitlab-ci", - "key_name": "gauntlet-wFQOOzXB51R3ebr-gitlab-ci", - "key_name_prefix": "", - "key_pair_id": "key-03d84801e3ffaeec7", - "public_key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC91hhpVHNonAG7ku2ugpxEskf9KHeyHJPQJT26OHrMUw7R+T5A8TjqSzTau07sXQ/E9SO3ebV8SJ5PqeaQOnQB8VEvVNK0DjQH7ppvNg1Rfs42FZT9ttzTMvOjsSbK3vZTHXdoKQEdC9NxBwSkFIRGQojK1HUOq9xGrw31fA1OjSwlpLcbx7yyg18lcqW6UOptnVR8U9Yy9qQ5jZF1HtkQ6L9J+gv4o1UyNAUK2bopeGiXpBc3PQ/CFaFT2h/aqLBP66qAHsHVyAFD3PIRtplC5EHa8jXDgLacEls0uF7Q3kRPxvzcuo4g4VkOn1rDy9qH3vd2hT3aKVnM73FIDUiL", - "tags": null, - "tags_all": {} - }, - "sensitive_attributes": [], - "private": "eyJzY2hlbWFfdmVyc2lvbiI6IjEifQ==" - } - ] - }, - { - "module": "module.samsung-cluster", - "mode": "managed", - "type": "aws_security_group", - "name": "featurebase", - "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", - "instances": [ - { - "schema_version": 1, - "attributes": { - "arn": "arn:aws:ec2:us-east-2:977373308795:security-group/sg-029350ea6d628a8b7", - "description": "Allow featurebase inbound traffic", - "egress": [ - { - "cidr_blocks": [ - "0.0.0.0/0" - ], - "description": "", - "from_port": 0, - "ipv6_cidr_blocks": [ - "::/0" - ], - "prefix_list_ids": [], - "protocol": "-1", - "security_groups": [], - "self": false, - "to_port": 0 - } - ], - "id": "sg-029350ea6d628a8b7", - "ingress": [ - { - "cidr_blocks": [ - "0.0.0.0/0" - ], - "description": "SSH", - "from_port": 22, - "ipv6_cidr_blocks": [ - "::/0" - ], - "prefix_list_ids": [], - "protocol": "tcp", - "security_groups": [], - "self": false, - "to_port": 22 - }, - { - "cidr_blocks": [ - "0.0.0.0/0" - ], - "description": "icmp from Anywhere", - "from_port": -1, - "ipv6_cidr_blocks": [], - "prefix_list_ids": [], - "protocol": "icmp", - "security_groups": [], - "self": false, - "to_port": -1 - }, - { - "cidr_blocks": [ - "10.0.0.0/16" - ], - "description": "etcd from internal 2", - "from_port": 10401, - "ipv6_cidr_blocks": [], - "prefix_list_ids": [], - "protocol": "tcp", - "security_groups": [], - "self": false, - "to_port": 10401 - }, - { - "cidr_blocks": [ - "10.0.0.0/16" - ], - "description": "etcd from internal", - "from_port": 10301, - "ipv6_cidr_blocks": [], - "prefix_list_ids": [], - "protocol": "tcp", - "security_groups": [], - "self": false, - "to_port": 10301 - }, - { - "cidr_blocks": [ - "10.0.0.0/8", - "172.31.0.0/16" - ], - "description": "GRPC from Internal", - "from_port": 20101, - "ipv6_cidr_blocks": [], - "prefix_list_ids": [], - "protocol": "tcp", - "security_groups": [], - "self": false, - "to_port": 20101 - }, - { - "cidr_blocks": [ - "10.0.0.0/8", - "172.31.0.0/16" - ], - "description": "HTTP from Internal", - "from_port": 10101, - "ipv6_cidr_blocks": [], - "prefix_list_ids": [], - "protocol": "tcp", - "security_groups": [], - "self": false, - "to_port": 10101 - }, - { - "cidr_blocks": [ - "10.0.0.0/8", - "172.31.0.0/16" - ], - "description": "PostgreSQL from Internal", - "from_port": 55432, - "ipv6_cidr_blocks": [], - "prefix_list_ids": [], - "protocol": "tcp", - "security_groups": [], - "self": false, - "to_port": 55432 - } - ], - "name": "gauntlet-wFQOOzXB51R3ebr-allow_featurebase", - "name_prefix": "", - "owner_id": "977373308795", - "revoke_rules_on_delete": false, - "tags": { - "Name": "allow_featurebase" - }, - "tags_all": { - "Name": "allow_featurebase" - }, - "timeouts": null, - "vpc_id": "vpc-05a26a122f961dc2b" - }, - "sensitive_attributes": [], - "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjo2MDAwMDAwMDAwMDAsImRlbGV0ZSI6OTAwMDAwMDAwMDAwfSwic2NoZW1hX3ZlcnNpb24iOiIxIn0=" - } - ] - }, - { - "module": "module.samsung-cluster", - "mode": "managed", - "type": "aws_security_group", - "name": "ingest", - "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", - "instances": [ - { - "schema_version": 1, - "attributes": { - "arn": "arn:aws:ec2:us-east-2:977373308795:security-group/sg-0a66d187ba05935ce", - "description": "Allow ingest inbound traffic", - "egress": [ - { - "cidr_blocks": [ - "0.0.0.0/0" - ], - "description": "", - "from_port": 0, - "ipv6_cidr_blocks": [ - "::/0" - ], - "prefix_list_ids": [], - "protocol": "-1", - "security_groups": [], - "self": false, - "to_port": 0 - } - ], - "id": "sg-0a66d187ba05935ce", - "ingress": [ - { - "cidr_blocks": [ - "0.0.0.0/0" - ], - "description": "", - "from_port": 10101, - "ipv6_cidr_blocks": [ - "::/0" - ], - "prefix_list_ids": [], - "protocol": "tcp", - "security_groups": [], - "self": false, - "to_port": 10101 - }, - { - "cidr_blocks": [ - "0.0.0.0/0" - ], - "description": "SSH", - "from_port": 22, - "ipv6_cidr_blocks": [ - "::/0" - ], - "prefix_list_ids": [], - "protocol": "tcp", - "security_groups": [], - "self": false, - "to_port": 22 - }, - { - "cidr_blocks": [ - "0.0.0.0/0" - ], - "description": "icmp from Anywhere", - "from_port": -1, - "ipv6_cidr_blocks": [], - "prefix_list_ids": [], - "protocol": "icmp", - "security_groups": [], - "self": false, - "to_port": -1 - } - ], - "name": "gauntlet-wFQOOzXB51R3ebr-allow_ingest", - "name_prefix": "", - "owner_id": "977373308795", - "revoke_rules_on_delete": false, - "tags": { - "Name": "allow_ingest" - }, - "tags_all": { - "Name": "allow_ingest" - }, - "timeouts": null, - "vpc_id": "vpc-05a26a122f961dc2b" - }, - "sensitive_attributes": [], - "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjo2MDAwMDAwMDAwMDAsImRlbGV0ZSI6OTAwMDAwMDAwMDAwfSwic2NoZW1hX3ZlcnNpb24iOiIxIn0=" - } - ] - } - ] -} From 994ba6f597f1712abc05831d16b3013e1684e20b Mon Sep 17 00:00:00 2001 From: pokeeffe-molecula Date: Tue, 11 Jan 2022 10:30:28 -0600 Subject: [PATCH 11/13] ignore in sonarcloud --- qa/simulacraData/simulacra_data.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/qa/simulacraData/simulacra_data.go b/qa/simulacraData/simulacra_data.go index a6af37b67..74d21a406 100644 --- a/qa/simulacraData/simulacra_data.go +++ b/qa/simulacraData/simulacra_data.go @@ -3,6 +3,9 @@ // INPUT: none // OUTPUT: 6 csv files, containing approx 1 billion lines of data associated to 200 million unique records (approx 28BGB of data) +// this code is not shipped to customers +// @@com.molecula.sonarcloud.ignore + package main import ( From 5056a8ea4d603e5470053570886e2370d7121050 Mon Sep 17 00:00:00 2001 From: pokeeffe-molecula Date: Tue, 11 Jan 2022 11:39:45 -0600 Subject: [PATCH 12/13] Update .gitlab-ci.yml --- .gitlab/.gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 6b964daef..3f1b4eb0a 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -219,7 +219,7 @@ smoke test: - docker - fbsmoke rules: - - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"' + - if: '$CI_PIPELINE_SOURCE == "push"' before_script: - apt-get update && apt-get install -y gnupg software-properties-common curl git - curl -fsSL https://apt.releases.hashicorp.com/gpg | apt-key add - From 7e8d2172087e7c140c7a6b349cf344f0e5f251d5 Mon Sep 17 00:00:00 2001 From: pokeeffe-molecula Date: Tue, 11 Jan 2022 12:21:38 -0600 Subject: [PATCH 13/13] so much fail... --- qa/simulacraData/simulacra_data.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/qa/simulacraData/simulacra_data.go b/qa/simulacraData/simulacra_data.go index 74d21a406..a6af37b67 100644 --- a/qa/simulacraData/simulacra_data.go +++ b/qa/simulacraData/simulacra_data.go @@ -3,9 +3,6 @@ // INPUT: none // OUTPUT: 6 csv files, containing approx 1 billion lines of data associated to 200 million unique records (approx 28BGB of data) -// this code is not shipped to customers -// @@com.molecula.sonarcloud.ignore - package main import (