From 664ec83524f30ab66d80d7204ead7262ee45ff43 Mon Sep 17 00:00:00 2001 From: Garrison Davis Date: Wed, 14 Dec 2022 11:30:39 -0700 Subject: [PATCH] Remove manual jobs from CI These jobs all need TLC and to be moved to the new ansible platform. For now, we're removing them because if they accidentally get triggered, they cost a lot of money very quickly, and don't necessarily get us anything useful if they pass or fail. Revert "Updating any AWS shape instance to use" ...our existing reserved instance types. This simply ensures if we ever do run one of these tests it is against existing reserved instances. If the tests get removed thats OK also." This reverts commit 69be968d6dc12fffd712abdb513ea97773d906da. --- .gitlab/.gitlab-ci.yml | 286 ------------------ qa/cft/FeatureBaseClusterCFTTremor.yaml | 6 +- .../.modules/featurebase-cluster/variables.tf | 4 +- qa/tf/ci/deletetest/delete_cloud_init.sh | 1 - qa/tf/ci/deletetest/main.tf | 4 +- qa/tf/perf/able/main.tf | 4 +- 6 files changed, 9 insertions(+), 296 deletions(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 07ed85117..98ab48286 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -689,217 +689,6 @@ smoke test: reports: junit: report.xml -tremor-delete-test: - stage: integration - image: registry.gitlab.com/gitlab-org/cloud-deploy/aws-base:latest - timeout: 6h - variables: - PROFILE: "service-terraform" - AWS_SSH_PRIVATE_KEY: $AWS_FBCI_SSH_KEY - AWS_ACCESS_KEY_ID: $AWS_FBCI_ACCESS_KEY_ID - AWS_SECRET_ACCESS_KEY: $AWS_FBCI_SECRET_ACCESS_KEY - STACK_PREFIX: $CI_JOB_NAME - FB_INSTANCE_COUNT: 5 - REPLICA_COUNT: 1 - CF_VAR_cluster_prefix: "" - tags: - - aws - - docker - - fbsmoke - # rules: - # - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && ($CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web")' - when: manual - before_script: - - apt-get update && apt-get install -y gnupg software-properties-common curl git - - curl -fsSL https://apt.releases.hashicorp.com/gpg | apt-key add - - - apt-add-repository "deb [arch=amd64] https://apt.releases.hashicorp.com $(lsb_release -cs) main" - - apt-get update && apt-get install terraform - - aws configure set aws_access_key_id $AWS_FBCI_ACCESS_KEY_ID --profile $PROFILE - - aws configure set aws_secret_access_key $AWS_FBCI_SECRET_ACCESS_KEY --profile $PROFILE - - aws configure set region "us-east-2" --profile $PROFILE - - aws configure set aws_profile $PROFILE - - echo $AWS_FBCI_SSH_KEY > gitlab-featurebase-ci.pem - - chmod 400 gitlab-featurebase-ci.pem - - "which ssh-agent || ( apt-get update -y && apt-get install openssh-client -y )" - - eval $(ssh-agent -s) - - mkdir -p ~/.ssh - - echo $AWS_FBCI_SSH_KEY > /root/.ssh/gitlab-featurebase-ci.pem - - chmod 400 /root/.ssh/gitlab-featurebase-ci.pem - - echo "$AWS_FBCI_SSH_KEY" | ssh-add - - - chmod 700 /root/.ssh - - '[[ -f /.dockerenv ]] && echo -e "Host *\n\tStrictHostKeyChecking no\n\n" > ~/.ssh/config' - - apt update && apt -y install jq wget - - wget -q https://go.dev/dl/go$GOVERSION.linux-amd64.tar.gz - - tar -C /usr/local -xzf go$GOVERSION.linux-amd64.tar.gz - - export PATH=$PATH:/usr/local/go/bin - - CF_STACK_NAME="${STACK_PREFIX}-${CI_JOB_ID}" - - echo "Cloudformation Stack Name --> $CF_STACK_NAME" - - echo $CF_STACK_NAME > /tmp/stackname-{$CI_JOB_ID} - - CF_VAR_cluster_prefix="pipeline-$CI_PIPELINE_ID-delete-$CI_JOB_ID" - - echo "Cluster Prefix --> $CF_VAR_cluster_prefix" - script: - - echo "Start CloudFormation deploy..." - - > - aws cloudformation deploy - --stack-name $CF_STACK_NAME - --template-file ./qa/cft/FeatureBaseClusterCFTTremor.yaml - --parameter-overrides - FBInstanceCount=$FB_INSTANCE_COUNT - EBSVolumeSize=1200 - TestName=$CI_JOB_NAME - TestID=$CI_JOB_ID - Name=$CF_VAR_cluster_prefix - Prefix=$CF_VAR_cluster_prefix - TestCommitSHA=$CI_COMMIT_SHA - --capabilities CAPABILITY_NAMED_IAM - --profile $PROFILE - - echo "CloudFormation deploy done." - - echo "Start configuring and start featurebase..." - - ./qa/scripts/setupTremorDeleteCFT.sh $CF_STACK_NAME $PROFILE $REPLICA_COUNT $FB_INSTANCE_COUNT $CI_COMMIT_BRANCH - # producer.log, consumer.log, queries.log are generated when runTremorDeleteCFT.sh is ran - - ./qa/scripts/runTremorDeleteCFT.sh $CF_STACK_NAME $PROFILE > tremor-delete-test.log 2>&1 - - echo "Featurebase configuration done." - - echo "Start running tests..." - - echo "Done with tests!" - artifacts: - when: always - paths: - - tremor-delete-test.log - after_script: - # note that we have to read the file to get the $CF_STACK_NAME - # because variables from the previous script & before_script do not get passed to after_script - - CF_STACK_NAME=$(cat /tmp/stackname-{$CI_JOB_ID}) - - echo "Deleting stack after test, stack name = ${CF_STACK_NAME}" - - aws cloudformation delete-stack --stack-name $CF_STACK_NAME --profile $PROFILE --retain-resources DeploymentEC2Role - - echo "Delete stack complete!" - needs: - - job: s3 dump - - job: idk s3 dump - -samsung-gauntlet: - stage: gauntlet - timeout: 4h - image: registry.gitlab.com/gitlab-org/cloud-deploy/aws-base:latest - variables: - FBCI_PROFILE: "service-terraform" - INFRA_PROFILE: "service-gitlab" - AWS_SSH_PRIVATE_KEY: $AWS_FBCI_SSH_KEY - AWS_ACCESS_KEY_ID: $AWS_FBCI_ACCESS_KEY_ID - AWS_SECRET_ACCESS_KEY: $AWS_FBCI_SECRET_ACCESS_KEY - ASG_NAME: "gitlab-runners" - TF_VAR_cluster_prefix: "" - tags: - - aws - - docker - - fbsmoke - # rules: - # - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && ($CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web")' - when: manual - before_script: - - apt-get update && apt-get install -y gnupg software-properties-common curl git - - curl -fsSL https://apt.releases.hashicorp.com/gpg | apt-key add - - - apt-add-repository "deb [arch=amd64] https://apt.releases.hashicorp.com $(lsb_release -cs) main" - - apt-get update && apt-get install terraform - - aws configure set aws_access_key_id $AWS_FBCI_ACCESS_KEY_ID --profile $FBCI_PROFILE - - aws configure set aws_secret_access_key $AWS_FBCI_SECRET_ACCESS_KEY --profile $FBCI_PROFILE - - aws configure set region "us-east-2" --profile $FBCI_PROFILE - - aws configure set aws_profile $FBCI_PROFILE - - aws configure set aws_access_key_id $AWS_INFRA_ACCESS_KEY_ID --profile $INFRA_PROFILE - - aws configure set aws_secret_access_key $AWS_INFRA_SECRET_ACCESS_KEY --profile $INFRA_PROFILE - - aws configure set region "us-east-2" --profile $INFRA_PROFILE - - echo $AWS_FBCI_SSH_KEY > gitlab-featurebase-ci.pem - - chmod 400 gitlab-featurebase-ci.pem - - "which ssh-agent || ( apt-get update -y && apt-get install openssh-client -y )" - - eval $(ssh-agent -s) - - mkdir -p ~/.ssh - - echo $AWS_FBCI_SSH_KEY > /root/.ssh/gitlab-featurebase-ci.pem - - chmod 400 /root/.ssh/gitlab-featurebase-ci.pem - - echo "$AWS_FBCI_SSH_KEY" | ssh-add - - - chmod 700 /root/.ssh - - '[[ -f /.dockerenv ]] && echo -e "Host *\n\tStrictHostKeyChecking no\n\n" > ~/.ssh/config' - - apt update && apt -y install jq wget - - wget -q "https://go.dev/dl/go$GOVERSION.linux-amd64.tar.gz" - - tar -C /usr/local -xzf "go$GOVERSION.linux-amd64.tar.gz" - - export PATH=$PATH:/usr/local/go/bin - - TF_VAR_cluster_prefix="pipeline-$CI_PIPELINE_ID-gauntlet-$CI_JOB_ID" - - echo "Cluster Prefix --> $TF_VAR_cluster_prefix" - - export INSTANCE_ID=$(curl --silent --fail "http://169.254.169.254/latest/meta-data/instance-id" | tee instance_id) - - aws autoscaling set-instance-protection --instance-ids "$INSTANCE_ID" --auto-scaling-group-name $ASG_NAME --protected-from-scale-in --profile $INFRA_PROFILE - - aws s3 cp s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/_latest/featurebase_linux_arm64 ./ - - chmod +x ./featurebase_linux_arm64 - script: - - ./qa/scripts/setupSamsungGauntlet.sh $CI_COMMIT_BRANCH - - ./qa/scripts/testSamsungGauntlet.sh - after_script: - - ./qa/scripts/teardownSamsungGauntlet.sh || true # leaving dangling resources is better than dangling ASG instances that can't be terminated - - export INSTANCE_ID=$(cat instance_id) - - aws autoscaling set-instance-protection --instance-ids "$INSTANCE_ID" --auto-scaling-group-name $ASG_NAME --no-protected-from-scale-in --profile $INFRA_PROFILE - needs: - - job: s3 dump - - job: idk s3 dump - -backup-restore-gauntlet: - stage: gauntlet - timeout: 4h - image: registry.gitlab.com/gitlab-org/cloud-deploy/aws-base:latest - variables: - FBCI_PROFILE: "service-terraform" - INFRA_PROFILE: "service-gitlab" - AWS_SSH_PRIVATE_KEY: $AWS_FBCI_SSH_KEY - AWS_ACCESS_KEY_ID: $AWS_FBCI_ACCESS_KEY_ID - AWS_SECRET_ACCESS_KEY: $AWS_FBCI_SECRET_ACCESS_KEY - ASG_NAME: "gitlab-runners" - TF_VAR_cluster_prefix: "" - # rules: - # - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && ($CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web")' - when: manual - tags: - - aws - - docker - - fbsmoke - before_script: - - apt-get update && apt-get install -y gnupg software-properties-common curl git - - curl -fsSL https://apt.releases.hashicorp.com/gpg | apt-key add - - - apt-add-repository "deb [arch=amd64] https://apt.releases.hashicorp.com $(lsb_release -cs) main" - - apt-get update && apt-get install terraform - - aws configure set aws_access_key_id $AWS_FBCI_ACCESS_KEY_ID --profile $FBCI_PROFILE - - aws configure set aws_secret_access_key $AWS_FBCI_SECRET_ACCESS_KEY --profile $FBCI_PROFILE - - aws configure set region "us-east-2" --profile $FBCI_PROFILE - - aws configure set aws_profile $FBCI_PROFILE - - aws configure set aws_access_key_id $AWS_INFRA_ACCESS_KEY_ID --profile $INFRA_PROFILE - - aws configure set aws_secret_access_key $AWS_INFRA_SECRET_ACCESS_KEY --profile $INFRA_PROFILE - - aws configure set region "us-east-2" --profile $INFRA_PROFILE - - echo $AWS_FBCI_SSH_KEY > gitlab-featurebase-ci.pem - - chmod 400 gitlab-featurebase-ci.pem - - "which ssh-agent || ( apt-get update -y && apt-get install openssh-client -y )" - - eval $(ssh-agent -s) - - mkdir -p ~/.ssh - - echo $AWS_FBCI_SSH_KEY > /root/.ssh/gitlab-featurebase-ci.pem - - chmod 400 /root/.ssh/gitlab-featurebase-ci.pem - - echo "$AWS_FBCI_SSH_KEY" | ssh-add - - - chmod 700 /root/.ssh - - '[[ -f /.dockerenv ]] && echo -e "Host *\n\tStrictHostKeyChecking no\n\n" > ~/.ssh/config' - - apt update && apt -y install jq wget - - wget -q "https://go.dev/dl/go$GOVERSION.linux-amd64.tar.gz" - - tar -C /usr/local -xzf "go$GOVERSION.linux-amd64.tar.gz" - - export PATH=$PATH:/usr/local/go/bin - - TF_VAR_cluster_prefix="pipeline-$CI_PIPELINE_ID-gauntlet-$CI_JOB_ID" - - echo "Cluster Prefix --> $TF_VAR_cluster_prefix" - - export INSTANCE_ID=$(curl --silent --fail "http://169.254.169.254/latest/meta-data/instance-id" | tee instance_id) - - aws autoscaling set-instance-protection --instance-ids "$INSTANCE_ID" --auto-scaling-group-name $ASG_NAME --protected-from-scale-in --profile $INFRA_PROFILE - - aws s3 cp s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/_latest/featurebase_linux_arm64 ./ - - chmod +x ./featurebase_linux_arm64 - script: - - ./qa/scripts/setupBackupRestoreGauntlet.sh $CI_COMMIT_BRANCH - - ./qa/scripts/testBackupRestoreGauntlet.sh $CI_COMMIT_BRANCH - after_script: - - ./qa/scripts/teardownBackupRestoreGauntlet.sh || true # leaving dangling resources is better than dangling ASG instances that can't be terminated - - export INSTANCE_ID=$(cat instance_id) - - aws autoscaling set-instance-protection --instance-ids "$INSTANCE_ID" --auto-scaling-group-name $ASG_NAME --no-protected-from-scale-in --profile $INFRA_PROFILE - needs: - - job: s3 dump - - job: idk s3 dump - s3 dump: stage: post build variables: @@ -927,81 +716,6 @@ s3 dump: needs: - job: build featurebase -perf_able: - stage: performance - # rules: - # - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $CI_PIPELINE_SOURCE == "push"' - when: manual - trigger: - include: .gitlab/.perf-able-gitlab-ci.yml - variables: - PARENT_PIPELINE_ID: $CI_PIPELINE_ID - -# only run on merge to master, the whole process takes about 10 minutes in total -# (which is a long time) -perf_delete: - stage: performance - # rules: - # - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $CI_PIPELINE_SOURCE == "push"' - when: manual - timeout: 2h - image: registry.gitlab.com/gitlab-org/cloud-deploy/aws-base:latest - variables: - PROFILE: "service-terraform" - INFRA_PROFILE: "service-gitlab" - AWS_SSH_PRIVATE_KEY: $AWS_FBCI_SSH_KEY - AWS_ACCESS_KEY_ID: $AWS_FBCI_ACCESS_KEY_ID - AWS_SECRET_ACCESS_KEY: $AWS_FBCI_SECRET_ACCESS_KEY - ASG_NAME: "gitlab-runners" - TF_VAR_cluster_prefix: "" - tags: - - aws - - docker - - fbsmoke - before_script: - - apt-get update && apt-get install -y gnupg software-properties-common curl git - - curl -fsSL https://apt.releases.hashicorp.com/gpg | apt-key add - - - apt-add-repository "deb [arch=amd64] https://apt.releases.hashicorp.com $(lsb_release -cs) main" - - apt-get update && apt-get install terraform - - apt-get update && apt-get install python3.7 - - aws configure set aws_access_key_id $AWS_FBCI_ACCESS_KEY_ID --profile $PROFILE - - aws configure set aws_secret_access_key $AWS_FBCI_SECRET_ACCESS_KEY --profile $PROFILE - - aws configure set region "us-east-2" --profile $PROFILE - - aws configure set aws_profile $PROFILE - - aws configure set aws_access_key_id $AWS_INFRA_ACCESS_KEY_ID --profile $INFRA_PROFILE - - aws configure set aws_secret_access_key $AWS_INFRA_SECRET_ACCESS_KEY --profile $INFRA_PROFILE - - aws configure set region "us-east-2" --profile $INFRA_PROFILE - - echo $AWS_FBCI_SSH_KEY > gitlab-featurebase-ci.pem - - chmod 400 gitlab-featurebase-ci.pem - - "which ssh-agent || ( apt-get update -y && apt-get install openssh-client -y )" - - eval $(ssh-agent -s) - - mkdir -p ~/.ssh - - echo $AWS_FBCI_SSH_KEY > /root/.ssh/gitlab-featurebase-ci.pem - - chmod 400 /root/.ssh/gitlab-featurebase-ci.pem - - echo "$AWS_FBCI_SSH_KEY" | ssh-add - - - chmod 700 /root/.ssh - - '[[ -f /.dockerenv ]] && echo -e "Host *\n\tStrictHostKeyChecking no\n\n" > ~/.ssh/config' - - apt update && apt -y install jq wget - - wget -q "https://go.dev/dl/go$GOVERSION.linux-amd64.tar.gz" - - tar -C /usr/local -xzf "go$GOVERSION.linux-amd64.tar.gz" - - export PATH=$PATH:/usr/local/go/bin - - TF_VAR_cluster_prefix="pipeline-$CI_PIPELINE_ID-delete-$CI_JOB_ID" - - echo "Cluster Prefix --> $TF_VAR_cluster_prefix" - - export INSTANCE_ID=$(curl --silent --fail "http://169.254.169.254/latest/meta-data/instance-id" | tee instance_id) - - aws autoscaling set-instance-protection --instance-ids "$INSTANCE_ID" --auto-scaling-group-name $ASG_NAME --protected-from-scale-in --profile $INFRA_PROFILE - - aws s3 cp s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/_latest/featurebase_linux_arm64 ./ - - chmod +x ./featurebase_linux_arm64 - script: - - ./qa/scripts/perf/delete/deleteSetup.sh $CI_COMMIT_BRANCH - - ./qa/scripts/perf/delete/deleteTest.sh $CI_COMMIT_BRANCH - after_script: - - ./qa/scripts/perf/delete/deleteTeardown.sh || true - - export INSTANCE_ID=$(cat instance_id) - - aws autoscaling set-instance-protection --instance-ids "$INSTANCE_ID" --auto-scaling-group-name $ASG_NAME --no-protected-from-scale-in --profile $INFRA_PROFILE - needs: - - job: s3 dump - - job: idk s3 dump - s3 dump tag: stage: post build variables: diff --git a/qa/cft/FeatureBaseClusterCFTTremor.yaml b/qa/cft/FeatureBaseClusterCFTTremor.yaml index 7820148c5..3cd001869 100644 --- a/qa/cft/FeatureBaseClusterCFTTremor.yaml +++ b/qa/cft/FeatureBaseClusterCFTTremor.yaml @@ -44,7 +44,7 @@ Parameters: Description: The number of EC2 instances for the FeatureBase cluster FBInstanceSize: Type: String - Default: m6i.16xlarge + Default: m5ad.16xlarge Description: Class of EC2 instance used to host the deployment. # note that we are using a producer, which is also the kafka host # and the consumer uses the kafka-static binary for ingesting data @@ -52,10 +52,10 @@ Parameters: # producer/consumer related params ProducerInstanceSize: Type: String - Default: m6i.16xlarge + Default: m4.16xlarge ConsumerInstanceSize: Type: String - Default: m6i.16xlarge + Default: m4.16xlarge ProducerInstanceCount: Type: Number Default: 1 diff --git a/qa/tf/.modules/featurebase-cluster/variables.tf b/qa/tf/.modules/featurebase-cluster/variables.tf index 5a743f703..5e9d59dd4 100644 --- a/qa/tf/.modules/featurebase-cluster/variables.tf +++ b/qa/tf/.modules/featurebase-cluster/variables.tf @@ -10,7 +10,7 @@ variable "fb_cluster_arch" { variable "fb_ingest_type" { type = string - default = "m6g.2xlarge" + default = "c6g.2xlarge" } variable "fb_ingest_node_count" { @@ -20,7 +20,7 @@ variable "fb_ingest_node_count" { variable "fb_data_node_type" { type = string - default = "m6g.16xlarge" + default = "c6g.16xlarge" } variable "fb_data_node_count" { diff --git a/qa/tf/ci/deletetest/delete_cloud_init.sh b/qa/tf/ci/deletetest/delete_cloud_init.sh index 34ca89135..6c4c48a0d 100644 --- a/qa/tf/ci/deletetest/delete_cloud_init.sh +++ b/qa/tf/ci/deletetest/delete_cloud_init.sh @@ -7,7 +7,6 @@ echo "Running delete_cloud_init.sh" # create and initialize a raid0 array # for m5ad.16xlarge instance type, there are 4 NVMe SSDs. -# Note JALLISON 20221214 - we are now using m6i instance types for cost savings sudo mdadm --create --verbose /dev/md0 --level=0 --raid-devices=4 /dev/nvme1n1 /dev/nvme2n1 /dev/nvme3n1 /dev/nvme4n1 # format filesystem diff --git a/qa/tf/ci/deletetest/main.tf b/qa/tf/ci/deletetest/main.tf index aa1188822..3d6bd2c77 100644 --- a/qa/tf/ci/deletetest/main.tf +++ b/qa/tf/ci/deletetest/main.tf @@ -4,10 +4,10 @@ module "ci-cluster" { region = var.region profile = var.profile fb_cluster_arch = ["x86_64"] - fb_data_node_type = "m6i.16xlarge" + fb_data_node_type = "m5ad.16xlarge" fb_data_node_count = 5 ebs_volumes = [] - fb_ingest_type = "m6i.16xlarge" + fb_ingest_type = "m4.16xlarge" fb_ingest_node_count = 2 fb_ingest_disk_size_gb = 1200 vpc_id = "vpc-05a26a122f961dc2b" diff --git a/qa/tf/perf/able/main.tf b/qa/tf/perf/able/main.tf index 10a9054b7..508153d21 100644 --- a/qa/tf/perf/able/main.tf +++ b/qa/tf/perf/able/main.tf @@ -3,10 +3,10 @@ module "able-cluster" { cluster_prefix = var.cluster_prefix region = var.region profile = var.profile - fb_data_node_type = "m6g.16xlarge" + fb_data_node_type = "c6g.16xlarge" fb_data_disk_iops = 10000 fb_data_node_count = 3 - fb_ingest_type = "m6g.2xlarge" + fb_ingest_type = "c6g.2xlarge" fb_ingest_disk_iops = 10000 fb_ingest_disk_size_gb = 500 fb_ingest_node_count = 1