diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 07ed85117..98ab48286 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -689,217 +689,6 @@ smoke test: reports: junit: report.xml -tremor-delete-test: - stage: integration - image: registry.gitlab.com/gitlab-org/cloud-deploy/aws-base:latest - timeout: 6h - variables: - PROFILE: "service-terraform" - AWS_SSH_PRIVATE_KEY: $AWS_FBCI_SSH_KEY - AWS_ACCESS_KEY_ID: $AWS_FBCI_ACCESS_KEY_ID - AWS_SECRET_ACCESS_KEY: $AWS_FBCI_SECRET_ACCESS_KEY - STACK_PREFIX: $CI_JOB_NAME - FB_INSTANCE_COUNT: 5 - REPLICA_COUNT: 1 - CF_VAR_cluster_prefix: "" - tags: - - aws - - docker - - fbsmoke - # rules: - # - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && ($CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web")' - when: manual - before_script: - - apt-get update && apt-get install -y gnupg software-properties-common curl git - - curl -fsSL https://apt.releases.hashicorp.com/gpg | apt-key add - - - apt-add-repository "deb [arch=amd64] https://apt.releases.hashicorp.com $(lsb_release -cs) main" - - apt-get update && apt-get install terraform - - aws configure set aws_access_key_id $AWS_FBCI_ACCESS_KEY_ID --profile $PROFILE - - aws configure set aws_secret_access_key $AWS_FBCI_SECRET_ACCESS_KEY --profile $PROFILE - - aws configure set region "us-east-2" --profile $PROFILE - - aws configure set aws_profile $PROFILE - - echo $AWS_FBCI_SSH_KEY > gitlab-featurebase-ci.pem - - chmod 400 gitlab-featurebase-ci.pem - - "which ssh-agent || ( apt-get update -y && apt-get install openssh-client -y )" - - eval $(ssh-agent -s) - - mkdir -p ~/.ssh - - echo $AWS_FBCI_SSH_KEY > /root/.ssh/gitlab-featurebase-ci.pem - - chmod 400 /root/.ssh/gitlab-featurebase-ci.pem - - echo "$AWS_FBCI_SSH_KEY" | ssh-add - - - chmod 700 /root/.ssh - - '[[ -f /.dockerenv ]] && echo -e "Host *\n\tStrictHostKeyChecking no\n\n" > ~/.ssh/config' - - apt update && apt -y install jq wget - - wget -q https://go.dev/dl/go$GOVERSION.linux-amd64.tar.gz - - tar -C /usr/local -xzf go$GOVERSION.linux-amd64.tar.gz - - export PATH=$PATH:/usr/local/go/bin - - CF_STACK_NAME="${STACK_PREFIX}-${CI_JOB_ID}" - - echo "Cloudformation Stack Name --> $CF_STACK_NAME" - - echo $CF_STACK_NAME > /tmp/stackname-{$CI_JOB_ID} - - CF_VAR_cluster_prefix="pipeline-$CI_PIPELINE_ID-delete-$CI_JOB_ID" - - echo "Cluster Prefix --> $CF_VAR_cluster_prefix" - script: - - echo "Start CloudFormation deploy..." - - > - aws cloudformation deploy - --stack-name $CF_STACK_NAME - --template-file ./qa/cft/FeatureBaseClusterCFTTremor.yaml - --parameter-overrides - FBInstanceCount=$FB_INSTANCE_COUNT - EBSVolumeSize=1200 - TestName=$CI_JOB_NAME - TestID=$CI_JOB_ID - Name=$CF_VAR_cluster_prefix - Prefix=$CF_VAR_cluster_prefix - TestCommitSHA=$CI_COMMIT_SHA - --capabilities CAPABILITY_NAMED_IAM - --profile $PROFILE - - echo "CloudFormation deploy done." - - echo "Start configuring and start featurebase..." - - ./qa/scripts/setupTremorDeleteCFT.sh $CF_STACK_NAME $PROFILE $REPLICA_COUNT $FB_INSTANCE_COUNT $CI_COMMIT_BRANCH - # producer.log, consumer.log, queries.log are generated when runTremorDeleteCFT.sh is ran - - ./qa/scripts/runTremorDeleteCFT.sh $CF_STACK_NAME $PROFILE > tremor-delete-test.log 2>&1 - - echo "Featurebase configuration done." - - echo "Start running tests..." - - echo "Done with tests!" - artifacts: - when: always - paths: - - tremor-delete-test.log - after_script: - # note that we have to read the file to get the $CF_STACK_NAME - # because variables from the previous script & before_script do not get passed to after_script - - CF_STACK_NAME=$(cat /tmp/stackname-{$CI_JOB_ID}) - - echo "Deleting stack after test, stack name = ${CF_STACK_NAME}" - - aws cloudformation delete-stack --stack-name $CF_STACK_NAME --profile $PROFILE --retain-resources DeploymentEC2Role - - echo "Delete stack complete!" - needs: - - job: s3 dump - - job: idk s3 dump - -samsung-gauntlet: - stage: gauntlet - timeout: 4h - image: registry.gitlab.com/gitlab-org/cloud-deploy/aws-base:latest - variables: - FBCI_PROFILE: "service-terraform" - INFRA_PROFILE: "service-gitlab" - AWS_SSH_PRIVATE_KEY: $AWS_FBCI_SSH_KEY - AWS_ACCESS_KEY_ID: $AWS_FBCI_ACCESS_KEY_ID - AWS_SECRET_ACCESS_KEY: $AWS_FBCI_SECRET_ACCESS_KEY - ASG_NAME: "gitlab-runners" - TF_VAR_cluster_prefix: "" - tags: - - aws - - docker - - fbsmoke - # rules: - # - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && ($CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web")' - when: manual - before_script: - - apt-get update && apt-get install -y gnupg software-properties-common curl git - - curl -fsSL https://apt.releases.hashicorp.com/gpg | apt-key add - - - apt-add-repository "deb [arch=amd64] https://apt.releases.hashicorp.com $(lsb_release -cs) main" - - apt-get update && apt-get install terraform - - aws configure set aws_access_key_id $AWS_FBCI_ACCESS_KEY_ID --profile $FBCI_PROFILE - - aws configure set aws_secret_access_key $AWS_FBCI_SECRET_ACCESS_KEY --profile $FBCI_PROFILE - - aws configure set region "us-east-2" --profile $FBCI_PROFILE - - aws configure set aws_profile $FBCI_PROFILE - - aws configure set aws_access_key_id $AWS_INFRA_ACCESS_KEY_ID --profile $INFRA_PROFILE - - aws configure set aws_secret_access_key $AWS_INFRA_SECRET_ACCESS_KEY --profile $INFRA_PROFILE - - aws configure set region "us-east-2" --profile $INFRA_PROFILE - - echo $AWS_FBCI_SSH_KEY > gitlab-featurebase-ci.pem - - chmod 400 gitlab-featurebase-ci.pem - - "which ssh-agent || ( apt-get update -y && apt-get install openssh-client -y )" - - eval $(ssh-agent -s) - - mkdir -p ~/.ssh - - echo $AWS_FBCI_SSH_KEY > /root/.ssh/gitlab-featurebase-ci.pem - - chmod 400 /root/.ssh/gitlab-featurebase-ci.pem - - echo "$AWS_FBCI_SSH_KEY" | ssh-add - - - chmod 700 /root/.ssh - - '[[ -f /.dockerenv ]] && echo -e "Host *\n\tStrictHostKeyChecking no\n\n" > ~/.ssh/config' - - apt update && apt -y install jq wget - - wget -q "https://go.dev/dl/go$GOVERSION.linux-amd64.tar.gz" - - tar -C /usr/local -xzf "go$GOVERSION.linux-amd64.tar.gz" - - export PATH=$PATH:/usr/local/go/bin - - TF_VAR_cluster_prefix="pipeline-$CI_PIPELINE_ID-gauntlet-$CI_JOB_ID" - - echo "Cluster Prefix --> $TF_VAR_cluster_prefix" - - export INSTANCE_ID=$(curl --silent --fail "http://169.254.169.254/latest/meta-data/instance-id" | tee instance_id) - - aws autoscaling set-instance-protection --instance-ids "$INSTANCE_ID" --auto-scaling-group-name $ASG_NAME --protected-from-scale-in --profile $INFRA_PROFILE - - aws s3 cp s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/_latest/featurebase_linux_arm64 ./ - - chmod +x ./featurebase_linux_arm64 - script: - - ./qa/scripts/setupSamsungGauntlet.sh $CI_COMMIT_BRANCH - - ./qa/scripts/testSamsungGauntlet.sh - after_script: - - ./qa/scripts/teardownSamsungGauntlet.sh || true # leaving dangling resources is better than dangling ASG instances that can't be terminated - - export INSTANCE_ID=$(cat instance_id) - - aws autoscaling set-instance-protection --instance-ids "$INSTANCE_ID" --auto-scaling-group-name $ASG_NAME --no-protected-from-scale-in --profile $INFRA_PROFILE - needs: - - job: s3 dump - - job: idk s3 dump - -backup-restore-gauntlet: - stage: gauntlet - timeout: 4h - image: registry.gitlab.com/gitlab-org/cloud-deploy/aws-base:latest - variables: - FBCI_PROFILE: "service-terraform" - INFRA_PROFILE: "service-gitlab" - AWS_SSH_PRIVATE_KEY: $AWS_FBCI_SSH_KEY - AWS_ACCESS_KEY_ID: $AWS_FBCI_ACCESS_KEY_ID - AWS_SECRET_ACCESS_KEY: $AWS_FBCI_SECRET_ACCESS_KEY - ASG_NAME: "gitlab-runners" - TF_VAR_cluster_prefix: "" - # rules: - # - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && ($CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web")' - when: manual - tags: - - aws - - docker - - fbsmoke - before_script: - - apt-get update && apt-get install -y gnupg software-properties-common curl git - - curl -fsSL https://apt.releases.hashicorp.com/gpg | apt-key add - - - apt-add-repository "deb [arch=amd64] https://apt.releases.hashicorp.com $(lsb_release -cs) main" - - apt-get update && apt-get install terraform - - aws configure set aws_access_key_id $AWS_FBCI_ACCESS_KEY_ID --profile $FBCI_PROFILE - - aws configure set aws_secret_access_key $AWS_FBCI_SECRET_ACCESS_KEY --profile $FBCI_PROFILE - - aws configure set region "us-east-2" --profile $FBCI_PROFILE - - aws configure set aws_profile $FBCI_PROFILE - - aws configure set aws_access_key_id $AWS_INFRA_ACCESS_KEY_ID --profile $INFRA_PROFILE - - aws configure set aws_secret_access_key $AWS_INFRA_SECRET_ACCESS_KEY --profile $INFRA_PROFILE - - aws configure set region "us-east-2" --profile $INFRA_PROFILE - - echo $AWS_FBCI_SSH_KEY > gitlab-featurebase-ci.pem - - chmod 400 gitlab-featurebase-ci.pem - - "which ssh-agent || ( apt-get update -y && apt-get install openssh-client -y )" - - eval $(ssh-agent -s) - - mkdir -p ~/.ssh - - echo $AWS_FBCI_SSH_KEY > /root/.ssh/gitlab-featurebase-ci.pem - - chmod 400 /root/.ssh/gitlab-featurebase-ci.pem - - echo "$AWS_FBCI_SSH_KEY" | ssh-add - - - chmod 700 /root/.ssh - - '[[ -f /.dockerenv ]] && echo -e "Host *\n\tStrictHostKeyChecking no\n\n" > ~/.ssh/config' - - apt update && apt -y install jq wget - - wget -q "https://go.dev/dl/go$GOVERSION.linux-amd64.tar.gz" - - tar -C /usr/local -xzf "go$GOVERSION.linux-amd64.tar.gz" - - export PATH=$PATH:/usr/local/go/bin - - TF_VAR_cluster_prefix="pipeline-$CI_PIPELINE_ID-gauntlet-$CI_JOB_ID" - - echo "Cluster Prefix --> $TF_VAR_cluster_prefix" - - export INSTANCE_ID=$(curl --silent --fail "http://169.254.169.254/latest/meta-data/instance-id" | tee instance_id) - - aws autoscaling set-instance-protection --instance-ids "$INSTANCE_ID" --auto-scaling-group-name $ASG_NAME --protected-from-scale-in --profile $INFRA_PROFILE - - aws s3 cp s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/_latest/featurebase_linux_arm64 ./ - - chmod +x ./featurebase_linux_arm64 - script: - - ./qa/scripts/setupBackupRestoreGauntlet.sh $CI_COMMIT_BRANCH - - ./qa/scripts/testBackupRestoreGauntlet.sh $CI_COMMIT_BRANCH - after_script: - - ./qa/scripts/teardownBackupRestoreGauntlet.sh || true # leaving dangling resources is better than dangling ASG instances that can't be terminated - - export INSTANCE_ID=$(cat instance_id) - - aws autoscaling set-instance-protection --instance-ids "$INSTANCE_ID" --auto-scaling-group-name $ASG_NAME --no-protected-from-scale-in --profile $INFRA_PROFILE - needs: - - job: s3 dump - - job: idk s3 dump - s3 dump: stage: post build variables: @@ -927,81 +716,6 @@ s3 dump: needs: - job: build featurebase -perf_able: - stage: performance - # rules: - # - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $CI_PIPELINE_SOURCE == "push"' - when: manual - trigger: - include: .gitlab/.perf-able-gitlab-ci.yml - variables: - PARENT_PIPELINE_ID: $CI_PIPELINE_ID - -# only run on merge to master, the whole process takes about 10 minutes in total -# (which is a long time) -perf_delete: - stage: performance - # rules: - # - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $CI_PIPELINE_SOURCE == "push"' - when: manual - timeout: 2h - image: registry.gitlab.com/gitlab-org/cloud-deploy/aws-base:latest - variables: - PROFILE: "service-terraform" - INFRA_PROFILE: "service-gitlab" - AWS_SSH_PRIVATE_KEY: $AWS_FBCI_SSH_KEY - AWS_ACCESS_KEY_ID: $AWS_FBCI_ACCESS_KEY_ID - AWS_SECRET_ACCESS_KEY: $AWS_FBCI_SECRET_ACCESS_KEY - ASG_NAME: "gitlab-runners" - TF_VAR_cluster_prefix: "" - tags: - - aws - - docker - - fbsmoke - before_script: - - apt-get update && apt-get install -y gnupg software-properties-common curl git - - curl -fsSL https://apt.releases.hashicorp.com/gpg | apt-key add - - - apt-add-repository "deb [arch=amd64] https://apt.releases.hashicorp.com $(lsb_release -cs) main" - - apt-get update && apt-get install terraform - - apt-get update && apt-get install python3.7 - - aws configure set aws_access_key_id $AWS_FBCI_ACCESS_KEY_ID --profile $PROFILE - - aws configure set aws_secret_access_key $AWS_FBCI_SECRET_ACCESS_KEY --profile $PROFILE - - aws configure set region "us-east-2" --profile $PROFILE - - aws configure set aws_profile $PROFILE - - aws configure set aws_access_key_id $AWS_INFRA_ACCESS_KEY_ID --profile $INFRA_PROFILE - - aws configure set aws_secret_access_key $AWS_INFRA_SECRET_ACCESS_KEY --profile $INFRA_PROFILE - - aws configure set region "us-east-2" --profile $INFRA_PROFILE - - echo $AWS_FBCI_SSH_KEY > gitlab-featurebase-ci.pem - - chmod 400 gitlab-featurebase-ci.pem - - "which ssh-agent || ( apt-get update -y && apt-get install openssh-client -y )" - - eval $(ssh-agent -s) - - mkdir -p ~/.ssh - - echo $AWS_FBCI_SSH_KEY > /root/.ssh/gitlab-featurebase-ci.pem - - chmod 400 /root/.ssh/gitlab-featurebase-ci.pem - - echo "$AWS_FBCI_SSH_KEY" | ssh-add - - - chmod 700 /root/.ssh - - '[[ -f /.dockerenv ]] && echo -e "Host *\n\tStrictHostKeyChecking no\n\n" > ~/.ssh/config' - - apt update && apt -y install jq wget - - wget -q "https://go.dev/dl/go$GOVERSION.linux-amd64.tar.gz" - - tar -C /usr/local -xzf "go$GOVERSION.linux-amd64.tar.gz" - - export PATH=$PATH:/usr/local/go/bin - - TF_VAR_cluster_prefix="pipeline-$CI_PIPELINE_ID-delete-$CI_JOB_ID" - - echo "Cluster Prefix --> $TF_VAR_cluster_prefix" - - export INSTANCE_ID=$(curl --silent --fail "http://169.254.169.254/latest/meta-data/instance-id" | tee instance_id) - - aws autoscaling set-instance-protection --instance-ids "$INSTANCE_ID" --auto-scaling-group-name $ASG_NAME --protected-from-scale-in --profile $INFRA_PROFILE - - aws s3 cp s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/_latest/featurebase_linux_arm64 ./ - - chmod +x ./featurebase_linux_arm64 - script: - - ./qa/scripts/perf/delete/deleteSetup.sh $CI_COMMIT_BRANCH - - ./qa/scripts/perf/delete/deleteTest.sh $CI_COMMIT_BRANCH - after_script: - - ./qa/scripts/perf/delete/deleteTeardown.sh || true - - export INSTANCE_ID=$(cat instance_id) - - aws autoscaling set-instance-protection --instance-ids "$INSTANCE_ID" --auto-scaling-group-name $ASG_NAME --no-protected-from-scale-in --profile $INFRA_PROFILE - needs: - - job: s3 dump - - job: idk s3 dump - s3 dump tag: stage: post build variables: diff --git a/qa/cft/FeatureBaseClusterCFTTremor.yaml b/qa/cft/FeatureBaseClusterCFTTremor.yaml index 7820148c5..3cd001869 100644 --- a/qa/cft/FeatureBaseClusterCFTTremor.yaml +++ b/qa/cft/FeatureBaseClusterCFTTremor.yaml @@ -44,7 +44,7 @@ Parameters: Description: The number of EC2 instances for the FeatureBase cluster FBInstanceSize: Type: String - Default: m6i.16xlarge + Default: m5ad.16xlarge Description: Class of EC2 instance used to host the deployment. # note that we are using a producer, which is also the kafka host # and the consumer uses the kafka-static binary for ingesting data @@ -52,10 +52,10 @@ Parameters: # producer/consumer related params ProducerInstanceSize: Type: String - Default: m6i.16xlarge + Default: m4.16xlarge ConsumerInstanceSize: Type: String - Default: m6i.16xlarge + Default: m4.16xlarge ProducerInstanceCount: Type: Number Default: 1 diff --git a/qa/tf/.modules/featurebase-cluster/variables.tf b/qa/tf/.modules/featurebase-cluster/variables.tf index 5a743f703..5e9d59dd4 100644 --- a/qa/tf/.modules/featurebase-cluster/variables.tf +++ b/qa/tf/.modules/featurebase-cluster/variables.tf @@ -10,7 +10,7 @@ variable "fb_cluster_arch" { variable "fb_ingest_type" { type = string - default = "m6g.2xlarge" + default = "c6g.2xlarge" } variable "fb_ingest_node_count" { @@ -20,7 +20,7 @@ variable "fb_ingest_node_count" { variable "fb_data_node_type" { type = string - default = "m6g.16xlarge" + default = "c6g.16xlarge" } variable "fb_data_node_count" { diff --git a/qa/tf/ci/deletetest/delete_cloud_init.sh b/qa/tf/ci/deletetest/delete_cloud_init.sh index 34ca89135..6c4c48a0d 100644 --- a/qa/tf/ci/deletetest/delete_cloud_init.sh +++ b/qa/tf/ci/deletetest/delete_cloud_init.sh @@ -7,7 +7,6 @@ echo "Running delete_cloud_init.sh" # create and initialize a raid0 array # for m5ad.16xlarge instance type, there are 4 NVMe SSDs. -# Note JALLISON 20221214 - we are now using m6i instance types for cost savings sudo mdadm --create --verbose /dev/md0 --level=0 --raid-devices=4 /dev/nvme1n1 /dev/nvme2n1 /dev/nvme3n1 /dev/nvme4n1 # format filesystem diff --git a/qa/tf/ci/deletetest/main.tf b/qa/tf/ci/deletetest/main.tf index aa1188822..3d6bd2c77 100644 --- a/qa/tf/ci/deletetest/main.tf +++ b/qa/tf/ci/deletetest/main.tf @@ -4,10 +4,10 @@ module "ci-cluster" { region = var.region profile = var.profile fb_cluster_arch = ["x86_64"] - fb_data_node_type = "m6i.16xlarge" + fb_data_node_type = "m5ad.16xlarge" fb_data_node_count = 5 ebs_volumes = [] - fb_ingest_type = "m6i.16xlarge" + fb_ingest_type = "m4.16xlarge" fb_ingest_node_count = 2 fb_ingest_disk_size_gb = 1200 vpc_id = "vpc-05a26a122f961dc2b" diff --git a/qa/tf/perf/able/main.tf b/qa/tf/perf/able/main.tf index 10a9054b7..508153d21 100644 --- a/qa/tf/perf/able/main.tf +++ b/qa/tf/perf/able/main.tf @@ -3,10 +3,10 @@ module "able-cluster" { cluster_prefix = var.cluster_prefix region = var.region profile = var.profile - fb_data_node_type = "m6g.16xlarge" + fb_data_node_type = "c6g.16xlarge" fb_data_disk_iops = 10000 fb_data_node_count = 3 - fb_ingest_type = "m6g.2xlarge" + fb_ingest_type = "c6g.2xlarge" fb_ingest_disk_iops = 10000 fb_ingest_disk_size_gb = 500 fb_ingest_node_count = 1