Remove manual jobs from CI

These jobs all need TLC and to be moved to the new ansible platform. For
now, we're removing them because if they accidentally get triggered,
they cost a lot of money very quickly, and don't necessarily get us
anything useful if they pass or fail.

Revert "Updating any AWS shape instance to use"
...our existing reserved instance types.  This simply ensures if we ever
do run one of these tests it is against existing reserved instances.  If
the tests get removed thats OK also."

This reverts commit 69be968d6d.
This commit is contained in:
Garrison Davis 2022-12-14 11:30:39 -07:00 committed by GitHub
parent c1dbc48fb2
commit 664ec83524
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
6 changed files with 9 additions and 296 deletions

View file

@ -689,217 +689,6 @@ smoke test:
reports:
junit: report.xml
tremor-delete-test:
stage: integration
image: registry.gitlab.com/gitlab-org/cloud-deploy/aws-base:latest
timeout: 6h
variables:
PROFILE: "service-terraform"
AWS_SSH_PRIVATE_KEY: $AWS_FBCI_SSH_KEY
AWS_ACCESS_KEY_ID: $AWS_FBCI_ACCESS_KEY_ID
AWS_SECRET_ACCESS_KEY: $AWS_FBCI_SECRET_ACCESS_KEY
STACK_PREFIX: $CI_JOB_NAME
FB_INSTANCE_COUNT: 5
REPLICA_COUNT: 1
CF_VAR_cluster_prefix: ""
tags:
- aws
- docker
- fbsmoke
# rules:
# - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && ($CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web")'
when: manual
before_script:
- apt-get update && apt-get install -y gnupg software-properties-common curl git
- curl -fsSL https://apt.releases.hashicorp.com/gpg | apt-key add -
- apt-add-repository "deb [arch=amd64] https://apt.releases.hashicorp.com $(lsb_release -cs) main"
- apt-get update && apt-get install terraform
- aws configure set aws_access_key_id $AWS_FBCI_ACCESS_KEY_ID --profile $PROFILE
- aws configure set aws_secret_access_key $AWS_FBCI_SECRET_ACCESS_KEY --profile $PROFILE
- aws configure set region "us-east-2" --profile $PROFILE
- aws configure set aws_profile $PROFILE
- echo $AWS_FBCI_SSH_KEY > gitlab-featurebase-ci.pem
- chmod 400 gitlab-featurebase-ci.pem
- "which ssh-agent || ( apt-get update -y && apt-get install openssh-client -y )"
- eval $(ssh-agent -s)
- mkdir -p ~/.ssh
- echo $AWS_FBCI_SSH_KEY > /root/.ssh/gitlab-featurebase-ci.pem
- chmod 400 /root/.ssh/gitlab-featurebase-ci.pem
- echo "$AWS_FBCI_SSH_KEY" | ssh-add -
- chmod 700 /root/.ssh
- '[[ -f /.dockerenv ]] && echo -e "Host *\n\tStrictHostKeyChecking no\n\n" > ~/.ssh/config'
- apt update && apt -y install jq wget
- wget -q https://go.dev/dl/go$GOVERSION.linux-amd64.tar.gz
- tar -C /usr/local -xzf go$GOVERSION.linux-amd64.tar.gz
- export PATH=$PATH:/usr/local/go/bin
- CF_STACK_NAME="${STACK_PREFIX}-${CI_JOB_ID}"
- echo "Cloudformation Stack Name --> $CF_STACK_NAME"
- echo $CF_STACK_NAME > /tmp/stackname-{$CI_JOB_ID}
- CF_VAR_cluster_prefix="pipeline-$CI_PIPELINE_ID-delete-$CI_JOB_ID"
- echo "Cluster Prefix --> $CF_VAR_cluster_prefix"
script:
- echo "Start CloudFormation deploy..."
- >
aws cloudformation deploy
--stack-name $CF_STACK_NAME
--template-file ./qa/cft/FeatureBaseClusterCFTTremor.yaml
--parameter-overrides
FBInstanceCount=$FB_INSTANCE_COUNT
EBSVolumeSize=1200
TestName=$CI_JOB_NAME
TestID=$CI_JOB_ID
Name=$CF_VAR_cluster_prefix
Prefix=$CF_VAR_cluster_prefix
TestCommitSHA=$CI_COMMIT_SHA
--capabilities CAPABILITY_NAMED_IAM
--profile $PROFILE
- echo "CloudFormation deploy done."
- echo "Start configuring and start featurebase..."
- ./qa/scripts/setupTremorDeleteCFT.sh $CF_STACK_NAME $PROFILE $REPLICA_COUNT $FB_INSTANCE_COUNT $CI_COMMIT_BRANCH
# producer.log, consumer.log, queries.log are generated when runTremorDeleteCFT.sh is ran
- ./qa/scripts/runTremorDeleteCFT.sh $CF_STACK_NAME $PROFILE > tremor-delete-test.log 2>&1
- echo "Featurebase configuration done."
- echo "Start running tests..."
- echo "Done with tests!"
artifacts:
when: always
paths:
- tremor-delete-test.log
after_script:
# note that we have to read the file to get the $CF_STACK_NAME
# because variables from the previous script & before_script do not get passed to after_script
- CF_STACK_NAME=$(cat /tmp/stackname-{$CI_JOB_ID})
- echo "Deleting stack after test, stack name = ${CF_STACK_NAME}"
- aws cloudformation delete-stack --stack-name $CF_STACK_NAME --profile $PROFILE --retain-resources DeploymentEC2Role
- echo "Delete stack complete!"
needs:
- job: s3 dump
- job: idk s3 dump
samsung-gauntlet:
stage: gauntlet
timeout: 4h
image: registry.gitlab.com/gitlab-org/cloud-deploy/aws-base:latest
variables:
FBCI_PROFILE: "service-terraform"
INFRA_PROFILE: "service-gitlab"
AWS_SSH_PRIVATE_KEY: $AWS_FBCI_SSH_KEY
AWS_ACCESS_KEY_ID: $AWS_FBCI_ACCESS_KEY_ID
AWS_SECRET_ACCESS_KEY: $AWS_FBCI_SECRET_ACCESS_KEY
ASG_NAME: "gitlab-runners"
TF_VAR_cluster_prefix: ""
tags:
- aws
- docker
- fbsmoke
# rules:
# - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && ($CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web")'
when: manual
before_script:
- apt-get update && apt-get install -y gnupg software-properties-common curl git
- curl -fsSL https://apt.releases.hashicorp.com/gpg | apt-key add -
- apt-add-repository "deb [arch=amd64] https://apt.releases.hashicorp.com $(lsb_release -cs) main"
- apt-get update && apt-get install terraform
- aws configure set aws_access_key_id $AWS_FBCI_ACCESS_KEY_ID --profile $FBCI_PROFILE
- aws configure set aws_secret_access_key $AWS_FBCI_SECRET_ACCESS_KEY --profile $FBCI_PROFILE
- aws configure set region "us-east-2" --profile $FBCI_PROFILE
- aws configure set aws_profile $FBCI_PROFILE
- aws configure set aws_access_key_id $AWS_INFRA_ACCESS_KEY_ID --profile $INFRA_PROFILE
- aws configure set aws_secret_access_key $AWS_INFRA_SECRET_ACCESS_KEY --profile $INFRA_PROFILE
- aws configure set region "us-east-2" --profile $INFRA_PROFILE
- echo $AWS_FBCI_SSH_KEY > gitlab-featurebase-ci.pem
- chmod 400 gitlab-featurebase-ci.pem
- "which ssh-agent || ( apt-get update -y && apt-get install openssh-client -y )"
- eval $(ssh-agent -s)
- mkdir -p ~/.ssh
- echo $AWS_FBCI_SSH_KEY > /root/.ssh/gitlab-featurebase-ci.pem
- chmod 400 /root/.ssh/gitlab-featurebase-ci.pem
- echo "$AWS_FBCI_SSH_KEY" | ssh-add -
- chmod 700 /root/.ssh
- '[[ -f /.dockerenv ]] && echo -e "Host *\n\tStrictHostKeyChecking no\n\n" > ~/.ssh/config'
- apt update && apt -y install jq wget
- wget -q "https://go.dev/dl/go$GOVERSION.linux-amd64.tar.gz"
- tar -C /usr/local -xzf "go$GOVERSION.linux-amd64.tar.gz"
- export PATH=$PATH:/usr/local/go/bin
- TF_VAR_cluster_prefix="pipeline-$CI_PIPELINE_ID-gauntlet-$CI_JOB_ID"
- echo "Cluster Prefix --> $TF_VAR_cluster_prefix"
- export INSTANCE_ID=$(curl --silent --fail "http://169.254.169.254/latest/meta-data/instance-id" | tee instance_id)
- aws autoscaling set-instance-protection --instance-ids "$INSTANCE_ID" --auto-scaling-group-name $ASG_NAME --protected-from-scale-in --profile $INFRA_PROFILE
- aws s3 cp s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/_latest/featurebase_linux_arm64 ./
- chmod +x ./featurebase_linux_arm64
script:
- ./qa/scripts/setupSamsungGauntlet.sh $CI_COMMIT_BRANCH
- ./qa/scripts/testSamsungGauntlet.sh
after_script:
- ./qa/scripts/teardownSamsungGauntlet.sh || true # leaving dangling resources is better than dangling ASG instances that can't be terminated
- export INSTANCE_ID=$(cat instance_id)
- aws autoscaling set-instance-protection --instance-ids "$INSTANCE_ID" --auto-scaling-group-name $ASG_NAME --no-protected-from-scale-in --profile $INFRA_PROFILE
needs:
- job: s3 dump
- job: idk s3 dump
backup-restore-gauntlet:
stage: gauntlet
timeout: 4h
image: registry.gitlab.com/gitlab-org/cloud-deploy/aws-base:latest
variables:
FBCI_PROFILE: "service-terraform"
INFRA_PROFILE: "service-gitlab"
AWS_SSH_PRIVATE_KEY: $AWS_FBCI_SSH_KEY
AWS_ACCESS_KEY_ID: $AWS_FBCI_ACCESS_KEY_ID
AWS_SECRET_ACCESS_KEY: $AWS_FBCI_SECRET_ACCESS_KEY
ASG_NAME: "gitlab-runners"
TF_VAR_cluster_prefix: ""
# rules:
# - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && ($CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web")'
when: manual
tags:
- aws
- docker
- fbsmoke
before_script:
- apt-get update && apt-get install -y gnupg software-properties-common curl git
- curl -fsSL https://apt.releases.hashicorp.com/gpg | apt-key add -
- apt-add-repository "deb [arch=amd64] https://apt.releases.hashicorp.com $(lsb_release -cs) main"
- apt-get update && apt-get install terraform
- aws configure set aws_access_key_id $AWS_FBCI_ACCESS_KEY_ID --profile $FBCI_PROFILE
- aws configure set aws_secret_access_key $AWS_FBCI_SECRET_ACCESS_KEY --profile $FBCI_PROFILE
- aws configure set region "us-east-2" --profile $FBCI_PROFILE
- aws configure set aws_profile $FBCI_PROFILE
- aws configure set aws_access_key_id $AWS_INFRA_ACCESS_KEY_ID --profile $INFRA_PROFILE
- aws configure set aws_secret_access_key $AWS_INFRA_SECRET_ACCESS_KEY --profile $INFRA_PROFILE
- aws configure set region "us-east-2" --profile $INFRA_PROFILE
- echo $AWS_FBCI_SSH_KEY > gitlab-featurebase-ci.pem
- chmod 400 gitlab-featurebase-ci.pem
- "which ssh-agent || ( apt-get update -y && apt-get install openssh-client -y )"
- eval $(ssh-agent -s)
- mkdir -p ~/.ssh
- echo $AWS_FBCI_SSH_KEY > /root/.ssh/gitlab-featurebase-ci.pem
- chmod 400 /root/.ssh/gitlab-featurebase-ci.pem
- echo "$AWS_FBCI_SSH_KEY" | ssh-add -
- chmod 700 /root/.ssh
- '[[ -f /.dockerenv ]] && echo -e "Host *\n\tStrictHostKeyChecking no\n\n" > ~/.ssh/config'
- apt update && apt -y install jq wget
- wget -q "https://go.dev/dl/go$GOVERSION.linux-amd64.tar.gz"
- tar -C /usr/local -xzf "go$GOVERSION.linux-amd64.tar.gz"
- export PATH=$PATH:/usr/local/go/bin
- TF_VAR_cluster_prefix="pipeline-$CI_PIPELINE_ID-gauntlet-$CI_JOB_ID"
- echo "Cluster Prefix --> $TF_VAR_cluster_prefix"
- export INSTANCE_ID=$(curl --silent --fail "http://169.254.169.254/latest/meta-data/instance-id" | tee instance_id)
- aws autoscaling set-instance-protection --instance-ids "$INSTANCE_ID" --auto-scaling-group-name $ASG_NAME --protected-from-scale-in --profile $INFRA_PROFILE
- aws s3 cp s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/_latest/featurebase_linux_arm64 ./
- chmod +x ./featurebase_linux_arm64
script:
- ./qa/scripts/setupBackupRestoreGauntlet.sh $CI_COMMIT_BRANCH
- ./qa/scripts/testBackupRestoreGauntlet.sh $CI_COMMIT_BRANCH
after_script:
- ./qa/scripts/teardownBackupRestoreGauntlet.sh || true # leaving dangling resources is better than dangling ASG instances that can't be terminated
- export INSTANCE_ID=$(cat instance_id)
- aws autoscaling set-instance-protection --instance-ids "$INSTANCE_ID" --auto-scaling-group-name $ASG_NAME --no-protected-from-scale-in --profile $INFRA_PROFILE
needs:
- job: s3 dump
- job: idk s3 dump
s3 dump:
stage: post build
variables:
@ -927,81 +716,6 @@ s3 dump:
needs:
- job: build featurebase
perf_able:
stage: performance
# rules:
# - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $CI_PIPELINE_SOURCE == "push"'
when: manual
trigger:
include: .gitlab/.perf-able-gitlab-ci.yml
variables:
PARENT_PIPELINE_ID: $CI_PIPELINE_ID
# only run on merge to master, the whole process takes about 10 minutes in total
# (which is a long time)
perf_delete:
stage: performance
# rules:
# - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $CI_PIPELINE_SOURCE == "push"'
when: manual
timeout: 2h
image: registry.gitlab.com/gitlab-org/cloud-deploy/aws-base:latest
variables:
PROFILE: "service-terraform"
INFRA_PROFILE: "service-gitlab"
AWS_SSH_PRIVATE_KEY: $AWS_FBCI_SSH_KEY
AWS_ACCESS_KEY_ID: $AWS_FBCI_ACCESS_KEY_ID
AWS_SECRET_ACCESS_KEY: $AWS_FBCI_SECRET_ACCESS_KEY
ASG_NAME: "gitlab-runners"
TF_VAR_cluster_prefix: ""
tags:
- aws
- docker
- fbsmoke
before_script:
- apt-get update && apt-get install -y gnupg software-properties-common curl git
- curl -fsSL https://apt.releases.hashicorp.com/gpg | apt-key add -
- apt-add-repository "deb [arch=amd64] https://apt.releases.hashicorp.com $(lsb_release -cs) main"
- apt-get update && apt-get install terraform
- apt-get update && apt-get install python3.7
- aws configure set aws_access_key_id $AWS_FBCI_ACCESS_KEY_ID --profile $PROFILE
- aws configure set aws_secret_access_key $AWS_FBCI_SECRET_ACCESS_KEY --profile $PROFILE
- aws configure set region "us-east-2" --profile $PROFILE
- aws configure set aws_profile $PROFILE
- aws configure set aws_access_key_id $AWS_INFRA_ACCESS_KEY_ID --profile $INFRA_PROFILE
- aws configure set aws_secret_access_key $AWS_INFRA_SECRET_ACCESS_KEY --profile $INFRA_PROFILE
- aws configure set region "us-east-2" --profile $INFRA_PROFILE
- echo $AWS_FBCI_SSH_KEY > gitlab-featurebase-ci.pem
- chmod 400 gitlab-featurebase-ci.pem
- "which ssh-agent || ( apt-get update -y && apt-get install openssh-client -y )"
- eval $(ssh-agent -s)
- mkdir -p ~/.ssh
- echo $AWS_FBCI_SSH_KEY > /root/.ssh/gitlab-featurebase-ci.pem
- chmod 400 /root/.ssh/gitlab-featurebase-ci.pem
- echo "$AWS_FBCI_SSH_KEY" | ssh-add -
- chmod 700 /root/.ssh
- '[[ -f /.dockerenv ]] && echo -e "Host *\n\tStrictHostKeyChecking no\n\n" > ~/.ssh/config'
- apt update && apt -y install jq wget
- wget -q "https://go.dev/dl/go$GOVERSION.linux-amd64.tar.gz"
- tar -C /usr/local -xzf "go$GOVERSION.linux-amd64.tar.gz"
- export PATH=$PATH:/usr/local/go/bin
- TF_VAR_cluster_prefix="pipeline-$CI_PIPELINE_ID-delete-$CI_JOB_ID"
- echo "Cluster Prefix --> $TF_VAR_cluster_prefix"
- export INSTANCE_ID=$(curl --silent --fail "http://169.254.169.254/latest/meta-data/instance-id" | tee instance_id)
- aws autoscaling set-instance-protection --instance-ids "$INSTANCE_ID" --auto-scaling-group-name $ASG_NAME --protected-from-scale-in --profile $INFRA_PROFILE
- aws s3 cp s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/_latest/featurebase_linux_arm64 ./
- chmod +x ./featurebase_linux_arm64
script:
- ./qa/scripts/perf/delete/deleteSetup.sh $CI_COMMIT_BRANCH
- ./qa/scripts/perf/delete/deleteTest.sh $CI_COMMIT_BRANCH
after_script:
- ./qa/scripts/perf/delete/deleteTeardown.sh || true
- export INSTANCE_ID=$(cat instance_id)
- aws autoscaling set-instance-protection --instance-ids "$INSTANCE_ID" --auto-scaling-group-name $ASG_NAME --no-protected-from-scale-in --profile $INFRA_PROFILE
needs:
- job: s3 dump
- job: idk s3 dump
s3 dump tag:
stage: post build
variables:

View file

@ -44,7 +44,7 @@ Parameters:
Description: The number of EC2 instances for the FeatureBase cluster
FBInstanceSize:
Type: String
Default: m6i.16xlarge
Default: m5ad.16xlarge
Description: Class of EC2 instance used to host the deployment.
# note that we are using a producer, which is also the kafka host
# and the consumer uses the kafka-static binary for ingesting data
@ -52,10 +52,10 @@ Parameters:
# producer/consumer related params
ProducerInstanceSize:
Type: String
Default: m6i.16xlarge
Default: m4.16xlarge
ConsumerInstanceSize:
Type: String
Default: m6i.16xlarge
Default: m4.16xlarge
ProducerInstanceCount:
Type: Number
Default: 1

View file

@ -10,7 +10,7 @@ variable "fb_cluster_arch" {
variable "fb_ingest_type" {
type = string
default = "m6g.2xlarge"
default = "c6g.2xlarge"
}
variable "fb_ingest_node_count" {
@ -20,7 +20,7 @@ variable "fb_ingest_node_count" {
variable "fb_data_node_type" {
type = string
default = "m6g.16xlarge"
default = "c6g.16xlarge"
}
variable "fb_data_node_count" {

View file

@ -7,7 +7,6 @@ echo "Running delete_cloud_init.sh"
# create and initialize a raid0 array
# for m5ad.16xlarge instance type, there are 4 NVMe SSDs.
# Note JALLISON 20221214 - we are now using m6i instance types for cost savings
sudo mdadm --create --verbose /dev/md0 --level=0 --raid-devices=4 /dev/nvme1n1 /dev/nvme2n1 /dev/nvme3n1 /dev/nvme4n1
# format filesystem

View file

@ -4,10 +4,10 @@ module "ci-cluster" {
region = var.region
profile = var.profile
fb_cluster_arch = ["x86_64"]
fb_data_node_type = "m6i.16xlarge"
fb_data_node_type = "m5ad.16xlarge"
fb_data_node_count = 5
ebs_volumes = []
fb_ingest_type = "m6i.16xlarge"
fb_ingest_type = "m4.16xlarge"
fb_ingest_node_count = 2
fb_ingest_disk_size_gb = 1200
vpc_id = "vpc-05a26a122f961dc2b"

View file

@ -3,10 +3,10 @@ module "able-cluster" {
cluster_prefix = var.cluster_prefix
region = var.region
profile = var.profile
fb_data_node_type = "m6g.16xlarge"
fb_data_node_type = "c6g.16xlarge"
fb_data_disk_iops = 10000
fb_data_node_count = 3
fb_ingest_type = "m6g.2xlarge"
fb_ingest_type = "c6g.2xlarge"
fb_ingest_disk_iops = 10000
fb_ingest_disk_size_gb = 500
fb_ingest_node_count = 1