From 6db05d615035febd5555e80099a2a37a48271071 Mon Sep 17 00:00:00 2001 From: pokeeffe-molecula Date: Tue, 1 Feb 2022 14:22:24 -0600 Subject: [PATCH 1/3] don't use gitlab api to get binaries --- qa/scripts/utilCluster.sh | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/qa/scripts/utilCluster.sh b/qa/scripts/utilCluster.sh index e460f1f48..8d1717b92 100644 --- a/qa/scripts/utilCluster.sh +++ b/qa/scripts/utilCluster.sh @@ -125,15 +125,22 @@ executeGeneralNodeConfigCommands() { ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "sudo mkdir -p /data/featurebase" ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "sudo chown molecula /data/featurebase" - # TODO handle different archs - echo "Getting featurebase binary (https://gitlab.com/api/v4/projects/molecula%2Ffeaturebase/jobs/artifacts/${TF_VAR_branch}/raw/featurebase_linux_arm64?job=build%20for%20linux%20arm64)..." - ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "curl --fail --header 'PRIVATE-TOKEN: ${TF_VAR_gitlab_token}' -o /home/ec2-user/featurebase_linux_arm64 https://gitlab.com/api/v4/projects/molecula%2Ffeaturebase/jobs/artifacts/${TF_VAR_branch}/raw/featurebase_linux_arm64?job=build%20for%20linux%20arm64" + scp -i ~/.ssh/gitlab-featurebase-ci.pem -o "StrictHostKeyChecking no" featurebase_linux_arm64 ec2-user@${NODEIP}: if (( $? != 0 )) then - echo "Unable to get featurebase binary" + echo "featurebase binary copy failed" exit 1 fi + + # TODO handle different archs +# echo "Getting featurebase binary (https://gitlab.com/api/v4/projects/molecula%2Ffeaturebase/jobs/artifacts/${TF_VAR_branch}/raw/featurebase_linux_arm64?job=build%20for%20linux%20arm64)..." +# ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "curl --fail --header 'PRIVATE-TOKEN: ${TF_VAR_gitlab_token}' -o /home/ec2-user/featurebase_linux_arm64 https://gitlab.com/api/v4/projects/molecula%2Ffeaturebase/jobs/artifacts/${TF_VAR_branch}/raw/featurebase_linux_arm64?job=build%20for%20linux%20arm64" +# if (( $? != 0 )) +# then +# echo "Unable to get featurebase binary" +# exit 1 +# fi ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "chown ec2-user:ec2-user /home/ec2-user/featurebase_linux_arm64" ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "chmod ugo+x /home/ec2-user/featurebase_linux_arm64" ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "sudo mv /home/ec2-user/featurebase_linux_arm64 /usr/local/bin/featurebase" From 1e4b3321ff33f5408dd196821b488ddea528ca41 Mon Sep 17 00:00:00 2001 From: pokeeffe-molecula Date: Tue, 1 Feb 2022 14:40:38 -0600 Subject: [PATCH 2/3] remove reliance on gitlab token --- qa/scripts/runSamsungGauntlet.sh | 3 --- qa/scripts/runSmokeTest.sh | 2 -- qa/scripts/setupSamsungGauntlet.sh | 3 --- qa/scripts/setupSmokeTest.sh | 3 --- qa/scripts/teardownSamsungGauntlet.sh | 1 - qa/scripts/teardownSmokeTest.sh | 2 -- qa/scripts/testSmokeTest.sh | 3 --- qa/scripts/utilCluster.sh | 8 -------- qa/tf/ci/smoketest/variables.tf | 5 ----- qa/tf/gauntlet/samsung/variables.tf | 5 ----- 10 files changed, 35 deletions(-) diff --git a/qa/scripts/runSamsungGauntlet.sh b/qa/scripts/runSamsungGauntlet.sh index 086bf0874..37cc0a930 100644 --- a/qa/scripts/runSamsungGauntlet.sh +++ b/qa/scripts/runSamsungGauntlet.sh @@ -2,9 +2,6 @@ SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) -# requires TF_VAR_gitlab_token env var to be set -if [ -z ${TF_VAR_gitlab_token+x} ]; then echo "TF_VAR_gitlab_token is unset"; else echo "TF_VAR_gitlab_token is set to '$TF_VAR_gitlab_token'"; fi - # requires TF_VAR_branch env var to be set if [ -z ${TF_VAR_branch+x} ]; then echo "TF_VAR_branch is unset"; else echo "TF_VAR_branch is set to '$TF_VAR_branch'"; fi diff --git a/qa/scripts/runSmokeTest.sh b/qa/scripts/runSmokeTest.sh index 96866fd6d..6b598995e 100755 --- a/qa/scripts/runSmokeTest.sh +++ b/qa/scripts/runSmokeTest.sh @@ -2,8 +2,6 @@ SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) -# requires TF_VAR_gitlab_token env var to be set -if [ -z ${TF_VAR_gitlab_token+x} ]; then echo "TF_VAR_gitlab_token is unset"; else echo "TF_VAR_gitlab_token is set to '$TF_VAR_gitlab_token'"; fi # requires TF_VAR_branch env var to be set if [ -z ${TF_VAR_branch+x} ]; then echo "TF_VAR_branch is unset"; else echo "TF_VAR_branch is set to '$TF_VAR_branch'"; fi diff --git a/qa/scripts/setupSamsungGauntlet.sh b/qa/scripts/setupSamsungGauntlet.sh index 1229c0044..e7929ea9f 100755 --- a/qa/scripts/setupSamsungGauntlet.sh +++ b/qa/scripts/setupSamsungGauntlet.sh @@ -3,9 +3,6 @@ # To run script: ./setupSamsungGauntlet.sh export TF_IN_AUTOMATION=1 -# requires TF_VAR_gitlab_token env var to be set -if [ -z ${TF_VAR_gitlab_token+x} ]; then echo "TF_VAR_gitlab_token is unset"; else echo "TF_VAR_gitlab_token is set to '$TF_VAR_gitlab_token'"; fi - # requires TF_VAR_branch env var to be set if [ -z ${TF_VAR_branch+x} ]; then echo "TF_VAR_branch is unset"; else echo "TF_VAR_branch is set to '$TF_VAR_branch'"; fi diff --git a/qa/scripts/setupSmokeTest.sh b/qa/scripts/setupSmokeTest.sh index fac0d7111..1059e5f29 100755 --- a/qa/scripts/setupSmokeTest.sh +++ b/qa/scripts/setupSmokeTest.sh @@ -3,9 +3,6 @@ # To run script: ./setupSmokeTest.sh export TF_IN_AUTOMATION=1 -# requires TF_VAR_gitlab_token env var to be set -if [ -z ${TF_VAR_gitlab_token+x} ]; then echo "TF_VAR_gitlab_token is unset"; else echo "TF_VAR_gitlab_token is set to '$TF_VAR_gitlab_token'"; fi - # requires TF_VAR_branch env var to be set if [ -z ${TF_VAR_branch+x} ]; then echo "TF_VAR_branch is unset"; else echo "TF_VAR_branch is set to '$TF_VAR_branch'"; fi diff --git a/qa/scripts/teardownSamsungGauntlet.sh b/qa/scripts/teardownSamsungGauntlet.sh index ae614d99b..0e5598569 100755 --- a/qa/scripts/teardownSamsungGauntlet.sh +++ b/qa/scripts/teardownSamsungGauntlet.sh @@ -1,7 +1,6 @@ #!/bin/bash # To run script: ./teardownSamsungGauntlet.sh -# requires TF_VAR_gitlab_token env var to be set cd qa/tf/gauntlet/samsung export TF_IN_AUTOMATION=1 diff --git a/qa/scripts/teardownSmokeTest.sh b/qa/scripts/teardownSmokeTest.sh index 6215d419e..76eeb564b 100755 --- a/qa/scripts/teardownSmokeTest.sh +++ b/qa/scripts/teardownSmokeTest.sh @@ -1,8 +1,6 @@ #!/bin/bash # To run script: ./teardownSmokeTest.sh -# requires TF_VAR_gitlab_token env var to be set -if [ -z ${TF_VAR_gitlab_token+x} ]; then echo "TF_VAR_gitlab_token is unset"; else echo "TF_VAR_gitlab_token is set to '$TF_VAR_gitlab_token'"; fi # requires TF_VAR_branch env var to be set if [ -z ${TF_VAR_branch+x} ]; then echo "TF_VAR_branch is unset"; else echo "TF_VAR_branch is set to '$TF_VAR_branch'"; fi diff --git a/qa/scripts/testSmokeTest.sh b/qa/scripts/testSmokeTest.sh index 31ef6f49d..b6ba34e7f 100755 --- a/qa/scripts/testSmokeTest.sh +++ b/qa/scripts/testSmokeTest.sh @@ -1,8 +1,5 @@ #!/bin/bash -# requires TF_VAR_gitlab_token env var to be set -if [ -z ${TF_VAR_gitlab_token+x} ]; then echo "TF_VAR_gitlab_token is unset"; else echo "TF_VAR_gitlab_token is set to '$TF_VAR_gitlab_token'"; fi - # requires TF_VAR_branch env var to be set if [ -z ${TF_VAR_branch+x} ]; then echo "TF_VAR_branch is unset"; else echo "TF_VAR_branch is set to '$TF_VAR_branch'"; fi diff --git a/qa/scripts/utilCluster.sh b/qa/scripts/utilCluster.sh index 8d1717b92..629a63aeb 100644 --- a/qa/scripts/utilCluster.sh +++ b/qa/scripts/utilCluster.sh @@ -133,14 +133,6 @@ executeGeneralNodeConfigCommands() { exit 1 fi - # TODO handle different archs -# echo "Getting featurebase binary (https://gitlab.com/api/v4/projects/molecula%2Ffeaturebase/jobs/artifacts/${TF_VAR_branch}/raw/featurebase_linux_arm64?job=build%20for%20linux%20arm64)..." -# ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "curl --fail --header 'PRIVATE-TOKEN: ${TF_VAR_gitlab_token}' -o /home/ec2-user/featurebase_linux_arm64 https://gitlab.com/api/v4/projects/molecula%2Ffeaturebase/jobs/artifacts/${TF_VAR_branch}/raw/featurebase_linux_arm64?job=build%20for%20linux%20arm64" -# if (( $? != 0 )) -# then -# echo "Unable to get featurebase binary" -# exit 1 -# fi ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "chown ec2-user:ec2-user /home/ec2-user/featurebase_linux_arm64" ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "chmod ugo+x /home/ec2-user/featurebase_linux_arm64" ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "sudo mv /home/ec2-user/featurebase_linux_arm64 /usr/local/bin/featurebase" diff --git a/qa/tf/ci/smoketest/variables.tf b/qa/tf/ci/smoketest/variables.tf index bab877497..578305341 100644 --- a/qa/tf/ci/smoketest/variables.tf +++ b/qa/tf/ci/smoketest/variables.tf @@ -8,11 +8,6 @@ variable "profile" { type = string } -variable "gitlab_token" { - description = "The API token for taking to Gitlab API - expected to come from an env variable." - type = string -} - variable "cluster_prefix" { type = string description = "This is a identifier that will be prefixed to created resources" diff --git a/qa/tf/gauntlet/samsung/variables.tf b/qa/tf/gauntlet/samsung/variables.tf index bab877497..578305341 100644 --- a/qa/tf/gauntlet/samsung/variables.tf +++ b/qa/tf/gauntlet/samsung/variables.tf @@ -8,11 +8,6 @@ variable "profile" { type = string } -variable "gitlab_token" { - description = "The API token for taking to Gitlab API - expected to come from an env variable." - type = string -} - variable "cluster_prefix" { type = string description = "This is a identifier that will be prefixed to created resources" From ca2def7389ab53399f27f430b20b517c5770651d Mon Sep 17 00:00:00 2001 From: pokeeffe-molecula Date: Tue, 1 Feb 2022 14:51:06 -0600 Subject: [PATCH 3/3] dump to s3 --- .gitlab/.gitlab-ci.yml | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index da44fa181..a44f9e30c 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -12,6 +12,7 @@ stages: - build - integration - gauntlet + - post build golangci-lint: image: golangci/golangci-lint:v1.39.0 @@ -407,3 +408,40 @@ gauntlet: after_script: - ./qa/scripts/teardownSamsungGauntlet.sh +s3 dump: + stage: post build + variables: + PROFILE: "service-fb-ci" + AWS_SSH_PRIVATE_KEY: $AWS_FBCI_SSH_KEY + AWS_ACCESS_KEY_ID: $AWS_FBCI_ACCESS_KEY_ID + AWS_SECRET_ACCESS_KEY: $AWS_FBCI_SECRET_ACCESS_KEY + tags: + - shell + rules: + - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"' + script: + - aws configure set aws_access_key_id $AWS_FBCI_ACCESS_KEY_ID + - aws configure set aws_secret_access_key $AWS_FBCI_SECRET_ACCESS_KEY + - aws configure set region "us-east-2" + - aws configure set aws_profile $PROFILE + - aws s3 cp featurebase_linux_amd64 s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/${CI_COMMIT_SHORT_SHA}/featurebase_linux_amd64 + - aws s3 cp featurebase_linux_amd64 s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/_latest/featurebase_linux_amd64 + - aws s3 cp roaring-migrate_linux_amd64 s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/${CI_COMMIT_SHORT_SHA}/roaring-migrate_linux_amd64 + - aws s3 cp roaring-migrate_linux_amd64 s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/_latest/roaring-migrate_linux_amd64 + - aws s3 cp featurebase_linux_arm64 s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/${CI_COMMIT_SHORT_SHA}/featurebase_linux_arm64 + - aws s3 cp featurebase_linux_arm64 s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/_latest/featurebase_linux_arm64 + - aws s3 cp roaring-migrate_linux_arm64 s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/${CI_COMMIT_SHORT_SHA}/roaring-migrate_linux_arm64 + - aws s3 cp roaring-migrate_linux_arm64 s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/_latest/roaring-migrate_linux_arm64 + - aws s3 cp featurebase_darwin_amd64 s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/${CI_COMMIT_SHORT_SHA}/featurebase_darwin_amd64 + - aws s3 cp featurebase_darwin_amd64 s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/_latest/featurebase_darwin_amd64 + - aws s3 cp roaring-migrate_darwin_amd64 s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/${CI_COMMIT_SHORT_SHA}/roaring-migrate_darwin_amd64 + - aws s3 cp roaring-migrate_darwin_amd64 s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/_latest/roaring-migrate_darwin_amd64 + - aws s3 cp featurebase_darwin_arm64 s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/${CI_COMMIT_SHORT_SHA}/featurebase_darwin_arm64 + - aws s3 cp featurebase_darwin_arm64 s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/_latest/featurebase_darwin_arm64 + - aws s3 cp roaring-migrate_darwin_arm64 s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/${CI_COMMIT_SHORT_SHA}/roaring-migrate_darwin_arm64 + - aws s3 cp roaring-migrate_darwin_arm64 s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/_latest/roaring-migrate_darwin_arm64 + needs: + - job: build for darwin amd64 + - job: build for darwin arm64 + - job: build for linux amd64 + - job: build for linux arm64 \ No newline at end of file