From 12a215a006261c4356f5e03dbbd16d0791b8fa87 Mon Sep 17 00:00:00 2001 From: Matthew Jaffee Date: Mon, 14 Feb 2022 11:58:33 -0600 Subject: [PATCH 1/3] add separate S3 dump step for tags --- .gitlab/.gitlab-ci.yml | 35 +++++++++++++++++++++++++++++++++-- 1 file changed, 33 insertions(+), 2 deletions(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 48e92cf3a..c331367ca 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -450,7 +450,7 @@ s3 dump: tags: - shell rules: - - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"' + - if: '$CI_COMMIT_TAG == "" && ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web")' script: - aws configure set aws_access_key_id $AWS_FBCI_ACCESS_KEY_ID - aws configure set aws_secret_access_key $AWS_FBCI_SECRET_ACCESS_KEY @@ -484,4 +484,35 @@ perf_able: include: .gitlab/.perf-able-gitlab-ci.yml rules: - changes: - - featurebase/* \ No newline at end of file + - featurebase/* + +s3 dump tag: + stage: post build + variables: + PROFILE: "service-fb-ci" + AWS_SSH_PRIVATE_KEY: $AWS_FBCI_SSH_KEY + AWS_ACCESS_KEY_ID: $AWS_FBCI_ACCESS_KEY_ID + AWS_SECRET_ACCESS_KEY: $AWS_FBCI_SECRET_ACCESS_KEY + LOCATION: molecula-artifact-storage/featurebase/_tags + tags: + - shell + rules: + - if: '$CI_COMMIT_TAG != "" && ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web")' + script: + - aws configure set aws_access_key_id $AWS_FBCI_ACCESS_KEY_ID + - aws configure set aws_secret_access_key $AWS_FBCI_SECRET_ACCESS_KEY + - aws configure set region "us-east-2" + - aws configure set aws_profile $PROFILE + - aws s3 cp featurebase_linux_amd64 s3://${LOCATION}/${CI_COMMIT_TAG}/featurebase_linux_amd64 + - aws s3 cp roaring-migrate_linux_amd64 s3://${LOCATION}/${CI_COMMIT_TAG}/${CI_COMMIT_SHORT_SHA}/roaring-migrate_linux_amd64 + - aws s3 cp featurebase_linux_arm64 s3://${LOCATION}/${CI_COMMIT_TAG}/${CI_COMMIT_SHORT_SHA}/featurebase_linux_arm64 + - aws s3 cp roaring-migrate_linux_arm64 s3://${LOCATION}/${CI_COMMIT_TAG}/${CI_COMMIT_SHORT_SHA}/roaring-migrate_linux_arm64 + - aws s3 cp featurebase_darwin_amd64 s3://${LOCATION}/${CI_COMMIT_TAG}/${CI_COMMIT_SHORT_SHA}/featurebase_darwin_amd64 + - aws s3 cp roaring-migrate_darwin_amd64 s3://${LOCATION}/${CI_COMMIT_TAG}/${CI_COMMIT_SHORT_SHA}/roaring-migrate_darwin_amd64 + - aws s3 cp featurebase_darwin_arm64 s3://${LOCATION}/${CI_COMMIT_TAG}/${CI_COMMIT_SHORT_SHA}/featurebase_darwin_arm64 + - aws s3 cp roaring-migrate_darwin_arm64 s3://${LOCATION}/${CI_COMMIT_TAG}/${CI_COMMIT_SHORT_SHA}/roaring-migrate_darwin_arm64 + needs: + - job: build for darwin amd64 + - job: build for darwin arm64 + - job: build for linux amd64 + - job: build for linux arm64 From 307aefc05ecd6c654ae33bb3fca88eadaf9beb78 Mon Sep 17 00:00:00 2001 From: Matthew Jaffee Date: Mon, 14 Feb 2022 15:08:58 -0600 Subject: [PATCH 2/3] fix up S3 release dump - remove commit SHA nesting - add NOTICE, .service files, and .conf --- .gitlab/.gitlab-ci.yml | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index c331367ca..09860b40a 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -504,13 +504,17 @@ s3 dump tag: - aws configure set region "us-east-2" - aws configure set aws_profile $PROFILE - aws s3 cp featurebase_linux_amd64 s3://${LOCATION}/${CI_COMMIT_TAG}/featurebase_linux_amd64 - - aws s3 cp roaring-migrate_linux_amd64 s3://${LOCATION}/${CI_COMMIT_TAG}/${CI_COMMIT_SHORT_SHA}/roaring-migrate_linux_amd64 - - aws s3 cp featurebase_linux_arm64 s3://${LOCATION}/${CI_COMMIT_TAG}/${CI_COMMIT_SHORT_SHA}/featurebase_linux_arm64 - - aws s3 cp roaring-migrate_linux_arm64 s3://${LOCATION}/${CI_COMMIT_TAG}/${CI_COMMIT_SHORT_SHA}/roaring-migrate_linux_arm64 - - aws s3 cp featurebase_darwin_amd64 s3://${LOCATION}/${CI_COMMIT_TAG}/${CI_COMMIT_SHORT_SHA}/featurebase_darwin_amd64 - - aws s3 cp roaring-migrate_darwin_amd64 s3://${LOCATION}/${CI_COMMIT_TAG}/${CI_COMMIT_SHORT_SHA}/roaring-migrate_darwin_amd64 - - aws s3 cp featurebase_darwin_arm64 s3://${LOCATION}/${CI_COMMIT_TAG}/${CI_COMMIT_SHORT_SHA}/featurebase_darwin_arm64 - - aws s3 cp roaring-migrate_darwin_arm64 s3://${LOCATION}/${CI_COMMIT_TAG}/${CI_COMMIT_SHORT_SHA}/roaring-migrate_darwin_arm64 + - aws s3 cp roaring-migrate_linux_amd64 s3://${LOCATION}/${CI_COMMIT_TAG}/roaring-migrate_linux_amd64 + - aws s3 cp featurebase_linux_arm64 s3://${LOCATION}/${CI_COMMIT_TAG}/featurebase_linux_arm64 + - aws s3 cp roaring-migrate_linux_arm64 s3://${LOCATION}/${CI_COMMIT_TAG}/roaring-migrate_linux_arm64 + - aws s3 cp featurebase_darwin_amd64 s3://${LOCATION}/${CI_COMMIT_TAG}/featurebase_darwin_amd64 + - aws s3 cp roaring-migrate_darwin_amd64 s3://${LOCATION}/${CI_COMMIT_TAG}/roaring-migrate_darwin_amd64 + - aws s3 cp featurebase_darwin_arm64 s3://${LOCATION}/${CI_COMMIT_TAG}/featurebase_darwin_arm64 + - aws s3 cp roaring-migrate_darwin_arm64 s3://${LOCATION}/${CI_COMMIT_TAG}/roaring-migrate_darwin_arm64 + - aws s3 cp NOTICE s3://${LOCATION}/${CI_COMMIT_TAG}/NOTICE + - aws s3 cp install/featurebase.debian.service s3://${LOCATION}/${CI_COMMIT_TAG}/featurebase.debian.service + - aws s3 cp install/featurebase.redhat.service s3://${LOCATION}/${CI_COMMIT_TAG}/featurebase.redhat.service + - aws s3 cp install/featurebase.conf s3://${LOCATION}/${CI_COMMIT_TAG}/featurebase.conf needs: - job: build for darwin amd64 - job: build for darwin arm64 From e7241ac024772c04aefe10f41663796cafdff7ca Mon Sep 17 00:00:00 2001 From: Matthew Jaffee Date: Tue, 15 Feb 2022 14:20:58 -0600 Subject: [PATCH 3/3] fix tag check to check against null empty string doesn't work because gitlab doesn't set the variable at all. How do I know that "null" is correct? Because Fletcher told me... apparently it's a ruby-ism --- .gitlab/.gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 09860b40a..9d8d5c420 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -450,7 +450,7 @@ s3 dump: tags: - shell rules: - - if: '$CI_COMMIT_TAG == "" && ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web")' + - if: '$CI_COMMIT_TAG == null && ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web")' script: - aws configure set aws_access_key_id $AWS_FBCI_ACCESS_KEY_ID - aws configure set aws_secret_access_key $AWS_FBCI_SECRET_ACCESS_KEY @@ -497,7 +497,7 @@ s3 dump tag: tags: - shell rules: - - if: '$CI_COMMIT_TAG != "" && ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web")' + - if: '$CI_COMMIT_TAG != null && ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web")' script: - aws configure set aws_access_key_id $AWS_FBCI_ACCESS_KEY_ID - aws configure set aws_secret_access_key $AWS_FBCI_SECRET_ACCESS_KEY