From 39f85fe2a4fd2439040f6ac7af07642d1daf966f Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Tue, 2 Nov 2021 12:25:01 -0500 Subject: [PATCH 01/98] added terraform config for aws single node deployment --- .gitlab/.gitlab-ci.yml | 34 ++++++++++++++++++++++++ .terraform/main.tf | 58 +++++++++++++++++++++++++++++++++++++++++ .terraform/server.yml | 5 ++++ .terraform/variables.tf | 10 +++++++ 4 files changed, 107 insertions(+) create mode 100644 .terraform/main.tf create mode 100644 .terraform/server.yml create mode 100644 .terraform/variables.tf diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index bb734ca27..599a8d65a 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -2,10 +2,12 @@ include: - template: Security/SAST.gitlab-ci.yml - template: Security/License-Scanning.gitlab-ci.yml - template: Security/Dependency-Scanning.gitlab-ci.yml + stages: - test - build + - deploy install lattice: stage: test @@ -137,3 +139,35 @@ build for darwin arm64: artifacts: paths: - featurebase_darwin_arm64 + +init terraform: + stage: deploy + image: registry.gitlab.com/gitlab-org/terraform-images/stable:latest + before_script: + - cd .terraform + script: + - gitlab-terraform init + +validate terraform: + stage: deploy + image: registry.gitlab.com/gitlab-org/terraform-images/stable:latest + script: + - gitlab-terraform validate + needs: + - job: init terraform + +plan terraform: + stage: deploy + image: registry.gitlab.com/gitlab-org/terraform-images/stable:latest + script: + - gitlab-terraform plan + needs: + - job: validate terraform + +apply terraform: + stage: deploy + image: registry.gitlab.com/gitlab-org/terraform-images/stable:latest + script: + - gitlab-terraform apply + needs: + - job: plan terraform \ No newline at end of file diff --git a/.terraform/main.tf b/.terraform/main.tf new file mode 100644 index 000000000..377853714 --- /dev/null +++ b/.terraform/main.tf @@ -0,0 +1,58 @@ +terraform { + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 3.27" + } + } + + required_version = ">= 1.0.0" +} + +provider "aws" { + region = "us-east-2" +} + +resource "aws_security_group" "server_sg" { + name = "Load Balancer Security Group" + + ingress { + from_port = 80 + to_port = 80 + protocol = "tcp" + cidr_blocks = ["0.0.0.0/0"] + } + + egress { + from_port = 0 + to_port = 0 + protocol = -1 + cidr_blocks = ["0.0.0.0/0"] + } +} + +data "cloudinit_config" "server_config" { + gzip = true + base64_encode = true + part { + content_type = "text/cloud-config" + content = templatefile("${path.module}/server.yml", { + header : aws_security_group.server_sg.id + }) + } +} + +resource "aws_instance" "server_instance1" { + ami = var.ami_linux_amd + instance_type = "t2.micro" + + key_name = "souhaila-rsa" + + vpc_security_group_ids = [aws_security_group.server_sg.id] + user_data = data.cloudinit_config.server_config.rendered + associate_public_ip_address = true + + tags = { + Name = var.instance_name + } +} \ No newline at end of file diff --git a/.terraform/server.yml b/.terraform/server.yml new file mode 100644 index 000000000..5c7571b83 --- /dev/null +++ b/.terraform/server.yml @@ -0,0 +1,5 @@ +#cloud-config +runcmd: + - sudo yum update + - sudo yum install postgresql14 + - sudo yum install golang1.16 \ No newline at end of file diff --git a/.terraform/variables.tf b/.terraform/variables.tf new file mode 100644 index 000000000..2d1859c98 --- /dev/null +++ b/.terraform/variables.tf @@ -0,0 +1,10 @@ +variable "instance_name" { + description = "Value of the Name tag for the EC2 instance" + type = string + default = "single-node-deployment-1" +} + +variable "ami_linux_amd" { + type = string + default = "ami-0f19d220602031aed" +} From d565303160f28483d4b53f000517ace6a7c3865c Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Tue, 2 Nov 2021 13:28:33 -0500 Subject: [PATCH 02/98] fix script not running in correct directory --- .gitlab/.gitlab-ci.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 599a8d65a..67f2250a8 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -151,6 +151,8 @@ init terraform: validate terraform: stage: deploy image: registry.gitlab.com/gitlab-org/terraform-images/stable:latest + before_script: + - cd .terraform script: - gitlab-terraform validate needs: @@ -159,6 +161,8 @@ validate terraform: plan terraform: stage: deploy image: registry.gitlab.com/gitlab-org/terraform-images/stable:latest + before_script: + - cd .terraform script: - gitlab-terraform plan needs: @@ -167,6 +171,8 @@ plan terraform: apply terraform: stage: deploy image: registry.gitlab.com/gitlab-org/terraform-images/stable:latest + before_script: + - cd .terraform script: - gitlab-terraform apply needs: From 01e4ae40700a3f583c41e7d3d8f422b6aa033b0f Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Tue, 2 Nov 2021 14:13:53 -0500 Subject: [PATCH 03/98] fix missing plan file for terraform apply --- .gitlab/.gitlab-ci.yml | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 67f2250a8..fc415a28b 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -3,7 +3,6 @@ include: - template: Security/License-Scanning.gitlab-ci.yml - template: Security/Dependency-Scanning.gitlab-ci.yml - stages: - test - build @@ -147,6 +146,9 @@ init terraform: - cd .terraform script: - gitlab-terraform init + cache: + paths: + - .terraform validate terraform: stage: deploy @@ -157,6 +159,9 @@ validate terraform: - gitlab-terraform validate needs: - job: init terraform + cache: + paths: + - .terraform plan terraform: stage: deploy @@ -164,9 +169,15 @@ plan terraform: before_script: - cd .terraform script: - - gitlab-terraform plan + - gitlab-terraform plan -out=plan.tfplan needs: - job: validate terraform + cache: + paths: + - .terraform + artifacts: + paths: + - plan.tfplan apply terraform: stage: deploy @@ -174,6 +185,9 @@ apply terraform: before_script: - cd .terraform script: - - gitlab-terraform apply + - gitlab-terraform apply -input=false plan.tfplan needs: - - job: plan terraform \ No newline at end of file + - job: plan terraform + cache: + paths: + - .terraform \ No newline at end of file From f3884e21bafdd094daa81572d3bdb78f79d65e1b Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Tue, 2 Nov 2021 14:40:06 -0500 Subject: [PATCH 04/98] changed directory name for terraform files --- .gitlab/.gitlab-ci.yml | 8 ++++---- {.terraform => molecular-terraform}/main.tf | 0 {.terraform => molecular-terraform}/server.yml | 0 {.terraform => molecular-terraform}/variables.tf | 0 4 files changed, 4 insertions(+), 4 deletions(-) rename {.terraform => molecular-terraform}/main.tf (100%) rename {.terraform => molecular-terraform}/server.yml (100%) rename {.terraform => molecular-terraform}/variables.tf (100%) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index fc415a28b..d109853f0 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -143,7 +143,7 @@ init terraform: stage: deploy image: registry.gitlab.com/gitlab-org/terraform-images/stable:latest before_script: - - cd .terraform + - cd molecular-terraform script: - gitlab-terraform init cache: @@ -154,7 +154,7 @@ validate terraform: stage: deploy image: registry.gitlab.com/gitlab-org/terraform-images/stable:latest before_script: - - cd .terraform + - cd molecular-terraform script: - gitlab-terraform validate needs: @@ -167,7 +167,7 @@ plan terraform: stage: deploy image: registry.gitlab.com/gitlab-org/terraform-images/stable:latest before_script: - - cd .terraform + - cd molecular-terraform script: - gitlab-terraform plan -out=plan.tfplan needs: @@ -183,7 +183,7 @@ apply terraform: stage: deploy image: registry.gitlab.com/gitlab-org/terraform-images/stable:latest before_script: - - cd .terraform + - cd molecular-terraform script: - gitlab-terraform apply -input=false plan.tfplan needs: diff --git a/.terraform/main.tf b/molecular-terraform/main.tf similarity index 100% rename from .terraform/main.tf rename to molecular-terraform/main.tf diff --git a/.terraform/server.yml b/molecular-terraform/server.yml similarity index 100% rename from .terraform/server.yml rename to molecular-terraform/server.yml diff --git a/.terraform/variables.tf b/molecular-terraform/variables.tf similarity index 100% rename from .terraform/variables.tf rename to molecular-terraform/variables.tf From a8d3d1c4d4a0790bf7471ae7b6f235de53180e95 Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Tue, 2 Nov 2021 15:00:58 -0500 Subject: [PATCH 05/98] fix terraform apply inputs --- .gitlab/.gitlab-ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index d109853f0..68be7fb55 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -143,6 +143,7 @@ init terraform: stage: deploy image: registry.gitlab.com/gitlab-org/terraform-images/stable:latest before_script: + - rm -rf .terraform - cd molecular-terraform script: - gitlab-terraform init @@ -185,7 +186,7 @@ apply terraform: before_script: - cd molecular-terraform script: - - gitlab-terraform apply -input=false plan.tfplan + - gitlab-terraform apply needs: - job: plan terraform cache: From c29dcddca6846e205b15d59c78afe8bdf5f7c0f5 Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Thu, 4 Nov 2021 08:57:10 -0500 Subject: [PATCH 06/98] replace terraform with awscli for integration test --- .gitlab/.gitlab-ci.yml | 70 +++++++-------------- {molecular-terraform => .gitlab}/server.yml | 0 molecular-terraform/main.tf | 58 ----------------- molecular-terraform/variables.tf | 10 --- 4 files changed, 21 insertions(+), 117 deletions(-) rename {molecular-terraform => .gitlab}/server.yml (100%) mode change 100644 => 100755 delete mode 100644 molecular-terraform/main.tf delete mode 100644 molecular-terraform/variables.tf diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 68be7fb55..f12c60249 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -6,7 +6,7 @@ include: stages: - test - build - - deploy + - integration install lattice: stage: test @@ -139,56 +139,28 @@ build for darwin arm64: paths: - featurebase_darwin_arm64 -init terraform: - stage: deploy - image: registry.gitlab.com/gitlab-org/terraform-images/stable:latest +deploy ec2 instance: + stage: integration + image: registry.gitlab.com/gitlab-org/cloud-deploy/aws-base:latest + variables: + AMI: "" + INSTANCE: "t2_micro" + SECURITY_GROUP: "" + SUBNET_ID: "" + SSH_KEY: "" + PUBLIC_IP: "" before_script: - - rm -rf .terraform - - cd molecular-terraform + - aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID + - aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY + - aws configure set region "us-east-2" + - export AWS_PROFILE=gitlab-cicd script: - - gitlab-terraform init - cache: - paths: - - .terraform + - AMI=$(aws ssm get-parameters --names /aws/service/ami-amazon-linux-latest/amzn2-ami-hvm-x86_64-ebs --query 'Parameters[0].[Value]' --output text --profile gitlab-cicd) -validate terraform: - stage: deploy - image: registry.gitlab.com/gitlab-org/terraform-images/stable:latest - before_script: - - cd molecular-terraform - script: - - gitlab-terraform validate - needs: - - job: init terraform - cache: - paths: - - .terraform + - SECURITY_GROUP=$(aws ec2 describe-security-groups --filter Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978 --query 'SecurityGroups[*].[GroupId]' --output text --profile gitlab-cicd) -plan terraform: - stage: deploy - image: registry.gitlab.com/gitlab-org/terraform-images/stable:latest - before_script: - - cd molecular-terraform - script: - - gitlab-terraform plan -out=plan.tfplan - needs: - - job: validate terraform - cache: - paths: - - .terraform - artifacts: - paths: - - plan.tfplan + - SUBNET_ID=$(aws ec2 describe-subnets --filters 'Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978' 'Name=availability-zone,Values=us-east-2a' --query 'Subnets[0].SubnetId' --output text --profile gitlab-cicd) -apply terraform: - stage: deploy - image: registry.gitlab.com/gitlab-org/terraform-images/stable:latest - before_script: - - cd molecular-terraform - script: - - gitlab-terraform apply - needs: - - job: plan terraform - cache: - paths: - - .terraform \ No newline at end of file + - aws ec2 run-instances --image-id $AMI --count 1 --instance-type $INSTANCE --security-group-ids $SECURITY_GROUP --subnet-id $SUBNET_ID —user-data file://.gitlab/server.yml + + - PUBLIC_IP=$(aws ec2 describe-instances --filters 'Name=tag:Name, Values=Single_Node_Deployment' 'Name=instance-state-name, Values=running' --query 'Reservations[*].Instances[*].PublicIpAddress' --output text --profile gitlab-cicd) \ No newline at end of file diff --git a/molecular-terraform/server.yml b/.gitlab/server.yml old mode 100644 new mode 100755 similarity index 100% rename from molecular-terraform/server.yml rename to .gitlab/server.yml diff --git a/molecular-terraform/main.tf b/molecular-terraform/main.tf deleted file mode 100644 index 377853714..000000000 --- a/molecular-terraform/main.tf +++ /dev/null @@ -1,58 +0,0 @@ -terraform { - required_providers { - aws = { - source = "hashicorp/aws" - version = "~> 3.27" - } - } - - required_version = ">= 1.0.0" -} - -provider "aws" { - region = "us-east-2" -} - -resource "aws_security_group" "server_sg" { - name = "Load Balancer Security Group" - - ingress { - from_port = 80 - to_port = 80 - protocol = "tcp" - cidr_blocks = ["0.0.0.0/0"] - } - - egress { - from_port = 0 - to_port = 0 - protocol = -1 - cidr_blocks = ["0.0.0.0/0"] - } -} - -data "cloudinit_config" "server_config" { - gzip = true - base64_encode = true - part { - content_type = "text/cloud-config" - content = templatefile("${path.module}/server.yml", { - header : aws_security_group.server_sg.id - }) - } -} - -resource "aws_instance" "server_instance1" { - ami = var.ami_linux_amd - instance_type = "t2.micro" - - key_name = "souhaila-rsa" - - vpc_security_group_ids = [aws_security_group.server_sg.id] - user_data = data.cloudinit_config.server_config.rendered - associate_public_ip_address = true - - tags = { - Name = var.instance_name - } -} \ No newline at end of file diff --git a/molecular-terraform/variables.tf b/molecular-terraform/variables.tf deleted file mode 100644 index 2d1859c98..000000000 --- a/molecular-terraform/variables.tf +++ /dev/null @@ -1,10 +0,0 @@ -variable "instance_name" { - description = "Value of the Name tag for the EC2 instance" - type = string - default = "single-node-deployment-1" -} - -variable "ami_linux_amd" { - type = string - default = "ami-0f19d220602031aed" -} From e1ca09a7b5dd995910e59d09d4c16e07dab2a991 Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Thu, 4 Nov 2021 09:02:17 -0500 Subject: [PATCH 07/98] comment other parts of script to speed up testing --- .gitlab/.gitlab-ci.yml | 244 ++++++++++++++++++++--------------------- 1 file changed, 122 insertions(+), 122 deletions(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index f12c60249..39f12afcb 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -4,140 +4,140 @@ include: - template: Security/Dependency-Scanning.gitlab-ci.yml stages: - - test - - build + # - test + # - build - integration -install lattice: - stage: test - image: node:14 - script: - - echo "Installing lattice dependencies..." - - cd lattice - - yarn install +# install lattice: +# stage: test +# image: node:14 +# script: +# - echo "Installing lattice dependencies..." +# - cd lattice +# - yarn install -build lattice: - stage: test - image: node:14 - variables: - CI: "true" - script: - - echo "Building lattice..." - - cd lattice - - yarn build - - tar -czvf lattice.tar.gz lattice/build - artifacts: - paths: - - lattice/lattice.tar.gz - needs: - - job: install lattice - allow_failure: true +# build lattice: +# stage: test +# image: node:14 +# variables: +# CI: "true" +# script: +# - echo "Building lattice..." +# - cd lattice +# - yarn build +# - tar -czvf lattice.tar.gz lattice/build +# artifacts: +# paths: +# - lattice/lattice.tar.gz +# needs: +# - job: install lattice +# allow_failure: true -run jest tests: - stage: test - image: node:14 - variables: - CI: "true" - script: - - echo "Testing lattice..." - - cd lattice - - npm install --force - - npm test -- --coverage --testResultsProcessor=jest-sonar-reporter - artifacts: - paths: - - lattice/coverage/lcov.info +# run jest tests: +# stage: test +# image: node:14 +# variables: +# CI: "true" +# script: +# - echo "Testing lattice..." +# - cd lattice +# - npm install --force +# - npm test -- --coverage --testResultsProcessor=jest-sonar-reporter +# artifacts: +# paths: +# - lattice/coverage/lcov.info -run go tests: - stage: test - image: golang:1.16.9 - script: - - echo "Running featurebase unit tests..." - - go test -covermode=atomic -coverprofile=coverage.out - artifacts: - paths: - - coverage.out +# run go tests: +# stage: test +# image: golang:1.16.9 +# script: +# - echo "Running featurebase unit tests..." +# - go test -covermode=atomic -coverprofile=coverage.out +# artifacts: +# paths: +# - coverage.out -run go tests with output: - stage: test - image: golang:1.16.9 - script: - - echo "Running featurebase unit tests to capture JSON output..." - - go test -json > test-report.out - artifacts: - paths: - - test-report.out +# run go tests with output: +# stage: test +# image: golang:1.16.9 +# script: +# - echo "Running featurebase unit tests to capture JSON output..." +# - go test -json > test-report.out +# artifacts: +# paths: +# - test-report.out -generate coverage html: - stage: test - image: golang:1.16.9 - script: - - echo "Generating coverage HTML.." - - go tool cover -html coverage.out -o coverage.html - artifacts: - paths: - - coverage.html - needs: - - job: run go tests +# generate coverage html: +# stage: test +# image: golang:1.16.9 +# script: +# - echo "Generating coverage HTML.." +# - go tool cover -html coverage.out -o coverage.html +# artifacts: +# paths: +# - coverage.html +# needs: +# - job: run go tests -upload to sonarcloud: - stage: test - image: sonarsource/sonar-scanner-cli:4.6 - variables: - SONAR_TOKEN: $SONAR_TOKEN - script: - - sonar-scanner -Dsonar.projectKey=molecula_featurebase -Dsonar.organization=molecula -Dsonar.sources=. -Dsonar.host.url=https://sonarcloud.io -Dsonar.go.coverage.reportPaths=coverage.out -Dsonar.go.tests.reportPaths=test-report.out -Dsonar.javascript.lcov.reportPaths=lattice/coverage/lcov.info - needs: - - job: run go tests - - job: run go tests with output - - job: run jest tests +# upload to sonarcloud: +# stage: test +# image: sonarsource/sonar-scanner-cli:4.6 +# variables: +# SONAR_TOKEN: $SONAR_TOKEN +# script: +# - sonar-scanner -Dsonar.projectKey=molecula_featurebase -Dsonar.organization=molecula -Dsonar.sources=. -Dsonar.host.url=https://sonarcloud.io -Dsonar.go.coverage.reportPaths=coverage.out -Dsonar.go.tests.reportPaths=test-report.out -Dsonar.javascript.lcov.reportPaths=lattice/coverage/lcov.info +# needs: +# - job: run go tests +# - job: run go tests with output +# - job: run jest tests -build for linux amd64: - stage: build - image: registry.gitlab.com/molecula/featurebase/builder:0.0.3 - variables: - GOOS: "linux" - GOARCH: "amd64" - script: - - go build -o featurebase_linux_amd64 ./cmd/featurebase - artifacts: - paths: - - featurebase_linux_amd64 +# build for linux amd64: +# stage: build +# image: registry.gitlab.com/molecula/featurebase/builder:0.0.3 +# variables: +# GOOS: "linux" +# GOARCH: "amd64" +# script: +# - go build -o featurebase_linux_amd64 ./cmd/featurebase +# artifacts: +# paths: +# - featurebase_linux_amd64 -build for linux arm64: - stage: build - image: registry.gitlab.com/molecula/featurebase/builder:0.0.3 - variables: - GOOS: "linux" - GOARCH: "arm64" - script: - - go build -o featurebase_linux_arm64 ./cmd/featurebase - artifacts: - paths: - - featurebase_linux_arm64 +# build for linux arm64: +# stage: build +# image: registry.gitlab.com/molecula/featurebase/builder:0.0.3 +# variables: +# GOOS: "linux" +# GOARCH: "arm64" +# script: +# - go build -o featurebase_linux_arm64 ./cmd/featurebase +# artifacts: +# paths: +# - featurebase_linux_arm64 -build for darwin amd64: - stage: build - image: registry.gitlab.com/molecula/featurebase/builder:0.0.3 - variables: - GOOS: "darwin" - GOARCH: "amd64" - script: - - go build -o featurebase_darwin_amd64 ./cmd/featurebase - artifacts: - paths: - - featurebase_darwin_amd64 +# build for darwin amd64: +# stage: build +# image: registry.gitlab.com/molecula/featurebase/builder:0.0.3 +# variables: +# GOOS: "darwin" +# GOARCH: "amd64" +# script: +# - go build -o featurebase_darwin_amd64 ./cmd/featurebase +# artifacts: +# paths: +# - featurebase_darwin_amd64 -build for darwin arm64: - stage: build - image: registry.gitlab.com/molecula/featurebase/builder:0.0.3 - variables: - GOOS: "darwin" - GOARCH: "arm64" - script: - - go build -o featurebase_darwin_arm64 ./cmd/featurebase - artifacts: - paths: - - featurebase_darwin_arm64 +# build for darwin arm64: +# stage: build +# image: registry.gitlab.com/molecula/featurebase/builder:0.0.3 +# variables: +# GOOS: "darwin" +# GOARCH: "arm64" +# script: +# - go build -o featurebase_darwin_arm64 ./cmd/featurebase +# artifacts: +# paths: +# - featurebase_darwin_arm64 deploy ec2 instance: stage: integration From 6ee2341c8810f15b17339e56edb39cd1907d31c6 Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Thu, 4 Nov 2021 09:04:43 -0500 Subject: [PATCH 08/98] undo last commit changes --- .gitlab/.gitlab-ci.yml | 244 ++++++++++++++++++++--------------------- 1 file changed, 122 insertions(+), 122 deletions(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 39f12afcb..f12c60249 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -4,140 +4,140 @@ include: - template: Security/Dependency-Scanning.gitlab-ci.yml stages: - # - test - # - build + - test + - build - integration -# install lattice: -# stage: test -# image: node:14 -# script: -# - echo "Installing lattice dependencies..." -# - cd lattice -# - yarn install +install lattice: + stage: test + image: node:14 + script: + - echo "Installing lattice dependencies..." + - cd lattice + - yarn install -# build lattice: -# stage: test -# image: node:14 -# variables: -# CI: "true" -# script: -# - echo "Building lattice..." -# - cd lattice -# - yarn build -# - tar -czvf lattice.tar.gz lattice/build -# artifacts: -# paths: -# - lattice/lattice.tar.gz -# needs: -# - job: install lattice -# allow_failure: true +build lattice: + stage: test + image: node:14 + variables: + CI: "true" + script: + - echo "Building lattice..." + - cd lattice + - yarn build + - tar -czvf lattice.tar.gz lattice/build + artifacts: + paths: + - lattice/lattice.tar.gz + needs: + - job: install lattice + allow_failure: true -# run jest tests: -# stage: test -# image: node:14 -# variables: -# CI: "true" -# script: -# - echo "Testing lattice..." -# - cd lattice -# - npm install --force -# - npm test -- --coverage --testResultsProcessor=jest-sonar-reporter -# artifacts: -# paths: -# - lattice/coverage/lcov.info +run jest tests: + stage: test + image: node:14 + variables: + CI: "true" + script: + - echo "Testing lattice..." + - cd lattice + - npm install --force + - npm test -- --coverage --testResultsProcessor=jest-sonar-reporter + artifacts: + paths: + - lattice/coverage/lcov.info -# run go tests: -# stage: test -# image: golang:1.16.9 -# script: -# - echo "Running featurebase unit tests..." -# - go test -covermode=atomic -coverprofile=coverage.out -# artifacts: -# paths: -# - coverage.out +run go tests: + stage: test + image: golang:1.16.9 + script: + - echo "Running featurebase unit tests..." + - go test -covermode=atomic -coverprofile=coverage.out + artifacts: + paths: + - coverage.out -# run go tests with output: -# stage: test -# image: golang:1.16.9 -# script: -# - echo "Running featurebase unit tests to capture JSON output..." -# - go test -json > test-report.out -# artifacts: -# paths: -# - test-report.out +run go tests with output: + stage: test + image: golang:1.16.9 + script: + - echo "Running featurebase unit tests to capture JSON output..." + - go test -json > test-report.out + artifacts: + paths: + - test-report.out -# generate coverage html: -# stage: test -# image: golang:1.16.9 -# script: -# - echo "Generating coverage HTML.." -# - go tool cover -html coverage.out -o coverage.html -# artifacts: -# paths: -# - coverage.html -# needs: -# - job: run go tests +generate coverage html: + stage: test + image: golang:1.16.9 + script: + - echo "Generating coverage HTML.." + - go tool cover -html coverage.out -o coverage.html + artifacts: + paths: + - coverage.html + needs: + - job: run go tests -# upload to sonarcloud: -# stage: test -# image: sonarsource/sonar-scanner-cli:4.6 -# variables: -# SONAR_TOKEN: $SONAR_TOKEN -# script: -# - sonar-scanner -Dsonar.projectKey=molecula_featurebase -Dsonar.organization=molecula -Dsonar.sources=. -Dsonar.host.url=https://sonarcloud.io -Dsonar.go.coverage.reportPaths=coverage.out -Dsonar.go.tests.reportPaths=test-report.out -Dsonar.javascript.lcov.reportPaths=lattice/coverage/lcov.info -# needs: -# - job: run go tests -# - job: run go tests with output -# - job: run jest tests +upload to sonarcloud: + stage: test + image: sonarsource/sonar-scanner-cli:4.6 + variables: + SONAR_TOKEN: $SONAR_TOKEN + script: + - sonar-scanner -Dsonar.projectKey=molecula_featurebase -Dsonar.organization=molecula -Dsonar.sources=. -Dsonar.host.url=https://sonarcloud.io -Dsonar.go.coverage.reportPaths=coverage.out -Dsonar.go.tests.reportPaths=test-report.out -Dsonar.javascript.lcov.reportPaths=lattice/coverage/lcov.info + needs: + - job: run go tests + - job: run go tests with output + - job: run jest tests -# build for linux amd64: -# stage: build -# image: registry.gitlab.com/molecula/featurebase/builder:0.0.3 -# variables: -# GOOS: "linux" -# GOARCH: "amd64" -# script: -# - go build -o featurebase_linux_amd64 ./cmd/featurebase -# artifacts: -# paths: -# - featurebase_linux_amd64 +build for linux amd64: + stage: build + image: registry.gitlab.com/molecula/featurebase/builder:0.0.3 + variables: + GOOS: "linux" + GOARCH: "amd64" + script: + - go build -o featurebase_linux_amd64 ./cmd/featurebase + artifacts: + paths: + - featurebase_linux_amd64 -# build for linux arm64: -# stage: build -# image: registry.gitlab.com/molecula/featurebase/builder:0.0.3 -# variables: -# GOOS: "linux" -# GOARCH: "arm64" -# script: -# - go build -o featurebase_linux_arm64 ./cmd/featurebase -# artifacts: -# paths: -# - featurebase_linux_arm64 +build for linux arm64: + stage: build + image: registry.gitlab.com/molecula/featurebase/builder:0.0.3 + variables: + GOOS: "linux" + GOARCH: "arm64" + script: + - go build -o featurebase_linux_arm64 ./cmd/featurebase + artifacts: + paths: + - featurebase_linux_arm64 -# build for darwin amd64: -# stage: build -# image: registry.gitlab.com/molecula/featurebase/builder:0.0.3 -# variables: -# GOOS: "darwin" -# GOARCH: "amd64" -# script: -# - go build -o featurebase_darwin_amd64 ./cmd/featurebase -# artifacts: -# paths: -# - featurebase_darwin_amd64 +build for darwin amd64: + stage: build + image: registry.gitlab.com/molecula/featurebase/builder:0.0.3 + variables: + GOOS: "darwin" + GOARCH: "amd64" + script: + - go build -o featurebase_darwin_amd64 ./cmd/featurebase + artifacts: + paths: + - featurebase_darwin_amd64 -# build for darwin arm64: -# stage: build -# image: registry.gitlab.com/molecula/featurebase/builder:0.0.3 -# variables: -# GOOS: "darwin" -# GOARCH: "arm64" -# script: -# - go build -o featurebase_darwin_arm64 ./cmd/featurebase -# artifacts: -# paths: -# - featurebase_darwin_arm64 +build for darwin arm64: + stage: build + image: registry.gitlab.com/molecula/featurebase/builder:0.0.3 + variables: + GOOS: "darwin" + GOARCH: "arm64" + script: + - go build -o featurebase_darwin_arm64 ./cmd/featurebase + artifacts: + paths: + - featurebase_darwin_arm64 deploy ec2 instance: stage: integration From df2064a860f266781e35cad1f20f4306183bcc05 Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Thu, 4 Nov 2021 09:31:31 -0500 Subject: [PATCH 09/98] define aws credential variables --- .gitlab/.gitlab-ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index f12c60249..fc97e4e78 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -149,6 +149,8 @@ deploy ec2 instance: SUBNET_ID: "" SSH_KEY: "" PUBLIC_IP: "" + AWS_ACCESS_KEY_ID: $AWS_ACCESS_KEY_ID + AWS_SECRET_ACCESS_KEY: $AWS_SECRET_ACCESS_KEY before_script: - aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID - aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY From 50f409771fe7fbe7bf9b9a9eda0fb5beaa5c8502 Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Thu, 4 Nov 2021 09:59:11 -0500 Subject: [PATCH 10/98] updated aws profile --- .gitlab/.gitlab-ci.yml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index fc97e4e78..8091bffdc 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -151,18 +151,19 @@ deploy ec2 instance: PUBLIC_IP: "" AWS_ACCESS_KEY_ID: $AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY: $AWS_SECRET_ACCESS_KEY + AWS_PROFILE: "" before_script: - aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID - aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY - aws configure set region "us-east-2" - - export AWS_PROFILE=gitlab-cicd + - AWS_PROFILE=$(aws configure list-profiles) script: - - AMI=$(aws ssm get-parameters --names /aws/service/ami-amazon-linux-latest/amzn2-ami-hvm-x86_64-ebs --query 'Parameters[0].[Value]' --output text --profile gitlab-cicd) + - AMI=$(aws ssm get-parameters --names /aws/service/ami-amazon-linux-latest/amzn2-ami-hvm-x86_64-ebs --query 'Parameters[0].[Value]' --output text --profile $AWS_PROFILE) - - SECURITY_GROUP=$(aws ec2 describe-security-groups --filter Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978 --query 'SecurityGroups[*].[GroupId]' --output text --profile gitlab-cicd) + - SECURITY_GROUP=$(aws ec2 describe-security-groups --filter Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978 --query 'SecurityGroups[*].[GroupId]' --output text --profile $AWS_PROFILE) - - SUBNET_ID=$(aws ec2 describe-subnets --filters 'Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978' 'Name=availability-zone,Values=us-east-2a' --query 'Subnets[0].SubnetId' --output text --profile gitlab-cicd) + - SUBNET_ID=$(aws ec2 describe-subnets --filters 'Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978' 'Name=availability-zone,Values=us-east-2a' --query 'Subnets[0].SubnetId' --output text --profile $AWS_PROFILE) - aws ec2 run-instances --image-id $AMI --count 1 --instance-type $INSTANCE --security-group-ids $SECURITY_GROUP --subnet-id $SUBNET_ID —user-data file://.gitlab/server.yml - - PUBLIC_IP=$(aws ec2 describe-instances --filters 'Name=tag:Name, Values=Single_Node_Deployment' 'Name=instance-state-name, Values=running' --query 'Reservations[*].Instances[*].PublicIpAddress' --output text --profile gitlab-cicd) \ No newline at end of file + - PUBLIC_IP=$(aws ec2 describe-instances --filters 'Name=tag:Name, Values=Single_Node_Deployment' 'Name=instance-state-name, Values=running' --query 'Reservations[*].Instances[*].PublicIpAddress' --output text --profile $AWS_PROFILE) \ No newline at end of file From b95d013e14bb74bd2b04f5aa3ee777b2d0ff1a80 Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Thu, 4 Nov 2021 10:12:26 -0500 Subject: [PATCH 11/98] configure aws profile --- .gitlab/.gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 8091bffdc..d7f5cf3c1 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -151,12 +151,12 @@ deploy ec2 instance: PUBLIC_IP: "" AWS_ACCESS_KEY_ID: $AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY: $AWS_SECRET_ACCESS_KEY - AWS_PROFILE: "" + AWS_PROFILE: "AWS-PROFILE" before_script: - aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID - aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY - aws configure set region "us-east-2" - - AWS_PROFILE=$(aws configure list-profiles) + - aws configure --profile $AWS_PROFILE script: - AMI=$(aws ssm get-parameters --names /aws/service/ami-amazon-linux-latest/amzn2-ami-hvm-x86_64-ebs --query 'Parameters[0].[Value]' --output text --profile $AWS_PROFILE) From c19aa7d6b31f83d1215b39709aaef5385d2975c8 Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Thu, 4 Nov 2021 10:27:13 -0500 Subject: [PATCH 12/98] changed profile name --- .gitlab/.gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index d7f5cf3c1..aba3baf19 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -151,7 +151,7 @@ deploy ec2 instance: PUBLIC_IP: "" AWS_ACCESS_KEY_ID: $AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY: $AWS_SECRET_ACCESS_KEY - AWS_PROFILE: "AWS-PROFILE" + AWS_PROFILE: "integration" before_script: - aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID - aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY From 6e673bafa37218879fc736f376ccae837a4035d1 Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Thu, 4 Nov 2021 10:58:08 -0500 Subject: [PATCH 13/98] configure profile --- .gitlab/.gitlab-ci.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index aba3baf19..8cf121587 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -151,19 +151,19 @@ deploy ec2 instance: PUBLIC_IP: "" AWS_ACCESS_KEY_ID: $AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY: $AWS_SECRET_ACCESS_KEY - AWS_PROFILE: "integration" + PROFILE: "integration" before_script: - aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID - aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY - aws configure set region "us-east-2" - - aws configure --profile $AWS_PROFILE + - aws configure set aws_profile $PROFILE script: - - AMI=$(aws ssm get-parameters --names /aws/service/ami-amazon-linux-latest/amzn2-ami-hvm-x86_64-ebs --query 'Parameters[0].[Value]' --output text --profile $AWS_PROFILE) + - AMI=$(aws ssm get-parameters --names /aws/service/ami-amazon-linux-latest/amzn2-ami-hvm-x86_64-ebs --query 'Parameters[0].[Value]' --output text --profile $PROFILE) - - SECURITY_GROUP=$(aws ec2 describe-security-groups --filter Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978 --query 'SecurityGroups[*].[GroupId]' --output text --profile $AWS_PROFILE) + - SECURITY_GROUP=$(aws ec2 describe-security-groups --filter Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978 --query 'SecurityGroups[*].[GroupId]' --output text --profile $PROFILE) - - SUBNET_ID=$(aws ec2 describe-subnets --filters 'Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978' 'Name=availability-zone,Values=us-east-2a' --query 'Subnets[0].SubnetId' --output text --profile $AWS_PROFILE) + - SUBNET_ID=$(aws ec2 describe-subnets --filters 'Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978' 'Name=availability-zone,Values=us-east-2a' --query 'Subnets[0].SubnetId' --output text --profile $PROFILE) - aws ec2 run-instances --image-id $AMI --count 1 --instance-type $INSTANCE --security-group-ids $SECURITY_GROUP --subnet-id $SUBNET_ID —user-data file://.gitlab/server.yml - - PUBLIC_IP=$(aws ec2 describe-instances --filters 'Name=tag:Name, Values=Single_Node_Deployment' 'Name=instance-state-name, Values=running' --query 'Reservations[*].Instances[*].PublicIpAddress' --output text --profile $AWS_PROFILE) \ No newline at end of file + - PUBLIC_IP=$(aws ec2 describe-instances --filters 'Name=tag:Name, Values=Single_Node_Deployment' 'Name=instance-state-name, Values=running' --query 'Reservations[*].Instances[*].PublicIpAddress' --output text --profile $PROFILE) \ No newline at end of file From 67da6d9ffa817fb2132a62c00c1982c912a617fc Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Thu, 4 Nov 2021 11:12:30 -0500 Subject: [PATCH 14/98] debug profile --- .gitlab/.gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 8cf121587..e2b33934c 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -157,6 +157,7 @@ deploy ec2 instance: - aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY - aws configure set region "us-east-2" - aws configure set aws_profile $PROFILE + - aws configure list-profiles script: - AMI=$(aws ssm get-parameters --names /aws/service/ami-amazon-linux-latest/amzn2-ami-hvm-x86_64-ebs --query 'Parameters[0].[Value]' --output text --profile $PROFILE) From 502f40593e599fdbbfe121f32f93ce9eca913cee Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Thu, 4 Nov 2021 11:26:05 -0500 Subject: [PATCH 15/98] updated profile --- .gitlab/.gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index e2b33934c..d653d362c 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -151,13 +151,13 @@ deploy ec2 instance: PUBLIC_IP: "" AWS_ACCESS_KEY_ID: $AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY: $AWS_SECRET_ACCESS_KEY - PROFILE: "integration" + PROFILE: "" before_script: - aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID - aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY - aws configure set region "us-east-2" - aws configure set aws_profile $PROFILE - - aws configure list-profiles + - PROFILE=$(aws configure list-profiles) script: - AMI=$(aws ssm get-parameters --names /aws/service/ami-amazon-linux-latest/amzn2-ami-hvm-x86_64-ebs --query 'Parameters[0].[Value]' --output text --profile $PROFILE) From 01260cfcced82431a16502c07bc073e5476dc7af Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Thu, 4 Nov 2021 11:38:58 -0500 Subject: [PATCH 16/98] update profile --- .gitlab/.gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index d653d362c..421a7b5ff 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -151,7 +151,7 @@ deploy ec2 instance: PUBLIC_IP: "" AWS_ACCESS_KEY_ID: $AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY: $AWS_SECRET_ACCESS_KEY - PROFILE: "" + PROFILE: "default" before_script: - aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID - aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY From 7a6e7c2fd5625fb0cf648a78f8b162b407677f68 Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Thu, 4 Nov 2021 11:53:43 -0500 Subject: [PATCH 17/98] set profile --- .gitlab/.gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 421a7b5ff..f6a8ed451 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -158,6 +158,7 @@ deploy ec2 instance: - aws configure set region "us-east-2" - aws configure set aws_profile $PROFILE - PROFILE=$(aws configure list-profiles) + - aws sts get-caller-identity --profile $PROFILE script: - AMI=$(aws ssm get-parameters --names /aws/service/ami-amazon-linux-latest/amzn2-ami-hvm-x86_64-ebs --query 'Parameters[0].[Value]' --output text --profile $PROFILE) From 8274a321277a99f2573042934c0b342e0e13657c Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Thu, 4 Nov 2021 12:09:20 -0500 Subject: [PATCH 18/98] debug profile --- .gitlab/.gitlab-ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index f6a8ed451..2f3810c62 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -157,7 +157,8 @@ deploy ec2 instance: - aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY - aws configure set region "us-east-2" - aws configure set aws_profile $PROFILE - - PROFILE=$(aws configure list-profiles) + - PROFILE=$(aws configure list-profiles --output text) + - aws configure list-profiles --output text - aws sts get-caller-identity --profile $PROFILE script: - AMI=$(aws ssm get-parameters --names /aws/service/ami-amazon-linux-latest/amzn2-ami-hvm-x86_64-ebs --query 'Parameters[0].[Value]' --output text --profile $PROFILE) From 20224b1df5afac0683cd76baa787bd54180a1843 Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Thu, 4 Nov 2021 13:28:48 -0500 Subject: [PATCH 19/98] updated aws config --- .gitlab/.gitlab-ci.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 2f3810c62..303d69af0 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -149,8 +149,8 @@ deploy ec2 instance: SUBNET_ID: "" SSH_KEY: "" PUBLIC_IP: "" - AWS_ACCESS_KEY_ID: $AWS_ACCESS_KEY_ID - AWS_SECRET_ACCESS_KEY: $AWS_SECRET_ACCESS_KEY + AWS_ACCESS_KEY_ID: $AWS_CLOUD_DEV_KEY + AWS_SECRET_ACCESS_KEY: $AWS_CLOUD_DEV_SECRET PROFILE: "default" before_script: - aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID @@ -159,7 +159,6 @@ deploy ec2 instance: - aws configure set aws_profile $PROFILE - PROFILE=$(aws configure list-profiles --output text) - aws configure list-profiles --output text - - aws sts get-caller-identity --profile $PROFILE script: - AMI=$(aws ssm get-parameters --names /aws/service/ami-amazon-linux-latest/amzn2-ami-hvm-x86_64-ebs --query 'Parameters[0].[Value]' --output text --profile $PROFILE) From 08161a366af47079b3c6219263fd43bd1fe0305b Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Thu, 4 Nov 2021 13:42:03 -0500 Subject: [PATCH 20/98] undo last commit --- .gitlab/.gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 303d69af0..8c6bf9c1c 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -149,8 +149,8 @@ deploy ec2 instance: SUBNET_ID: "" SSH_KEY: "" PUBLIC_IP: "" - AWS_ACCESS_KEY_ID: $AWS_CLOUD_DEV_KEY - AWS_SECRET_ACCESS_KEY: $AWS_CLOUD_DEV_SECRET + AWS_ACCESS_KEY_ID: $AWS_ACCESS_KEY_ID + AWS_SECRET_ACCESS_KEY: $AWS_SECRET_ACCESS_KEY PROFILE: "default" before_script: - aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID From f80f2fc702b66f7af81b1a11a0cb0e55540893f6 Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Thu, 4 Nov 2021 14:34:19 -0500 Subject: [PATCH 21/98] added rsa key --- .gitlab/.gitlab-ci.yml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 8c6bf9c1c..59bf3488e 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -152,6 +152,7 @@ deploy ec2 instance: AWS_ACCESS_KEY_ID: $AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY: $AWS_SECRET_ACCESS_KEY PROFILE: "default" + RSA_KEY: $AWS_RSA_Private_Key before_script: - aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID - aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY @@ -161,11 +162,8 @@ deploy ec2 instance: - aws configure list-profiles --output text script: - AMI=$(aws ssm get-parameters --names /aws/service/ami-amazon-linux-latest/amzn2-ami-hvm-x86_64-ebs --query 'Parameters[0].[Value]' --output text --profile $PROFILE) - - SECURITY_GROUP=$(aws ec2 describe-security-groups --filter Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978 --query 'SecurityGroups[*].[GroupId]' --output text --profile $PROFILE) - - SUBNET_ID=$(aws ec2 describe-subnets --filters 'Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978' 'Name=availability-zone,Values=us-east-2a' --query 'Subnets[0].SubnetId' --output text --profile $PROFILE) - - - aws ec2 run-instances --image-id $AMI --count 1 --instance-type $INSTANCE --security-group-ids $SECURITY_GROUP --subnet-id $SUBNET_ID —user-data file://.gitlab/server.yml - + - aws ec2 run-instances --image-id $AMI --count 1 --instance-type $INSTANCE --security-group-ids $SECURITY_GROUP --subnet-id $SUBNET_ID -—user-data file://.gitlab/server.yml --key-name $RSA_KEY + - sleep 30 - PUBLIC_IP=$(aws ec2 describe-instances --filters 'Name=tag:Name, Values=Single_Node_Deployment' 'Name=instance-state-name, Values=running' --query 'Reservations[*].Instances[*].PublicIpAddress' --output text --profile $PROFILE) \ No newline at end of file From 82b97022ff9559322aea3222df02df29a238372f Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Thu, 4 Nov 2021 14:57:41 -0500 Subject: [PATCH 22/98] removed sleep time --- .gitlab/.gitlab-ci.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index a434f30b6..95a9c0af8 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -202,5 +202,4 @@ deploy ec2 instance: - SECURITY_GROUP=$(aws ec2 describe-security-groups --filter Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978 --query 'SecurityGroups[*].[GroupId]' --output text --profile $PROFILE) - SUBNET_ID=$(aws ec2 describe-subnets --filters 'Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978' 'Name=availability-zone,Values=us-east-2a' --query 'Subnets[0].SubnetId' --output text --profile $PROFILE) - aws ec2 run-instances --image-id $AMI --count 1 --instance-type $INSTANCE --security-group-ids $SECURITY_GROUP --subnet-id $SUBNET_ID -—user-data file://.gitlab/server.yml --key-name $RSA_KEY - - sleep 30 - PUBLIC_IP=$(aws ec2 describe-instances --filters 'Name=tag:Name, Values=Single_Node_Deployment' 'Name=instance-state-name, Values=running' --query 'Reservations[*].Instances[*].PublicIpAddress' --output text --profile $PROFILE) From 321d273dce080593b92fd8c0dd1eb723a7b8bab0 Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Fri, 5 Nov 2021 09:03:51 -0500 Subject: [PATCH 23/98] fixed typo in instance name --- .gitlab/.gitlab-ci.yml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 95a9c0af8..11182d7c8 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -177,11 +177,11 @@ build container fb: - echo Created docker featurebase image with tag "$tag" deploy ec2 instance: - stage: integration + stage: lint image: registry.gitlab.com/gitlab-org/cloud-deploy/aws-base:latest variables: AMI: "" - INSTANCE: "t2_micro" + INSTANCE: "t2.micro" SECURITY_GROUP: "" SUBNET_ID: "" SSH_KEY: "" @@ -189,7 +189,6 @@ deploy ec2 instance: AWS_ACCESS_KEY_ID: $AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY: $AWS_SECRET_ACCESS_KEY PROFILE: "default" - RSA_KEY: $AWS_RSA_Private_Key before_script: - aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID - aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY @@ -201,5 +200,5 @@ deploy ec2 instance: - AMI=$(aws ssm get-parameters --names /aws/service/ami-amazon-linux-latest/amzn2-ami-hvm-x86_64-ebs --query 'Parameters[0].[Value]' --output text --profile $PROFILE) - SECURITY_GROUP=$(aws ec2 describe-security-groups --filter Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978 --query 'SecurityGroups[*].[GroupId]' --output text --profile $PROFILE) - SUBNET_ID=$(aws ec2 describe-subnets --filters 'Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978' 'Name=availability-zone,Values=us-east-2a' --query 'Subnets[0].SubnetId' --output text --profile $PROFILE) - - aws ec2 run-instances --image-id $AMI --count 1 --instance-type $INSTANCE --security-group-ids $SECURITY_GROUP --subnet-id $SUBNET_ID -—user-data file://.gitlab/server.yml --key-name $RSA_KEY - - PUBLIC_IP=$(aws ec2 describe-instances --filters 'Name=tag:Name, Values=Single_Node_Deployment' 'Name=instance-state-name, Values=running' --query 'Reservations[*].Instances[*].PublicIpAddress' --output text --profile $PROFILE) + - aws ec2 run-instances --image-id $AMI --count 1 --instance-type $INSTANCE --security-group-ids $SECURITY_GROUP --subnet-id $SUBNET_ID + - PUBLIC_IP=$(aws ec2 describe-instances --filters 'Name=tag:Name, Values=Single_Node_Deployment' 'Name=instance-state-name, Values=running' --query 'Reservations[*].Instances[*].PublicIpAddress' --output text --profile $PROFILE) \ No newline at end of file From 383663673ba55de953fbe2c334266eddb4368c5d Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Fri, 5 Nov 2021 09:11:27 -0500 Subject: [PATCH 24/98] configure ssh --- .gitlab/.gitlab-ci.yml | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 11182d7c8..c80bef661 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -13,6 +13,15 @@ include: variables: GOVERSION: "1.16.9" +before_script: + - 'which ssh-agent || ( apt-get update -y && apt-get install openssh-client -y )' + - eval `ssh-agent -s` + - mkdir -p /root/.ssh + - chmod 700 /root/.ssh + - cp $AWS_RSA_PRIVATE_KEY /root/.ssh/id_rsa + - chmod 600 /root/.ssh/id_rsa + - ssh-add /root/.ssh/id_rsa + stages: - lint - test @@ -200,5 +209,6 @@ deploy ec2 instance: - AMI=$(aws ssm get-parameters --names /aws/service/ami-amazon-linux-latest/amzn2-ami-hvm-x86_64-ebs --query 'Parameters[0].[Value]' --output text --profile $PROFILE) - SECURITY_GROUP=$(aws ec2 describe-security-groups --filter Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978 --query 'SecurityGroups[*].[GroupId]' --output text --profile $PROFILE) - SUBNET_ID=$(aws ec2 describe-subnets --filters 'Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978' 'Name=availability-zone,Values=us-east-2a' --query 'Subnets[0].SubnetId' --output text --profile $PROFILE) - - aws ec2 run-instances --image-id $AMI --count 1 --instance-type $INSTANCE --security-group-ids $SECURITY_GROUP --subnet-id $SUBNET_ID - - PUBLIC_IP=$(aws ec2 describe-instances --filters 'Name=tag:Name, Values=Single_Node_Deployment' 'Name=instance-state-name, Values=running' --query 'Reservations[*].Instances[*].PublicIpAddress' --output text --profile $PROFILE) \ No newline at end of file + - aws ec2 run-instances --image-id $AMI --count 1 --instance-type $INSTANCE --security-group-ids $SECURITY_GROUP --subnet-id $SUBNET_ID --key-pair "AWS_RSA_PRIVATE_KEY" + - PUBLIC_IP=$(aws ec2 describe-instances --filters 'Name=tag:Name, Values=Single_Node_Deployment' 'Name=instance-state-name, Values=running' --query 'Reservations[*].Instances[*].PublicIpAddress' --output text --profile $PROFILE) + - echo $PUBLIC_IP \ No newline at end of file From f9ff3973d0675ee439a29ab376aa14cb340bb1db Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Fri, 5 Nov 2021 09:17:19 -0500 Subject: [PATCH 25/98] fixed argument name --- .gitlab/.gitlab-ci.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index c80bef661..fde2c6c56 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -198,6 +198,7 @@ deploy ec2 instance: AWS_ACCESS_KEY_ID: $AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY: $AWS_SECRET_ACCESS_KEY PROFILE: "default" + INSTANCE_NAME: "Single_Node_Deployment" before_script: - aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID - aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY @@ -209,6 +210,7 @@ deploy ec2 instance: - AMI=$(aws ssm get-parameters --names /aws/service/ami-amazon-linux-latest/amzn2-ami-hvm-x86_64-ebs --query 'Parameters[0].[Value]' --output text --profile $PROFILE) - SECURITY_GROUP=$(aws ec2 describe-security-groups --filter Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978 --query 'SecurityGroups[*].[GroupId]' --output text --profile $PROFILE) - SUBNET_ID=$(aws ec2 describe-subnets --filters 'Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978' 'Name=availability-zone,Values=us-east-2a' --query 'Subnets[0].SubnetId' --output text --profile $PROFILE) - - aws ec2 run-instances --image-id $AMI --count 1 --instance-type $INSTANCE --security-group-ids $SECURITY_GROUP --subnet-id $SUBNET_ID --key-pair "AWS_RSA_PRIVATE_KEY" + - aws ec2 run-instances --image-id $AMI --count 1 --instance-type $INSTANCE --security-group-ids $SECURITY_GROUP --subnet-id $SUBNET_ID --key-name "AWS_RSA_PRIVATE_KEY" --tag-specifications \ + 'ResourceType=instance,Tags=[{Key=Name,Value=$INSTANCE_NAME}]' - PUBLIC_IP=$(aws ec2 describe-instances --filters 'Name=tag:Name, Values=Single_Node_Deployment' 'Name=instance-state-name, Values=running' --query 'Reservations[*].Instances[*].PublicIpAddress' --output text --profile $PROFILE) - echo $PUBLIC_IP \ No newline at end of file From 14a9662494a868a7b779d2f7b9116ff581897e6c Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Fri, 5 Nov 2021 09:24:20 -0500 Subject: [PATCH 26/98] fixed yaml syntax error --- .gitlab/.gitlab-ci.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index fde2c6c56..2b559da53 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -210,7 +210,6 @@ deploy ec2 instance: - AMI=$(aws ssm get-parameters --names /aws/service/ami-amazon-linux-latest/amzn2-ami-hvm-x86_64-ebs --query 'Parameters[0].[Value]' --output text --profile $PROFILE) - SECURITY_GROUP=$(aws ec2 describe-security-groups --filter Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978 --query 'SecurityGroups[*].[GroupId]' --output text --profile $PROFILE) - SUBNET_ID=$(aws ec2 describe-subnets --filters 'Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978' 'Name=availability-zone,Values=us-east-2a' --query 'Subnets[0].SubnetId' --output text --profile $PROFILE) - - aws ec2 run-instances --image-id $AMI --count 1 --instance-type $INSTANCE --security-group-ids $SECURITY_GROUP --subnet-id $SUBNET_ID --key-name "AWS_RSA_PRIVATE_KEY" --tag-specifications \ - 'ResourceType=instance,Tags=[{Key=Name,Value=$INSTANCE_NAME}]' + - aws ec2 run-instances --image-id $AMI --count 1 --instance-type $INSTANCE --security-group-ids $SECURITY_GROUP --subnet-id $SUBNET_ID --key-name "AWS_RSA_PRIVATE_KEY" --tag-specifications 'ResourceType=instance,Tags=[{Key=Name,Value=$INSTANCE_NAME}]' - PUBLIC_IP=$(aws ec2 describe-instances --filters 'Name=tag:Name, Values=Single_Node_Deployment' 'Name=instance-state-name, Values=running' --query 'Reservations[*].Instances[*].PublicIpAddress' --output text --profile $PROFILE) - echo $PUBLIC_IP \ No newline at end of file From 7271821ebea0250d2eb437914b1b5c2fb924fc33 Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Fri, 5 Nov 2021 09:28:25 -0500 Subject: [PATCH 27/98] fixed path for ssh keys --- .gitlab/.gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 2b559da53..2b4c8cc9d 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -210,6 +210,6 @@ deploy ec2 instance: - AMI=$(aws ssm get-parameters --names /aws/service/ami-amazon-linux-latest/amzn2-ami-hvm-x86_64-ebs --query 'Parameters[0].[Value]' --output text --profile $PROFILE) - SECURITY_GROUP=$(aws ec2 describe-security-groups --filter Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978 --query 'SecurityGroups[*].[GroupId]' --output text --profile $PROFILE) - SUBNET_ID=$(aws ec2 describe-subnets --filters 'Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978' 'Name=availability-zone,Values=us-east-2a' --query 'Subnets[0].SubnetId' --output text --profile $PROFILE) - - aws ec2 run-instances --image-id $AMI --count 1 --instance-type $INSTANCE --security-group-ids $SECURITY_GROUP --subnet-id $SUBNET_ID --key-name "AWS_RSA_PRIVATE_KEY" --tag-specifications 'ResourceType=instance,Tags=[{Key=Name,Value=$INSTANCE_NAME}]' + - aws ec2 run-instances --image-id $AMI --count 1 --instance-type $INSTANCE --security-group-ids $SECURITY_GROUP --subnet-id $SUBNET_ID --key-name /root/.ssh/id_rsa --tag-specifications 'ResourceType=instance,Tags=[{Key=Name,Value=$INSTANCE_NAME}]' - PUBLIC_IP=$(aws ec2 describe-instances --filters 'Name=tag:Name, Values=Single_Node_Deployment' 'Name=instance-state-name, Values=running' --query 'Reservations[*].Instances[*].PublicIpAddress' --output text --profile $PROFILE) - echo $PUBLIC_IP \ No newline at end of file From d147c03e227f10765a0a04079c5b681af5752f00 Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Fri, 5 Nov 2021 09:32:53 -0500 Subject: [PATCH 28/98] add ssh config in section with using ssh keys --- .gitlab/.gitlab-ci.yml | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 2b4c8cc9d..ef3ac1d37 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -13,15 +13,6 @@ include: variables: GOVERSION: "1.16.9" -before_script: - - 'which ssh-agent || ( apt-get update -y && apt-get install openssh-client -y )' - - eval `ssh-agent -s` - - mkdir -p /root/.ssh - - chmod 700 /root/.ssh - - cp $AWS_RSA_PRIVATE_KEY /root/.ssh/id_rsa - - chmod 600 /root/.ssh/id_rsa - - ssh-add /root/.ssh/id_rsa - stages: - lint - test @@ -193,7 +184,6 @@ deploy ec2 instance: INSTANCE: "t2.micro" SECURITY_GROUP: "" SUBNET_ID: "" - SSH_KEY: "" PUBLIC_IP: "" AWS_ACCESS_KEY_ID: $AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY: $AWS_SECRET_ACCESS_KEY @@ -206,6 +196,13 @@ deploy ec2 instance: - aws configure set aws_profile $PROFILE - PROFILE=$(aws configure list-profiles --output text) - aws configure list-profiles --output text + - 'which ssh-agent || ( apt-get update -y && apt-get install openssh-client -y )' + - eval `ssh-agent -s` + - mkdir -p /root/.ssh + - chmod 700 /root/.ssh + - cp $AWS_RSA_PRIVATE_KEY /root/.ssh/id_rsa + - chmod 600 /root/.ssh/id_rsa + - ssh-add /root/.ssh/id_rsa script: - AMI=$(aws ssm get-parameters --names /aws/service/ami-amazon-linux-latest/amzn2-ami-hvm-x86_64-ebs --query 'Parameters[0].[Value]' --output text --profile $PROFILE) - SECURITY_GROUP=$(aws ec2 describe-security-groups --filter Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978 --query 'SecurityGroups[*].[GroupId]' --output text --profile $PROFILE) From bd440f4c23baf61ebd77eb385bc8f02f086e4cca Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Fri, 5 Nov 2021 09:43:19 -0500 Subject: [PATCH 29/98] updated permissions for ssh key --- .gitlab/.gitlab-ci.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index ef3ac1d37..634e07765 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -200,13 +200,11 @@ deploy ec2 instance: - eval `ssh-agent -s` - mkdir -p /root/.ssh - chmod 700 /root/.ssh - - cp $AWS_RSA_PRIVATE_KEY /root/.ssh/id_rsa - - chmod 600 /root/.ssh/id_rsa - - ssh-add /root/.ssh/id_rsa + - chmod 400 $AWS_SSH_PRIVATE_KEY script: - AMI=$(aws ssm get-parameters --names /aws/service/ami-amazon-linux-latest/amzn2-ami-hvm-x86_64-ebs --query 'Parameters[0].[Value]' --output text --profile $PROFILE) - SECURITY_GROUP=$(aws ec2 describe-security-groups --filter Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978 --query 'SecurityGroups[*].[GroupId]' --output text --profile $PROFILE) - SUBNET_ID=$(aws ec2 describe-subnets --filters 'Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978' 'Name=availability-zone,Values=us-east-2a' --query 'Subnets[0].SubnetId' --output text --profile $PROFILE) - - aws ec2 run-instances --image-id $AMI --count 1 --instance-type $INSTANCE --security-group-ids $SECURITY_GROUP --subnet-id $SUBNET_ID --key-name /root/.ssh/id_rsa --tag-specifications 'ResourceType=instance,Tags=[{Key=Name,Value=$INSTANCE_NAME}]' + - aws ec2 run-instances --image-id $AMI --count 1 --instance-type $INSTANCE --security-group-ids $SECURITY_GROUP --subnet-id $SUBNET_ID --key-name $AWS_SSH_PRIVATE_KEY --tag-specifications 'ResourceType=instance,Tags=[{Key=Name,Value=$INSTANCE_NAME}]' - PUBLIC_IP=$(aws ec2 describe-instances --filters 'Name=tag:Name, Values=Single_Node_Deployment' 'Name=instance-state-name, Values=running' --query 'Reservations[*].Instances[*].PublicIpAddress' --output text --profile $PROFILE) - echo $PUBLIC_IP \ No newline at end of file From 04b64c0f301c835cc254ea5f6e507df3e44e82d7 Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Fri, 5 Nov 2021 09:54:37 -0500 Subject: [PATCH 30/98] add path for ssh keys --- .gitlab/.gitlab-ci.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 634e07765..66e04b2a9 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -200,11 +200,12 @@ deploy ec2 instance: - eval `ssh-agent -s` - mkdir -p /root/.ssh - chmod 700 /root/.ssh - - chmod 400 $AWS_SSH_PRIVATE_KEY + - cp $AWS_SSH_PRIVATE_KEY /builds/molecula/featurebase.tmp/ + - chmod 400 /builds/molecula/featurebase.tmp/$AWS_SSH_PRIVATE_KEY script: - AMI=$(aws ssm get-parameters --names /aws/service/ami-amazon-linux-latest/amzn2-ami-hvm-x86_64-ebs --query 'Parameters[0].[Value]' --output text --profile $PROFILE) - SECURITY_GROUP=$(aws ec2 describe-security-groups --filter Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978 --query 'SecurityGroups[*].[GroupId]' --output text --profile $PROFILE) - SUBNET_ID=$(aws ec2 describe-subnets --filters 'Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978' 'Name=availability-zone,Values=us-east-2a' --query 'Subnets[0].SubnetId' --output text --profile $PROFILE) - - aws ec2 run-instances --image-id $AMI --count 1 --instance-type $INSTANCE --security-group-ids $SECURITY_GROUP --subnet-id $SUBNET_ID --key-name $AWS_SSH_PRIVATE_KEY --tag-specifications 'ResourceType=instance,Tags=[{Key=Name,Value=$INSTANCE_NAME}]' + - aws ec2 run-instances --image-id $AMI --count 1 --instance-type $INSTANCE --security-group-ids $SECURITY_GROUP --subnet-id $SUBNET_ID --key-name /builds/molecula/featurebase.tmp/$AWS_SSH_PRIVATE_KEY --tag-specifications 'ResourceType=instance,Tags=[{Key=Name,Value=$INSTANCE_NAME}]' - PUBLIC_IP=$(aws ec2 describe-instances --filters 'Name=tag:Name, Values=Single_Node_Deployment' 'Name=instance-state-name, Values=running' --query 'Reservations[*].Instances[*].PublicIpAddress' --output text --profile $PROFILE) - echo $PUBLIC_IP \ No newline at end of file From 6643e3963e82516f5f8549e2dcfb5a9f28f2aed1 Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Fri, 5 Nov 2021 10:28:36 -0500 Subject: [PATCH 31/98] reverted back to variable for ssh --- .gitlab/.gitlab-ci.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 66e04b2a9..8591d9728 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -189,6 +189,7 @@ deploy ec2 instance: AWS_SECRET_ACCESS_KEY: $AWS_SECRET_ACCESS_KEY PROFILE: "default" INSTANCE_NAME: "Single_Node_Deployment" + AWS_SSH_PRIVATE_KEY: $AWS_SSH_PRIVATE_KEY before_script: - aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID - aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY @@ -200,12 +201,10 @@ deploy ec2 instance: - eval `ssh-agent -s` - mkdir -p /root/.ssh - chmod 700 /root/.ssh - - cp $AWS_SSH_PRIVATE_KEY /builds/molecula/featurebase.tmp/ - - chmod 400 /builds/molecula/featurebase.tmp/$AWS_SSH_PRIVATE_KEY script: - AMI=$(aws ssm get-parameters --names /aws/service/ami-amazon-linux-latest/amzn2-ami-hvm-x86_64-ebs --query 'Parameters[0].[Value]' --output text --profile $PROFILE) - SECURITY_GROUP=$(aws ec2 describe-security-groups --filter Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978 --query 'SecurityGroups[*].[GroupId]' --output text --profile $PROFILE) - SUBNET_ID=$(aws ec2 describe-subnets --filters 'Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978' 'Name=availability-zone,Values=us-east-2a' --query 'Subnets[0].SubnetId' --output text --profile $PROFILE) - - aws ec2 run-instances --image-id $AMI --count 1 --instance-type $INSTANCE --security-group-ids $SECURITY_GROUP --subnet-id $SUBNET_ID --key-name /builds/molecula/featurebase.tmp/$AWS_SSH_PRIVATE_KEY --tag-specifications 'ResourceType=instance,Tags=[{Key=Name,Value=$INSTANCE_NAME}]' + - aws ec2 run-instances --image-id $AMI --count 1 --instance-type $INSTANCE --security-group-ids $SECURITY_GROUP --subnet-id $SUBNET_ID --key-name $AWS_SSH_PRIVATE_KEY --tag-specifications 'ResourceType=instance,Tags=[{Key=Name,Value=$INSTANCE_NAME}]' - PUBLIC_IP=$(aws ec2 describe-instances --filters 'Name=tag:Name, Values=Single_Node_Deployment' 'Name=instance-state-name, Values=running' --query 'Reservations[*].Instances[*].PublicIpAddress' --output text --profile $PROFILE) - echo $PUBLIC_IP \ No newline at end of file From f7497bfe8ffba6ec1f8fed1b404c6ce8ba25fdef Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Fri, 5 Nov 2021 10:35:15 -0500 Subject: [PATCH 32/98] pass ssh key to a file --- .gitlab/.gitlab-ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 8591d9728..397e6ba0d 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -201,10 +201,11 @@ deploy ec2 instance: - eval `ssh-agent -s` - mkdir -p /root/.ssh - chmod 700 /root/.ssh + - echo $AWS_SSH_PRIVATE_KEY > aws_ssh_key script: - AMI=$(aws ssm get-parameters --names /aws/service/ami-amazon-linux-latest/amzn2-ami-hvm-x86_64-ebs --query 'Parameters[0].[Value]' --output text --profile $PROFILE) - SECURITY_GROUP=$(aws ec2 describe-security-groups --filter Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978 --query 'SecurityGroups[*].[GroupId]' --output text --profile $PROFILE) - SUBNET_ID=$(aws ec2 describe-subnets --filters 'Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978' 'Name=availability-zone,Values=us-east-2a' --query 'Subnets[0].SubnetId' --output text --profile $PROFILE) - - aws ec2 run-instances --image-id $AMI --count 1 --instance-type $INSTANCE --security-group-ids $SECURITY_GROUP --subnet-id $SUBNET_ID --key-name $AWS_SSH_PRIVATE_KEY --tag-specifications 'ResourceType=instance,Tags=[{Key=Name,Value=$INSTANCE_NAME}]' + - aws ec2 run-instances --image-id $AMI --count 1 --instance-type $INSTANCE --security-group-ids $SECURITY_GROUP --subnet-id $SUBNET_ID --key-name aws_ssh_key --tag-specifications 'ResourceType=instance,Tags=[{Key=Name,Value=$INSTANCE_NAME}]' - PUBLIC_IP=$(aws ec2 describe-instances --filters 'Name=tag:Name, Values=Single_Node_Deployment' 'Name=instance-state-name, Values=running' --query 'Reservations[*].Instances[*].PublicIpAddress' --output text --profile $PROFILE) - echo $PUBLIC_IP \ No newline at end of file From 550a610b5be50fba4de67cc4aef36b7e490fa1a9 Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Fri, 5 Nov 2021 10:46:54 -0500 Subject: [PATCH 33/98] sanity checks --- .gitlab/.gitlab-ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 397e6ba0d..5e3343f4a 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -195,8 +195,7 @@ deploy ec2 instance: - aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY - aws configure set region "us-east-2" - aws configure set aws_profile $PROFILE - - PROFILE=$(aws configure list-profiles --output text) - - aws configure list-profiles --output text + - echo $PROFILE - 'which ssh-agent || ( apt-get update -y && apt-get install openssh-client -y )' - eval `ssh-agent -s` - mkdir -p /root/.ssh @@ -206,6 +205,7 @@ deploy ec2 instance: - AMI=$(aws ssm get-parameters --names /aws/service/ami-amazon-linux-latest/amzn2-ami-hvm-x86_64-ebs --query 'Parameters[0].[Value]' --output text --profile $PROFILE) - SECURITY_GROUP=$(aws ec2 describe-security-groups --filter Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978 --query 'SecurityGroups[*].[GroupId]' --output text --profile $PROFILE) - SUBNET_ID=$(aws ec2 describe-subnets --filters 'Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978' 'Name=availability-zone,Values=us-east-2a' --query 'Subnets[0].SubnetId' --output text --profile $PROFILE) - - aws ec2 run-instances --image-id $AMI --count 1 --instance-type $INSTANCE --security-group-ids $SECURITY_GROUP --subnet-id $SUBNET_ID --key-name aws_ssh_key --tag-specifications 'ResourceType=instance,Tags=[{Key=Name,Value=$INSTANCE_NAME}]' + - echo $AMI, $SECURITY_GROUP, $SUBNET_ID + - aws ec2 run-instances --image-id $AMI --instance-type $INSTANCE --security-group-ids $SECURITY_GROUP --subnet-id $SUBNET_ID --key-name aws_ssh_key --tag-specifications 'ResourceType=instance,Tags=[{Key=Name,Value=$INSTANCE_NAME}]' - PUBLIC_IP=$(aws ec2 describe-instances --filters 'Name=tag:Name, Values=Single_Node_Deployment' 'Name=instance-state-name, Values=running' --query 'Reservations[*].Instances[*].PublicIpAddress' --output text --profile $PROFILE) - echo $PUBLIC_IP \ No newline at end of file From 0cfd5f116d171f029d90bf72dac6f3a471d4467b Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Fri, 5 Nov 2021 10:50:59 -0500 Subject: [PATCH 34/98] echo ssh file --- .gitlab/.gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 5e3343f4a..065e20bbc 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -195,7 +195,6 @@ deploy ec2 instance: - aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY - aws configure set region "us-east-2" - aws configure set aws_profile $PROFILE - - echo $PROFILE - 'which ssh-agent || ( apt-get update -y && apt-get install openssh-client -y )' - eval `ssh-agent -s` - mkdir -p /root/.ssh @@ -206,6 +205,7 @@ deploy ec2 instance: - SECURITY_GROUP=$(aws ec2 describe-security-groups --filter Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978 --query 'SecurityGroups[*].[GroupId]' --output text --profile $PROFILE) - SUBNET_ID=$(aws ec2 describe-subnets --filters 'Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978' 'Name=availability-zone,Values=us-east-2a' --query 'Subnets[0].SubnetId' --output text --profile $PROFILE) - echo $AMI, $SECURITY_GROUP, $SUBNET_ID + - cat aws_ssh_key - aws ec2 run-instances --image-id $AMI --instance-type $INSTANCE --security-group-ids $SECURITY_GROUP --subnet-id $SUBNET_ID --key-name aws_ssh_key --tag-specifications 'ResourceType=instance,Tags=[{Key=Name,Value=$INSTANCE_NAME}]' - PUBLIC_IP=$(aws ec2 describe-instances --filters 'Name=tag:Name, Values=Single_Node_Deployment' 'Name=instance-state-name, Values=running' --query 'Reservations[*].Instances[*].PublicIpAddress' --output text --profile $PROFILE) - echo $PUBLIC_IP \ No newline at end of file From 021af2c5e5073b06b016e5e683ab1a1f56b2522c Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Fri, 5 Nov 2021 10:56:10 -0500 Subject: [PATCH 35/98] changed ssh key name to match aws --- .gitlab/.gitlab-ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 065e20bbc..897119510 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -199,13 +199,13 @@ deploy ec2 instance: - eval `ssh-agent -s` - mkdir -p /root/.ssh - chmod 700 /root/.ssh - - echo $AWS_SSH_PRIVATE_KEY > aws_ssh_key + - echo $AWS_SSH_PRIVATE_KEY > gitlab-featurebase-dev script: - AMI=$(aws ssm get-parameters --names /aws/service/ami-amazon-linux-latest/amzn2-ami-hvm-x86_64-ebs --query 'Parameters[0].[Value]' --output text --profile $PROFILE) - SECURITY_GROUP=$(aws ec2 describe-security-groups --filter Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978 --query 'SecurityGroups[*].[GroupId]' --output text --profile $PROFILE) - SUBNET_ID=$(aws ec2 describe-subnets --filters 'Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978' 'Name=availability-zone,Values=us-east-2a' --query 'Subnets[0].SubnetId' --output text --profile $PROFILE) - echo $AMI, $SECURITY_GROUP, $SUBNET_ID - - cat aws_ssh_key - - aws ec2 run-instances --image-id $AMI --instance-type $INSTANCE --security-group-ids $SECURITY_GROUP --subnet-id $SUBNET_ID --key-name aws_ssh_key --tag-specifications 'ResourceType=instance,Tags=[{Key=Name,Value=$INSTANCE_NAME}]' + - cat gitlab-featurebase-dev + - aws ec2 run-instances --image-id $AMI --instance-type $INSTANCE --security-group-ids $SECURITY_GROUP --subnet-id $SUBNET_ID --key-name gitlab-featurebase-dev --tag-specifications 'ResourceType=instance,Tags=[{Key=Name,Value=$INSTANCE_NAME}]' - PUBLIC_IP=$(aws ec2 describe-instances --filters 'Name=tag:Name, Values=Single_Node_Deployment' 'Name=instance-state-name, Values=running' --query 'Reservations[*].Instances[*].PublicIpAddress' --output text --profile $PROFILE) - echo $PUBLIC_IP \ No newline at end of file From a9ab597d28cba8e39e995580794416d99ac51ed5 Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Fri, 5 Nov 2021 11:20:21 -0500 Subject: [PATCH 36/98] added profile to start-instance --- .gitlab/.gitlab-ci.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 897119510..6f846c13a 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -204,8 +204,6 @@ deploy ec2 instance: - AMI=$(aws ssm get-parameters --names /aws/service/ami-amazon-linux-latest/amzn2-ami-hvm-x86_64-ebs --query 'Parameters[0].[Value]' --output text --profile $PROFILE) - SECURITY_GROUP=$(aws ec2 describe-security-groups --filter Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978 --query 'SecurityGroups[*].[GroupId]' --output text --profile $PROFILE) - SUBNET_ID=$(aws ec2 describe-subnets --filters 'Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978' 'Name=availability-zone,Values=us-east-2a' --query 'Subnets[0].SubnetId' --output text --profile $PROFILE) - - echo $AMI, $SECURITY_GROUP, $SUBNET_ID - - cat gitlab-featurebase-dev - - aws ec2 run-instances --image-id $AMI --instance-type $INSTANCE --security-group-ids $SECURITY_GROUP --subnet-id $SUBNET_ID --key-name gitlab-featurebase-dev --tag-specifications 'ResourceType=instance,Tags=[{Key=Name,Value=$INSTANCE_NAME}]' + - aws ec2 run-instances --image-id $AMI --instance-type $INSTANCE --security-group-ids $SECURITY_GROUP --subnet-id $SUBNET_ID --key-name gitlab-featurebase-dev --tag-specifications 'ResourceType=instance,Tags=[{Key=Name,Value=$INSTANCE_NAME}]' --profile $PROFILE - PUBLIC_IP=$(aws ec2 describe-instances --filters 'Name=tag:Name, Values=Single_Node_Deployment' 'Name=instance-state-name, Values=running' --query 'Reservations[*].Instances[*].PublicIpAddress' --output text --profile $PROFILE) - echo $PUBLIC_IP \ No newline at end of file From 6e6d7a77db68baffe0d52fbe8f0a7cf19c42ec82 Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Fri, 5 Nov 2021 11:26:09 -0500 Subject: [PATCH 37/98] debug echo statements --- .gitlab/.gitlab-ci.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 6f846c13a..d9470df4e 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -195,6 +195,9 @@ deploy ec2 instance: - aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY - aws configure set region "us-east-2" - aws configure set aws_profile $PROFILE + - cat .aws/credentials + - cat .aws/config + - cat .aws/sso - 'which ssh-agent || ( apt-get update -y && apt-get install openssh-client -y )' - eval `ssh-agent -s` - mkdir -p /root/.ssh From ab3679bef797c3b8a8a9d1a2596acb8e21485e69 Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Fri, 5 Nov 2021 11:33:17 -0500 Subject: [PATCH 38/98] sanity check --- .gitlab/.gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index d9470df4e..8ced2aada 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -195,9 +195,9 @@ deploy ec2 instance: - aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY - aws configure set region "us-east-2" - aws configure set aws_profile $PROFILE - - cat .aws/credentials - cat .aws/config - - cat .aws/sso + - cat .aws/sso + - cat .aws/credentials - 'which ssh-agent || ( apt-get update -y && apt-get install openssh-client -y )' - eval `ssh-agent -s` - mkdir -p /root/.ssh From d080164a49ae689ca8f77ee6f8fcbb5da5620b44 Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Fri, 5 Nov 2021 11:35:54 -0500 Subject: [PATCH 39/98] echo aws config --- .gitlab/.gitlab-ci.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 8ced2aada..38e0615e5 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -195,9 +195,7 @@ deploy ec2 instance: - aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY - aws configure set region "us-east-2" - aws configure set aws_profile $PROFILE - - cat .aws/config - - cat .aws/sso - - cat .aws/credentials + - aws configure list - 'which ssh-agent || ( apt-get update -y && apt-get install openssh-client -y )' - eval `ssh-agent -s` - mkdir -p /root/.ssh From 7c02e9c447fed82c02988dfd020f0b3cb1cbe682 Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Fri, 5 Nov 2021 12:37:15 -0500 Subject: [PATCH 40/98] added scp --- .gitlab/.gitlab-ci.yml | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 38e0615e5..3c39fa448 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -176,8 +176,8 @@ build container fb: - docker push $tag - echo Created docker featurebase image with tag "$tag" -deploy ec2 instance: - stage: lint +deploy node for linux arm64: + stage: integration image: registry.gitlab.com/gitlab-org/cloud-deploy/aws-base:latest variables: AMI: "" @@ -188,14 +188,12 @@ deploy ec2 instance: AWS_ACCESS_KEY_ID: $AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY: $AWS_SECRET_ACCESS_KEY PROFILE: "default" - INSTANCE_NAME: "Single_Node_Deployment" AWS_SSH_PRIVATE_KEY: $AWS_SSH_PRIVATE_KEY before_script: - aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID - aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY - aws configure set region "us-east-2" - aws configure set aws_profile $PROFILE - - aws configure list - 'which ssh-agent || ( apt-get update -y && apt-get install openssh-client -y )' - eval `ssh-agent -s` - mkdir -p /root/.ssh @@ -205,6 +203,9 @@ deploy ec2 instance: - AMI=$(aws ssm get-parameters --names /aws/service/ami-amazon-linux-latest/amzn2-ami-hvm-x86_64-ebs --query 'Parameters[0].[Value]' --output text --profile $PROFILE) - SECURITY_GROUP=$(aws ec2 describe-security-groups --filter Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978 --query 'SecurityGroups[*].[GroupId]' --output text --profile $PROFILE) - SUBNET_ID=$(aws ec2 describe-subnets --filters 'Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978' 'Name=availability-zone,Values=us-east-2a' --query 'Subnets[0].SubnetId' --output text --profile $PROFILE) - - aws ec2 run-instances --image-id $AMI --instance-type $INSTANCE --security-group-ids $SECURITY_GROUP --subnet-id $SUBNET_ID --key-name gitlab-featurebase-dev --tag-specifications 'ResourceType=instance,Tags=[{Key=Name,Value=$INSTANCE_NAME}]' --profile $PROFILE + - aws ec2 run-instances --image-id $AMI --instance-type $INSTANCE --security-group-ids $SECURITY_GROUP --subnet-id $SUBNET_ID --key-name gitlab-featurebase-dev --tag-specifications 'ResourceType=instance,Tags=[{Key=Name,Value=Single_Node_Deployment}]' --profile $PROFILE - PUBLIC_IP=$(aws ec2 describe-instances --filters 'Name=tag:Name, Values=Single_Node_Deployment' 'Name=instance-state-name, Values=running' --query 'Reservations[*].Instances[*].PublicIpAddress' --output text --profile $PROFILE) - - echo $PUBLIC_IP \ No newline at end of file + - echo $PUBLIC_IP + - scp featurebase_linux_amd64 ec2-user@$PUBLIC_IP + dependencies: + - build for linux arm64 \ No newline at end of file From 86407ebcabeb012efb1f3c66796c111d877d8d7c Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Fri, 5 Nov 2021 12:39:11 -0500 Subject: [PATCH 41/98] sanity check for scp --- .gitlab/.gitlab-ci.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 3c39fa448..a65a9c0f8 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -177,7 +177,7 @@ build container fb: - echo Created docker featurebase image with tag "$tag" deploy node for linux arm64: - stage: integration + stage: lint image: registry.gitlab.com/gitlab-org/cloud-deploy/aws-base:latest variables: AMI: "" @@ -206,6 +206,5 @@ deploy node for linux arm64: - aws ec2 run-instances --image-id $AMI --instance-type $INSTANCE --security-group-ids $SECURITY_GROUP --subnet-id $SUBNET_ID --key-name gitlab-featurebase-dev --tag-specifications 'ResourceType=instance,Tags=[{Key=Name,Value=Single_Node_Deployment}]' --profile $PROFILE - PUBLIC_IP=$(aws ec2 describe-instances --filters 'Name=tag:Name, Values=Single_Node_Deployment' 'Name=instance-state-name, Values=running' --query 'Reservations[*].Instances[*].PublicIpAddress' --output text --profile $PROFILE) - echo $PUBLIC_IP - - scp featurebase_linux_amd64 ec2-user@$PUBLIC_IP - dependencies: - - build for linux arm64 \ No newline at end of file + - echo "scp test" > test.txt + - scp test.txt ec2-user@$PUBLIC_IP From 4f964aeef4f4c1a381fb17b9f9875689dc3163d6 Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Fri, 5 Nov 2021 12:53:39 -0500 Subject: [PATCH 42/98] updated ssh config --- .gitlab/.gitlab-ci.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index a65a9c0f8..6ea55f1d8 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -196,8 +196,9 @@ deploy node for linux arm64: - aws configure set aws_profile $PROFILE - 'which ssh-agent || ( apt-get update -y && apt-get install openssh-client -y )' - eval `ssh-agent -s` - - mkdir -p /root/.ssh - - chmod 700 /root/.ssh + - ssh-add <(echo "$SSH_PRIVATE_KEY") + - mkdir -p ~/.ssh + - '[[ -f /.dockerenv ]] && echo -e "Host *\n\tStrictHostKeyChecking no\n\n" > ~/.ssh/config' - echo $AWS_SSH_PRIVATE_KEY > gitlab-featurebase-dev script: - AMI=$(aws ssm get-parameters --names /aws/service/ami-amazon-linux-latest/amzn2-ami-hvm-x86_64-ebs --query 'Parameters[0].[Value]' --output text --profile $PROFILE) @@ -207,4 +208,4 @@ deploy node for linux arm64: - PUBLIC_IP=$(aws ec2 describe-instances --filters 'Name=tag:Name, Values=Single_Node_Deployment' 'Name=instance-state-name, Values=running' --query 'Reservations[*].Instances[*].PublicIpAddress' --output text --profile $PROFILE) - echo $PUBLIC_IP - echo "scp test" > test.txt - - scp test.txt ec2-user@$PUBLIC_IP + - scp -i gitlab-featurebase-dev test.txt ec2-user@$PUBLIC_IP From 642d4fcb868aeca817fa942ccf3ff9f0dfef0da1 Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Fri, 5 Nov 2021 13:27:03 -0500 Subject: [PATCH 43/98] fixed variable name --- .gitlab/.gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 6ea55f1d8..8b63d04f2 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -196,7 +196,7 @@ deploy node for linux arm64: - aws configure set aws_profile $PROFILE - 'which ssh-agent || ( apt-get update -y && apt-get install openssh-client -y )' - eval `ssh-agent -s` - - ssh-add <(echo "$SSH_PRIVATE_KEY") + - ssh-add <(echo "$AWS_SSH_PRIVATE_KEY") - mkdir -p ~/.ssh - '[[ -f /.dockerenv ]] && echo -e "Host *\n\tStrictHostKeyChecking no\n\n" > ~/.ssh/config' - echo $AWS_SSH_PRIVATE_KEY > gitlab-featurebase-dev From fae8d2d1163ebaf46378782c4ecccf4d458c3768 Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Fri, 5 Nov 2021 13:32:15 -0500 Subject: [PATCH 44/98] ssh add command --- .gitlab/.gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 8b63d04f2..69bce4706 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -196,7 +196,7 @@ deploy node for linux arm64: - aws configure set aws_profile $PROFILE - 'which ssh-agent || ( apt-get update -y && apt-get install openssh-client -y )' - eval `ssh-agent -s` - - ssh-add <(echo "$AWS_SSH_PRIVATE_KEY") + - echo -n "$AWS_SSH_PRIVATE_KEY" | tr -d '\r' | ssh-add - > /dev/null - mkdir -p ~/.ssh - '[[ -f /.dockerenv ]] && echo -e "Host *\n\tStrictHostKeyChecking no\n\n" > ~/.ssh/config' - echo $AWS_SSH_PRIVATE_KEY > gitlab-featurebase-dev From 4b2322160bee6a0d669939a617fee5193434f351 Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Fri, 5 Nov 2021 13:41:13 -0500 Subject: [PATCH 45/98] sanity --- .gitlab/.gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 69bce4706..6685ccdcc 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -196,7 +196,7 @@ deploy node for linux arm64: - aws configure set aws_profile $PROFILE - 'which ssh-agent || ( apt-get update -y && apt-get install openssh-client -y )' - eval `ssh-agent -s` - - echo -n "$AWS_SSH_PRIVATE_KEY" | tr -d '\r' | ssh-add - > /dev/null + - echo "$AWS_SSH_PRIVATE_KEY" | tr -d '\r' | ssh-add - - mkdir -p ~/.ssh - '[[ -f /.dockerenv ]] && echo -e "Host *\n\tStrictHostKeyChecking no\n\n" > ~/.ssh/config' - echo $AWS_SSH_PRIVATE_KEY > gitlab-featurebase-dev From 98e5f6deaf6ad57c63f74b89cbe5904b308d28e9 Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Fri, 5 Nov 2021 13:45:11 -0500 Subject: [PATCH 46/98] ssh key --- .gitlab/server.sh | 5 +++++ 1 file changed, 5 insertions(+) create mode 100755 .gitlab/server.sh diff --git a/.gitlab/server.sh b/.gitlab/server.sh new file mode 100755 index 000000000..5c7571b83 --- /dev/null +++ b/.gitlab/server.sh @@ -0,0 +1,5 @@ +#cloud-config +runcmd: + - sudo yum update + - sudo yum install postgresql14 + - sudo yum install golang1.16 \ No newline at end of file From cf365db944e7426b0f0a79b30d2a27361bf01905 Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Fri, 5 Nov 2021 13:47:47 -0500 Subject: [PATCH 47/98] changed ssh config --- .gitlab/.gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 6685ccdcc..60bd54c3f 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -194,12 +194,12 @@ deploy node for linux arm64: - aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY - aws configure set region "us-east-2" - aws configure set aws_profile $PROFILE + - echo $AWS_SSH_PRIVATE_KEY > gitlab-featurebase-dev - 'which ssh-agent || ( apt-get update -y && apt-get install openssh-client -y )' - eval `ssh-agent -s` - - echo "$AWS_SSH_PRIVATE_KEY" | tr -d '\r' | ssh-add - + - echo gitlab-featurebase-dev | tr -d '\r' | ssh-add - - mkdir -p ~/.ssh - '[[ -f /.dockerenv ]] && echo -e "Host *\n\tStrictHostKeyChecking no\n\n" > ~/.ssh/config' - - echo $AWS_SSH_PRIVATE_KEY > gitlab-featurebase-dev script: - AMI=$(aws ssm get-parameters --names /aws/service/ami-amazon-linux-latest/amzn2-ami-hvm-x86_64-ebs --query 'Parameters[0].[Value]' --output text --profile $PROFILE) - SECURITY_GROUP=$(aws ec2 describe-security-groups --filter Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978 --query 'SecurityGroups[*].[GroupId]' --output text --profile $PROFILE) From 7a5ba0e5d6a452572c3a2dc89ffde7ee22610db4 Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Fri, 5 Nov 2021 13:52:26 -0500 Subject: [PATCH 48/98] Revert "sanity check for scp" This reverts commit 86407ebcabeb012efb1f3c66796c111d877d8d7c. --- .gitlab/.gitlab-ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 60bd54c3f..876185e57 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -205,7 +205,8 @@ deploy node for linux arm64: - SECURITY_GROUP=$(aws ec2 describe-security-groups --filter Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978 --query 'SecurityGroups[*].[GroupId]' --output text --profile $PROFILE) - SUBNET_ID=$(aws ec2 describe-subnets --filters 'Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978' 'Name=availability-zone,Values=us-east-2a' --query 'Subnets[0].SubnetId' --output text --profile $PROFILE) - aws ec2 run-instances --image-id $AMI --instance-type $INSTANCE --security-group-ids $SECURITY_GROUP --subnet-id $SUBNET_ID --key-name gitlab-featurebase-dev --tag-specifications 'ResourceType=instance,Tags=[{Key=Name,Value=Single_Node_Deployment}]' --profile $PROFILE + - sleep 30 - PUBLIC_IP=$(aws ec2 describe-instances --filters 'Name=tag:Name, Values=Single_Node_Deployment' 'Name=instance-state-name, Values=running' --query 'Reservations[*].Instances[*].PublicIpAddress' --output text --profile $PROFILE) - echo $PUBLIC_IP - echo "scp test" > test.txt - - scp -i gitlab-featurebase-dev test.txt ec2-user@$PUBLIC_IP + - scp -o StrictHostKeyChecking=no -i gitlab-featurebase-dev test.txt ec2-user@$PUBLIC_IP From 590e5edc57b12740be233b901d0ffca88bde17cf Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Fri, 5 Nov 2021 13:57:37 -0500 Subject: [PATCH 49/98] reverted to last working version --- .gitlab/.gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 876185e57..04dbe2178 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -197,8 +197,8 @@ deploy node for linux arm64: - echo $AWS_SSH_PRIVATE_KEY > gitlab-featurebase-dev - 'which ssh-agent || ( apt-get update -y && apt-get install openssh-client -y )' - eval `ssh-agent -s` - - echo gitlab-featurebase-dev | tr -d '\r' | ssh-add - - mkdir -p ~/.ssh + - chmod 700 /root/.ssh - '[[ -f /.dockerenv ]] && echo -e "Host *\n\tStrictHostKeyChecking no\n\n" > ~/.ssh/config' script: - AMI=$(aws ssm get-parameters --names /aws/service/ami-amazon-linux-latest/amzn2-ami-hvm-x86_64-ebs --query 'Parameters[0].[Value]' --output text --profile $PROFILE) From 179a0e7624cb52fbe7d1c19f9f3fd349e4cdb74b Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Fri, 5 Nov 2021 14:03:14 -0500 Subject: [PATCH 50/98] fixed typo for copy path --- .gitlab/.gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 04dbe2178..a3ba3f4e1 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -209,4 +209,4 @@ deploy node for linux arm64: - PUBLIC_IP=$(aws ec2 describe-instances --filters 'Name=tag:Name, Values=Single_Node_Deployment' 'Name=instance-state-name, Values=running' --query 'Reservations[*].Instances[*].PublicIpAddress' --output text --profile $PROFILE) - echo $PUBLIC_IP - echo "scp test" > test.txt - - scp -o StrictHostKeyChecking=no -i gitlab-featurebase-dev test.txt ec2-user@$PUBLIC_IP + - scp -o StrictHostKeyChecking=no -i gitlab-featurebase-dev test.txt ec2-user@$PUBLIC_IP:. From bf93278ca30c370e5e9630b6d4e76be7db8b6e37 Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Fri, 5 Nov 2021 14:24:02 -0500 Subject: [PATCH 51/98] scp binary --- .gitlab/.gitlab-ci.yml | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index a3ba3f4e1..beedce17b 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -176,8 +176,8 @@ build container fb: - docker push $tag - echo Created docker featurebase image with tag "$tag" -deploy node for linux arm64: - stage: lint +deploy node for linux amd64: + stage: integration image: registry.gitlab.com/gitlab-org/cloud-deploy/aws-base:latest variables: AMI: "" @@ -204,9 +204,16 @@ deploy node for linux arm64: - AMI=$(aws ssm get-parameters --names /aws/service/ami-amazon-linux-latest/amzn2-ami-hvm-x86_64-ebs --query 'Parameters[0].[Value]' --output text --profile $PROFILE) - SECURITY_GROUP=$(aws ec2 describe-security-groups --filter Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978 --query 'SecurityGroups[*].[GroupId]' --output text --profile $PROFILE) - SUBNET_ID=$(aws ec2 describe-subnets --filters 'Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978' 'Name=availability-zone,Values=us-east-2a' --query 'Subnets[0].SubnetId' --output text --profile $PROFILE) - - aws ec2 run-instances --image-id $AMI --instance-type $INSTANCE --security-group-ids $SECURITY_GROUP --subnet-id $SUBNET_ID --key-name gitlab-featurebase-dev --tag-specifications 'ResourceType=instance,Tags=[{Key=Name,Value=Single_Node_Deployment}]' --profile $PROFILE + - aws ec2 run-instances --image-id $AMI --instance-type $INSTANCE --security-group-ids $SECURITY_GROUP --subnet-id $SUBNET_ID --key-name gitlab-featurebase-dev --tag-specifications 'ResourceType=instance,Tags=[{Key=Name,Value=linux-amd64-node}]' --profile $PROFILE - sleep 30 - - PUBLIC_IP=$(aws ec2 describe-instances --filters 'Name=tag:Name, Values=Single_Node_Deployment' 'Name=instance-state-name, Values=running' --query 'Reservations[*].Instances[*].PublicIpAddress' --output text --profile $PROFILE) + - PUBLIC_IP=$(aws ec2 describe-instances --filters 'Name=tag:Name, Values=linux-amd64-node' 'Name=instance-state-name, Values=running' --query 'Reservations[*].Instances[*].PublicIpAddress' --output text --profile $PROFILE) - echo $PUBLIC_IP - - echo "scp test" > test.txt - - scp -o StrictHostKeyChecking=no -i gitlab-featurebase-dev test.txt ec2-user@$PUBLIC_IP:. + - INSTANCE_ID=$(aws ec2 describe-instances --filters 'Name=tag:Name, Values=linux-amd64-node' 'Name=instance-state-name, Values=running' --query 'Reservations[*].Instances[*].InstanceId' --output text --profile $PROFILE) + - echo $INSTANCE_ID > linux-amd64-instance.txt + - echo $INSTANCE_ID + - scp -o StrictHostKeyChecking=no -i gitlab-featurebase-dev featurebase_linux_amd64 ec2-user@$PUBLIC_IP:. + dependencies: + jobs: build for linux amd64 + artifacts: + paths: + - linux-amd64-instance.txt \ No newline at end of file From 6531f907472cd02399a0de62a82a37792111642b Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Fri, 5 Nov 2021 14:26:20 -0500 Subject: [PATCH 52/98] fixed syntax error --- .gitlab/.gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index beedce17b..5621f38cc 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -212,8 +212,8 @@ deploy node for linux amd64: - echo $INSTANCE_ID > linux-amd64-instance.txt - echo $INSTANCE_ID - scp -o StrictHostKeyChecking=no -i gitlab-featurebase-dev featurebase_linux_amd64 ec2-user@$PUBLIC_IP:. - dependencies: - jobs: build for linux amd64 + needs: + job: build for linux amd64 artifacts: paths: - linux-amd64-instance.txt \ No newline at end of file From 103312be8f9fd53a268788f8672d50e643f674a5 Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Fri, 5 Nov 2021 15:04:09 -0500 Subject: [PATCH 53/98] added cloutinit --- .gitlab/.gitlab-ci.yml | 6 +++--- .gitlab/server.sh | 5 ----- .gitlab/server.yml | 9 ++++++--- 3 files changed, 9 insertions(+), 11 deletions(-) delete mode 100755 .gitlab/server.sh diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 5621f38cc..29a6e5b64 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -109,7 +109,7 @@ upload to sonarcloud: - job: run jest tests build for linux amd64: - stage: build + stage: lint image: registry.gitlab.com/molecula/featurebase/builder:0.0.3 extends: .go-cache variables: @@ -177,7 +177,7 @@ build container fb: - echo Created docker featurebase image with tag "$tag" deploy node for linux amd64: - stage: integration + stage: lint image: registry.gitlab.com/gitlab-org/cloud-deploy/aws-base:latest variables: AMI: "" @@ -204,7 +204,7 @@ deploy node for linux amd64: - AMI=$(aws ssm get-parameters --names /aws/service/ami-amazon-linux-latest/amzn2-ami-hvm-x86_64-ebs --query 'Parameters[0].[Value]' --output text --profile $PROFILE) - SECURITY_GROUP=$(aws ec2 describe-security-groups --filter Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978 --query 'SecurityGroups[*].[GroupId]' --output text --profile $PROFILE) - SUBNET_ID=$(aws ec2 describe-subnets --filters 'Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978' 'Name=availability-zone,Values=us-east-2a' --query 'Subnets[0].SubnetId' --output text --profile $PROFILE) - - aws ec2 run-instances --image-id $AMI --instance-type $INSTANCE --security-group-ids $SECURITY_GROUP --subnet-id $SUBNET_ID --key-name gitlab-featurebase-dev --tag-specifications 'ResourceType=instance,Tags=[{Key=Name,Value=linux-amd64-node}]' --profile $PROFILE + - aws ec2 run-instances --image-id $AMI --instance-type $INSTANCE --security-group-ids $SECURITY_GROUP --subnet-id $SUBNET_ID --key-name gitlab-featurebase-dev --tag-specifications 'ResourceType=instance,Tags=[{Key=Name,Value=linux-amd64-node}]' --profile $PROFILE --user-data-file server.yml - sleep 30 - PUBLIC_IP=$(aws ec2 describe-instances --filters 'Name=tag:Name, Values=linux-amd64-node' 'Name=instance-state-name, Values=running' --query 'Reservations[*].Instances[*].PublicIpAddress' --output text --profile $PROFILE) - echo $PUBLIC_IP diff --git a/.gitlab/server.sh b/.gitlab/server.sh deleted file mode 100755 index 5c7571b83..000000000 --- a/.gitlab/server.sh +++ /dev/null @@ -1,5 +0,0 @@ -#cloud-config -runcmd: - - sudo yum update - - sudo yum install postgresql14 - - sudo yum install golang1.16 \ No newline at end of file diff --git a/.gitlab/server.yml b/.gitlab/server.yml index 5c7571b83..3fe5243ff 100755 --- a/.gitlab/server.yml +++ b/.gitlab/server.yml @@ -1,5 +1,8 @@ #cloud-config +repo_update: true +repo_upgrade: all + runcmd: - - sudo yum update - - sudo yum install postgresql14 - - sudo yum install golang1.16 \ No newline at end of file + - yum update + - yum install postgresql + - yum install golang \ No newline at end of file From 78713c6a8c900f0efc3c2caa5ad6042e77e8ce11 Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Sun, 7 Nov 2021 08:39:43 -0600 Subject: [PATCH 54/98] test cloudinit --- .gitlab/cloud-init.sh | 24 ++++++++++++++++++++++++ .gitlab/server.yml | 8 -------- 2 files changed, 24 insertions(+), 8 deletions(-) create mode 100755 .gitlab/cloud-init.sh delete mode 100755 .gitlab/server.yml diff --git a/.gitlab/cloud-init.sh b/.gitlab/cloud-init.sh new file mode 100755 index 000000000..3ef2133ff --- /dev/null +++ b/.gitlab/cloud-init.sh @@ -0,0 +1,24 @@ +#!/bin/bash -ex +# generate log +exec > >(tee /var/log/user-data.log|logger -t user-data -s 2>/dev/console) 2>&1 + +# Install packages +yum update -y +yum install golang -y # latest verion in ec2 is 1.15.14 +yum install postgresql -y + +# Configure host system +# N=200000 +# sysctl -w vm.max_map_count=N +# echo N > /proc/sys/vm/max_map_count +ulimit -n -S 200000 + +# # install go 1.16.9 +# curl -O https://dl.google.com/go/go1.16.9.linux-amd64.tar.gz +# tar xvf go1.16.9.linux-amd64.tar.gz +# chown -R root:root ./go +# mv go /usr/local +# echo 'export GOPATH=$HOME/work' >> ~/.profile +# echo 'export PATH=$PATH:/usr/local/go/bin:$GOPATH/bin' >> ~/.profile +# chown ec2-user ~/.profile +# source ~/.profile diff --git a/.gitlab/server.yml b/.gitlab/server.yml deleted file mode 100755 index 3fe5243ff..000000000 --- a/.gitlab/server.yml +++ /dev/null @@ -1,8 +0,0 @@ -#cloud-config -repo_update: true -repo_upgrade: all - -runcmd: - - yum update - - yum install postgresql - - yum install golang \ No newline at end of file From 606b7184289b564322408e43fc06ac067ff1a0b0 Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Sun, 7 Nov 2021 08:42:40 -0600 Subject: [PATCH 55/98] fixed path for cloud-init --- .gitlab/.gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 29a6e5b64..92a82b0f3 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -204,7 +204,7 @@ deploy node for linux amd64: - AMI=$(aws ssm get-parameters --names /aws/service/ami-amazon-linux-latest/amzn2-ami-hvm-x86_64-ebs --query 'Parameters[0].[Value]' --output text --profile $PROFILE) - SECURITY_GROUP=$(aws ec2 describe-security-groups --filter Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978 --query 'SecurityGroups[*].[GroupId]' --output text --profile $PROFILE) - SUBNET_ID=$(aws ec2 describe-subnets --filters 'Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978' 'Name=availability-zone,Values=us-east-2a' --query 'Subnets[0].SubnetId' --output text --profile $PROFILE) - - aws ec2 run-instances --image-id $AMI --instance-type $INSTANCE --security-group-ids $SECURITY_GROUP --subnet-id $SUBNET_ID --key-name gitlab-featurebase-dev --tag-specifications 'ResourceType=instance,Tags=[{Key=Name,Value=linux-amd64-node}]' --profile $PROFILE --user-data-file server.yml + - aws ec2 run-instances --image-id $AMI --instance-type $INSTANCE --security-group-ids $SECURITY_GROUP --subnet-id $SUBNET_ID --key-name gitlab-featurebase-dev --tag-specifications 'ResourceType=instance,Tags=[{Key=Name,Value=linux-amd64-node}]' --profile $PROFILE --user-data file://featurebase/.gitlab/cloud-init.sh - sleep 30 - PUBLIC_IP=$(aws ec2 describe-instances --filters 'Name=tag:Name, Values=linux-amd64-node' 'Name=instance-state-name, Values=running' --query 'Reservations[*].Instances[*].PublicIpAddress' --output text --profile $PROFILE) - echo $PUBLIC_IP From 0e564a00e8b4d25163feb0de9bdef043c3ca538d Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Sun, 7 Nov 2021 08:47:10 -0600 Subject: [PATCH 56/98] path for cloud-init --- .gitlab/.gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 92a82b0f3..93714a717 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -204,7 +204,7 @@ deploy node for linux amd64: - AMI=$(aws ssm get-parameters --names /aws/service/ami-amazon-linux-latest/amzn2-ami-hvm-x86_64-ebs --query 'Parameters[0].[Value]' --output text --profile $PROFILE) - SECURITY_GROUP=$(aws ec2 describe-security-groups --filter Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978 --query 'SecurityGroups[*].[GroupId]' --output text --profile $PROFILE) - SUBNET_ID=$(aws ec2 describe-subnets --filters 'Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978' 'Name=availability-zone,Values=us-east-2a' --query 'Subnets[0].SubnetId' --output text --profile $PROFILE) - - aws ec2 run-instances --image-id $AMI --instance-type $INSTANCE --security-group-ids $SECURITY_GROUP --subnet-id $SUBNET_ID --key-name gitlab-featurebase-dev --tag-specifications 'ResourceType=instance,Tags=[{Key=Name,Value=linux-amd64-node}]' --profile $PROFILE --user-data file://featurebase/.gitlab/cloud-init.sh + - aws ec2 run-instances --image-id $AMI --instance-type $INSTANCE --security-group-ids $SECURITY_GROUP --subnet-id $SUBNET_ID --key-name gitlab-featurebase-dev --tag-specifications 'ResourceType=instance,Tags=[{Key=Name,Value=linux-amd64-node}]' --profile $PROFILE --user-data file://cloud-init.sh - sleep 30 - PUBLIC_IP=$(aws ec2 describe-instances --filters 'Name=tag:Name, Values=linux-amd64-node' 'Name=instance-state-name, Values=running' --query 'Reservations[*].Instances[*].PublicIpAddress' --output text --profile $PROFILE) - echo $PUBLIC_IP From 57d4eb45ded68ef704c1a6479e90342be5baf698 Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Sun, 7 Nov 2021 08:53:01 -0600 Subject: [PATCH 57/98] gitlab path test2 --- .gitlab/.gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 93714a717..26a786640 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -204,7 +204,7 @@ deploy node for linux amd64: - AMI=$(aws ssm get-parameters --names /aws/service/ami-amazon-linux-latest/amzn2-ami-hvm-x86_64-ebs --query 'Parameters[0].[Value]' --output text --profile $PROFILE) - SECURITY_GROUP=$(aws ec2 describe-security-groups --filter Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978 --query 'SecurityGroups[*].[GroupId]' --output text --profile $PROFILE) - SUBNET_ID=$(aws ec2 describe-subnets --filters 'Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978' 'Name=availability-zone,Values=us-east-2a' --query 'Subnets[0].SubnetId' --output text --profile $PROFILE) - - aws ec2 run-instances --image-id $AMI --instance-type $INSTANCE --security-group-ids $SECURITY_GROUP --subnet-id $SUBNET_ID --key-name gitlab-featurebase-dev --tag-specifications 'ResourceType=instance,Tags=[{Key=Name,Value=linux-amd64-node}]' --profile $PROFILE --user-data file://cloud-init.sh + - aws ec2 run-instances --image-id $AMI --instance-type $INSTANCE --security-group-ids $SECURITY_GROUP --subnet-id $SUBNET_ID --key-name gitlab-featurebase-dev --tag-specifications 'ResourceType=instance,Tags=[{Key=Name,Value=linux-amd64-node}]' --profile $PROFILE --user-data file://.gitlab/cloud-init.sh - sleep 30 - PUBLIC_IP=$(aws ec2 describe-instances --filters 'Name=tag:Name, Values=linux-amd64-node' 'Name=instance-state-name, Values=running' --query 'Reservations[*].Instances[*].PublicIpAddress' --output text --profile $PROFILE) - echo $PUBLIC_IP From dfc98e3000a7e642e18c76d640946eb61f4b7484 Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Sun, 7 Nov 2021 09:11:52 -0600 Subject: [PATCH 58/98] updated sleep time --- .gitlab/.gitlab-ci.yml | 2 +- .gitlab/cloud-init.sh | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 26a786640..47b571d2e 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -205,7 +205,7 @@ deploy node for linux amd64: - SECURITY_GROUP=$(aws ec2 describe-security-groups --filter Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978 --query 'SecurityGroups[*].[GroupId]' --output text --profile $PROFILE) - SUBNET_ID=$(aws ec2 describe-subnets --filters 'Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978' 'Name=availability-zone,Values=us-east-2a' --query 'Subnets[0].SubnetId' --output text --profile $PROFILE) - aws ec2 run-instances --image-id $AMI --instance-type $INSTANCE --security-group-ids $SECURITY_GROUP --subnet-id $SUBNET_ID --key-name gitlab-featurebase-dev --tag-specifications 'ResourceType=instance,Tags=[{Key=Name,Value=linux-amd64-node}]' --profile $PROFILE --user-data file://.gitlab/cloud-init.sh - - sleep 30 + - sleep 120 - PUBLIC_IP=$(aws ec2 describe-instances --filters 'Name=tag:Name, Values=linux-amd64-node' 'Name=instance-state-name, Values=running' --query 'Reservations[*].Instances[*].PublicIpAddress' --output text --profile $PROFILE) - echo $PUBLIC_IP - INSTANCE_ID=$(aws ec2 describe-instances --filters 'Name=tag:Name, Values=linux-amd64-node' 'Name=instance-state-name, Values=running' --query 'Reservations[*].Instances[*].InstanceId' --output text --profile $PROFILE) diff --git a/.gitlab/cloud-init.sh b/.gitlab/cloud-init.sh index 3ef2133ff..e192938ba 100755 --- a/.gitlab/cloud-init.sh +++ b/.gitlab/cloud-init.sh @@ -8,10 +8,10 @@ yum install golang -y # latest verion in ec2 is 1.15.14 yum install postgresql -y # Configure host system -# N=200000 -# sysctl -w vm.max_map_count=N -# echo N > /proc/sys/vm/max_map_count -ulimit -n -S 200000 +cat /proc/sys/fs/file-max +sysctl -w fs.file-max=262144 +sysctl -p +cat /proc/sys/fs/file-max # # install go 1.16.9 # curl -O https://dl.google.com/go/go1.16.9.linux-amd64.tar.gz From f0b2e6beeaba2d074161cce23fe707ff40ca7b78 Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Sun, 7 Nov 2021 09:17:02 -0600 Subject: [PATCH 59/98] updated permissions for ssh key --- .gitlab/.gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 47b571d2e..b9ef214ff 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -195,6 +195,7 @@ deploy node for linux amd64: - aws configure set region "us-east-2" - aws configure set aws_profile $PROFILE - echo $AWS_SSH_PRIVATE_KEY > gitlab-featurebase-dev + - chmod 400 gitlab-featurebase-dev - 'which ssh-agent || ( apt-get update -y && apt-get install openssh-client -y )' - eval `ssh-agent -s` - mkdir -p ~/.ssh From 46d6ced5b46251095a9297909da95cb6ef7fca57 Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Sun, 7 Nov 2021 09:24:40 -0600 Subject: [PATCH 60/98] updated ssh permissions --- .gitlab/.gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index b9ef214ff..efb828c44 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -200,6 +200,7 @@ deploy node for linux amd64: - eval `ssh-agent -s` - mkdir -p ~/.ssh - chmod 700 /root/.ssh + - chmod 600 /root/.ssh/authorized_keys - '[[ -f /.dockerenv ]] && echo -e "Host *\n\tStrictHostKeyChecking no\n\n" > ~/.ssh/config' script: - AMI=$(aws ssm get-parameters --names /aws/service/ami-amazon-linux-latest/amzn2-ami-hvm-x86_64-ebs --query 'Parameters[0].[Value]' --output text --profile $PROFILE) From 76ba07e806e17f9bf5d4fce10a731414c69f29f9 Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Sun, 7 Nov 2021 09:29:46 -0600 Subject: [PATCH 61/98] update ssh key filename --- .gitlab/.gitlab-ci.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index efb828c44..bdd73b7d5 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -194,13 +194,12 @@ deploy node for linux amd64: - aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY - aws configure set region "us-east-2" - aws configure set aws_profile $PROFILE - - echo $AWS_SSH_PRIVATE_KEY > gitlab-featurebase-dev + - echo $AWS_SSH_PRIVATE_KEY > gitlab-featurebase-dev.pem - chmod 400 gitlab-featurebase-dev - 'which ssh-agent || ( apt-get update -y && apt-get install openssh-client -y )' - eval `ssh-agent -s` - mkdir -p ~/.ssh - chmod 700 /root/.ssh - - chmod 600 /root/.ssh/authorized_keys - '[[ -f /.dockerenv ]] && echo -e "Host *\n\tStrictHostKeyChecking no\n\n" > ~/.ssh/config' script: - AMI=$(aws ssm get-parameters --names /aws/service/ami-amazon-linux-latest/amzn2-ami-hvm-x86_64-ebs --query 'Parameters[0].[Value]' --output text --profile $PROFILE) @@ -213,7 +212,7 @@ deploy node for linux amd64: - INSTANCE_ID=$(aws ec2 describe-instances --filters 'Name=tag:Name, Values=linux-amd64-node' 'Name=instance-state-name, Values=running' --query 'Reservations[*].Instances[*].InstanceId' --output text --profile $PROFILE) - echo $INSTANCE_ID > linux-amd64-instance.txt - echo $INSTANCE_ID - - scp -o StrictHostKeyChecking=no -i gitlab-featurebase-dev featurebase_linux_amd64 ec2-user@$PUBLIC_IP:. + - scp -o StrictHostKeyChecking=no -i gitlab-featurebase-dev.pem featurebase_linux_amd64 ec2-user@$PUBLIC_IP:. needs: job: build for linux amd64 artifacts: From df99dd7578c92bcabdba1887eb6ad8e6543cac0a Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Sun, 7 Nov 2021 09:31:34 -0600 Subject: [PATCH 62/98] fixed typo --- .gitlab/.gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index bdd73b7d5..576eb8e7a 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -195,7 +195,7 @@ deploy node for linux amd64: - aws configure set region "us-east-2" - aws configure set aws_profile $PROFILE - echo $AWS_SSH_PRIVATE_KEY > gitlab-featurebase-dev.pem - - chmod 400 gitlab-featurebase-dev + - chmod 400 gitlab-featurebase-dev.pem - 'which ssh-agent || ( apt-get update -y && apt-get install openssh-client -y )' - eval `ssh-agent -s` - mkdir -p ~/.ssh From 5b5c003b1ed666226a1086d0b1f1e6e9e2f749d5 Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Mon, 8 Nov 2021 07:43:31 -0600 Subject: [PATCH 63/98] private key fix --- .gitlab/.gitlab-ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 576eb8e7a..903cfb53f 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -199,6 +199,7 @@ deploy node for linux amd64: - 'which ssh-agent || ( apt-get update -y && apt-get install openssh-client -y )' - eval `ssh-agent -s` - mkdir -p ~/.ssh + - echo "$AWS_SSH_PRIVATE_KEY" | ssh-add - - chmod 700 /root/.ssh - '[[ -f /.dockerenv ]] && echo -e "Host *\n\tStrictHostKeyChecking no\n\n" > ~/.ssh/config' script: @@ -212,7 +213,7 @@ deploy node for linux amd64: - INSTANCE_ID=$(aws ec2 describe-instances --filters 'Name=tag:Name, Values=linux-amd64-node' 'Name=instance-state-name, Values=running' --query 'Reservations[*].Instances[*].InstanceId' --output text --profile $PROFILE) - echo $INSTANCE_ID > linux-amd64-instance.txt - echo $INSTANCE_ID - - scp -o StrictHostKeyChecking=no -i gitlab-featurebase-dev.pem featurebase_linux_amd64 ec2-user@$PUBLIC_IP:. + - scp -i gitlab-featurebase-dev.pem featurebase_linux_amd64 ec2-user@$PUBLIC_IP:. needs: job: build for linux amd64 artifacts: From 79d1a005ce6f0873e45c6e13161d3dab9a75d629 Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Mon, 8 Nov 2021 07:55:42 -0600 Subject: [PATCH 64/98] job to terminate ec2 instance --- .gitlab/.gitlab-ci.yml | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 903cfb53f..e8a320a7b 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -214,8 +214,21 @@ deploy node for linux amd64: - echo $INSTANCE_ID > linux-amd64-instance.txt - echo $INSTANCE_ID - scp -i gitlab-featurebase-dev.pem featurebase_linux_amd64 ec2-user@$PUBLIC_IP:. + - sleep 120 needs: job: build for linux amd64 artifacts: paths: - - linux-amd64-instance.txt \ No newline at end of file + - linux-amd64-instance.txt + +terminate node for linux amd64: + stage: lint + image: registry.gitlab.com/gitlab-org/cloud-deploy/aws-base:latest + variables: + INSTANCE_ID + before_script: + - INSTANCE_ID=$(cat linux-amd64-instance.txt) + script: + - aws ec2 terminate-instances --instance-ids $INSTANCE_ID + needs: + job: deploy node for linux amd64 From 4c2832ab31e37f0303a6a3c04793f8dbf2b34d9c Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Mon, 8 Nov 2021 07:59:15 -0600 Subject: [PATCH 65/98] fixed yaml syntax error --- .gitlab/.gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index e8a320a7b..6d698f258 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -225,7 +225,7 @@ terminate node for linux amd64: stage: lint image: registry.gitlab.com/gitlab-org/cloud-deploy/aws-base:latest variables: - INSTANCE_ID + INSTANCE_ID: "" before_script: - INSTANCE_ID=$(cat linux-amd64-instance.txt) script: From 1befe2ad320a833ecf4e818d8ab1ccc2c902c76c Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Mon, 8 Nov 2021 08:10:20 -0600 Subject: [PATCH 66/98] added aws config for terminate job --- .gitlab/.gitlab-ci.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 6d698f258..a3599bbe7 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -227,6 +227,9 @@ terminate node for linux amd64: variables: INSTANCE_ID: "" before_script: + - aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID + - aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY + - aws configure set region "us-east-2" - INSTANCE_ID=$(cat linux-amd64-instance.txt) script: - aws ec2 terminate-instances --instance-ids $INSTANCE_ID From aae537424851ad244c0eb7aff02d92f6d92fba52 Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Mon, 8 Nov 2021 08:18:37 -0600 Subject: [PATCH 67/98] fixed stages --- .gitlab/.gitlab-ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index a3599bbe7..bfb881491 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -109,7 +109,7 @@ upload to sonarcloud: - job: run jest tests build for linux amd64: - stage: lint + stage: build image: registry.gitlab.com/molecula/featurebase/builder:0.0.3 extends: .go-cache variables: @@ -177,7 +177,7 @@ build container fb: - echo Created docker featurebase image with tag "$tag" deploy node for linux amd64: - stage: lint + stage: integration image: registry.gitlab.com/gitlab-org/cloud-deploy/aws-base:latest variables: AMI: "" @@ -222,7 +222,7 @@ deploy node for linux amd64: - linux-amd64-instance.txt terminate node for linux amd64: - stage: lint + stage: integration image: registry.gitlab.com/gitlab-org/cloud-deploy/aws-base:latest variables: INSTANCE_ID: "" From e629e0165de388ab2fb44cc8032f42487d98105d Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Mon, 8 Nov 2021 08:42:49 -0600 Subject: [PATCH 68/98] install correct version of go --- .gitlab/.gitlab-ci.yml | 6 +++--- .gitlab/cloud-init.sh | 22 ++++++++++------------ 2 files changed, 13 insertions(+), 15 deletions(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index bfb881491..a3599bbe7 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -109,7 +109,7 @@ upload to sonarcloud: - job: run jest tests build for linux amd64: - stage: build + stage: lint image: registry.gitlab.com/molecula/featurebase/builder:0.0.3 extends: .go-cache variables: @@ -177,7 +177,7 @@ build container fb: - echo Created docker featurebase image with tag "$tag" deploy node for linux amd64: - stage: integration + stage: lint image: registry.gitlab.com/gitlab-org/cloud-deploy/aws-base:latest variables: AMI: "" @@ -222,7 +222,7 @@ deploy node for linux amd64: - linux-amd64-instance.txt terminate node for linux amd64: - stage: integration + stage: lint image: registry.gitlab.com/gitlab-org/cloud-deploy/aws-base:latest variables: INSTANCE_ID: "" diff --git a/.gitlab/cloud-init.sh b/.gitlab/cloud-init.sh index e192938ba..18257a90b 100755 --- a/.gitlab/cloud-init.sh +++ b/.gitlab/cloud-init.sh @@ -4,21 +4,19 @@ exec > >(tee /var/log/user-data.log|logger -t user-data -s 2>/dev/console) 2>&1 # Install packages yum update -y -yum install golang -y # latest verion in ec2 is 1.15.14 +# yum install golang -y # latest verion in ec2 is 1.15.14 yum install postgresql -y # Configure host system -cat /proc/sys/fs/file-max +echo 'cat /proc/sys/fs/file-max' sysctl -w fs.file-max=262144 sysctl -p -cat /proc/sys/fs/file-max +echo 'cat /proc/sys/fs/file-max' -# # install go 1.16.9 -# curl -O https://dl.google.com/go/go1.16.9.linux-amd64.tar.gz -# tar xvf go1.16.9.linux-amd64.tar.gz -# chown -R root:root ./go -# mv go /usr/local -# echo 'export GOPATH=$HOME/work' >> ~/.profile -# echo 'export PATH=$PATH:/usr/local/go/bin:$GOPATH/bin' >> ~/.profile -# chown ec2-user ~/.profile -# source ~/.profile +# install go 1.16.9 +curl -O https://dl.google.com/go/go1.16.9.linux-amd64.tar.gz +tar xvf go1.16.9.linux-amd64.tar.gz +chown -R root:root ./go +mv go /usr/local +echo "export PATH=$PATH:/usr/local/go/bin" | sudo tee -a /etc/profile > /dev/null +source /etc/profile From ebde0142da01c1c4d205a7b363c2a782c638fd42 Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Mon, 8 Nov 2021 09:14:47 -0600 Subject: [PATCH 69/98] add stage to configure featurebase --- .gitlab/.gitlab-ci.yml | 29 ++++++++++++++++++++--------- .gitlab/cloud-init.sh | 4 ++-- .gitlab/configureFeatureBase.json | 20 ++++++++++++++++++++ .gitlab/featurebase.conf | 30 ++++++++++++++++++++++++++++++ .gitlab/featurebase.service | 13 +++++++++++++ 5 files changed, 85 insertions(+), 11 deletions(-) create mode 100644 .gitlab/configureFeatureBase.json create mode 100644 .gitlab/featurebase.conf create mode 100644 .gitlab/featurebase.service diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index a3599bbe7..bf210f5a9 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -176,6 +176,11 @@ build container fb: - docker push $tag - echo Created docker featurebase image with tag "$tag" +before_script: + - aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID + - aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY + - aws configure set region "us-east-2" + deploy node for linux amd64: stage: lint image: registry.gitlab.com/gitlab-org/cloud-deploy/aws-base:latest @@ -185,14 +190,9 @@ deploy node for linux amd64: SECURITY_GROUP: "" SUBNET_ID: "" PUBLIC_IP: "" - AWS_ACCESS_KEY_ID: $AWS_ACCESS_KEY_ID - AWS_SECRET_ACCESS_KEY: $AWS_SECRET_ACCESS_KEY PROFILE: "default" AWS_SSH_PRIVATE_KEY: $AWS_SSH_PRIVATE_KEY before_script: - - aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID - - aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY - - aws configure set region "us-east-2" - aws configure set aws_profile $PROFILE - echo $AWS_SSH_PRIVATE_KEY > gitlab-featurebase-dev.pem - chmod 400 gitlab-featurebase-dev.pem @@ -214,6 +214,7 @@ deploy node for linux amd64: - echo $INSTANCE_ID > linux-amd64-instance.txt - echo $INSTANCE_ID - scp -i gitlab-featurebase-dev.pem featurebase_linux_amd64 ec2-user@$PUBLIC_IP:. + - scp -i gitlab-featurebase-dev.pem .gitlab/featurebase.conf .gitlab/featurebase.service ec2-user@$PUBLIC_IP:. - sleep 120 needs: job: build for linux amd64 @@ -221,17 +222,27 @@ deploy node for linux amd64: paths: - linux-amd64-instance.txt +run featurebase in linux amd64 node: + stage: lint + image: + variables: + INSTANCE_ID: "" + before_script: + - INSTANCE_ID=$(cat linux-amd64-instance.txt) + script: + - aws ssm send-command --document-name "AWS-RunShellScript" --targets InstanceIds=$INSTANCE_ID --cli-input-json file://.gitlab/configureFeatureBase.json + needs: + job: + deploy node for linux amd64 + terminate node for linux amd64: stage: lint image: registry.gitlab.com/gitlab-org/cloud-deploy/aws-base:latest variables: INSTANCE_ID: "" before_script: - - aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID - - aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY - - aws configure set region "us-east-2" - INSTANCE_ID=$(cat linux-amd64-instance.txt) script: - aws ec2 terminate-instances --instance-ids $INSTANCE_ID needs: - job: deploy node for linux amd64 + job: run featurebase in linux amd64 node diff --git a/.gitlab/cloud-init.sh b/.gitlab/cloud-init.sh index 18257a90b..f9f289582 100755 --- a/.gitlab/cloud-init.sh +++ b/.gitlab/cloud-init.sh @@ -4,7 +4,6 @@ exec > >(tee /var/log/user-data.log|logger -t user-data -s 2>/dev/console) 2>&1 # Install packages yum update -y -# yum install golang -y # latest verion in ec2 is 1.15.14 yum install postgresql -y # Configure host system @@ -13,7 +12,8 @@ sysctl -w fs.file-max=262144 sysctl -p echo 'cat /proc/sys/fs/file-max' -# install go 1.16.9 +# yum install golang -y # latest verion in ec2 is 1.15.14 +# install go 1.16.9 manually curl -O https://dl.google.com/go/go1.16.9.linux-amd64.tar.gz tar xvf go1.16.9.linux-amd64.tar.gz chown -R root:root ./go diff --git a/.gitlab/configureFeatureBase.json b/.gitlab/configureFeatureBase.json new file mode 100644 index 000000000..569b6d0bd --- /dev/null +++ b/.gitlab/configureFeatureBase.json @@ -0,0 +1,20 @@ +{ + "Parameters": { + "commands": [ + "#!/bin/bash", + "mv featurebase_linux_amd64 /usr/local/bin/", + "mv featurebase.conf /etc/", + "mv featurebase.service /etc/systemd/system/", + "adduser molecula", + "sudo mkdir /var/log/molecula", + "sudo chown molecula /var/log/molecula", + "sudo mkdir -p /opt/molecula/featurebase", + "sudo chown molecula /opt/molecula/featurebase", + "systemctl daemon-reload", + "sudo systemctl start featurebase", + "sudo systemctl enable featurebase", + "sudo systemctl status featurebase", + "curl localhost:10101" + ] + } +} \ No newline at end of file diff --git a/.gitlab/featurebase.conf b/.gitlab/featurebase.conf new file mode 100644 index 000000000..7716d0f6b --- /dev/null +++ b/.gitlab/featurebase.conf @@ -0,0 +1,30 @@ +name = "pilosa1" +bind = "0.0.0.0:10101" +bind-grpc = "0.0.0.0:20101" + +data-dir = "/opt/molecula/featurebase" +log-path = "/var/log/molecula/featurebase.log" + +max-file-count=900000 +max-map-count=900000 + +long-query-time = "10s" + +[postgres] + + bind = "localhost:55432" + +[cluster] + + name = "cluster1" + replicas = 1 + +[etcd] + + listen-client-address = "http://localhost:10401" + listen-peer-address = "http://localhost:10301" + initial-cluster = "pilosa1=http://localhost:10301" + +[metric] + + service = "prometheus" \ No newline at end of file diff --git a/.gitlab/featurebase.service b/.gitlab/featurebase.service new file mode 100644 index 000000000..a543b92af --- /dev/null +++ b/.gitlab/featurebase.service @@ -0,0 +1,13 @@ +# Not Ansible managed + +[Unit] +Description="Service for FeatureBase" + +[Service] +RestartSec=30 +Restart=on-failure +EnvironmentFile= +User=molecula +ExecStart=/usr/local/bin/featurebase server -c /etc/featurebase.conf + +[Install] \ No newline at end of file From 27e39b933fdcf7ff5f2b6510a02b60f8edebdc23 Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Mon, 8 Nov 2021 09:21:14 -0600 Subject: [PATCH 70/98] reverted change to aws configure --- .gitlab/.gitlab-ci.yml | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index bf210f5a9..02be1f147 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -176,11 +176,6 @@ build container fb: - docker push $tag - echo Created docker featurebase image with tag "$tag" -before_script: - - aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID - - aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY - - aws configure set region "us-east-2" - deploy node for linux amd64: stage: lint image: registry.gitlab.com/gitlab-org/cloud-deploy/aws-base:latest @@ -193,6 +188,9 @@ deploy node for linux amd64: PROFILE: "default" AWS_SSH_PRIVATE_KEY: $AWS_SSH_PRIVATE_KEY before_script: + - aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID + - aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY + - aws configure set region "us-east-2" - aws configure set aws_profile $PROFILE - echo $AWS_SSH_PRIVATE_KEY > gitlab-featurebase-dev.pem - chmod 400 gitlab-featurebase-dev.pem @@ -228,6 +226,9 @@ run featurebase in linux amd64 node: variables: INSTANCE_ID: "" before_script: + - aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID + - aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY + - aws configure set region "us-east-2" - INSTANCE_ID=$(cat linux-amd64-instance.txt) script: - aws ssm send-command --document-name "AWS-RunShellScript" --targets InstanceIds=$INSTANCE_ID --cli-input-json file://.gitlab/configureFeatureBase.json @@ -241,6 +242,9 @@ terminate node for linux amd64: variables: INSTANCE_ID: "" before_script: + - aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID + - aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY + - aws configure set region "us-east-2" - INSTANCE_ID=$(cat linux-amd64-instance.txt) script: - aws ec2 terminate-instances --instance-ids $INSTANCE_ID From 1cdc8e495921c158fbadb273033ac353d0c82fce Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Mon, 8 Nov 2021 09:33:34 -0600 Subject: [PATCH 71/98] added aws image --- .gitlab/.gitlab-ci.yml | 4 ++-- .gitlab/cloud-init.sh | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 02be1f147..6e941512d 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -213,7 +213,6 @@ deploy node for linux amd64: - echo $INSTANCE_ID - scp -i gitlab-featurebase-dev.pem featurebase_linux_amd64 ec2-user@$PUBLIC_IP:. - scp -i gitlab-featurebase-dev.pem .gitlab/featurebase.conf .gitlab/featurebase.service ec2-user@$PUBLIC_IP:. - - sleep 120 needs: job: build for linux amd64 artifacts: @@ -222,7 +221,7 @@ deploy node for linux amd64: run featurebase in linux amd64 node: stage: lint - image: + image: registry.gitlab.com/gitlab-org/cloud-deploy/aws-base:latest variables: INSTANCE_ID: "" before_script: @@ -232,6 +231,7 @@ run featurebase in linux amd64 node: - INSTANCE_ID=$(cat linux-amd64-instance.txt) script: - aws ssm send-command --document-name "AWS-RunShellScript" --targets InstanceIds=$INSTANCE_ID --cli-input-json file://.gitlab/configureFeatureBase.json + - sleep 120 needs: job: deploy node for linux amd64 diff --git a/.gitlab/cloud-init.sh b/.gitlab/cloud-init.sh index f9f289582..e87c6124a 100755 --- a/.gitlab/cloud-init.sh +++ b/.gitlab/cloud-init.sh @@ -18,5 +18,5 @@ curl -O https://dl.google.com/go/go1.16.9.linux-amd64.tar.gz tar xvf go1.16.9.linux-amd64.tar.gz chown -R root:root ./go mv go /usr/local -echo "export PATH=$PATH:/usr/local/go/bin" | sudo tee -a /etc/profile > /dev/null +echo "export PATH=/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/home/ec2-user/.local/bin:/home/ec2-user/bin:/usr/local/go/bin" | tee -a /etc/profile > /dev/null source /etc/profile From 38a17e2b941c3f0694096c5f6453cc3d28ba1346 Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Mon, 8 Nov 2021 09:40:14 -0600 Subject: [PATCH 72/98] fixed typo --- .gitlab/.gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 6e941512d..f614e052f 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -230,7 +230,7 @@ run featurebase in linux amd64 node: - aws configure set region "us-east-2" - INSTANCE_ID=$(cat linux-amd64-instance.txt) script: - - aws ssm send-command --document-name "AWS-RunShellScript" --targets InstanceIds=$INSTANCE_ID --cli-input-json file://.gitlab/configureFeatureBase.json + - aws ssm send-command --document-name "AWS-RunShellScript" --targets InstanceId=$INSTANCE_ID --cli-input-json file://.gitlab/configureFeatureBase.json - sleep 120 needs: job: From 14265090db186114637a0814e9abd195ed8d75fd Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Mon, 8 Nov 2021 10:20:10 -0600 Subject: [PATCH 73/98] fixed arguments --- .gitlab/.gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index f614e052f..9db55187a 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -230,7 +230,7 @@ run featurebase in linux amd64 node: - aws configure set region "us-east-2" - INSTANCE_ID=$(cat linux-amd64-instance.txt) script: - - aws ssm send-command --document-name "AWS-RunShellScript" --targets InstanceId=$INSTANCE_ID --cli-input-json file://.gitlab/configureFeatureBase.json + - aws ssm send-command --document-name "AWS-RunShellScript" --targets "Key=InstanceIds,Values={$INSTANCE_ID}" --cli-input-json file://.gitlab/configureFeatureBase.json - sleep 120 needs: job: From d3cc4d7f2a8c67d918b0a3f9c11f26cb39cd2b7f Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Mon, 8 Nov 2021 10:26:59 -0600 Subject: [PATCH 74/98] added argument to scp --- .gitlab/.gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 9db55187a..fb71ade61 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -211,8 +211,8 @@ deploy node for linux amd64: - INSTANCE_ID=$(aws ec2 describe-instances --filters 'Name=tag:Name, Values=linux-amd64-node' 'Name=instance-state-name, Values=running' --query 'Reservations[*].Instances[*].InstanceId' --output text --profile $PROFILE) - echo $INSTANCE_ID > linux-amd64-instance.txt - echo $INSTANCE_ID - - scp -i gitlab-featurebase-dev.pem featurebase_linux_amd64 ec2-user@$PUBLIC_IP:. - - scp -i gitlab-featurebase-dev.pem .gitlab/featurebase.conf .gitlab/featurebase.service ec2-user@$PUBLIC_IP:. + - scp -o StrictHostKeyChecking=no -i gitlab-featurebase-dev.pem featurebase_linux_amd64 ec2-user@$PUBLIC_IP:. + - scp -o StrictHostKeyChecking=no -i gitlab-featurebase-dev.pem .gitlab/featurebase.conf .gitlab/featurebase.service ec2-user@$PUBLIC_IP:. needs: job: build for linux amd64 artifacts: From 22c081528833aef34b259a230831eaa581e79372 Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Mon, 8 Nov 2021 10:31:57 -0600 Subject: [PATCH 75/98] fixed syntax error --- .gitlab/.gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index fb71ade61..b018ef816 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -230,7 +230,7 @@ run featurebase in linux amd64 node: - aws configure set region "us-east-2" - INSTANCE_ID=$(cat linux-amd64-instance.txt) script: - - aws ssm send-command --document-name "AWS-RunShellScript" --targets "Key=InstanceIds,Values={$INSTANCE_ID}" --cli-input-json file://.gitlab/configureFeatureBase.json + - aws ssm send-command --document-name "AWS-RunShellScript" --targets "Key=InstanceIds,Values=$INSTANCE_ID" --cli-input-json file://.gitlab/configureFeatureBase.json - sleep 120 needs: job: From 4a42571dd9cb81ecdb823503fb1e14ca57aaeea0 Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Mon, 8 Nov 2021 10:45:10 -0600 Subject: [PATCH 76/98] fix for go --- .gitlab/.gitlab-ci.yml | 1 + .gitlab/cloud-init.sh | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index b018ef816..d6b57420b 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -229,6 +229,7 @@ run featurebase in linux amd64 node: - aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY - aws configure set region "us-east-2" - INSTANCE_ID=$(cat linux-amd64-instance.txt) + - echo $INSTANCE_ID script: - aws ssm send-command --document-name "AWS-RunShellScript" --targets "Key=InstanceIds,Values=$INSTANCE_ID" --cli-input-json file://.gitlab/configureFeatureBase.json - sleep 120 diff --git a/.gitlab/cloud-init.sh b/.gitlab/cloud-init.sh index e87c6124a..c86818e48 100755 --- a/.gitlab/cloud-init.sh +++ b/.gitlab/cloud-init.sh @@ -18,5 +18,5 @@ curl -O https://dl.google.com/go/go1.16.9.linux-amd64.tar.gz tar xvf go1.16.9.linux-amd64.tar.gz chown -R root:root ./go mv go /usr/local -echo "export PATH=/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/home/ec2-user/.local/bin:/home/ec2-user/bin:/usr/local/go/bin" | tee -a /etc/profile > /dev/null -source /etc/profile +echo "export PATH=/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/home/ec2-user/.local/bin:/home/ec2-user/bin:/usr/local/go/bin" | tee -a /etc/profile.d > /dev/null +source /etc/profile.d From 8edfc0c9ff81d74311a51a9f21bc78b16454ed85 Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Mon, 8 Nov 2021 11:00:49 -0600 Subject: [PATCH 77/98] made instanceId a global variable --- .gitlab/.gitlab-ci.yml | 3 +-- .gitlab/cloud-init.sh | 4 ++-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index d6b57420b..8545a6f2f 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -12,6 +12,7 @@ include: - .go/pkg/mod/ variables: GOVERSION: "1.16.9" + INSTANCE_ID: "" stages: - lint @@ -228,8 +229,6 @@ run featurebase in linux amd64 node: - aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID - aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY - aws configure set region "us-east-2" - - INSTANCE_ID=$(cat linux-amd64-instance.txt) - - echo $INSTANCE_ID script: - aws ssm send-command --document-name "AWS-RunShellScript" --targets "Key=InstanceIds,Values=$INSTANCE_ID" --cli-input-json file://.gitlab/configureFeatureBase.json - sleep 120 diff --git a/.gitlab/cloud-init.sh b/.gitlab/cloud-init.sh index c86818e48..e87c6124a 100755 --- a/.gitlab/cloud-init.sh +++ b/.gitlab/cloud-init.sh @@ -18,5 +18,5 @@ curl -O https://dl.google.com/go/go1.16.9.linux-amd64.tar.gz tar xvf go1.16.9.linux-amd64.tar.gz chown -R root:root ./go mv go /usr/local -echo "export PATH=/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/home/ec2-user/.local/bin:/home/ec2-user/bin:/usr/local/go/bin" | tee -a /etc/profile.d > /dev/null -source /etc/profile.d +echo "export PATH=/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/home/ec2-user/.local/bin:/home/ec2-user/bin:/usr/local/go/bin" | tee -a /etc/profile > /dev/null +source /etc/profile From 26d77c51b53c0a4a2d172f6eeb80ccab2c52fc51 Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Mon, 8 Nov 2021 11:06:55 -0600 Subject: [PATCH 78/98] removed variable overwrite --- .gitlab/.gitlab-ci.yml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 8545a6f2f..e1cbaef21 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -223,12 +223,11 @@ deploy node for linux amd64: run featurebase in linux amd64 node: stage: lint image: registry.gitlab.com/gitlab-org/cloud-deploy/aws-base:latest - variables: - INSTANCE_ID: "" before_script: - aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID - aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY - aws configure set region "us-east-2" + - echo $INSTANCE_ID script: - aws ssm send-command --document-name "AWS-RunShellScript" --targets "Key=InstanceIds,Values=$INSTANCE_ID" --cli-input-json file://.gitlab/configureFeatureBase.json - sleep 120 @@ -239,13 +238,10 @@ run featurebase in linux amd64 node: terminate node for linux amd64: stage: lint image: registry.gitlab.com/gitlab-org/cloud-deploy/aws-base:latest - variables: - INSTANCE_ID: "" before_script: - aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID - aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY - aws configure set region "us-east-2" - - INSTANCE_ID=$(cat linux-amd64-instance.txt) script: - aws ec2 terminate-instances --instance-ids $INSTANCE_ID needs: From 2b315c6634eade3cf08b1987536239231ed99bd1 Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Mon, 8 Nov 2021 11:23:05 -0600 Subject: [PATCH 79/98] removed instance id file --- .gitlab/.gitlab-ci.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index e1cbaef21..c590c1885 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -210,15 +210,11 @@ deploy node for linux amd64: - PUBLIC_IP=$(aws ec2 describe-instances --filters 'Name=tag:Name, Values=linux-amd64-node' 'Name=instance-state-name, Values=running' --query 'Reservations[*].Instances[*].PublicIpAddress' --output text --profile $PROFILE) - echo $PUBLIC_IP - INSTANCE_ID=$(aws ec2 describe-instances --filters 'Name=tag:Name, Values=linux-amd64-node' 'Name=instance-state-name, Values=running' --query 'Reservations[*].Instances[*].InstanceId' --output text --profile $PROFILE) - - echo $INSTANCE_ID > linux-amd64-instance.txt - echo $INSTANCE_ID - scp -o StrictHostKeyChecking=no -i gitlab-featurebase-dev.pem featurebase_linux_amd64 ec2-user@$PUBLIC_IP:. - scp -o StrictHostKeyChecking=no -i gitlab-featurebase-dev.pem .gitlab/featurebase.conf .gitlab/featurebase.service ec2-user@$PUBLIC_IP:. needs: job: build for linux amd64 - artifacts: - paths: - - linux-amd64-instance.txt run featurebase in linux amd64 node: stage: lint @@ -242,6 +238,7 @@ terminate node for linux amd64: - aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID - aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY - aws configure set region "us-east-2" + - echo $INSTANCE_ID script: - aws ec2 terminate-instances --instance-ids $INSTANCE_ID needs: From 6d4ae4c345f4b65620a7db6dde2b27a355dc7f52 Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Mon, 8 Nov 2021 11:34:10 -0600 Subject: [PATCH 80/98] changed var assignment --- .gitlab/.gitlab-ci.yml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index c590c1885..c1675d40b 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -210,19 +210,25 @@ deploy node for linux amd64: - PUBLIC_IP=$(aws ec2 describe-instances --filters 'Name=tag:Name, Values=linux-amd64-node' 'Name=instance-state-name, Values=running' --query 'Reservations[*].Instances[*].PublicIpAddress' --output text --profile $PROFILE) - echo $PUBLIC_IP - INSTANCE_ID=$(aws ec2 describe-instances --filters 'Name=tag:Name, Values=linux-amd64-node' 'Name=instance-state-name, Values=running' --query 'Reservations[*].Instances[*].InstanceId' --output text --profile $PROFILE) - - echo $INSTANCE_ID + - echo $INSTANCE_ID > linux-amd64-instance.txt - scp -o StrictHostKeyChecking=no -i gitlab-featurebase-dev.pem featurebase_linux_amd64 ec2-user@$PUBLIC_IP:. - scp -o StrictHostKeyChecking=no -i gitlab-featurebase-dev.pem .gitlab/featurebase.conf .gitlab/featurebase.service ec2-user@$PUBLIC_IP:. needs: job: build for linux amd64 + artifacts: + paths: + - linux-amd64-instance.txt run featurebase in linux amd64 node: stage: lint image: registry.gitlab.com/gitlab-org/cloud-deploy/aws-base:latest + variables: + INSTANCE_ID: "" before_script: - aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID - aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY - aws configure set region "us-east-2" + - INSTANCE_ID=$(cat linux-amd64-instance.txt) - echo $INSTANCE_ID script: - aws ssm send-command --document-name "AWS-RunShellScript" --targets "Key=InstanceIds,Values=$INSTANCE_ID" --cli-input-json file://.gitlab/configureFeatureBase.json @@ -234,10 +240,13 @@ run featurebase in linux amd64 node: terminate node for linux amd64: stage: lint image: registry.gitlab.com/gitlab-org/cloud-deploy/aws-base:latest + variables: + INSTANCE_ID: "" before_script: - aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID - aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY - aws configure set region "us-east-2" + - INSTANCE_ID=$(echo "$( Date: Mon, 8 Nov 2021 11:39:17 -0600 Subject: [PATCH 81/98] updated var assignment --- .gitlab/.gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index c1675d40b..457aa0b4d 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -228,7 +228,7 @@ run featurebase in linux amd64 node: - aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID - aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY - aws configure set region "us-east-2" - - INSTANCE_ID=$(cat linux-amd64-instance.txt) + - INSTANCE_ID=$(echo "$( Date: Mon, 8 Nov 2021 11:45:41 -0600 Subject: [PATCH 82/98] access array value --- .gitlab/.gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 457aa0b4d..dba79b098 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -231,7 +231,7 @@ run featurebase in linux amd64 node: - INSTANCE_ID=$(echo "$( Date: Mon, 8 Nov 2021 11:55:17 -0600 Subject: [PATCH 83/98] fix var assignment --- .gitlab/.gitlab-ci.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index dba79b098..f238ff88f 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -229,9 +229,8 @@ run featurebase in linux amd64 node: - aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY - aws configure set region "us-east-2" - INSTANCE_ID=$(echo "$( Date: Mon, 8 Nov 2021 12:00:17 -0600 Subject: [PATCH 84/98] removed targets flag --- .gitlab/.gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index f238ff88f..a7b229c89 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -230,7 +230,7 @@ run featurebase in linux amd64 node: - aws configure set region "us-east-2" - INSTANCE_ID=$(echo "$( Date: Mon, 8 Nov 2021 13:21:07 -0600 Subject: [PATCH 85/98] added iam role --- .gitlab/.gitlab-ci.yml | 8 +++++--- .gitlab/cloud-init.sh | 4 ++++ 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index a7b229c89..3540c1a8c 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -213,6 +213,7 @@ deploy node for linux amd64: - echo $INSTANCE_ID > linux-amd64-instance.txt - scp -o StrictHostKeyChecking=no -i gitlab-featurebase-dev.pem featurebase_linux_amd64 ec2-user@$PUBLIC_IP:. - scp -o StrictHostKeyChecking=no -i gitlab-featurebase-dev.pem .gitlab/featurebase.conf .gitlab/featurebase.service ec2-user@$PUBLIC_IP:. + - aws ec2 associate-iam-instance-profile --instance-id $INSTANCE_ID --iam-instance-profile Name=featurebase-dev-ssm needs: job: build for linux amd64 artifacts: @@ -228,9 +229,10 @@ run featurebase in linux amd64 node: - aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID - aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY - aws configure set region "us-east-2" - - INSTANCE_ID=$(echo "$( /dev/null source /etc/profile + +# install aws session manager pluggin +curl "https://s3.amazonaws.com/session-manager-downloads/plugin/latest/linux_64bit/session-manager-plugin.rpm" -o "session-manager-plugin.rpm" +yum install -y session-manager-plugin.rpm \ No newline at end of file From 3d11f9cebf43bfd13883b7ec967e88aa7246af5b Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Mon, 8 Nov 2021 13:35:20 -0600 Subject: [PATCH 86/98] added profile --- .gitlab/.gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 3540c1a8c..8c1d2d0e4 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -225,6 +225,7 @@ run featurebase in linux amd64 node: image: registry.gitlab.com/gitlab-org/cloud-deploy/aws-base:latest variables: INSTANCE_ID: "" + PROFILE: "default" before_script: - aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID - aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY From 48295a5491be764dad3b9f7043cf15870f2218f5 Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Mon, 8 Nov 2021 13:50:47 -0600 Subject: [PATCH 87/98] added abs paths and add sleep time --- .gitlab/.gitlab-ci.yml | 1 + .gitlab/configureFeatureBase.json | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 8c1d2d0e4..bed04357b 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -214,6 +214,7 @@ deploy node for linux amd64: - scp -o StrictHostKeyChecking=no -i gitlab-featurebase-dev.pem featurebase_linux_amd64 ec2-user@$PUBLIC_IP:. - scp -o StrictHostKeyChecking=no -i gitlab-featurebase-dev.pem .gitlab/featurebase.conf .gitlab/featurebase.service ec2-user@$PUBLIC_IP:. - aws ec2 associate-iam-instance-profile --instance-id $INSTANCE_ID --iam-instance-profile Name=featurebase-dev-ssm + - sleep 120 needs: job: build for linux amd64 artifacts: diff --git a/.gitlab/configureFeatureBase.json b/.gitlab/configureFeatureBase.json index 569b6d0bd..6afd135d9 100644 --- a/.gitlab/configureFeatureBase.json +++ b/.gitlab/configureFeatureBase.json @@ -2,9 +2,9 @@ "Parameters": { "commands": [ "#!/bin/bash", - "mv featurebase_linux_amd64 /usr/local/bin/", - "mv featurebase.conf /etc/", - "mv featurebase.service /etc/systemd/system/", + "mv /home/ec2-user/featurebase_linux_amd64 /usr/local/bin/featurebase", + "mv /home/ec2-user/featurebase.conf /etc/", + "mv /home/ec2-user/featurebase.service /etc/systemd/system/", "adduser molecula", "sudo mkdir /var/log/molecula", "sudo chown molecula /var/log/molecula", From af4b420d661cfa7cd413c314e5c3d08d524b62d1 Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Mon, 8 Nov 2021 13:58:34 -0600 Subject: [PATCH 88/98] added region --- .gitlab/.gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index bed04357b..ae18e9f1d 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -234,7 +234,7 @@ run featurebase in linux amd64 node: - aws configure set aws_profile $PROFILE - INSTANCE_ID=$(cat linux-amd64-instance.txt) script: - - aws ssm send-command --document-name "AWS-RunShellScript" --instance-ids $INSTANCE_ID --cli-input-json file://.gitlab/configureFeatureBase.json --profile $PROFILE + - aws ssm send-command --document-name "AWS-RunShellScript" --instance-ids $INSTANCE_ID --cli-input-json file://.gitlab/configureFeatureBase.json --profile $PROFILE --region us-east-2 - sleep 120 needs: job: From d92bd37d71a87d8739932c640084508f98971939 Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Mon, 8 Nov 2021 14:20:04 -0600 Subject: [PATCH 89/98] added another sleep --- .gitlab/.gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index ae18e9f1d..b19fe0757 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -234,6 +234,7 @@ run featurebase in linux amd64 node: - aws configure set aws_profile $PROFILE - INSTANCE_ID=$(cat linux-amd64-instance.txt) script: + - sleep 60 - aws ssm send-command --document-name "AWS-RunShellScript" --instance-ids $INSTANCE_ID --cli-input-json file://.gitlab/configureFeatureBase.json --profile $PROFILE --region us-east-2 - sleep 120 needs: From 1b503a3ddfb1573c941c19d095f94860eb2b7e0c Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Mon, 8 Nov 2021 14:32:00 -0600 Subject: [PATCH 90/98] made the sleep time longer --- .gitlab/.gitlab-ci.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index b19fe0757..d28d3c5ce 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -214,7 +214,6 @@ deploy node for linux amd64: - scp -o StrictHostKeyChecking=no -i gitlab-featurebase-dev.pem featurebase_linux_amd64 ec2-user@$PUBLIC_IP:. - scp -o StrictHostKeyChecking=no -i gitlab-featurebase-dev.pem .gitlab/featurebase.conf .gitlab/featurebase.service ec2-user@$PUBLIC_IP:. - aws ec2 associate-iam-instance-profile --instance-id $INSTANCE_ID --iam-instance-profile Name=featurebase-dev-ssm - - sleep 120 needs: job: build for linux amd64 artifacts: @@ -234,7 +233,7 @@ run featurebase in linux amd64 node: - aws configure set aws_profile $PROFILE - INSTANCE_ID=$(cat linux-amd64-instance.txt) script: - - sleep 60 + - sleep 240 # needed to allow enough time for the ssm to take into effect before the next command can get called - aws ssm send-command --document-name "AWS-RunShellScript" --instance-ids $INSTANCE_ID --cli-input-json file://.gitlab/configureFeatureBase.json --profile $PROFILE --region us-east-2 - sleep 120 needs: From aab7a2e4be2627ac037d8ba8b01d735c7e243fb4 Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Mon, 8 Nov 2021 14:33:34 -0600 Subject: [PATCH 91/98] fixed stages --- .gitlab/.gitlab-ci.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index d28d3c5ce..e04dab61d 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -110,7 +110,7 @@ upload to sonarcloud: - job: run jest tests build for linux amd64: - stage: lint + stage: build image: registry.gitlab.com/molecula/featurebase/builder:0.0.3 extends: .go-cache variables: @@ -178,7 +178,7 @@ build container fb: - echo Created docker featurebase image with tag "$tag" deploy node for linux amd64: - stage: lint + stage: integration image: registry.gitlab.com/gitlab-org/cloud-deploy/aws-base:latest variables: AMI: "" @@ -237,11 +237,10 @@ run featurebase in linux amd64 node: - aws ssm send-command --document-name "AWS-RunShellScript" --instance-ids $INSTANCE_ID --cli-input-json file://.gitlab/configureFeatureBase.json --profile $PROFILE --region us-east-2 - sleep 120 needs: - job: - deploy node for linux amd64 + job: deploy node for linux amd64 terminate node for linux amd64: - stage: lint + stage: integration image: registry.gitlab.com/gitlab-org/cloud-deploy/aws-base:latest variables: INSTANCE_ID: "" @@ -254,4 +253,5 @@ terminate node for linux amd64: script: - aws ec2 terminate-instances --instance-ids $INSTANCE_ID needs: + job: deploy node for linux amd64 job: run featurebase in linux amd64 node From 4c291180b979aa638688b2defc535ae19adc5d14 Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Mon, 8 Nov 2021 14:38:35 -0600 Subject: [PATCH 92/98] added artifact paths --- .gitlab/.gitlab-ci.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index e04dab61d..57c0c98a0 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -238,6 +238,9 @@ run featurebase in linux amd64 node: - sleep 120 needs: job: deploy node for linux amd64 + artifacts: + paths: + - linux-amd64-instance.txt terminate node for linux amd64: stage: integration @@ -253,5 +256,4 @@ terminate node for linux amd64: script: - aws ec2 terminate-instances --instance-ids $INSTANCE_ID needs: - job: deploy node for linux amd64 job: run featurebase in linux amd64 node From d5f5340016e576ae5646780054379959cbd06f51 Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Mon, 8 Nov 2021 14:44:33 -0600 Subject: [PATCH 93/98] fixed syntax error --- .gitlab/.gitlab-ci.yml | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 57c0c98a0..16c5f4a11 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -12,7 +12,6 @@ include: - .go/pkg/mod/ variables: GOVERSION: "1.16.9" - INSTANCE_ID: "" stages: - lint @@ -215,7 +214,7 @@ deploy node for linux amd64: - scp -o StrictHostKeyChecking=no -i gitlab-featurebase-dev.pem .gitlab/featurebase.conf .gitlab/featurebase.service ec2-user@$PUBLIC_IP:. - aws ec2 associate-iam-instance-profile --instance-id $INSTANCE_ID --iam-instance-profile Name=featurebase-dev-ssm needs: - job: build for linux amd64 + - job: build for linux amd64 artifacts: paths: - linux-amd64-instance.txt @@ -237,10 +236,7 @@ run featurebase in linux amd64 node: - aws ssm send-command --document-name "AWS-RunShellScript" --instance-ids $INSTANCE_ID --cli-input-json file://.gitlab/configureFeatureBase.json --profile $PROFILE --region us-east-2 - sleep 120 needs: - job: deploy node for linux amd64 - artifacts: - paths: - - linux-amd64-instance.txt + - job: deploy node for linux amd64 terminate node for linux amd64: stage: integration @@ -256,4 +252,5 @@ terminate node for linux amd64: script: - aws ec2 terminate-instances --instance-ids $INSTANCE_ID needs: - job: run featurebase in linux amd64 node + - job: deploy node for linux amd64 + - job: run featurebase in linux amd64 node From 91e93cb975eaceb4a00a67573b0c7e8fa6193b8b Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Mon, 8 Nov 2021 14:47:14 -0600 Subject: [PATCH 94/98] fixed stage --- .gitlab/.gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 16c5f4a11..66e266a79 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -220,7 +220,7 @@ deploy node for linux amd64: - linux-amd64-instance.txt run featurebase in linux amd64 node: - stage: lint + stage: integration image: registry.gitlab.com/gitlab-org/cloud-deploy/aws-base:latest variables: INSTANCE_ID: "" From d48da50cf690686cfdbb486c9bd4b858a2578caa Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Mon, 8 Nov 2021 15:15:26 -0600 Subject: [PATCH 95/98] consolidated some commands --- .gitlab/.gitlab-ci.yml | 28 ++++++---------------------- 1 file changed, 6 insertions(+), 22 deletions(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 66e266a79..f34d6d93b 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -176,6 +176,7 @@ build container fb: - docker push $tag - echo Created docker featurebase image with tag "$tag" +# deploy EC2 instance and upload featurebase binary deploy node for linux amd64: stage: integration image: registry.gitlab.com/gitlab-org/cloud-deploy/aws-base:latest @@ -205,7 +206,7 @@ deploy node for linux amd64: - SECURITY_GROUP=$(aws ec2 describe-security-groups --filter Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978 --query 'SecurityGroups[*].[GroupId]' --output text --profile $PROFILE) - SUBNET_ID=$(aws ec2 describe-subnets --filters 'Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978' 'Name=availability-zone,Values=us-east-2a' --query 'Subnets[0].SubnetId' --output text --profile $PROFILE) - aws ec2 run-instances --image-id $AMI --instance-type $INSTANCE --security-group-ids $SECURITY_GROUP --subnet-id $SUBNET_ID --key-name gitlab-featurebase-dev --tag-specifications 'ResourceType=instance,Tags=[{Key=Name,Value=linux-amd64-node}]' --profile $PROFILE --user-data file://.gitlab/cloud-init.sh - - sleep 120 + - sleep 120 # Need to wait for the EC2 instance to launch and run the initialization commands passed through user-data - PUBLIC_IP=$(aws ec2 describe-instances --filters 'Name=tag:Name, Values=linux-amd64-node' 'Name=instance-state-name, Values=running' --query 'Reservations[*].Instances[*].PublicIpAddress' --output text --profile $PROFILE) - echo $PUBLIC_IP - INSTANCE_ID=$(aws ec2 describe-instances --filters 'Name=tag:Name, Values=linux-amd64-node' 'Name=instance-state-name, Values=running' --query 'Reservations[*].Instances[*].InstanceId' --output text --profile $PROFILE) @@ -213,31 +214,15 @@ deploy node for linux amd64: - scp -o StrictHostKeyChecking=no -i gitlab-featurebase-dev.pem featurebase_linux_amd64 ec2-user@$PUBLIC_IP:. - scp -o StrictHostKeyChecking=no -i gitlab-featurebase-dev.pem .gitlab/featurebase.conf .gitlab/featurebase.service ec2-user@$PUBLIC_IP:. - aws ec2 associate-iam-instance-profile --instance-id $INSTANCE_ID --iam-instance-profile Name=featurebase-dev-ssm - needs: + - sleep 240 # needed to allow enough time for the ssm to take into effect before the next command can get called + - aws ssm send-command --document-name "AWS-RunShellScript" --instance-ids $INSTANCE_ID --cli-input-json file://.gitlab/configureFeatureBase.json --profile $PROFILE --region us-east-2 + - sleep 120 + needs: - job: build for linux amd64 artifacts: paths: - linux-amd64-instance.txt -run featurebase in linux amd64 node: - stage: integration - image: registry.gitlab.com/gitlab-org/cloud-deploy/aws-base:latest - variables: - INSTANCE_ID: "" - PROFILE: "default" - before_script: - - aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID - - aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY - - aws configure set region "us-east-2" - - aws configure set aws_profile $PROFILE - - INSTANCE_ID=$(cat linux-amd64-instance.txt) - script: - - sleep 240 # needed to allow enough time for the ssm to take into effect before the next command can get called - - aws ssm send-command --document-name "AWS-RunShellScript" --instance-ids $INSTANCE_ID --cli-input-json file://.gitlab/configureFeatureBase.json --profile $PROFILE --region us-east-2 - - sleep 120 - needs: - - job: deploy node for linux amd64 - terminate node for linux amd64: stage: integration image: registry.gitlab.com/gitlab-org/cloud-deploy/aws-base:latest @@ -253,4 +238,3 @@ terminate node for linux amd64: - aws ec2 terminate-instances --instance-ids $INSTANCE_ID needs: - job: deploy node for linux amd64 - - job: run featurebase in linux amd64 node From 1bda8e439d0811a3c7a3ab5b4d147dc7a00227ce Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Mon, 8 Nov 2021 15:26:01 -0600 Subject: [PATCH 96/98] fixed indentation --- .gitlab/.gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 373993c79..8f317eed1 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -232,7 +232,7 @@ deploy node for linux amd64: - sleep 240 # needed to allow enough time for the ssm to take into effect before the next command can get called - aws ssm send-command --document-name "AWS-RunShellScript" --instance-ids $INSTANCE_ID --cli-input-json file://.gitlab/configureFeatureBase.json --profile $PROFILE --region us-east-2 - sleep 120 - needs: + needs: - job: build for linux amd64 artifacts: paths: From 05be45194a0ff106024a70dd35718d1d9e03eb3c Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Mon, 8 Nov 2021 15:54:12 -0600 Subject: [PATCH 97/98] combined iam role with run-instance --- .gitlab/.gitlab-ci.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 8f317eed1..904f6c5cf 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -220,7 +220,7 @@ deploy node for linux amd64: - AMI=$(aws ssm get-parameters --names /aws/service/ami-amazon-linux-latest/amzn2-ami-hvm-x86_64-ebs --query 'Parameters[0].[Value]' --output text --profile $PROFILE) - SECURITY_GROUP=$(aws ec2 describe-security-groups --filter Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978 --query 'SecurityGroups[*].[GroupId]' --output text --profile $PROFILE) - SUBNET_ID=$(aws ec2 describe-subnets --filters 'Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978' 'Name=availability-zone,Values=us-east-2a' --query 'Subnets[0].SubnetId' --output text --profile $PROFILE) - - aws ec2 run-instances --image-id $AMI --instance-type $INSTANCE --security-group-ids $SECURITY_GROUP --subnet-id $SUBNET_ID --key-name gitlab-featurebase-dev --tag-specifications 'ResourceType=instance,Tags=[{Key=Name,Value=linux-amd64-node}]' --profile $PROFILE --user-data file://.gitlab/cloud-init.sh + - aws ec2 run-instances --image-id $AMI --instance-type $INSTANCE --security-group-ids $SECURITY_GROUP --subnet-id $SUBNET_ID --key-name gitlab-featurebase-dev --tag-specifications 'ResourceType=instance,Tags=[{Key=Name,Value=linux-amd64-node}]' --profile $PROFILE --user-data file://.gitlab/cloud-init.sh --iam-instance-profile Name=featurebase-dev-ssm - sleep 120 # Need to wait for the EC2 instance to launch and run the initialization commands passed through user-data - PUBLIC_IP=$(aws ec2 describe-instances --filters 'Name=tag:Name, Values=linux-amd64-node' 'Name=instance-state-name, Values=running' --query 'Reservations[*].Instances[*].PublicIpAddress' --output text --profile $PROFILE) - echo $PUBLIC_IP @@ -228,10 +228,7 @@ deploy node for linux amd64: - echo $INSTANCE_ID > linux-amd64-instance.txt - scp -o StrictHostKeyChecking=no -i gitlab-featurebase-dev.pem featurebase_linux_amd64 ec2-user@$PUBLIC_IP:. - scp -o StrictHostKeyChecking=no -i gitlab-featurebase-dev.pem .gitlab/featurebase.conf .gitlab/featurebase.service ec2-user@$PUBLIC_IP:. - - aws ec2 associate-iam-instance-profile --instance-id $INSTANCE_ID --iam-instance-profile Name=featurebase-dev-ssm - - sleep 240 # needed to allow enough time for the ssm to take into effect before the next command can get called - aws ssm send-command --document-name "AWS-RunShellScript" --instance-ids $INSTANCE_ID --cli-input-json file://.gitlab/configureFeatureBase.json --profile $PROFILE --region us-east-2 - - sleep 120 needs: - job: build for linux amd64 artifacts: From a8707b55bddbbd09aab956834604061ed39d3fb2 Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Mon, 8 Nov 2021 16:14:25 -0600 Subject: [PATCH 98/98] updated instance type --- .gitlab/.gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 904f6c5cf..247e377bd 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -197,7 +197,7 @@ deploy node for linux amd64: image: registry.gitlab.com/gitlab-org/cloud-deploy/aws-base:latest variables: AMI: "" - INSTANCE: "t2.micro" + INSTANCE: "t3a.large" SECURITY_GROUP: "" SUBNET_ID: "" PUBLIC_IP: ""