diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 3afbafbdc..247e377bd 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -17,6 +17,7 @@ stages: - lint - test - build + - integration #before_script: #- echo "before_script" @@ -188,4 +189,64 @@ build container fb: - tag=${CI_REGISTRY_IMAGE}/server:${CI_COMMIT_REF_SLUG} - docker build --build-arg GO_VERSION=$GOVERSION -t $tag -f .gitlab/Dockerfile . - docker push $tag - - echo Created docker featurebase image with tag "$tag" \ No newline at end of file + - echo Created docker featurebase image with tag "$tag" + +# deploy EC2 instance and upload featurebase binary +deploy node for linux amd64: + stage: integration + image: registry.gitlab.com/gitlab-org/cloud-deploy/aws-base:latest + variables: + AMI: "" + INSTANCE: "t3a.large" + SECURITY_GROUP: "" + SUBNET_ID: "" + PUBLIC_IP: "" + PROFILE: "default" + AWS_SSH_PRIVATE_KEY: $AWS_SSH_PRIVATE_KEY + before_script: + - aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID + - aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY + - aws configure set region "us-east-2" + - aws configure set aws_profile $PROFILE + - echo $AWS_SSH_PRIVATE_KEY > gitlab-featurebase-dev.pem + - chmod 400 gitlab-featurebase-dev.pem + - 'which ssh-agent || ( apt-get update -y && apt-get install openssh-client -y )' + - eval `ssh-agent -s` + - mkdir -p ~/.ssh + - echo "$AWS_SSH_PRIVATE_KEY" | ssh-add - + - chmod 700 /root/.ssh + - '[[ -f /.dockerenv ]] && echo -e "Host *\n\tStrictHostKeyChecking no\n\n" > ~/.ssh/config' + script: + - AMI=$(aws ssm get-parameters --names /aws/service/ami-amazon-linux-latest/amzn2-ami-hvm-x86_64-ebs --query 'Parameters[0].[Value]' --output text --profile $PROFILE) + - SECURITY_GROUP=$(aws ec2 describe-security-groups --filter Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978 --query 'SecurityGroups[*].[GroupId]' --output text --profile $PROFILE) + - SUBNET_ID=$(aws ec2 describe-subnets --filters 'Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978' 'Name=availability-zone,Values=us-east-2a' --query 'Subnets[0].SubnetId' --output text --profile $PROFILE) + - aws ec2 run-instances --image-id $AMI --instance-type $INSTANCE --security-group-ids $SECURITY_GROUP --subnet-id $SUBNET_ID --key-name gitlab-featurebase-dev --tag-specifications 'ResourceType=instance,Tags=[{Key=Name,Value=linux-amd64-node}]' --profile $PROFILE --user-data file://.gitlab/cloud-init.sh --iam-instance-profile Name=featurebase-dev-ssm + - sleep 120 # Need to wait for the EC2 instance to launch and run the initialization commands passed through user-data + - PUBLIC_IP=$(aws ec2 describe-instances --filters 'Name=tag:Name, Values=linux-amd64-node' 'Name=instance-state-name, Values=running' --query 'Reservations[*].Instances[*].PublicIpAddress' --output text --profile $PROFILE) + - echo $PUBLIC_IP + - INSTANCE_ID=$(aws ec2 describe-instances --filters 'Name=tag:Name, Values=linux-amd64-node' 'Name=instance-state-name, Values=running' --query 'Reservations[*].Instances[*].InstanceId' --output text --profile $PROFILE) + - echo $INSTANCE_ID > linux-amd64-instance.txt + - scp -o StrictHostKeyChecking=no -i gitlab-featurebase-dev.pem featurebase_linux_amd64 ec2-user@$PUBLIC_IP:. + - scp -o StrictHostKeyChecking=no -i gitlab-featurebase-dev.pem .gitlab/featurebase.conf .gitlab/featurebase.service ec2-user@$PUBLIC_IP:. + - aws ssm send-command --document-name "AWS-RunShellScript" --instance-ids $INSTANCE_ID --cli-input-json file://.gitlab/configureFeatureBase.json --profile $PROFILE --region us-east-2 + needs: + - job: build for linux amd64 + artifacts: + paths: + - linux-amd64-instance.txt + +terminate node for linux amd64: + stage: integration + image: registry.gitlab.com/gitlab-org/cloud-deploy/aws-base:latest + variables: + INSTANCE_ID: "" + before_script: + - aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID + - aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY + - aws configure set region "us-east-2" + - INSTANCE_ID=$(cat linux-amd64-instance.txt) + - echo $INSTANCE_ID + script: + - aws ec2 terminate-instances --instance-ids $INSTANCE_ID + needs: + - job: deploy node for linux amd64 diff --git a/.gitlab/cloud-init.sh b/.gitlab/cloud-init.sh new file mode 100755 index 000000000..055d0f6aa --- /dev/null +++ b/.gitlab/cloud-init.sh @@ -0,0 +1,26 @@ +#!/bin/bash -ex +# generate log +exec > >(tee /var/log/user-data.log|logger -t user-data -s 2>/dev/console) 2>&1 + +# Install packages +yum update -y +yum install postgresql -y + +# Configure host system +echo 'cat /proc/sys/fs/file-max' +sysctl -w fs.file-max=262144 +sysctl -p +echo 'cat /proc/sys/fs/file-max' + +# yum install golang -y # latest verion in ec2 is 1.15.14 +# install go 1.16.9 manually +curl -O https://dl.google.com/go/go1.16.9.linux-amd64.tar.gz +tar xvf go1.16.9.linux-amd64.tar.gz +chown -R root:root ./go +mv go /usr/local +echo "export PATH=/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/home/ec2-user/.local/bin:/home/ec2-user/bin:/usr/local/go/bin" | tee -a /etc/profile > /dev/null +source /etc/profile + +# install aws session manager pluggin +curl "https://s3.amazonaws.com/session-manager-downloads/plugin/latest/linux_64bit/session-manager-plugin.rpm" -o "session-manager-plugin.rpm" +yum install -y session-manager-plugin.rpm \ No newline at end of file diff --git a/.gitlab/configureFeatureBase.json b/.gitlab/configureFeatureBase.json new file mode 100644 index 000000000..6afd135d9 --- /dev/null +++ b/.gitlab/configureFeatureBase.json @@ -0,0 +1,20 @@ +{ + "Parameters": { + "commands": [ + "#!/bin/bash", + "mv /home/ec2-user/featurebase_linux_amd64 /usr/local/bin/featurebase", + "mv /home/ec2-user/featurebase.conf /etc/", + "mv /home/ec2-user/featurebase.service /etc/systemd/system/", + "adduser molecula", + "sudo mkdir /var/log/molecula", + "sudo chown molecula /var/log/molecula", + "sudo mkdir -p /opt/molecula/featurebase", + "sudo chown molecula /opt/molecula/featurebase", + "systemctl daemon-reload", + "sudo systemctl start featurebase", + "sudo systemctl enable featurebase", + "sudo systemctl status featurebase", + "curl localhost:10101" + ] + } +} \ No newline at end of file diff --git a/.gitlab/featurebase.conf b/.gitlab/featurebase.conf new file mode 100644 index 000000000..7716d0f6b --- /dev/null +++ b/.gitlab/featurebase.conf @@ -0,0 +1,30 @@ +name = "pilosa1" +bind = "0.0.0.0:10101" +bind-grpc = "0.0.0.0:20101" + +data-dir = "/opt/molecula/featurebase" +log-path = "/var/log/molecula/featurebase.log" + +max-file-count=900000 +max-map-count=900000 + +long-query-time = "10s" + +[postgres] + + bind = "localhost:55432" + +[cluster] + + name = "cluster1" + replicas = 1 + +[etcd] + + listen-client-address = "http://localhost:10401" + listen-peer-address = "http://localhost:10301" + initial-cluster = "pilosa1=http://localhost:10301" + +[metric] + + service = "prometheus" \ No newline at end of file diff --git a/.gitlab/featurebase.service b/.gitlab/featurebase.service new file mode 100644 index 000000000..a543b92af --- /dev/null +++ b/.gitlab/featurebase.service @@ -0,0 +1,13 @@ +# Not Ansible managed + +[Unit] +Description="Service for FeatureBase" + +[Service] +RestartSec=30 +Restart=on-failure +EnvironmentFile= +User=molecula +ExecStart=/usr/local/bin/featurebase server -c /etc/featurebase.conf + +[Install] \ No newline at end of file