From cfacc72c539ed65158834708a993d1541b473148 Mon Sep 17 00:00:00 2001 From: pokeeffe-molecula Date: Tue, 4 Jan 2022 15:00:39 -0600 Subject: [PATCH] smoking or non-smoking? --- .gitlab/.gitlab-ci.yml | 68 +++++++------ qa/scripts/cloud-init.sh | 26 ----- qa/scripts/config.json | 0 qa/scripts/configureFeatureBase.json | 20 ---- qa/scripts/deployNode.sh | 98 ------------------- qa/scripts/deploySingleNodeCluster.sh | 28 ------ qa/scripts/featurebase.conf | 30 ------ qa/scripts/featurebase.service | 13 --- qa/scripts/ingestWorkload.sh | 84 ---------------- qa/scripts/perf.sh | 3 - qa/scripts/runSmokeTest.sh | 5 + qa/scripts/setupSmokeTest.sh | 46 +++++++++ qa/scripts/teardownSmokeTest.sh | 8 ++ qa/scripts/testSmokeTest.sh | 10 ++ qa/tf/ci/{singlenode => smoketest}/main.tf | 3 +- qa/tf/ci/{singlenode => smoketest}/outputs.tf | 0 .../ci/{singlenode => smoketest}/provider.tf | 0 .../{singlenode => smoketest}/tf.auto.tfvars | 0 .../ci/{singlenode => smoketest}/variables.tf | 0 19 files changed, 110 insertions(+), 332 deletions(-) delete mode 100755 qa/scripts/cloud-init.sh delete mode 100644 qa/scripts/config.json delete mode 100644 qa/scripts/configureFeatureBase.json delete mode 100755 qa/scripts/deployNode.sh delete mode 100644 qa/scripts/deploySingleNodeCluster.sh delete mode 100644 qa/scripts/featurebase.conf delete mode 100644 qa/scripts/featurebase.service delete mode 100755 qa/scripts/ingestWorkload.sh delete mode 100644 qa/scripts/perf.sh create mode 100644 qa/scripts/runSmokeTest.sh create mode 100755 qa/scripts/setupSmokeTest.sh create mode 100755 qa/scripts/teardownSmokeTest.sh create mode 100755 qa/scripts/testSmokeTest.sh rename qa/tf/ci/{singlenode => smoketest}/main.tf (75%) rename qa/tf/ci/{singlenode => smoketest}/outputs.tf (100%) rename qa/tf/ci/{singlenode => smoketest}/provider.tf (100%) rename qa/tf/ci/{singlenode => smoketest}/tf.auto.tfvars (100%) rename qa/tf/ci/{singlenode => smoketest}/variables.tf (100%) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 55bbde528..87b68da71 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -218,34 +218,46 @@ build container fb: - docker push $tag - echo Created docker featurebase image with tag "$tag" -# deploy EC2 instance, configure and run featurebase -# diabling for now - will come back and refactor POK -# deploy node for linux amd64: -# stage: integration -# image: registry.gitlab.com/gitlab-org/cloud-deploy/aws-base:latest -# variables: -# PROFILE: "default" -# AWS_SSH_PRIVATE_KEY: $AWS_FBCI_SSH_KEY -# rules: -# - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH' -# before_script: -# - aws configure set aws_access_key_id $AWS_FBCI_ACCESS_KEY_ID -# - aws configure set aws_secret_access_key $AWS_FBCI_SECRET_ACCESS_KEY -# - aws configure set region "us-east-2" -# - aws configure set aws_profile $PROFILE -# - echo $AWS_FBCI_SSH_KEY > gitlab-featurebase-ci.pem -# - chmod 400 gitlab-featurebase-ci.pem -# - 'which ssh-agent || ( apt-get update -y && apt-get install openssh-client -y )' -# - eval `ssh-agent -s` -# - mkdir -p ~/.ssh -# - echo "$AWS_FBCI_SSH_KEY" | ssh-add - -# - chmod 700 /root/.ssh -# - '[[ -f /.dockerenv ]] && echo -e "Host *\n\tStrictHostKeyChecking no\n\n" > ~/.ssh/config' -# - apt update && apt -y install jq -# script: -# - ./qa/scripts/deployNode.sh $PROFILE -# needs: -# - job: build for linux amd64 +smoke test: + stage: integration + image: registry.gitlab.com/gitlab-org/cloud-deploy/aws-base:latest + variables: + PROFILE: "service-terraform" + AWS_SSH_PRIVATE_KEY: $AWS_FBCI_SSH_KEY + AWS_ACCESS_KEY_ID: $AWS_FBCI_ACCESS_KEY_ID + AWS_SECRET_ACCESS_KEY: $AWS_FBCI_SECRET_ACCESS_KEY + rules: + - if: '$CI_PIPELINE_SOURCE == "push"' + before_script: + - apt-get update && apt-get install -y gnupg software-properties-common curl git + - curl -fsSL https://apt.releases.hashicorp.com/gpg | apt-key add - + - apt-add-repository "deb [arch=amd64] https://apt.releases.hashicorp.com $(lsb_release -cs) main" + - apt-get update && apt-get install terraform + - aws configure set aws_access_key_id $AWS_FBCI_ACCESS_KEY_ID + - aws configure set aws_secret_access_key $AWS_FBCI_SECRET_ACCESS_KEY + - aws configure set region "us-east-2" + - aws configure set aws_profile $PROFILE + - echo $AWS_FBCI_SSH_KEY > gitlab-featurebase-ci.pem + - chmod 400 gitlab-featurebase-ci.pem + - 'which ssh-agent || ( apt-get update -y && apt-get install openssh-client -y )' + - eval $(ssh-agent -s) + - mkdir -p ~/.ssh + - echo $AWS_FBCI_SSH_KEY > /root/.ssh/gitlab-featurebase-ci.pem + - chmod 400 /root/.ssh/gitlab-featurebase-ci.pem + - echo "$AWS_FBCI_SSH_KEY" | ssh-add - + - chmod 700 /root/.ssh + - '[[ -f /.dockerenv ]] && echo -e "Host *\n\tStrictHostKeyChecking no\n\n" > ~/.ssh/config' + - apt update && apt -y install jq wget + - wget https://go.dev/dl/go1.17.5.linux-amd64.tar.gz + - tar -C /usr/local -xzf go1.17.5.linux-amd64.tar.gz + - export PATH=$PATH:/usr/local/go/bin + script: + - ./qa/scripts/setupSmokeTest.sh + - ./qa/scripts/testSmokeTest.sh + after_script: + - ./qa/scripts/teardownSmokeTest.sh + needs: + - job: build for linux arm64 gauntlet: stage: gauntlet diff --git a/qa/scripts/cloud-init.sh b/qa/scripts/cloud-init.sh deleted file mode 100755 index 8d710d0b2..000000000 --- a/qa/scripts/cloud-init.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/bash -ex -# generate log -exec > >(tee /var/log/user-data.log|logger -t user-data -s 2>/dev/console) 2>&1 - -# Install packages -yum update -y -yum install postgresql -y - -# Configure host system -echo 'cat /proc/sys/fs/file-max' -sysctl -w fs.file-max=262144 -sysctl -p -echo 'cat /proc/sys/fs/file-max' - -# yum install golang -y # latest verion in ec2 is 1.15.14 -# install go 1.16.9 manually -curl -O https://dl.google.com/go/go1.16.10.linux-amd64.tar.gz -tar xvf go1.16.10.linux-amd64.tar.gz -chown -R root:root ./go -mv go /usr/local -echo "export PATH=/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/home/ec2-user/.local/bin:/home/ec2-user/bin:/usr/local/go/bin" | tee -a /etc/profile > /dev/null -source /etc/profile - -# install aws session manager pluggin -curl "https://s3.amazonaws.com/session-manager-downloads/plugin/latest/linux_64bit/session-manager-plugin.rpm" -o "session-manager-plugin.rpm" -yum install -y session-manager-plugin.rpm diff --git a/qa/scripts/config.json b/qa/scripts/config.json deleted file mode 100644 index e69de29bb..000000000 diff --git a/qa/scripts/configureFeatureBase.json b/qa/scripts/configureFeatureBase.json deleted file mode 100644 index 6afd135d9..000000000 --- a/qa/scripts/configureFeatureBase.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "Parameters": { - "commands": [ - "#!/bin/bash", - "mv /home/ec2-user/featurebase_linux_amd64 /usr/local/bin/featurebase", - "mv /home/ec2-user/featurebase.conf /etc/", - "mv /home/ec2-user/featurebase.service /etc/systemd/system/", - "adduser molecula", - "sudo mkdir /var/log/molecula", - "sudo chown molecula /var/log/molecula", - "sudo mkdir -p /opt/molecula/featurebase", - "sudo chown molecula /opt/molecula/featurebase", - "systemctl daemon-reload", - "sudo systemctl start featurebase", - "sudo systemctl enable featurebase", - "sudo systemctl status featurebase", - "curl localhost:10101" - ] - } -} \ No newline at end of file diff --git a/qa/scripts/deployNode.sh b/qa/scripts/deployNode.sh deleted file mode 100755 index 7b5f53796..000000000 --- a/qa/scripts/deployNode.sh +++ /dev/null @@ -1,98 +0,0 @@ -#!/bin/bash - -# To run script: ./deployNode.sh $PROFILE - -# default to the VPC initially created -VPC=${VPC:-vpc-0582f594d7d2ca2d4} - -function deploy_node() { - # get AMI, security group and subnet ID - AMI=$(aws ssm get-parameters --names /aws/service/ami-amazon-linux-latest/amzn2-ami-hvm-x86_64-ebs --query 'Parameters[0].[Value]' --output text --profile $PROFILE) - if [[ $? > 0 ]]; then - echo "aws session manager failed to find AMI" - exit 1 - fi - - SECURITY_GROUP=$(aws ec2 describe-security-groups --filters "Name=vpc-id,Values=$VPC" Name=group-name,Values=default --query 'SecurityGroups[*].[GroupId]' --output text --profile $PROFILE) - if [[ $? > 0 ]]; then - echo "aws session manager failed to find security group" - exit 1 - fi - - SUBNET_ID=$(aws ec2 describe-subnets --filters "Name=vpc-id,Values=$VPC" 'Name=availability-zone,Values=us-east-2a' --query 'Subnets[0].SubnetId' --output text --profile $PROFILE) - if [[ $? > 0 ]]; then - echo "aws session manager failed to find subnet ID" - exit 1 - fi - - # launch EC2 instance and get instance ID - aws ec2 run-instances --image-id $AMI --instance-type $INSTANCE --security-group-ids $SECURITY_GROUP --subnet-id $SUBNET_ID --key-name gitlab-featurebase-dev --tag-specifications 'ResourceType=instance,Tags=[{Key=Name,Value=linux-amd64-node}]' --profile $PROFILE --user-data file://./qa/scripts/cloud-init.sh --iam-instance-profile Name=featurebase-dev-ssm > config.json - if [[ $? > 0 ]]; then - echo "aws run-instances failed to launch a new EC2 instance" - exit 1 - fi - - INSTANCE_ID=$(jq '.Instances | .[0] |.InstanceId' config.json | tr -d '"') - echo "aws run-instances succeeded in launching a new EC2 instance with instance ID: " $INSTANCE_ID -} - -function initialize_featurebase() { - # get IP for node - for i in {0..24} - do - IP=$(aws ec2 describe-instances --instance-ids $INSTANCE_ID --filters 'Name=instance-state-name, Values=running' --query 'Reservations[*].Instances[*].PublicIpAddress' --output text --profile $PROFILE) - if [ -n "$IP" ]; then - echo "Public IP for EC2 instance found: " $IP - break - fi - - if [[ $? > 0 ]]; then - echo "aws cli describe-instances command failed to find public IP" - terminate_node - exit 1 - fi - - sleep 5 - done - - sleep 60 # to allow enough time for node to be ready for use - - # copy featurebase binary and files to ec2 instance - scp -o StrictHostKeyChecking=no -i gitlab-featurebase-dev.pem featurebase_linux_amd64 ./qa/scripts/featurebase.conf ./qa/scripts/featurebase.service ec2-user@$IP:. - if [[ $? > 0 ]]; then - echo "scp of featurebase binary, service and config files to EC2 instance failed" - terminate_node - exit 1 - fi - - # execute script to configure featurebase on the EC2 node - aws ssm send-command --document-name "AWS-RunShellScript" --instance-ids $INSTANCE_ID --cli-input-json file://./qa/scripts/configureFeatureBase.json --profile $PROFILE --region $REGION - if [[ $? > 0 ]]; then - echo "aws cli session manager send-command failed" - terminate_node - exit 1 - fi -} - -function terminate_node() { - aws ec2 terminate-instances --instance-ids $INSTANCE_ID --profile $PROFILE -} - -# Pass variables to shell script -PROFILE=$1 -shift - -# set some variables -INSTANCE="t3a.large" -REGION="us-east-2" - -# get AMI, security group and subnet for EC2 instance, -# launch instance, save instance Id and run cloud-init to set up node env -deploy_node - -# Get IP for instance, scp featurebase binary, config and service files; -# set up featurebase config in node -initialize_featurebase - -# terminate node -terminate_node diff --git a/qa/scripts/deploySingleNodeCluster.sh b/qa/scripts/deploySingleNodeCluster.sh deleted file mode 100644 index d4d1240ec..000000000 --- a/qa/scripts/deploySingleNodeCluster.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/bash - -# To run script: ./deploySingleNodeCluster.sh -# requires TF_VAR_gitlab_token env var to be set - -echo “$(pwd)” - -pushd ./qa/tf/ci/singlenode -export TF_IN_AUTOMATION=1 -terraform init -input=false -terraform apply -input=false -auto-approve -popd - -# configure Featurebase - -# step 1a: get IPs of the cluster - - - -# step 1b: get IPs of the ingest nodes - -# step 2: write a featurebase.conf file - -# step 3: write featurebase.service - -# step 4: start featurebase - -# step 5: verify featurebase running \ No newline at end of file diff --git a/qa/scripts/featurebase.conf b/qa/scripts/featurebase.conf deleted file mode 100644 index 7716d0f6b..000000000 --- a/qa/scripts/featurebase.conf +++ /dev/null @@ -1,30 +0,0 @@ -name = "pilosa1" -bind = "0.0.0.0:10101" -bind-grpc = "0.0.0.0:20101" - -data-dir = "/opt/molecula/featurebase" -log-path = "/var/log/molecula/featurebase.log" - -max-file-count=900000 -max-map-count=900000 - -long-query-time = "10s" - -[postgres] - - bind = "localhost:55432" - -[cluster] - - name = "cluster1" - replicas = 1 - -[etcd] - - listen-client-address = "http://localhost:10401" - listen-peer-address = "http://localhost:10301" - initial-cluster = "pilosa1=http://localhost:10301" - -[metric] - - service = "prometheus" \ No newline at end of file diff --git a/qa/scripts/featurebase.service b/qa/scripts/featurebase.service deleted file mode 100644 index a543b92af..000000000 --- a/qa/scripts/featurebase.service +++ /dev/null @@ -1,13 +0,0 @@ -# Not Ansible managed - -[Unit] -Description="Service for FeatureBase" - -[Service] -RestartSec=30 -Restart=on-failure -EnvironmentFile= -User=molecula -ExecStart=/usr/local/bin/featurebase server -c /etc/featurebase.conf - -[Install] \ No newline at end of file diff --git a/qa/scripts/ingestWorkload.sh b/qa/scripts/ingestWorkload.sh deleted file mode 100755 index 6b87fb762..000000000 --- a/qa/scripts/ingestWorkload.sh +++ /dev/null @@ -1,84 +0,0 @@ -#!/usr/bin/env bash - -# path for featurebase binary -FEATUREBASE_PATH=/usr/local/bin - -# path for directory with csv directory files for all fields to be ingested -CSV_DIR_PATH=/data - - -# To run: -# ./ingestWorkload.sh {Local host & port for featurebase} {initialize flag} - -function delete_field { - if (($INITIALIZE == 0)); - then - curl -XDELETE $HOST/index/$INDEX/field/$FIELD - fi -} - -# Script to replicate samsung workload of deleting and re-ingesting fields every night -# outline delete and re-ingest workload -function ingest_int_field { - delete_field - curl -XPOST $HOST/index/$INDEX/field/$FIELD -d '{"options": {"type": "int", "min": 0, "max":'$MAX'}}' - $FEATUREBASE_PATH/featurebase import --host $HOST -i $INDEX -f $FIELD $CSV_FILE -} - -function ingest_time_field { - delete_field - curl -XPOST $HOST/index/$INDEX/field/$FIELD -d '{"options": {"keys": true, "type": "time", "timeQuantum": "YMD"}}' - $FEATUREBASE_PATH/featurebase import --host $HOST -i $INDEX -f $FIELD $CSV_FILE -} - -function ingest_set_field { - delete_field - curl -XPOST $HOST/index/$INDEX/field/$FIELD -d '{"options": {"keys": true}}' - $FEATUREBASE_PATH/featurebase import --host $HOST -i $INDEX -f $FIELD $CSV_FILE -} - -# featurebase host & port -HOST=$1 -shift - -# intialize flag - 0:disabled, 1:enabled - creates the index and fields for testing -INITIALIZE=$1 -shift - -# get a list of csv files in the directory -CSV_FILES=`ls $CSV_DIR_PATH/*.csv` - -# assign index name -INDEX="samsung" -if (($INITIALIZE == 1)); -then - curl -XPOST $HOST/index/$INDEX -fi - -# perform delete and re-ingest for all fields -for CSV_FILE in ${CSV_FILES[@]} - do - # get field name from csv file path - FIELD="$(basename $CSV_FILE .csv)" - if [[ "$FIELD" == *"age"* ]]; - then - MAX=100 - ingest_int_field - elif [[ "$FIELD" == *"identifier"* ]]; - then - MAX=$((2**63 - 1)) # compute max value for 64bit - ingest_int_field - elif [[ "$FIELD" == *"ip"* ]]; - then - MAX=$((2**31 - 1)) # compute max value for 32bit - ingest_int_field - elif [[ "$FIELD" == *"time"* ]]; - then - ingest_time_field - else - ingest_set_field - fi - done - - - diff --git a/qa/scripts/perf.sh b/qa/scripts/perf.sh deleted file mode 100644 index 41a734d59..000000000 --- a/qa/scripts/perf.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash -echo >&2 "performance testing" -time ./simulacraData diff --git a/qa/scripts/runSmokeTest.sh b/qa/scripts/runSmokeTest.sh new file mode 100644 index 000000000..554a234cc --- /dev/null +++ b/qa/scripts/runSmokeTest.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +./setupSmokeTest.sh +./testSmokeTest.sh +./teardownSmokeTest.sh diff --git a/qa/scripts/setupSmokeTest.sh b/qa/scripts/setupSmokeTest.sh new file mode 100755 index 000000000..853d4caf6 --- /dev/null +++ b/qa/scripts/setupSmokeTest.sh @@ -0,0 +1,46 @@ +#!/bin/bash + +# To run script: ./setupSmokeTest.sh +# requires TF_VAR_gitlab_token env var to be set + +pushd ./qa/tf/ci/smoketest +export TF_IN_AUTOMATION=1 +echo "Running terraform init..." +terraform init -input=false +echo "Running terraform apply..." +terraform apply -input=false -auto-approve +terraform output -json > samsung-gauntlet.json +popd + +# get the bastion host +BASTION=$(cat ./qa/tf/ci/smoketest/smoketest.json | jq -r '[.ingest_ips][0]["value"][0]') +echo "using bastion ${BASTION}" + +NODE=$(cat ./qa/tf/ci/smoketest/smoketest.json | jq -r '[.data_node_ips][0]["value"][0]') +echo "using node ${NODE}" + +# remember that the nodes will take at least 2 mins to be up and going and finish cloud-init +echo "Waiting for cluster to become available..." +while true +do + ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o "StrictHostKeyChecking no" ec2-user@${BASTION} "curl -s http://${NODE}:10101/status" + if [ $? -eq 0 ] + then + break + fi + sleep 20 +done + + +# verify featurebase running +echo "Verifying featurebase cluster running..." +ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o "StrictHostKeyChecking no" ec2-user@${BASTION} "curl -s http://${NODE}:10101/status" +if (( $? != 0 )) +then + echo "Featurebase cluster not running" + exit 1 +fi + +echo "Cluster running." + + diff --git a/qa/scripts/teardownSmokeTest.sh b/qa/scripts/teardownSmokeTest.sh new file mode 100755 index 000000000..1e67d2895 --- /dev/null +++ b/qa/scripts/teardownSmokeTest.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +# To run script: ./teardownSmokeTest.sh +# requires TF_VAR_gitlab_token env var to be set + +cd qa/tf/ci/smoketest +export TF_IN_AUTOMATION=1 +terraform destroy -auto-approve diff --git a/qa/scripts/testSmokeTest.sh b/qa/scripts/testSmokeTest.sh new file mode 100755 index 000000000..3d1f615f0 --- /dev/null +++ b/qa/scripts/testSmokeTest.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +# get the bastion host +BASTION=$(cat ./qa/tf/ci/smoketest/smoketest.json | jq -r '[.ingest_ips][0]["value"][0]') +echo "using bastion ${BASTION}" + +NODE=$(cat ./qa/tf/ci/smoketest/smoketest.json | jq -r '[.data_node_ips][0]["value"][0]') +echo "using node ${NODE}" + +echo "Smoke test complete" \ No newline at end of file diff --git a/qa/tf/ci/singlenode/main.tf b/qa/tf/ci/smoketest/main.tf similarity index 75% rename from qa/tf/ci/singlenode/main.tf rename to qa/tf/ci/smoketest/main.tf index d50f41182..397da3391 100644 --- a/qa/tf/ci/singlenode/main.tf +++ b/qa/tf/ci/smoketest/main.tf @@ -1,10 +1,9 @@ module "ci-cluster" { source = "../../.modules/featurebase-cluster" - cluster_prefix = "ci-single-node" + cluster_prefix = "smoke" region = var.region profile = var.profile fb_data_node_type = "m6g.large" - fb_data_node_count = 1 gitlab_token = var.gitlab_token } diff --git a/qa/tf/ci/singlenode/outputs.tf b/qa/tf/ci/smoketest/outputs.tf similarity index 100% rename from qa/tf/ci/singlenode/outputs.tf rename to qa/tf/ci/smoketest/outputs.tf diff --git a/qa/tf/ci/singlenode/provider.tf b/qa/tf/ci/smoketest/provider.tf similarity index 100% rename from qa/tf/ci/singlenode/provider.tf rename to qa/tf/ci/smoketest/provider.tf diff --git a/qa/tf/ci/singlenode/tf.auto.tfvars b/qa/tf/ci/smoketest/tf.auto.tfvars similarity index 100% rename from qa/tf/ci/singlenode/tf.auto.tfvars rename to qa/tf/ci/smoketest/tf.auto.tfvars diff --git a/qa/tf/ci/singlenode/variables.tf b/qa/tf/ci/smoketest/variables.tf similarity index 100% rename from qa/tf/ci/singlenode/variables.tf rename to qa/tf/ci/smoketest/variables.tf