mirror of
https://github.com/featurebasedb/featurebase.git
synced 2026-10-07 03:17:50 +00:00
smoking or non-smoking?
This commit is contained in:
pokeeffe-molecula
parent
094f70a91a
commit
cfacc72c53
19 changed files with 110 additions and 332 deletions
|
|
@ -218,34 +218,46 @@ build container fb:
|
|||
- docker push $tag
|
||||
- echo Created docker featurebase image with tag "$tag"
|
||||
|
||||
# deploy EC2 instance, configure and run featurebase
|
||||
# diabling for now - will come back and refactor POK
|
||||
# deploy node for linux amd64:
|
||||
# stage: integration
|
||||
# image: registry.gitlab.com/gitlab-org/cloud-deploy/aws-base:latest
|
||||
# variables:
|
||||
# PROFILE: "default"
|
||||
# AWS_SSH_PRIVATE_KEY: $AWS_FBCI_SSH_KEY
|
||||
# rules:
|
||||
# - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'
|
||||
# before_script:
|
||||
# - aws configure set aws_access_key_id $AWS_FBCI_ACCESS_KEY_ID
|
||||
# - aws configure set aws_secret_access_key $AWS_FBCI_SECRET_ACCESS_KEY
|
||||
# - aws configure set region "us-east-2"
|
||||
# - aws configure set aws_profile $PROFILE
|
||||
# - echo $AWS_FBCI_SSH_KEY > gitlab-featurebase-ci.pem
|
||||
# - chmod 400 gitlab-featurebase-ci.pem
|
||||
# - 'which ssh-agent || ( apt-get update -y && apt-get install openssh-client -y )'
|
||||
# - eval `ssh-agent -s`
|
||||
# - mkdir -p ~/.ssh
|
||||
# - echo "$AWS_FBCI_SSH_KEY" | ssh-add -
|
||||
# - chmod 700 /root/.ssh
|
||||
# - '[[ -f /.dockerenv ]] && echo -e "Host *\n\tStrictHostKeyChecking no\n\n" > ~/.ssh/config'
|
||||
# - apt update && apt -y install jq
|
||||
# script:
|
||||
# - ./qa/scripts/deployNode.sh $PROFILE
|
||||
# needs:
|
||||
# - job: build for linux amd64
|
||||
smoke test:
|
||||
stage: integration
|
||||
image: registry.gitlab.com/gitlab-org/cloud-deploy/aws-base:latest
|
||||
variables:
|
||||
PROFILE: "service-terraform"
|
||||
AWS_SSH_PRIVATE_KEY: $AWS_FBCI_SSH_KEY
|
||||
AWS_ACCESS_KEY_ID: $AWS_FBCI_ACCESS_KEY_ID
|
||||
AWS_SECRET_ACCESS_KEY: $AWS_FBCI_SECRET_ACCESS_KEY
|
||||
rules:
|
||||
- if: '$CI_PIPELINE_SOURCE == "push"'
|
||||
before_script:
|
||||
- apt-get update && apt-get install -y gnupg software-properties-common curl git
|
||||
- curl -fsSL https://apt.releases.hashicorp.com/gpg | apt-key add -
|
||||
- apt-add-repository "deb [arch=amd64] https://apt.releases.hashicorp.com $(lsb_release -cs) main"
|
||||
- apt-get update && apt-get install terraform
|
||||
- aws configure set aws_access_key_id $AWS_FBCI_ACCESS_KEY_ID
|
||||
- aws configure set aws_secret_access_key $AWS_FBCI_SECRET_ACCESS_KEY
|
||||
- aws configure set region "us-east-2"
|
||||
- aws configure set aws_profile $PROFILE
|
||||
- echo $AWS_FBCI_SSH_KEY > gitlab-featurebase-ci.pem
|
||||
- chmod 400 gitlab-featurebase-ci.pem
|
||||
- 'which ssh-agent || ( apt-get update -y && apt-get install openssh-client -y )'
|
||||
- eval $(ssh-agent -s)
|
||||
- mkdir -p ~/.ssh
|
||||
- echo $AWS_FBCI_SSH_KEY > /root/.ssh/gitlab-featurebase-ci.pem
|
||||
- chmod 400 /root/.ssh/gitlab-featurebase-ci.pem
|
||||
- echo "$AWS_FBCI_SSH_KEY" | ssh-add -
|
||||
- chmod 700 /root/.ssh
|
||||
- '[[ -f /.dockerenv ]] && echo -e "Host *\n\tStrictHostKeyChecking no\n\n" > ~/.ssh/config'
|
||||
- apt update && apt -y install jq wget
|
||||
- wget https://go.dev/dl/go1.17.5.linux-amd64.tar.gz
|
||||
- tar -C /usr/local -xzf go1.17.5.linux-amd64.tar.gz
|
||||
- export PATH=$PATH:/usr/local/go/bin
|
||||
script:
|
||||
- ./qa/scripts/setupSmokeTest.sh
|
||||
- ./qa/scripts/testSmokeTest.sh
|
||||
after_script:
|
||||
- ./qa/scripts/teardownSmokeTest.sh
|
||||
needs:
|
||||
- job: build for linux arm64
|
||||
|
||||
gauntlet:
|
||||
stage: gauntlet
|
||||
|
|
|
|||
|
|
@ -1,26 +0,0 @@
|
|||
#!/bin/bash -ex
|
||||
# generate log
|
||||
exec > >(tee /var/log/user-data.log|logger -t user-data -s 2>/dev/console) 2>&1
|
||||
|
||||
# Install packages
|
||||
yum update -y
|
||||
yum install postgresql -y
|
||||
|
||||
# Configure host system
|
||||
echo 'cat /proc/sys/fs/file-max'
|
||||
sysctl -w fs.file-max=262144
|
||||
sysctl -p
|
||||
echo 'cat /proc/sys/fs/file-max'
|
||||
|
||||
# yum install golang -y # latest verion in ec2 is 1.15.14
|
||||
# install go 1.16.9 manually
|
||||
curl -O https://dl.google.com/go/go1.16.10.linux-amd64.tar.gz
|
||||
tar xvf go1.16.10.linux-amd64.tar.gz
|
||||
chown -R root:root ./go
|
||||
mv go /usr/local
|
||||
echo "export PATH=/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/home/ec2-user/.local/bin:/home/ec2-user/bin:/usr/local/go/bin" | tee -a /etc/profile > /dev/null
|
||||
source /etc/profile
|
||||
|
||||
# install aws session manager pluggin
|
||||
curl "https://s3.amazonaws.com/session-manager-downloads/plugin/latest/linux_64bit/session-manager-plugin.rpm" -o "session-manager-plugin.rpm"
|
||||
yum install -y session-manager-plugin.rpm
|
||||
|
|
@ -1,20 +0,0 @@
|
|||
{
|
||||
"Parameters": {
|
||||
"commands": [
|
||||
"#!/bin/bash",
|
||||
"mv /home/ec2-user/featurebase_linux_amd64 /usr/local/bin/featurebase",
|
||||
"mv /home/ec2-user/featurebase.conf /etc/",
|
||||
"mv /home/ec2-user/featurebase.service /etc/systemd/system/",
|
||||
"adduser molecula",
|
||||
"sudo mkdir /var/log/molecula",
|
||||
"sudo chown molecula /var/log/molecula",
|
||||
"sudo mkdir -p /opt/molecula/featurebase",
|
||||
"sudo chown molecula /opt/molecula/featurebase",
|
||||
"systemctl daemon-reload",
|
||||
"sudo systemctl start featurebase",
|
||||
"sudo systemctl enable featurebase",
|
||||
"sudo systemctl status featurebase",
|
||||
"curl localhost:10101"
|
||||
]
|
||||
}
|
||||
}
|
||||
|
|
@ -1,98 +0,0 @@
|
|||
#!/bin/bash
|
||||
|
||||
# To run script: ./deployNode.sh $PROFILE
|
||||
|
||||
# default to the VPC initially created
|
||||
VPC=${VPC:-vpc-0582f594d7d2ca2d4}
|
||||
|
||||
function deploy_node() {
|
||||
# get AMI, security group and subnet ID
|
||||
AMI=$(aws ssm get-parameters --names /aws/service/ami-amazon-linux-latest/amzn2-ami-hvm-x86_64-ebs --query 'Parameters[0].[Value]' --output text --profile $PROFILE)
|
||||
if [[ $? > 0 ]]; then
|
||||
echo "aws session manager failed to find AMI"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
SECURITY_GROUP=$(aws ec2 describe-security-groups --filters "Name=vpc-id,Values=$VPC" Name=group-name,Values=default --query 'SecurityGroups[*].[GroupId]' --output text --profile $PROFILE)
|
||||
if [[ $? > 0 ]]; then
|
||||
echo "aws session manager failed to find security group"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
SUBNET_ID=$(aws ec2 describe-subnets --filters "Name=vpc-id,Values=$VPC" 'Name=availability-zone,Values=us-east-2a' --query 'Subnets[0].SubnetId' --output text --profile $PROFILE)
|
||||
if [[ $? > 0 ]]; then
|
||||
echo "aws session manager failed to find subnet ID"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# launch EC2 instance and get instance ID
|
||||
aws ec2 run-instances --image-id $AMI --instance-type $INSTANCE --security-group-ids $SECURITY_GROUP --subnet-id $SUBNET_ID --key-name gitlab-featurebase-dev --tag-specifications 'ResourceType=instance,Tags=[{Key=Name,Value=linux-amd64-node}]' --profile $PROFILE --user-data file://./qa/scripts/cloud-init.sh --iam-instance-profile Name=featurebase-dev-ssm > config.json
|
||||
if [[ $? > 0 ]]; then
|
||||
echo "aws run-instances failed to launch a new EC2 instance"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
INSTANCE_ID=$(jq '.Instances | .[0] |.InstanceId' config.json | tr -d '"')
|
||||
echo "aws run-instances succeeded in launching a new EC2 instance with instance ID: " $INSTANCE_ID
|
||||
}
|
||||
|
||||
function initialize_featurebase() {
|
||||
# get IP for node
|
||||
for i in {0..24}
|
||||
do
|
||||
IP=$(aws ec2 describe-instances --instance-ids $INSTANCE_ID --filters 'Name=instance-state-name, Values=running' --query 'Reservations[*].Instances[*].PublicIpAddress' --output text --profile $PROFILE)
|
||||
if [ -n "$IP" ]; then
|
||||
echo "Public IP for EC2 instance found: " $IP
|
||||
break
|
||||
fi
|
||||
|
||||
if [[ $? > 0 ]]; then
|
||||
echo "aws cli describe-instances command failed to find public IP"
|
||||
terminate_node
|
||||
exit 1
|
||||
fi
|
||||
|
||||
sleep 5
|
||||
done
|
||||
|
||||
sleep 60 # to allow enough time for node to be ready for use
|
||||
|
||||
# copy featurebase binary and files to ec2 instance
|
||||
scp -o StrictHostKeyChecking=no -i gitlab-featurebase-dev.pem featurebase_linux_amd64 ./qa/scripts/featurebase.conf ./qa/scripts/featurebase.service ec2-user@$IP:.
|
||||
if [[ $? > 0 ]]; then
|
||||
echo "scp of featurebase binary, service and config files to EC2 instance failed"
|
||||
terminate_node
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# execute script to configure featurebase on the EC2 node
|
||||
aws ssm send-command --document-name "AWS-RunShellScript" --instance-ids $INSTANCE_ID --cli-input-json file://./qa/scripts/configureFeatureBase.json --profile $PROFILE --region $REGION
|
||||
if [[ $? > 0 ]]; then
|
||||
echo "aws cli session manager send-command failed"
|
||||
terminate_node
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
function terminate_node() {
|
||||
aws ec2 terminate-instances --instance-ids $INSTANCE_ID --profile $PROFILE
|
||||
}
|
||||
|
||||
# Pass variables to shell script
|
||||
PROFILE=$1
|
||||
shift
|
||||
|
||||
# set some variables
|
||||
INSTANCE="t3a.large"
|
||||
REGION="us-east-2"
|
||||
|
||||
# get AMI, security group and subnet for EC2 instance,
|
||||
# launch instance, save instance Id and run cloud-init to set up node env
|
||||
deploy_node
|
||||
|
||||
# Get IP for instance, scp featurebase binary, config and service files;
|
||||
# set up featurebase config in node
|
||||
initialize_featurebase
|
||||
|
||||
# terminate node
|
||||
terminate_node
|
||||
|
|
@ -1,28 +0,0 @@
|
|||
#!/bin/bash
|
||||
|
||||
# To run script: ./deploySingleNodeCluster.sh
|
||||
# requires TF_VAR_gitlab_token env var to be set
|
||||
|
||||
echo “$(pwd)”
|
||||
|
||||
pushd ./qa/tf/ci/singlenode
|
||||
export TF_IN_AUTOMATION=1
|
||||
terraform init -input=false
|
||||
terraform apply -input=false -auto-approve
|
||||
popd
|
||||
|
||||
# configure Featurebase
|
||||
|
||||
# step 1a: get IPs of the cluster
|
||||
|
||||
|
||||
|
||||
# step 1b: get IPs of the ingest nodes
|
||||
|
||||
# step 2: write a featurebase.conf file
|
||||
|
||||
# step 3: write featurebase.service
|
||||
|
||||
# step 4: start featurebase
|
||||
|
||||
# step 5: verify featurebase running
|
||||
|
|
@ -1,30 +0,0 @@
|
|||
name = "pilosa1"
|
||||
bind = "0.0.0.0:10101"
|
||||
bind-grpc = "0.0.0.0:20101"
|
||||
|
||||
data-dir = "/opt/molecula/featurebase"
|
||||
log-path = "/var/log/molecula/featurebase.log"
|
||||
|
||||
max-file-count=900000
|
||||
max-map-count=900000
|
||||
|
||||
long-query-time = "10s"
|
||||
|
||||
[postgres]
|
||||
|
||||
bind = "localhost:55432"
|
||||
|
||||
[cluster]
|
||||
|
||||
name = "cluster1"
|
||||
replicas = 1
|
||||
|
||||
[etcd]
|
||||
|
||||
listen-client-address = "http://localhost:10401"
|
||||
listen-peer-address = "http://localhost:10301"
|
||||
initial-cluster = "pilosa1=http://localhost:10301"
|
||||
|
||||
[metric]
|
||||
|
||||
service = "prometheus"
|
||||
|
|
@ -1,13 +0,0 @@
|
|||
# Not Ansible managed
|
||||
|
||||
[Unit]
|
||||
Description="Service for FeatureBase"
|
||||
|
||||
[Service]
|
||||
RestartSec=30
|
||||
Restart=on-failure
|
||||
EnvironmentFile=
|
||||
User=molecula
|
||||
ExecStart=/usr/local/bin/featurebase server -c /etc/featurebase.conf
|
||||
|
||||
[Install]
|
||||
|
|
@ -1,84 +0,0 @@
|
|||
#!/usr/bin/env bash
|
||||
|
||||
# path for featurebase binary
|
||||
FEATUREBASE_PATH=/usr/local/bin
|
||||
|
||||
# path for directory with csv directory files for all fields to be ingested
|
||||
CSV_DIR_PATH=/data
|
||||
|
||||
|
||||
# To run:
|
||||
# ./ingestWorkload.sh {Local host & port for featurebase} {initialize flag}
|
||||
|
||||
function delete_field {
|
||||
if (($INITIALIZE == 0));
|
||||
then
|
||||
curl -XDELETE $HOST/index/$INDEX/field/$FIELD
|
||||
fi
|
||||
}
|
||||
|
||||
# Script to replicate samsung workload of deleting and re-ingesting fields every night
|
||||
# outline delete and re-ingest workload
|
||||
function ingest_int_field {
|
||||
delete_field
|
||||
curl -XPOST $HOST/index/$INDEX/field/$FIELD -d '{"options": {"type": "int", "min": 0, "max":'$MAX'}}'
|
||||
$FEATUREBASE_PATH/featurebase import --host $HOST -i $INDEX -f $FIELD $CSV_FILE
|
||||
}
|
||||
|
||||
function ingest_time_field {
|
||||
delete_field
|
||||
curl -XPOST $HOST/index/$INDEX/field/$FIELD -d '{"options": {"keys": true, "type": "time", "timeQuantum": "YMD"}}'
|
||||
$FEATUREBASE_PATH/featurebase import --host $HOST -i $INDEX -f $FIELD $CSV_FILE
|
||||
}
|
||||
|
||||
function ingest_set_field {
|
||||
delete_field
|
||||
curl -XPOST $HOST/index/$INDEX/field/$FIELD -d '{"options": {"keys": true}}'
|
||||
$FEATUREBASE_PATH/featurebase import --host $HOST -i $INDEX -f $FIELD $CSV_FILE
|
||||
}
|
||||
|
||||
# featurebase host & port
|
||||
HOST=$1
|
||||
shift
|
||||
|
||||
# intialize flag - 0:disabled, 1:enabled - creates the index and fields for testing
|
||||
INITIALIZE=$1
|
||||
shift
|
||||
|
||||
# get a list of csv files in the directory
|
||||
CSV_FILES=`ls $CSV_DIR_PATH/*.csv`
|
||||
|
||||
# assign index name
|
||||
INDEX="samsung"
|
||||
if (($INITIALIZE == 1));
|
||||
then
|
||||
curl -XPOST $HOST/index/$INDEX
|
||||
fi
|
||||
|
||||
# perform delete and re-ingest for all fields
|
||||
for CSV_FILE in ${CSV_FILES[@]}
|
||||
do
|
||||
# get field name from csv file path
|
||||
FIELD="$(basename $CSV_FILE .csv)"
|
||||
if [[ "$FIELD" == *"age"* ]];
|
||||
then
|
||||
MAX=100
|
||||
ingest_int_field
|
||||
elif [[ "$FIELD" == *"identifier"* ]];
|
||||
then
|
||||
MAX=$((2**63 - 1)) # compute max value for 64bit
|
||||
ingest_int_field
|
||||
elif [[ "$FIELD" == *"ip"* ]];
|
||||
then
|
||||
MAX=$((2**31 - 1)) # compute max value for 32bit
|
||||
ingest_int_field
|
||||
elif [[ "$FIELD" == *"time"* ]];
|
||||
then
|
||||
ingest_time_field
|
||||
else
|
||||
ingest_set_field
|
||||
fi
|
||||
done
|
||||
|
||||
|
||||
|
||||
|
|
@ -1,3 +0,0 @@
|
|||
#!/bin/bash
|
||||
echo >&2 "performance testing"
|
||||
time ./simulacraData
|
||||
5
qa/scripts/runSmokeTest.sh
Normal file
5
qa/scripts/runSmokeTest.sh
Normal file
|
|
@ -0,0 +1,5 @@
|
|||
#!/bin/bash
|
||||
|
||||
./setupSmokeTest.sh
|
||||
./testSmokeTest.sh
|
||||
./teardownSmokeTest.sh
|
||||
46
qa/scripts/setupSmokeTest.sh
Executable file
46
qa/scripts/setupSmokeTest.sh
Executable file
|
|
@ -0,0 +1,46 @@
|
|||
#!/bin/bash
|
||||
|
||||
# To run script: ./setupSmokeTest.sh
|
||||
# requires TF_VAR_gitlab_token env var to be set
|
||||
|
||||
pushd ./qa/tf/ci/smoketest
|
||||
export TF_IN_AUTOMATION=1
|
||||
echo "Running terraform init..."
|
||||
terraform init -input=false
|
||||
echo "Running terraform apply..."
|
||||
terraform apply -input=false -auto-approve
|
||||
terraform output -json > samsung-gauntlet.json
|
||||
popd
|
||||
|
||||
# get the bastion host
|
||||
BASTION=$(cat ./qa/tf/ci/smoketest/smoketest.json | jq -r '[.ingest_ips][0]["value"][0]')
|
||||
echo "using bastion ${BASTION}"
|
||||
|
||||
NODE=$(cat ./qa/tf/ci/smoketest/smoketest.json | jq -r '[.data_node_ips][0]["value"][0]')
|
||||
echo "using node ${NODE}"
|
||||
|
||||
# remember that the nodes will take at least 2 mins to be up and going and finish cloud-init
|
||||
echo "Waiting for cluster to become available..."
|
||||
while true
|
||||
do
|
||||
ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o "StrictHostKeyChecking no" ec2-user@${BASTION} "curl -s http://${NODE}:10101/status"
|
||||
if [ $? -eq 0 ]
|
||||
then
|
||||
break
|
||||
fi
|
||||
sleep 20
|
||||
done
|
||||
|
||||
|
||||
# verify featurebase running
|
||||
echo "Verifying featurebase cluster running..."
|
||||
ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o "StrictHostKeyChecking no" ec2-user@${BASTION} "curl -s http://${NODE}:10101/status"
|
||||
if (( $? != 0 ))
|
||||
then
|
||||
echo "Featurebase cluster not running"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Cluster running."
|
||||
|
||||
|
||||
8
qa/scripts/teardownSmokeTest.sh
Executable file
8
qa/scripts/teardownSmokeTest.sh
Executable file
|
|
@ -0,0 +1,8 @@
|
|||
#!/bin/bash
|
||||
|
||||
# To run script: ./teardownSmokeTest.sh
|
||||
# requires TF_VAR_gitlab_token env var to be set
|
||||
|
||||
cd qa/tf/ci/smoketest
|
||||
export TF_IN_AUTOMATION=1
|
||||
terraform destroy -auto-approve
|
||||
10
qa/scripts/testSmokeTest.sh
Executable file
10
qa/scripts/testSmokeTest.sh
Executable file
|
|
@ -0,0 +1,10 @@
|
|||
#!/bin/bash
|
||||
|
||||
# get the bastion host
|
||||
BASTION=$(cat ./qa/tf/ci/smoketest/smoketest.json | jq -r '[.ingest_ips][0]["value"][0]')
|
||||
echo "using bastion ${BASTION}"
|
||||
|
||||
NODE=$(cat ./qa/tf/ci/smoketest/smoketest.json | jq -r '[.data_node_ips][0]["value"][0]')
|
||||
echo "using node ${NODE}"
|
||||
|
||||
echo "Smoke test complete"
|
||||
|
|
@ -1,10 +1,9 @@
|
|||
|
||||
module "ci-cluster" {
|
||||
source = "../../.modules/featurebase-cluster"
|
||||
cluster_prefix = "ci-single-node"
|
||||
cluster_prefix = "smoke"
|
||||
region = var.region
|
||||
profile = var.profile
|
||||
fb_data_node_type = "m6g.large"
|
||||
fb_data_node_count = 1
|
||||
gitlab_token = var.gitlab_token
|
||||
}
|
||||
Loading…
Add table
Reference in a new issue