mirror of
https://github.com/featurebasedb/featurebase.git
synced 2026-09-07 09:05:55 +00:00
refactored scripts for imperative setup and execution
This commit is contained in:
pokeeffe-molecula
parent
bb434639ad
commit
4743624210
13 changed files with 384 additions and 757 deletions
|
|
@ -1,10 +1,19 @@
|
|||
#!/bin/bash
|
||||
|
||||
# To run script: ./setupSamsungGauntlet.sh
|
||||
export TF_IN_AUTOMATION=1
|
||||
|
||||
# requires TF_VAR_gitlab_token env var to be set
|
||||
if [ -z ${TF_VAR_gitlab_token+x} ]; then echo "TF_VAR_gitlab_token is unset"; else echo "TF_VAR_gitlab_token is set to '$TF_VAR_gitlab_token'"; fi
|
||||
|
||||
# requires TF_VAR_branch env var to be set
|
||||
if [ -z ${TF_VAR_branch+x} ]; then echo "TF_VAR_branch is unset"; else echo "TF_VAR_branch is set to '$TF_VAR_branch'"; fi
|
||||
|
||||
SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
|
||||
source $SCRIPT_DIR/utilCluster.sh
|
||||
|
||||
|
||||
pushd ./qa/tf/gauntlet/samsung
|
||||
export TF_IN_AUTOMATION=1
|
||||
echo "Running terraform init..."
|
||||
terraform init -input=false
|
||||
echo "Running terraform apply..."
|
||||
|
|
@ -12,30 +21,45 @@ terraform apply -input=false -auto-approve
|
|||
terraform output -json > outputs.json
|
||||
popd
|
||||
|
||||
# get the bastion host
|
||||
BASTION=$(cat ./qa/tf/gauntlet/samsung/outputs.json | jq -r '[.ingest_ips][0]["value"][0]')
|
||||
echo "using bastion ${BASTION}"
|
||||
# get the first ingest host
|
||||
INGESTNODE0=$(cat ./qa/tf/ci/smoketest/outputs.json | jq -r '[.ingest_ips][0]["value"][0]')
|
||||
echo "using INGESTNODE0 ${INGESTNODE0}"
|
||||
|
||||
NODE=$(cat ./qa/tf/gauntlet/samsung/outputs.json | jq -r '[.data_node_ips][0]["value"][0]')
|
||||
echo "using node ${NODE}"
|
||||
# get the first data host
|
||||
DATANODE0=$(cat ./qa/tf/ci/smoketest/outputs.json | jq -r '[.data_node_ips][0]["value"][0]')
|
||||
echo "using DATANODE0 ${DATANODE0}"
|
||||
|
||||
# remember that the nodes will take at least 2 mins to be up and going and finish cloud-init
|
||||
#while true
|
||||
#do
|
||||
# nc -G 2 -w 1 $BASTION 22
|
||||
# if [ $? -eq 0 ]
|
||||
# then
|
||||
# break
|
||||
# fi
|
||||
#done
|
||||
sleep 150
|
||||
#wait until we can connect to one of the hosts
|
||||
for i in {0..24}
|
||||
do
|
||||
ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no -o ConnectTimeout=10 ec2-user@${DATANODE0} "pwd"
|
||||
if [ $? -eq 0 ]
|
||||
then
|
||||
echo "Cluster is up after $${i} tries."
|
||||
break
|
||||
fi
|
||||
sleep 10s
|
||||
done
|
||||
|
||||
ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no -o ConnectTimeout=10 ec2-user@${DATANODE0} "pwd"
|
||||
if [ $? -ne 0 ]
|
||||
then
|
||||
echo "Unable to connect to cluster - giving up"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
setupClusterNodes
|
||||
|
||||
# verify featurebase running
|
||||
ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o "StrictHostKeyChecking no" ec2-user@${BASTION} "curl -s http://${NODE}:10101/status"
|
||||
echo "Verifying featurebase cluster running..."
|
||||
curl -s http://${DATANODE0}:10101/status
|
||||
if (( $? != 0 ))
|
||||
then
|
||||
echo "Featurebase cluster not running"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Cluster running."
|
||||
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -1,10 +1,18 @@
|
|||
#!/bin/bash
|
||||
|
||||
# To run script: ./setupSmokeTest.sh
|
||||
export TF_IN_AUTOMATION=1
|
||||
|
||||
# requires TF_VAR_gitlab_token env var to be set
|
||||
if [ -z ${TF_VAR_gitlab_token+x} ]; then echo "TF_VAR_gitlab_token is unset"; else echo "TF_VAR_gitlab_token is set to '$TF_VAR_gitlab_token'"; fi
|
||||
|
||||
# requires TF_VAR_branch env var to be set
|
||||
if [ -z ${TF_VAR_branch+x} ]; then echo "TF_VAR_branch is unset"; else echo "TF_VAR_branch is set to '$TF_VAR_branch'"; fi
|
||||
|
||||
SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
|
||||
source $SCRIPT_DIR/utilCluster.sh
|
||||
|
||||
pushd ./qa/tf/ci/smoketest
|
||||
export TF_IN_AUTOMATION=1
|
||||
echo "Running terraform init..."
|
||||
terraform init -input=false
|
||||
echo "Running terraform apply..."
|
||||
|
|
@ -12,21 +20,38 @@ terraform apply -input=false -auto-approve
|
|||
terraform output -json > outputs.json
|
||||
popd
|
||||
|
||||
# get the bastion host
|
||||
BASTION=$(cat ./qa/tf/ci/smoketest/outputs.json | jq -r '[.ingest_ips][0]["value"][0]')
|
||||
echo "using bastion ${BASTION}"
|
||||
# get the first ingest host
|
||||
INGESTNODE0=$(cat ./qa/tf/ci/smoketest/outputs.json | jq -r '[.ingest_ips][0]["value"][0]')
|
||||
echo "using INGESTNODE0 ${INGESTNODE0}"
|
||||
|
||||
NODE=$(cat ./qa/tf/ci/smoketest/outputs.json | jq -r '[.data_node_ips][0]["value"][0]')
|
||||
echo "using node ${NODE}"
|
||||
# get the first data host
|
||||
DATANODE0=$(cat ./qa/tf/ci/smoketest/outputs.json | jq -r '[.data_node_ips][0]["value"][0]')
|
||||
echo "using DATANODE0 ${DATANODE0}"
|
||||
|
||||
# remember that the nodes will take at least 2 mins to be up and going and finish cloud-init
|
||||
echo "Waiting for cluster to become available..."
|
||||
# jaffee - I do wanna do a loop here, but I give up, and am running home to sleep... -POK
|
||||
sleep 150
|
||||
#wait until we can connect to one of the hosts
|
||||
for i in {0..24}
|
||||
do
|
||||
ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no -o ConnectTimeout=10 ec2-user@${DATANODE0} "pwd"
|
||||
if [ $? -eq 0 ]
|
||||
then
|
||||
echo "Cluster is up after $${i} tries."
|
||||
break
|
||||
fi
|
||||
sleep 10
|
||||
done
|
||||
|
||||
ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no -o ConnectTimeout=10 ec2-user@${DATANODE0} "pwd"
|
||||
if [ $? -ne 0 ]
|
||||
then
|
||||
echo "Unable to connect to cluster - giving up"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
setupClusterNodes
|
||||
|
||||
# verify featurebase running
|
||||
echo "Verifying featurebase cluster running..."
|
||||
ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${BASTION} "curl -s http://${NODE}:10101/status"
|
||||
curl -s http://${DATANODE0}:10101/status
|
||||
if (( $? != 0 ))
|
||||
then
|
||||
echo "Featurebase cluster not running"
|
||||
|
|
|
|||
|
|
@ -2,6 +2,10 @@
|
|||
|
||||
# To run script: ./teardownSmokeTest.sh
|
||||
# requires TF_VAR_gitlab_token env var to be set
|
||||
if [ -z ${TF_VAR_gitlab_token+x} ]; then echo "TF_VAR_gitlab_token is unset"; else echo "TF_VAR_gitlab_token is set to '$TF_VAR_gitlab_token'"; fi
|
||||
|
||||
# requires TF_VAR_branch env var to be set
|
||||
if [ -z ${TF_VAR_branch+x} ]; then echo "TF_VAR_branch is unset"; else echo "TF_VAR_branch is set to '$TF_VAR_branch'"; fi
|
||||
|
||||
cd qa/tf/ci/smoketest
|
||||
export TF_IN_AUTOMATION=1
|
||||
|
|
|
|||
|
|
@ -1,14 +1,25 @@
|
|||
#!/bin/bash
|
||||
|
||||
# get the bastion host
|
||||
BASTION=$(cat ./qa/tf/ci/smoketest/outputs.json | jq -r '[.ingest_ips][0]["value"][0]')
|
||||
echo "using bastion ${BASTION}"
|
||||
# requires TF_VAR_gitlab_token env var to be set
|
||||
if [ -z ${TF_VAR_gitlab_token+x} ]; then echo "TF_VAR_gitlab_token is unset"; else echo "TF_VAR_gitlab_token is set to '$TF_VAR_gitlab_token'"; fi
|
||||
|
||||
# requires TF_VAR_branch env var to be set
|
||||
if [ -z ${TF_VAR_branch+x} ]; then echo "TF_VAR_branch is unset"; else echo "TF_VAR_branch is set to '$TF_VAR_branch'"; fi
|
||||
|
||||
SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
|
||||
source $SCRIPT_DIR/utilCluster.sh
|
||||
|
||||
# get the first ingest host
|
||||
INGESTNODE0=$(cat ./qa/tf/ci/smoketest/outputs.json | jq -r '[.ingest_ips][0]["value"][0]')
|
||||
echo "using INGESTNODE0 ${INGESTNODE0}"
|
||||
|
||||
# get the first data host
|
||||
DATANODE0=$(cat ./qa/tf/ci/smoketest/outputs.json | jq -r '[.data_node_ips][0]["value"][0]')
|
||||
echo "using DATANODE0 ${DATANODE0}"
|
||||
|
||||
NODE=$(cat ./qa/tf/ci/smoketest/outputs.json | jq -r '[.data_node_ips][0]["value"][0]')
|
||||
echo "using node ${NODE}"
|
||||
|
||||
echo "Copying tests to remote"
|
||||
scp -r -i ~/.ssh/gitlab-featurebase-ci.pem ./qa/testcases/smoketest/*.py ec2-user@${BASTION}:/data
|
||||
scp -r -i ~/.ssh/gitlab-featurebase-ci.pem ./qa/testcases/smoketest/*.py ec2-user@${INGESTNODE0}:/data
|
||||
if (( $? != 0 ))
|
||||
then
|
||||
echo "Copy failed"
|
||||
|
|
@ -17,7 +28,7 @@ fi
|
|||
|
||||
# run smoke test
|
||||
echo "Running smoke test..."
|
||||
ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o "StrictHostKeyChecking no" ec2-user@${BASTION} " pushd /data; pytest --junitxml=report.xml; popd"
|
||||
ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o "StrictHostKeyChecking no" ec2-user@${INGESTNODE0} "pushd /data; ~/.local/bin/pytest --junitxml=report.xml; popd"
|
||||
if (( $? != 0 ))
|
||||
then
|
||||
echo "Unable to run smoke test"
|
||||
|
|
@ -25,7 +36,7 @@ then
|
|||
fi
|
||||
|
||||
echo "Copying test report to local"
|
||||
scp -r -i ~/.ssh/gitlab-featurebase-ci.pem ec2-user@${BASTION}:/data/report.xml report.xml
|
||||
scp -r -i ~/.ssh/gitlab-featurebase-ci.pem ec2-user@${INGESTNODE0}:/data/report.xml report.xml
|
||||
if (( $? != 0 ))
|
||||
then
|
||||
echo "Copy failed"
|
||||
|
|
|
|||
252
qa/scripts/utilCluster.sh
Normal file
252
qa/scripts/utilCluster.sh
Normal file
|
|
@ -0,0 +1,252 @@
|
|||
#!/bin/bash
|
||||
|
||||
#path to the featurebase.conf file
|
||||
CONFIG_FILE_PATH="/etc/featurebase.conf"
|
||||
#path to the featurebase.service file
|
||||
SERVICE_FILE_PATH="/etc/systemd/system/featurebase.service"
|
||||
|
||||
#cluster prefix that was used
|
||||
DEPLOYED_CLUSTER_PREFIX=""
|
||||
|
||||
#cluster replica count that was used
|
||||
DEPLOYED_CLUSTER_REPLICA_COUNT=""
|
||||
|
||||
#List of deployed IPs for data nodes
|
||||
DEPLOYED_DATA_IPS=""
|
||||
DEPLOYED_DATA_IPS_LEN=0
|
||||
|
||||
#List of deployed IPs for ingest nodes
|
||||
DEPLOYED_INGEST_IPS=""
|
||||
DEPLOYED_INGEST_IPS_LEN=0
|
||||
|
||||
#Initial cluster string
|
||||
INITIAL_CLUSTER=""
|
||||
|
||||
writeFeatureBaseNodeServiceFile() {
|
||||
echo "Writing featurebase.service file...index: $1, ip:$2"
|
||||
NODEIDX=$1
|
||||
NODEIP=$2
|
||||
cat << EOT > featurebase.service
|
||||
# Not Ansible managed
|
||||
|
||||
[Unit]
|
||||
Description="Service for FeatureBase"
|
||||
|
||||
[Service]
|
||||
RestartSec=30
|
||||
Restart=on-failure
|
||||
EnvironmentFile=
|
||||
User=molecula
|
||||
ExecStart=/usr/local/bin/featurebase server -c /etc/featurebase.conf
|
||||
|
||||
[Install]
|
||||
EOT
|
||||
|
||||
#echo "featurebase.service >>"
|
||||
#cat featurebase.service
|
||||
#echo "featurebase.service <<"
|
||||
|
||||
scp -i ~/.ssh/gitlab-featurebase-ci.pem -o "StrictHostKeyChecking no" featurebase.service ec2-user@${NODEIP}:
|
||||
if (( $? != 0 ))
|
||||
then
|
||||
echo "featurebase.service copy failed"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
rm -f featurebase.service
|
||||
|
||||
ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "sudo mv featurebase.service ${SERVICE_FILE_PATH}"
|
||||
}
|
||||
|
||||
writeFeatureBaseNodeConfigFile() {
|
||||
echo "Writing featurebase.conf file...index: $1, ip:$2"
|
||||
NODEIDX=$1
|
||||
NODEIP=$2
|
||||
cat << EOT > featurebase.conf
|
||||
name = "p${NODEIDX}"
|
||||
bind = "0.0.0.0:10101"
|
||||
bind-grpc = "0.0.0.0:20101"
|
||||
|
||||
data-dir = "/data/featurebase"
|
||||
log-path = "/var/log/molecula/featurebase.log"
|
||||
|
||||
max-file-count=900000
|
||||
max-map-count=900000
|
||||
|
||||
long-query-time = "10s"
|
||||
|
||||
[postgres]
|
||||
|
||||
bind = "localhost:55432"
|
||||
|
||||
[cluster]
|
||||
|
||||
name = "${DEPLOYED_CLUSTER_PREFIX}"
|
||||
replicas = ${DEPLOYED_CLUSTER_REPLICA_COUNT}
|
||||
|
||||
[etcd]
|
||||
|
||||
listen-client-address = "http://${NODEIP}:10401"
|
||||
listen-peer-address = "http://${NODEIP}:10301"
|
||||
initial-cluster = "${INITIAL_CLUSTER}"
|
||||
|
||||
[metric]
|
||||
|
||||
service = "prometheus"
|
||||
EOT
|
||||
|
||||
#echo "featurebase.conf >>"
|
||||
#cat featurebase.conf
|
||||
#echo "featurebase.conf <<"
|
||||
|
||||
scp -i ~/.ssh/gitlab-featurebase-ci.pem -o "StrictHostKeyChecking no" featurebase.conf ec2-user@${NODEIP}:
|
||||
if (( $? != 0 ))
|
||||
then
|
||||
echo "featurebase.conf copy failed"
|
||||
exit 1
|
||||
fi
|
||||
rm -f featurebase.conf
|
||||
|
||||
ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "sudo mv featurebase.conf ${CONFIG_FILE_PATH}"
|
||||
}
|
||||
|
||||
executeGeneralNodeConfigCommands() {
|
||||
echo "Executing node config...index: $1, ip:$2"
|
||||
NODEIDX=$1
|
||||
NODEIP=$2
|
||||
|
||||
ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "sudo mkdir /data"
|
||||
ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "sudo mkfs.ext4 /dev/nvme1n1"
|
||||
ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "sudo mount /dev/nvme1n1 /data"
|
||||
|
||||
ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "sudo adduser molecula"
|
||||
ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "sudo mkdir /var/log/molecula"
|
||||
ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "sudo chown molecula /var/log/molecula"
|
||||
ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "sudo mkdir -p /data/featurebase"
|
||||
ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "sudo chown molecula /data/featurebase"
|
||||
|
||||
|
||||
# TODO handle different archs
|
||||
echo "Getting featurebase binary..."
|
||||
ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "curl --fail --header 'PRIVATE-TOKEN: ${TF_VAR_gitlab_token}' -o /home/ec2-user/featurebase_linux_arm64 https://gitlab.com/api/v4/projects/molecula%2Ffeaturebase/jobs/artifacts/${TF_VAR_branch}/raw/featurebase_linux_arm64?job=build%20for%20linux%20arm64"
|
||||
if (( $? != 0 ))
|
||||
then
|
||||
echo "Unable to get featurebase binary"
|
||||
exit 1
|
||||
fi
|
||||
ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "chown ec2-user:ec2-user /home/ec2-user/featurebase_linux_arm64"
|
||||
ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "chmod ugo+x /home/ec2-user/featurebase_linux_arm64"
|
||||
ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "sudo mv /home/ec2-user/featurebase_linux_arm64 /usr/local/bin/featurebase"
|
||||
|
||||
echo "featurebase binary copied."
|
||||
}
|
||||
|
||||
executeDataStartCommands() {
|
||||
echo "executeDataStartCommands...index: $1, ip:$2"
|
||||
NODEIDX=$1
|
||||
NODEIP=$2
|
||||
|
||||
ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "sudo systemctl daemon-reload"
|
||||
ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "sudo systemctl start featurebase"
|
||||
ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "sudo systemctl enable featurebase"
|
||||
ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "sudo systemctl status featurebase"
|
||||
}
|
||||
|
||||
startDataNodes() {
|
||||
#now go thru loop again to start up each node
|
||||
cnt=0
|
||||
for ip in $DEPLOYED_DATA_IPS
|
||||
do
|
||||
executeDataStartCommands $cnt $ip
|
||||
cnt=$((cnt+1))
|
||||
done
|
||||
}
|
||||
|
||||
setupDataNode() {
|
||||
echo "setting up node $1 at $2"
|
||||
|
||||
writeFeatureBaseNodeConfigFile $1 $2
|
||||
writeFeatureBaseNodeServiceFile $1 $2
|
||||
executeGeneralNodeConfigCommands $1 $2
|
||||
}
|
||||
|
||||
setupIngestNode() {
|
||||
echo "setting up ingest node $1 at $2"
|
||||
NODEIDX=$1
|
||||
NODEIP=$2
|
||||
|
||||
executeGeneralNodeConfigCommands $1 $2
|
||||
|
||||
ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "sudo chown -R ec2-user /data"
|
||||
ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "pip3 install -U pytest"
|
||||
ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "pip3 install -U requests"
|
||||
}
|
||||
|
||||
setupDataNodes() {
|
||||
cnt=0
|
||||
for ip in $DEPLOYED_DATA_IPS
|
||||
do
|
||||
setupDataNode $cnt $ip
|
||||
cnt=$((cnt+1))
|
||||
done
|
||||
}
|
||||
|
||||
setupIngestNodes() {
|
||||
cnt=0
|
||||
for ip in $DEPLOYED_INGEST_IPS
|
||||
do
|
||||
setupIngestNode $cnt $ip
|
||||
cnt=$((cnt+1))
|
||||
done
|
||||
}
|
||||
|
||||
generateInitialClusterString() {
|
||||
IFS=$'\n'
|
||||
cnt=0
|
||||
for ip in $DEPLOYED_DATA_IPS
|
||||
do
|
||||
if (($cnt + 1 != $DEPLOYED_DATA_IPS_LEN))
|
||||
then
|
||||
INITIAL_CLUSTER="${INITIAL_CLUSTER}p${cnt}=http://$ip:10301,"
|
||||
else
|
||||
INITIAL_CLUSTER="${INITIAL_CLUSTER}p${cnt}=http://$ip:10301"
|
||||
fi
|
||||
cnt=$((cnt+1))
|
||||
done
|
||||
|
||||
echo "INITIAL_CLUSTER: ${INITIAL_CLUSTER}"
|
||||
}
|
||||
|
||||
setupClusterNodes() {
|
||||
DEPLOYED_CLUSTER_PREFIX=$(cat ./qa/tf/ci/smoketest/outputs.json | jq -r '[.cluster_prefix][0]["value"]')
|
||||
echo "Using DEPLOYED_CLUSTER_PREFIX: ${DEPLOYED_CLUSTER_PREFIX}"
|
||||
|
||||
DEPLOYED_CLUSTER_REPLICA_COUNT=$(cat ./qa/tf/ci/smoketest/outputs.json | jq -r '[.fb_cluster_replica_count][0]["value"]')
|
||||
echo "Using DEPLOYED_CLUSTER_REPLICA_COUNT: ${DEPLOYED_CLUSTDEPLOYED_CLUSTER_REPLICA_COUNTER_PREFIX}"
|
||||
|
||||
DEPLOYED_DATA_IPS=$(cat ./qa/tf/ci/smoketest/outputs.json | jq -r '[.data_node_ips][0]["value"][]')
|
||||
echo "DEPLOYED_DATA_IPS: {"
|
||||
echo "${DEPLOYED_DATA_IPS}"
|
||||
echo "}"
|
||||
|
||||
DEPLOYED_DATA_IPS_LEN=`echo "$DEPLOYED_DATA_IPS" | wc -l`
|
||||
|
||||
DEPLOYED_INGEST_IPS=$(cat ./qa/tf/ci/smoketest/outputs.json | jq -r '[.ingest_ips][0]["value"][]')
|
||||
echo "DEPLOYED_INGEST_IPS: {"
|
||||
echo "${DEPLOYED_INGEST_IPS}"
|
||||
echo "}"
|
||||
|
||||
DEPLOYED_INGEST_IPS_LEN=`echo "$DEPLOYED_INGEST_IPS" | wc -l`
|
||||
|
||||
|
||||
#data nodes
|
||||
generateInitialClusterString
|
||||
|
||||
setupDataNodes
|
||||
|
||||
startDataNodes
|
||||
|
||||
#ingest nodes
|
||||
setupIngestNodes
|
||||
|
||||
}
|
||||
|
|
@ -46,7 +46,6 @@ resource "aws_instance" "fb_cluster_nodes" {
|
|||
Role = "cluster_node"
|
||||
}
|
||||
|
||||
user_data = base64encode(templatefile("${path.module}/setup_cluster_node.sh.tpl", { gitlab_token = var.gitlab_token, branch = var.branch, cluster_prefix = var.cluster_prefix, node_count = var.fb_data_node_count, fb_cluster_replica_count = var.fb_cluster_replica_count, region = var.region }))
|
||||
}
|
||||
|
||||
resource "aws_instance" "fb_ingest" {
|
||||
|
|
@ -79,7 +78,6 @@ resource "aws_instance" "fb_ingest" {
|
|||
Role = "ingest_node"
|
||||
}
|
||||
|
||||
user_data = base64encode(templatefile("${path.module}/setup_ingest_node.sh.tpl", { gitlab_token = var.gitlab_token, branch = var.branch, cluster_prefix = var.cluster_prefix, node_count = var.fb_ingest_node_count, this_node = count.index, region = var.region }))
|
||||
}
|
||||
|
||||
resource "aws_key_pair" "gitlab-featurebase-ci" {
|
||||
|
|
|
|||
|
|
@ -6,6 +6,10 @@ output "data_node_ips" {
|
|||
value = aws_instance.fb_cluster_nodes.*.private_ip
|
||||
}
|
||||
|
||||
output "vpc_id" {
|
||||
value = var.vpc_id
|
||||
}
|
||||
output "cluster_prefix" {
|
||||
value = var.cluster_prefix
|
||||
}
|
||||
|
||||
output "fb_cluster_replica_count" {
|
||||
value = var.fb_cluster_replica_count
|
||||
}
|
||||
|
|
|
|||
|
|
@ -87,15 +87,6 @@ variable "profile" {
|
|||
type = string
|
||||
}
|
||||
|
||||
variable "gitlab_token" {
|
||||
description = "Gitlab API token"
|
||||
type = string
|
||||
}
|
||||
|
||||
variable "branch" {
|
||||
description = "The branch we are on"
|
||||
type = string
|
||||
}
|
||||
|
||||
variable "vpc_id" {
|
||||
description = "The VPC in which we will build the cluster"
|
||||
|
|
|
|||
|
|
@ -6,8 +6,6 @@ module "ci-cluster" {
|
|||
profile = var.profile
|
||||
fb_data_node_type = "m6g.large"
|
||||
fb_data_node_count = 1
|
||||
gitlab_token = var.gitlab_token
|
||||
branch = var.branch
|
||||
vpc_id = "vpc-05a26a122f961dc2b"
|
||||
vpc_cidr_block = "10.0.0.0/16"
|
||||
vpc_public_subnets = ["subnet-066b4b922b54e51a2","subnet-037b8884269a69025","subnet-08482631514426210",]
|
||||
|
|
|
|||
|
|
@ -6,4 +6,14 @@ output "ingest_ips" {
|
|||
output "data_node_ips" {
|
||||
description = "List of data node IPs"
|
||||
value = module.ci-cluster.data_node_ips
|
||||
}
|
||||
}
|
||||
|
||||
output "cluster_prefix" {
|
||||
description = "The cluster prefix used"
|
||||
value = module.ci-cluster.cluster_prefix
|
||||
}
|
||||
|
||||
output "fb_cluster_replica_count" {
|
||||
description = "The cluster replica count used"
|
||||
value = module.ci-cluster.fb_cluster_replica_count
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,706 +1,8 @@
|
|||
{
|
||||
"version": 4,
|
||||
"terraform_version": "1.1.2",
|
||||
"serial": 64,
|
||||
"serial": 178,
|
||||
"lineage": "0f5e8a05-0e94-e86f-f384-26086bd40585",
|
||||
"outputs": {
|
||||
"data_node_ips": {
|
||||
"value": [
|
||||
"10.0.1.92"
|
||||
],
|
||||
"type": [
|
||||
"tuple",
|
||||
[
|
||||
"string"
|
||||
]
|
||||
]
|
||||
},
|
||||
"ingest_ips": {
|
||||
"value": [
|
||||
"18.119.132.64"
|
||||
],
|
||||
"type": [
|
||||
"tuple",
|
||||
[
|
||||
"string"
|
||||
]
|
||||
]
|
||||
},
|
||||
"vpc_id": {
|
||||
"value": "vpc-05a26a122f961dc2b",
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"resources": [
|
||||
{
|
||||
"module": "module.ci-cluster",
|
||||
"mode": "data",
|
||||
"type": "aws_ami",
|
||||
"name": "amazon_linux_2",
|
||||
"provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
|
||||
"instances": [
|
||||
{
|
||||
"schema_version": 0,
|
||||
"attributes": {
|
||||
"architecture": "arm64",
|
||||
"arn": "arn:aws:ec2:us-east-2::image/ami-0b09f36be67d32fff",
|
||||
"block_device_mappings": [
|
||||
{
|
||||
"device_name": "/dev/xvda",
|
||||
"ebs": {
|
||||
"delete_on_termination": "true",
|
||||
"encrypted": "false",
|
||||
"iops": "0",
|
||||
"snapshot_id": "snap-0617b00e90bae012b",
|
||||
"throughput": "0",
|
||||
"volume_size": "8",
|
||||
"volume_type": "gp2"
|
||||
},
|
||||
"no_device": "",
|
||||
"virtual_name": ""
|
||||
}
|
||||
],
|
||||
"creation_date": "2021-12-01T19:36:11.000Z",
|
||||
"description": "Amazon Linux 2 LTS Arm64 AMI 2.0.20211201.0 arm64 HVM gp2",
|
||||
"ena_support": true,
|
||||
"executable_users": null,
|
||||
"filter": [
|
||||
{
|
||||
"name": "architecture",
|
||||
"values": [
|
||||
"arm64"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "name",
|
||||
"values": [
|
||||
"amzn2-ami-hvm-*"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "virtualization-type",
|
||||
"values": [
|
||||
"hvm"
|
||||
]
|
||||
}
|
||||
],
|
||||
"hypervisor": "xen",
|
||||
"id": "ami-0b09f36be67d32fff",
|
||||
"image_id": "ami-0b09f36be67d32fff",
|
||||
"image_location": "amazon/amzn2-ami-hvm-2.0.20211201.0-arm64-gp2",
|
||||
"image_owner_alias": "amazon",
|
||||
"image_type": "machine",
|
||||
"kernel_id": null,
|
||||
"most_recent": true,
|
||||
"name": "amzn2-ami-hvm-2.0.20211201.0-arm64-gp2",
|
||||
"name_regex": null,
|
||||
"owner_id": "137112412989",
|
||||
"owners": [
|
||||
"amazon"
|
||||
],
|
||||
"platform": null,
|
||||
"platform_details": "Linux/UNIX",
|
||||
"product_codes": [],
|
||||
"public": true,
|
||||
"ramdisk_id": null,
|
||||
"root_device_name": "/dev/xvda",
|
||||
"root_device_type": "ebs",
|
||||
"root_snapshot_id": "snap-0617b00e90bae012b",
|
||||
"sriov_net_support": "simple",
|
||||
"state": "available",
|
||||
"state_reason": {
|
||||
"code": "UNSET",
|
||||
"message": "UNSET"
|
||||
},
|
||||
"tags": {},
|
||||
"usage_operation": "RunInstances",
|
||||
"virtualization_type": "hvm"
|
||||
},
|
||||
"sensitive_attributes": []
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"module": "module.ci-cluster",
|
||||
"mode": "managed",
|
||||
"type": "aws_iam_instance_profile",
|
||||
"name": "fb_cluster_node_profile",
|
||||
"provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
|
||||
"instances": [
|
||||
{
|
||||
"schema_version": 0,
|
||||
"attributes": {
|
||||
"arn": "arn:aws:iam::977373308795:instance-profile/fb_cluster_node_profile",
|
||||
"create_date": "2022-01-07T22:01:54Z",
|
||||
"id": "fb_cluster_node_profile",
|
||||
"name": "fb_cluster_node_profile",
|
||||
"name_prefix": null,
|
||||
"path": "/",
|
||||
"role": "fb_cluster_node",
|
||||
"tags": null,
|
||||
"tags_all": {},
|
||||
"unique_id": "AIPA6HD75E55U4UWVZABB"
|
||||
},
|
||||
"sensitive_attributes": [],
|
||||
"private": "bnVsbA==",
|
||||
"dependencies": [
|
||||
"module.ci-cluster.aws_iam_role.fb_cluster_node_role"
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"module": "module.ci-cluster",
|
||||
"mode": "managed",
|
||||
"type": "aws_iam_role",
|
||||
"name": "fb_cluster_node_role",
|
||||
"provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
|
||||
"instances": [
|
||||
{
|
||||
"schema_version": 0,
|
||||
"attributes": {
|
||||
"arn": "arn:aws:iam::977373308795:role/fb_cluster_node",
|
||||
"assume_role_policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Sid\":\"\",\"Effect\":\"Allow\",\"Principal\":{\"Service\":\"ec2.amazonaws.com\"},\"Action\":\"sts:AssumeRole\"}]}",
|
||||
"create_date": "2022-01-07T22:01:52Z",
|
||||
"description": "",
|
||||
"force_detach_policies": false,
|
||||
"id": "fb_cluster_node",
|
||||
"inline_policy": [
|
||||
{
|
||||
"name": "ec2_read_all",
|
||||
"policy": "{\"Statement\":[{\"Action\":[\"ec2:Describe*\"],\"Effect\":\"Allow\",\"Resource\":\"*\"}],\"Version\":\"2012-10-17\"}"
|
||||
}
|
||||
],
|
||||
"managed_policy_arns": [],
|
||||
"max_session_duration": 3600,
|
||||
"name": "fb_cluster_node",
|
||||
"name_prefix": "",
|
||||
"path": "/",
|
||||
"permissions_boundary": null,
|
||||
"tags": null,
|
||||
"tags_all": {},
|
||||
"unique_id": "AROA6HD75E55VPUUN47EM"
|
||||
},
|
||||
"sensitive_attributes": [],
|
||||
"private": "bnVsbA=="
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"module": "module.ci-cluster",
|
||||
"mode": "managed",
|
||||
"type": "aws_instance",
|
||||
"name": "fb_cluster_nodes",
|
||||
"provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
|
||||
"instances": [
|
||||
{
|
||||
"index_key": 0,
|
||||
"schema_version": 1,
|
||||
"attributes": {
|
||||
"ami": "ami-0b09f36be67d32fff",
|
||||
"arn": "arn:aws:ec2:us-east-2:977373308795:instance/i-018646ae2723b9266",
|
||||
"associate_public_ip_address": false,
|
||||
"availability_zone": "us-east-2a",
|
||||
"capacity_reservation_specification": [
|
||||
{
|
||||
"capacity_reservation_preference": "open",
|
||||
"capacity_reservation_target": []
|
||||
}
|
||||
],
|
||||
"cpu_core_count": 2,
|
||||
"cpu_threads_per_core": 1,
|
||||
"credit_specification": [],
|
||||
"disable_api_termination": false,
|
||||
"ebs_block_device": [
|
||||
{
|
||||
"delete_on_termination": true,
|
||||
"device_name": "/dev/sdb",
|
||||
"encrypted": false,
|
||||
"iops": 3000,
|
||||
"kms_key_id": "",
|
||||
"snapshot_id": "",
|
||||
"tags": {},
|
||||
"throughput": 125,
|
||||
"volume_id": "vol-064dca608db210bf4",
|
||||
"volume_size": 100,
|
||||
"volume_type": "gp3"
|
||||
}
|
||||
],
|
||||
"ebs_optimized": false,
|
||||
"enclave_options": [
|
||||
{
|
||||
"enabled": false
|
||||
}
|
||||
],
|
||||
"ephemeral_block_device": [],
|
||||
"get_password_data": false,
|
||||
"hibernation": false,
|
||||
"host_id": null,
|
||||
"iam_instance_profile": "fb_cluster_node_profile",
|
||||
"id": "i-018646ae2723b9266",
|
||||
"instance_initiated_shutdown_behavior": "stop",
|
||||
"instance_state": "running",
|
||||
"instance_type": "m6g.large",
|
||||
"ipv6_address_count": 0,
|
||||
"ipv6_addresses": [],
|
||||
"key_name": "gitlab-featurebase-ci",
|
||||
"launch_template": [],
|
||||
"metadata_options": [
|
||||
{
|
||||
"http_endpoint": "enabled",
|
||||
"http_put_response_hop_limit": 1,
|
||||
"http_tokens": "optional"
|
||||
}
|
||||
],
|
||||
"monitoring": true,
|
||||
"network_interface": [],
|
||||
"outpost_arn": "",
|
||||
"password_data": "",
|
||||
"placement_group": "",
|
||||
"placement_partition_number": null,
|
||||
"primary_network_interface_id": "eni-013fb4dd776a4ca48",
|
||||
"private_dns": "ip-10-0-1-92.us-east-2.compute.internal",
|
||||
"private_ip": "10.0.1.92",
|
||||
"public_dns": "",
|
||||
"public_ip": "",
|
||||
"root_block_device": [
|
||||
{
|
||||
"delete_on_termination": true,
|
||||
"device_name": "/dev/xvda",
|
||||
"encrypted": false,
|
||||
"iops": 3000,
|
||||
"kms_key_id": "",
|
||||
"tags": null,
|
||||
"throughput": 125,
|
||||
"volume_id": "vol-008d2b71268dd84e3",
|
||||
"volume_size": 20,
|
||||
"volume_type": "gp3"
|
||||
}
|
||||
],
|
||||
"secondary_private_ips": [],
|
||||
"security_groups": [],
|
||||
"source_dest_check": true,
|
||||
"subnet_id": "subnet-050b1219d78f2db1b",
|
||||
"tags": {
|
||||
"Name": "smoke-X8A0attf2zz6EhnV-featurebase-cluster-0",
|
||||
"Prefix": "smoke-X8A0attf2zz6EhnV",
|
||||
"Role": "cluster_node"
|
||||
},
|
||||
"tags_all": {
|
||||
"Name": "smoke-X8A0attf2zz6EhnV-featurebase-cluster-0",
|
||||
"Prefix": "smoke-X8A0attf2zz6EhnV",
|
||||
"Role": "cluster_node"
|
||||
},
|
||||
"tenancy": "default",
|
||||
"timeouts": null,
|
||||
"user_data": "efedaaed014dc69321f5e633ebc5dc436baf7b1e",
|
||||
"user_data_base64": null,
|
||||
"volume_tags": null,
|
||||
"vpc_security_group_ids": [
|
||||
"sg-014eb096b1eee30a5"
|
||||
]
|
||||
},
|
||||
"sensitive_attributes": [],
|
||||
"private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjo2MDAwMDAwMDAwMDAsImRlbGV0ZSI6MTIwMDAwMDAwMDAwMCwidXBkYXRlIjo2MDAwMDAwMDAwMDB9LCJzY2hlbWFfdmVyc2lvbiI6IjEifQ==",
|
||||
"dependencies": [
|
||||
"module.ci-cluster.aws_iam_instance_profile.fb_cluster_node_profile",
|
||||
"module.ci-cluster.aws_iam_role.fb_cluster_node_role",
|
||||
"module.ci-cluster.aws_key_pair.gitlab-featurebase-ci",
|
||||
"module.ci-cluster.aws_security_group.featurebase",
|
||||
"module.ci-cluster.data.aws_ami.amazon_linux_2"
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"module": "module.ci-cluster",
|
||||
"mode": "managed",
|
||||
"type": "aws_instance",
|
||||
"name": "fb_ingest",
|
||||
"provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
|
||||
"instances": [
|
||||
{
|
||||
"index_key": 0,
|
||||
"schema_version": 1,
|
||||
"attributes": {
|
||||
"ami": "ami-0b09f36be67d32fff",
|
||||
"arn": "arn:aws:ec2:us-east-2:977373308795:instance/i-013fde620a82c4102",
|
||||
"associate_public_ip_address": true,
|
||||
"availability_zone": "us-east-2a",
|
||||
"capacity_reservation_specification": [
|
||||
{
|
||||
"capacity_reservation_preference": "open",
|
||||
"capacity_reservation_target": []
|
||||
}
|
||||
],
|
||||
"cpu_core_count": 8,
|
||||
"cpu_threads_per_core": 1,
|
||||
"credit_specification": [],
|
||||
"disable_api_termination": false,
|
||||
"ebs_block_device": [
|
||||
{
|
||||
"delete_on_termination": true,
|
||||
"device_name": "/dev/sdb",
|
||||
"encrypted": false,
|
||||
"iops": 3000,
|
||||
"kms_key_id": "",
|
||||
"snapshot_id": "",
|
||||
"tags": {},
|
||||
"throughput": 125,
|
||||
"volume_id": "vol-0bd5eaddda4e19de5",
|
||||
"volume_size": 100,
|
||||
"volume_type": "gp3"
|
||||
}
|
||||
],
|
||||
"ebs_optimized": false,
|
||||
"enclave_options": [
|
||||
{
|
||||
"enabled": false
|
||||
}
|
||||
],
|
||||
"ephemeral_block_device": [],
|
||||
"get_password_data": false,
|
||||
"hibernation": false,
|
||||
"host_id": null,
|
||||
"iam_instance_profile": "fb_cluster_node_profile",
|
||||
"id": "i-013fde620a82c4102",
|
||||
"instance_initiated_shutdown_behavior": "stop",
|
||||
"instance_state": "running",
|
||||
"instance_type": "c6g.2xlarge",
|
||||
"ipv6_address_count": 0,
|
||||
"ipv6_addresses": [],
|
||||
"key_name": "gitlab-featurebase-ci",
|
||||
"launch_template": [],
|
||||
"metadata_options": [
|
||||
{
|
||||
"http_endpoint": "enabled",
|
||||
"http_put_response_hop_limit": 1,
|
||||
"http_tokens": "optional"
|
||||
}
|
||||
],
|
||||
"monitoring": true,
|
||||
"network_interface": [],
|
||||
"outpost_arn": "",
|
||||
"password_data": "",
|
||||
"placement_group": "",
|
||||
"placement_partition_number": null,
|
||||
"primary_network_interface_id": "eni-0bfb2b924677bd7bd",
|
||||
"private_dns": "ip-10-0-101-31.us-east-2.compute.internal",
|
||||
"private_ip": "10.0.101.31",
|
||||
"public_dns": "",
|
||||
"public_ip": "18.119.132.64",
|
||||
"root_block_device": [
|
||||
{
|
||||
"delete_on_termination": true,
|
||||
"device_name": "/dev/xvda",
|
||||
"encrypted": false,
|
||||
"iops": 3000,
|
||||
"kms_key_id": "",
|
||||
"tags": null,
|
||||
"throughput": 125,
|
||||
"volume_id": "vol-0f2c177cb0a77aecc",
|
||||
"volume_size": 20,
|
||||
"volume_type": "gp3"
|
||||
}
|
||||
],
|
||||
"secondary_private_ips": [],
|
||||
"security_groups": [],
|
||||
"source_dest_check": true,
|
||||
"subnet_id": "subnet-066b4b922b54e51a2",
|
||||
"tags": {
|
||||
"Name": "smoke-X8A0attf2zz6EhnV-featurebase-ingest-0",
|
||||
"Prefix": "smoke-X8A0attf2zz6EhnV",
|
||||
"Role": "ingest_node"
|
||||
},
|
||||
"tags_all": {
|
||||
"Name": "smoke-X8A0attf2zz6EhnV-featurebase-ingest-0",
|
||||
"Prefix": "smoke-X8A0attf2zz6EhnV",
|
||||
"Role": "ingest_node"
|
||||
},
|
||||
"tenancy": "default",
|
||||
"timeouts": null,
|
||||
"user_data": "38da76e780fedb19ef551fe9d3c84540d1823453",
|
||||
"user_data_base64": null,
|
||||
"volume_tags": null,
|
||||
"vpc_security_group_ids": [
|
||||
"sg-04e315d7b9ace5146"
|
||||
]
|
||||
},
|
||||
"sensitive_attributes": [],
|
||||
"private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjo2MDAwMDAwMDAwMDAsImRlbGV0ZSI6MTIwMDAwMDAwMDAwMCwidXBkYXRlIjo2MDAwMDAwMDAwMDB9LCJzY2hlbWFfdmVyc2lvbiI6IjEifQ==",
|
||||
"dependencies": [
|
||||
"module.ci-cluster.aws_iam_instance_profile.fb_cluster_node_profile",
|
||||
"module.ci-cluster.aws_iam_role.fb_cluster_node_role",
|
||||
"module.ci-cluster.aws_key_pair.gitlab-featurebase-ci",
|
||||
"module.ci-cluster.aws_security_group.ingest",
|
||||
"module.ci-cluster.data.aws_ami.amazon_linux_2"
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"module": "module.ci-cluster",
|
||||
"mode": "managed",
|
||||
"type": "aws_key_pair",
|
||||
"name": "gitlab-featurebase-ci",
|
||||
"provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
|
||||
"instances": [
|
||||
{
|
||||
"schema_version": 1,
|
||||
"attributes": {
|
||||
"arn": "arn:aws:ec2:us-east-2:977373308795:key-pair/gitlab-featurebase-ci",
|
||||
"fingerprint": "69:e5:4b:15:8d:1f:22:61:de:08:a9:ee:f3:29:5c:69",
|
||||
"id": "gitlab-featurebase-ci",
|
||||
"key_name": "gitlab-featurebase-ci",
|
||||
"key_name_prefix": "",
|
||||
"key_pair_id": "key-059c3d744adf3c048",
|
||||
"public_key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC91hhpVHNonAG7ku2ugpxEskf9KHeyHJPQJT26OHrMUw7R+T5A8TjqSzTau07sXQ/E9SO3ebV8SJ5PqeaQOnQB8VEvVNK0DjQH7ppvNg1Rfs42FZT9ttzTMvOjsSbK3vZTHXdoKQEdC9NxBwSkFIRGQojK1HUOq9xGrw31fA1OjSwlpLcbx7yyg18lcqW6UOptnVR8U9Yy9qQ5jZF1HtkQ6L9J+gv4o1UyNAUK2bopeGiXpBc3PQ/CFaFT2h/aqLBP66qAHsHVyAFD3PIRtplC5EHa8jXDgLacEls0uF7Q3kRPxvzcuo4g4VkOn1rDy9qH3vd2hT3aKVnM73FIDUiL",
|
||||
"tags": null,
|
||||
"tags_all": {}
|
||||
},
|
||||
"sensitive_attributes": [],
|
||||
"private": "eyJzY2hlbWFfdmVyc2lvbiI6IjEifQ=="
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"module": "module.ci-cluster",
|
||||
"mode": "managed",
|
||||
"type": "aws_security_group",
|
||||
"name": "featurebase",
|
||||
"provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
|
||||
"instances": [
|
||||
{
|
||||
"schema_version": 1,
|
||||
"attributes": {
|
||||
"arn": "arn:aws:ec2:us-east-2:977373308795:security-group/sg-014eb096b1eee30a5",
|
||||
"description": "Allow featurebase inbound traffic",
|
||||
"egress": [
|
||||
{
|
||||
"cidr_blocks": [
|
||||
"0.0.0.0/0"
|
||||
],
|
||||
"description": "",
|
||||
"from_port": 0,
|
||||
"ipv6_cidr_blocks": [
|
||||
"::/0"
|
||||
],
|
||||
"prefix_list_ids": [],
|
||||
"protocol": "-1",
|
||||
"security_groups": [],
|
||||
"self": false,
|
||||
"to_port": 0
|
||||
}
|
||||
],
|
||||
"id": "sg-014eb096b1eee30a5",
|
||||
"ingress": [
|
||||
{
|
||||
"cidr_blocks": [
|
||||
"0.0.0.0/0"
|
||||
],
|
||||
"description": "SSH",
|
||||
"from_port": 22,
|
||||
"ipv6_cidr_blocks": [
|
||||
"::/0"
|
||||
],
|
||||
"prefix_list_ids": [],
|
||||
"protocol": "tcp",
|
||||
"security_groups": [],
|
||||
"self": false,
|
||||
"to_port": 22
|
||||
},
|
||||
{
|
||||
"cidr_blocks": [
|
||||
"0.0.0.0/0"
|
||||
],
|
||||
"description": "icmp from Anywhere",
|
||||
"from_port": -1,
|
||||
"ipv6_cidr_blocks": [],
|
||||
"prefix_list_ids": [],
|
||||
"protocol": "icmp",
|
||||
"security_groups": [],
|
||||
"self": false,
|
||||
"to_port": -1
|
||||
},
|
||||
{
|
||||
"cidr_blocks": [
|
||||
"10.0.0.0/16"
|
||||
],
|
||||
"description": "etcd from internal 2",
|
||||
"from_port": 10401,
|
||||
"ipv6_cidr_blocks": [],
|
||||
"prefix_list_ids": [],
|
||||
"protocol": "tcp",
|
||||
"security_groups": [],
|
||||
"self": false,
|
||||
"to_port": 10401
|
||||
},
|
||||
{
|
||||
"cidr_blocks": [
|
||||
"10.0.0.0/16"
|
||||
],
|
||||
"description": "etcd from internal",
|
||||
"from_port": 10301,
|
||||
"ipv6_cidr_blocks": [],
|
||||
"prefix_list_ids": [],
|
||||
"protocol": "tcp",
|
||||
"security_groups": [],
|
||||
"self": false,
|
||||
"to_port": 10301
|
||||
},
|
||||
{
|
||||
"cidr_blocks": [
|
||||
"10.0.0.0/8",
|
||||
"172.31.0.0/16"
|
||||
],
|
||||
"description": "GRPC from Internal",
|
||||
"from_port": 20101,
|
||||
"ipv6_cidr_blocks": [],
|
||||
"prefix_list_ids": [],
|
||||
"protocol": "tcp",
|
||||
"security_groups": [],
|
||||
"self": false,
|
||||
"to_port": 20101
|
||||
},
|
||||
{
|
||||
"cidr_blocks": [
|
||||
"10.0.0.0/8",
|
||||
"172.31.0.0/16"
|
||||
],
|
||||
"description": "HTTP from Internal",
|
||||
"from_port": 10101,
|
||||
"ipv6_cidr_blocks": [],
|
||||
"prefix_list_ids": [],
|
||||
"protocol": "tcp",
|
||||
"security_groups": [],
|
||||
"self": false,
|
||||
"to_port": 10101
|
||||
},
|
||||
{
|
||||
"cidr_blocks": [
|
||||
"10.0.0.0/8",
|
||||
"172.31.0.0/16"
|
||||
],
|
||||
"description": "PostgreSQL from Internal",
|
||||
"from_port": 55432,
|
||||
"ipv6_cidr_blocks": [],
|
||||
"prefix_list_ids": [],
|
||||
"protocol": "tcp",
|
||||
"security_groups": [],
|
||||
"self": false,
|
||||
"to_port": 55432
|
||||
}
|
||||
],
|
||||
"name": "allow_featurebase",
|
||||
"name_prefix": "",
|
||||
"owner_id": "977373308795",
|
||||
"revoke_rules_on_delete": false,
|
||||
"tags": {
|
||||
"Name": "allow_featurebase"
|
||||
},
|
||||
"tags_all": {
|
||||
"Name": "allow_featurebase"
|
||||
},
|
||||
"timeouts": null,
|
||||
"vpc_id": "vpc-05a26a122f961dc2b"
|
||||
},
|
||||
"sensitive_attributes": [],
|
||||
"private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjo2MDAwMDAwMDAwMDAsImRlbGV0ZSI6OTAwMDAwMDAwMDAwfSwic2NoZW1hX3ZlcnNpb24iOiIxIn0="
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"module": "module.ci-cluster",
|
||||
"mode": "managed",
|
||||
"type": "aws_security_group",
|
||||
"name": "ingest",
|
||||
"provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
|
||||
"instances": [
|
||||
{
|
||||
"schema_version": 1,
|
||||
"attributes": {
|
||||
"arn": "arn:aws:ec2:us-east-2:977373308795:security-group/sg-04e315d7b9ace5146",
|
||||
"description": "Allow ingest inbound traffic",
|
||||
"egress": [
|
||||
{
|
||||
"cidr_blocks": [
|
||||
"0.0.0.0/0"
|
||||
],
|
||||
"description": "",
|
||||
"from_port": 0,
|
||||
"ipv6_cidr_blocks": [
|
||||
"::/0"
|
||||
],
|
||||
"prefix_list_ids": [],
|
||||
"protocol": "-1",
|
||||
"security_groups": [],
|
||||
"self": false,
|
||||
"to_port": 0
|
||||
}
|
||||
],
|
||||
"id": "sg-04e315d7b9ace5146",
|
||||
"ingress": [
|
||||
{
|
||||
"cidr_blocks": [
|
||||
"0.0.0.0/0"
|
||||
],
|
||||
"description": "",
|
||||
"from_port": 10101,
|
||||
"ipv6_cidr_blocks": [
|
||||
"::/0"
|
||||
],
|
||||
"prefix_list_ids": [],
|
||||
"protocol": "tcp",
|
||||
"security_groups": [],
|
||||
"self": false,
|
||||
"to_port": 10101
|
||||
},
|
||||
{
|
||||
"cidr_blocks": [
|
||||
"0.0.0.0/0"
|
||||
],
|
||||
"description": "SSH",
|
||||
"from_port": 22,
|
||||
"ipv6_cidr_blocks": [
|
||||
"::/0"
|
||||
],
|
||||
"prefix_list_ids": [],
|
||||
"protocol": "tcp",
|
||||
"security_groups": [],
|
||||
"self": false,
|
||||
"to_port": 22
|
||||
},
|
||||
{
|
||||
"cidr_blocks": [
|
||||
"0.0.0.0/0"
|
||||
],
|
||||
"description": "icmp from Anywhere",
|
||||
"from_port": -1,
|
||||
"ipv6_cidr_blocks": [],
|
||||
"prefix_list_ids": [],
|
||||
"protocol": "icmp",
|
||||
"security_groups": [],
|
||||
"self": false,
|
||||
"to_port": -1
|
||||
}
|
||||
],
|
||||
"name": "allow_ingest",
|
||||
"name_prefix": "",
|
||||
"owner_id": "977373308795",
|
||||
"revoke_rules_on_delete": false,
|
||||
"tags": {
|
||||
"Name": "allow_ingest"
|
||||
},
|
||||
"tags_all": {
|
||||
"Name": "allow_ingest"
|
||||
},
|
||||
"timeouts": null,
|
||||
"vpc_id": "vpc-05a26a122f961dc2b"
|
||||
},
|
||||
"sensitive_attributes": [],
|
||||
"private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjo2MDAwMDAwMDAwMDAsImRlbGV0ZSI6OTAwMDAwMDAwMDAwfSwic2NoZW1hX3ZlcnNpb24iOiIxIn0="
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
"outputs": {},
|
||||
"resources": []
|
||||
}
|
||||
|
|
|
|||
|
|
@ -9,8 +9,6 @@ module "samsung-cluster" {
|
|||
fb_ingest_type = "m6g.large"
|
||||
fb_ingest_disk_iops = 10000
|
||||
fb_ingest_node_count = 1
|
||||
gitlab_token = var.gitlab_token
|
||||
branch = var.branch
|
||||
vpc_id = "vpc-05a26a122f961dc2b"
|
||||
vpc_cidr_block = "10.0.0.0/16"
|
||||
vpc_public_subnets = ["subnet-066b4b922b54e51a2","subnet-037b8884269a69025","subnet-08482631514426210",]
|
||||
|
|
|
|||
|
|
@ -6,4 +6,14 @@ output "ingest_ips" {
|
|||
output "data_node_ips" {
|
||||
description = "List of data node IPs"
|
||||
value = module.samsung-cluster.data_node_ips
|
||||
}
|
||||
}
|
||||
|
||||
output "cluster_prefix" {
|
||||
description = "The cluster prefix used"
|
||||
value = module.samsung-cluster.cluster_prefix
|
||||
}
|
||||
|
||||
output "fb_cluster_replica_count" {
|
||||
description = "The cluster replica count used"
|
||||
value = module.samsung-cluster.fb_cluster_replica_count
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue