mirror of
https://github.com/featurebasedb/featurebase.git
synced 2026-08-28 10:54:59 +00:00
782faa3354
This is a bit complicated and entangled, sorry. First, we squash the auth-based smoke tests into the regular smoke tests; we just run all the tests with auth on and that way we don't need to spin up an entire separate cluster of machines just to run a single query against them. We improve the error detection, and standardize the jq-to-get-config code. The purpose of this is to try to make sure that, if we actually hit a failure and get "null" for a host name, we report *that* as an error, rather than running ahead and producing 20+ separate reports that ssh failed because it couldn't find a host named null.
689 lines
No EOL
23 KiB
Bash
Executable file
689 lines
No EOL
23 KiB
Bash
Executable file
#!/bin/bash
|
|
|
|
#path to the featurebase.conf file
|
|
CONFIG_FILE_PATH="/etc/featurebase.conf"
|
|
#path to the featurebase.service file
|
|
SERVICE_FILE_PATH="/etc/systemd/system/featurebase.service"
|
|
|
|
#cluster prefix that was used
|
|
DEPLOYED_CLUSTER_PREFIX=""
|
|
|
|
#cluster replica count that was used
|
|
DEPLOYED_CLUSTER_REPLICA_COUNT=""
|
|
|
|
#List of deployed IPs for data nodes
|
|
DEPLOYED_DATA_IPS=""
|
|
DEPLOYED_DATA_IPS_LEN=0
|
|
|
|
#List of deployed IPs for ingest nodes
|
|
DEPLOYED_INGEST_IPS=""
|
|
DEPLOYED_INGEST_IPS_LEN=0
|
|
|
|
#Initial cluster string
|
|
INITIAL_CLUSTER=""
|
|
|
|
# ebs device name
|
|
EBS_DEVICE_NAME=""
|
|
|
|
# featurebase binary
|
|
FB_BINARY=""
|
|
|
|
# AWS Profile
|
|
AWS_PROFILE=""
|
|
ifErr() {
|
|
res=$?
|
|
if (( res != 0 )); then
|
|
echo "error: $1"
|
|
exit $res
|
|
fi
|
|
}
|
|
|
|
writeFeatureBaseNodeServiceFile() {
|
|
echo "Writing featurebase.service file...index: $1, ip:$2"
|
|
NODEIDX=$1
|
|
NODEIP=$2
|
|
cat << EOT > featurebase.service
|
|
# Not Ansible managed
|
|
|
|
[Unit]
|
|
Description="Service for FeatureBase"
|
|
|
|
[Service]
|
|
RestartSec=30
|
|
Restart=on-failure
|
|
EnvironmentFile=
|
|
User=molecula
|
|
ExecStart=/usr/local/bin/featurebase server -c /etc/featurebase.conf
|
|
|
|
[Install]
|
|
EOT
|
|
|
|
#echo "featurebase.service >>"
|
|
#cat featurebase.service
|
|
#echo "featurebase.service <<"
|
|
|
|
scp -i ~/.ssh/gitlab-featurebase-ci.pem -o "StrictHostKeyChecking no" featurebase.service ec2-user@${NODEIP}:
|
|
if (( $? != 0 ))
|
|
then
|
|
echo "featurebase.service copy failed"
|
|
exit 1
|
|
fi
|
|
|
|
rm -f featurebase.service
|
|
|
|
ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "sudo mv featurebase.service ${SERVICE_FILE_PATH}"
|
|
}
|
|
|
|
writeFeatureBaseNodeConfigFile() {
|
|
echo "Writing featurebase.conf file...index: $1, ip:$2"
|
|
NODEIDX=$1
|
|
NODEIP=$2
|
|
if [[ "$AUTH_ENABLED" = "1" ]]; then
|
|
echo "writing auth enabled featurebase.conf"
|
|
cat << EOT > featurebase.conf
|
|
name = "p${NODEIDX}"
|
|
bind = "https://0.0.0.0:10101"
|
|
bind-grpc = "https://0.0.0.0:20101"
|
|
|
|
data-dir = "/data/featurebase"
|
|
log-path = "/var/log/molecula/featurebase.log"
|
|
|
|
max-file-count=900000
|
|
max-map-count=900000
|
|
|
|
long-query-time = "10s"
|
|
|
|
[cluster]
|
|
name = "${DEPLOYED_CLUSTER_PREFIX}"
|
|
replicas = ${DEPLOYED_CLUSTER_REPLICA_COUNT}
|
|
|
|
[etcd]
|
|
listen-client-address = "http://${NODEIP}:10401"
|
|
listen-peer-address = "http://${NODEIP}:10301"
|
|
initial-cluster = "${INITIAL_CLUSTER}"
|
|
|
|
[metric]
|
|
service = "prometheus"
|
|
|
|
[tls]
|
|
certificate = "/data/cert.crt"
|
|
key = "/data/key.key"
|
|
|
|
[auth]
|
|
enable = true
|
|
client-id = "e9088663-eb08-41d7-8f65-efb5f54bbb71"
|
|
client-secret = "bex7Q~aiWeQ70iBXbEup-XydyQrrNc_Q5n8EW"
|
|
authorize-url = "http://localhost:12345/authorize"
|
|
redirect-base-url = "http://localhost:12345/redirect"
|
|
token-url = "http://localhost:12345/token"
|
|
group-endpoint-url = "http://localhost:12345/groups"
|
|
logout-url = "http://localhost:12345/logout"
|
|
scopes = ["openid", "https://graph.microsoft.com/.default", "offline_access"]
|
|
secret-key = "98995f0530eeba96da1d0a04311073c0abb7b6abbfb0f5f4ef3629527ff88428"
|
|
permissions = "/etc/permissions.yml"
|
|
query-log-path = "/data/featurebase/query.log"
|
|
configured-ips = ["${DEPLOYED_INGEST_IPS}"]
|
|
|
|
[sql]
|
|
endpoint-enabled = true
|
|
|
|
EOT
|
|
|
|
echo "writing the permissions file"
|
|
cat << EOT > permissions.yml
|
|
"user-groups":
|
|
"group-id-reader":
|
|
"user": "read"
|
|
"group-id-writer":
|
|
"user": "write"
|
|
admin: "group-id-admin"
|
|
EOT
|
|
else
|
|
cat << EOT > featurebase.conf
|
|
name = "p${NODEIDX}"
|
|
bind = "0.0.0.0:10101"
|
|
bind-grpc = "0.0.0.0:20101"
|
|
|
|
data-dir = "/data/featurebase"
|
|
log-path = "/var/log/molecula/featurebase.log"
|
|
|
|
max-file-count=900000
|
|
max-map-count=900000
|
|
|
|
long-query-time = "10s"
|
|
|
|
[cluster]
|
|
name = "${DEPLOYED_CLUSTER_PREFIX}"
|
|
replicas = ${DEPLOYED_CLUSTER_REPLICA_COUNT}
|
|
|
|
[etcd]
|
|
listen-client-address = "http://${NODEIP}:10401"
|
|
listen-peer-address = "http://${NODEIP}:10301"
|
|
initial-cluster = "${INITIAL_CLUSTER}"
|
|
|
|
[metric]
|
|
service = "prometheus"
|
|
|
|
[sql]
|
|
endpoint-enabled = true
|
|
|
|
EOT
|
|
fi
|
|
|
|
scp -i ~/.ssh/gitlab-featurebase-ci.pem -o "StrictHostKeyChecking no" featurebase.conf ec2-user@${NODEIP}:
|
|
if (( $? != 0 ))
|
|
then
|
|
echo "featurebase.conf copy failed"
|
|
exit 1
|
|
fi
|
|
rm -f featurebase.conf
|
|
|
|
ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "sudo mv featurebase.conf ${CONFIG_FILE_PATH}"
|
|
|
|
if [[ "$AUTH_ENABLED" = "1" ]]; then
|
|
scp -i ~/.ssh/gitlab-featurebase-ci.pem -o "StrictHostKeyChecking no" permissions.yml ec2-user@${NODEIP}:
|
|
ifErr "permissions.yml copy failed"
|
|
rm -f permissions.yml
|
|
|
|
ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "sudo mv ~/permissions.yml /etc/permissions.yml"
|
|
ifErr "mv permissions.yml failed"
|
|
fi
|
|
}
|
|
|
|
setupMkCertCA() {
|
|
# this function sets up a CA on the runner for use by all nodes in setupTLS
|
|
git clone https://github.com/FiloSottile/mkcert && cd mkcert
|
|
ifErr "cloning mkcert"
|
|
|
|
go build -ldflags "-X main.Version=$(git describe --tags)"
|
|
ifErr "building mkcert"
|
|
|
|
./mkcert -install
|
|
ifErr "installing root CA"
|
|
|
|
cp mkcert /usr/local/bin
|
|
|
|
cd ../
|
|
|
|
}
|
|
|
|
setupTLS() {
|
|
NODEIP=$1
|
|
scp -i ~/.ssh/gitlab-featurebase-ci.pem -o "StrictHostKeyChecking no" ./qa/scripts/setupTLS.sh ec2-user@${NODEIP}:
|
|
ifErr "setupTLS.sh copy failed"
|
|
|
|
scp -r -i ~/.ssh/gitlab-featurebase-ci.pem -o "StrictHostKeyChecking no" ./qa/fakeidp/ ec2-user@${NODEIP}:
|
|
ifErr "fakeidp copy failed"
|
|
|
|
ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "sudo mv ~/fakeidp /etc"
|
|
ifErr "mv fakeidp failed"
|
|
|
|
ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "sudo ./setupTLS.sh ${NODEIP}"
|
|
ifErr "error setting up TLS"
|
|
}
|
|
|
|
mountEBSVolume() {
|
|
NODEIP=$1
|
|
echo "in mountEBSVolume, EBS_DEVICE_NAME = $EBS_DEVICE_NAME, NODEIP = $NODEIP"
|
|
# if it is nvme, then skip the following 3 lines, because we mount the device in UserData
|
|
# in the CloudFormation template
|
|
if [[ "$EBS_DEVICE_NAME" != "nvme" ]]; then
|
|
echo "mounting EBS device"
|
|
ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "sudo mkdir /data"
|
|
ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "sudo mkfs.ext4 ${EBS_DEVICE_NAME}"
|
|
ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "sudo mount ${EBS_DEVICE_NAME} /data"
|
|
ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "sudo chown -R ec2-user:ec2-user /data"
|
|
fi
|
|
}
|
|
|
|
executeGeneralNodeConfigCommands() {
|
|
echo "Executing node config...index: $1, ip:$2"
|
|
NODEIDX=$1
|
|
NODEIP=$2
|
|
|
|
mountEBSVolume $NODEIP
|
|
|
|
ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "sudo adduser molecula"
|
|
ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "sudo mkdir /var/log/molecula"
|
|
ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "sudo chown molecula /var/log/molecula"
|
|
ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "sudo mkdir -p /data/featurebase"
|
|
ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "sudo chown molecula /data/featurebase"
|
|
|
|
# TODO handle different archs
|
|
scp -i ~/.ssh/gitlab-featurebase-ci.pem -o "StrictHostKeyChecking no" ${FB_BINARY} ec2-user@${NODEIP}:
|
|
if (( $? != 0 ))
|
|
then
|
|
echo "featurebase binary copy failed"
|
|
exit 1
|
|
fi
|
|
|
|
ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "chown ec2-user:ec2-user /home/ec2-user/${FB_BINARY}"
|
|
ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "chmod ugo+x /home/ec2-user/${FB_BINARY}"
|
|
ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "sudo mv /home/ec2-user/${FB_BINARY} /usr/local/bin/featurebase"
|
|
ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "sudo yum install jq htop tmux nc -y"
|
|
echo "featurebase binary copied."
|
|
|
|
if [[ "$AUTH_ENABLED" = "1" ]]; then
|
|
caroot=$(mkcert -CAROOT)
|
|
|
|
echo "copying root ca to ${NODEIP}"
|
|
scp -i ~/.ssh/gitlab-featurebase-ci.pem -o "StrictHostKeyChecking no" "${caroot}/rootCA.pem" ec2-user@${NODEIP}:
|
|
ifErr "copying CAROOT to ${NODEIP}"
|
|
|
|
scp -i ~/.ssh/gitlab-featurebase-ci.pem -o "StrictHostKeyChecking no" "${caroot}/rootCA-key.pem" ec2-user@${NODEIP}:
|
|
ifErr "copying CAROOT to ${NODEIP}"
|
|
|
|
ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "sudo mkdir -p /root/.local/share/mkcert && sudo cp ~/rootCA.pem /root/.local/share/mkcert && sudo cp ~/rootCA-key.pem /root/.local/share/mkcert"
|
|
ifErr "cp-ing rootCA.pem to /root/.local/share/mkcert on ${NODEIP}"
|
|
|
|
echo "setting up tls certificates"
|
|
setupTLS $NODEIP
|
|
fi
|
|
}
|
|
|
|
executeDataStartCommands() {
|
|
echo "executeDataStartCommands...index: $1, ip:$2"
|
|
NODEIDX=$1
|
|
NODEIP=$2
|
|
|
|
ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "sudo systemctl daemon-reload"
|
|
ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "sudo systemctl start featurebase"
|
|
ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "sudo systemctl enable featurebase"
|
|
ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "sudo systemctl status featurebase"
|
|
}
|
|
|
|
startDataNodes() {
|
|
#now go thru loop again to start up each node
|
|
cnt=0
|
|
for ip in ${DEPLOYED_DATA_IPS[@]}
|
|
do
|
|
executeDataStartCommands $cnt $ip
|
|
cnt=$((cnt+1))
|
|
done
|
|
}
|
|
|
|
setupDataNode() {
|
|
echo "setting up node $1 at $2"
|
|
|
|
writeFeatureBaseNodeConfigFile $1 $2
|
|
writeFeatureBaseNodeServiceFile $1 $2
|
|
executeGeneralNodeConfigCommands $1 $2
|
|
}
|
|
|
|
setupIngestNode() {
|
|
echo "setting up ingest node $1 at $2"
|
|
NODEIDX=$1
|
|
NODEIP=$2
|
|
BRANCH_NAME=$3
|
|
|
|
executeGeneralNodeConfigCommands $1 $2
|
|
|
|
ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "pip3 install -U pytest"
|
|
ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "pip3 install -U requests"
|
|
ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "pip3 install -U json"
|
|
ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "sudo yum install -y jq"
|
|
installDatagen $NODEIP $BRANCH_NAME
|
|
}
|
|
|
|
setupDataNodes() {
|
|
BRANCH_NAME=$1
|
|
cnt=0
|
|
for ip in ${DEPLOYED_DATA_IPS[@]}
|
|
do
|
|
setupDataNode $cnt $ip
|
|
setupDatadog $ip "featurebase" $BRANCH_NAME
|
|
cnt=$((cnt+1))
|
|
done
|
|
}
|
|
|
|
setupIngestNodes() {
|
|
BRANCH_NAME=$1
|
|
cnt=0
|
|
echo "setupIngestNodes: " ${DEPLOYED_INGEST_IPS[@]}
|
|
for ip in ${DEPLOYED_INGEST_IPS[@]}
|
|
do
|
|
setupIngestNode $cnt $ip $BRANCH_NAME
|
|
setupDatadog $ip "ingest" $BRANCH_NAME
|
|
cnt=$((cnt+1))
|
|
done
|
|
}
|
|
|
|
generateInitialClusterString() {
|
|
IFS=$'\n'
|
|
cnt=0
|
|
for ip in ${DEPLOYED_DATA_IPS[@]}
|
|
do
|
|
if (($cnt + 1 != $DEPLOYED_DATA_IPS_LEN))
|
|
then
|
|
INITIAL_CLUSTER="${INITIAL_CLUSTER}p${cnt}=http://$ip:10301,"
|
|
else
|
|
INITIAL_CLUSTER="${INITIAL_CLUSTER}p${cnt}=http://$ip:10301"
|
|
fi
|
|
cnt=$((cnt+1))
|
|
done
|
|
|
|
echo "INITIAL_CLUSTER: ${INITIAL_CLUSTER}"
|
|
}
|
|
|
|
setupClusterNodes() {
|
|
BRANCH_NAME=$1
|
|
|
|
#data nodes
|
|
generateInitialClusterString
|
|
|
|
if [[ "$AUTH_ENABLED" = "1" ]]; then
|
|
setupMkCertCA
|
|
fi
|
|
|
|
setupDataNodes $BRANCH_NAME
|
|
|
|
startDataNodes
|
|
|
|
#ingest nodes
|
|
setupIngestNodes $BRANCH_NAME
|
|
|
|
}
|
|
|
|
writeKafkaServerConfig() {
|
|
echo "Writing server.properties file"
|
|
NODEIP=$1
|
|
|
|
cat << EOT > server.properties
|
|
process.roles=broker,controller
|
|
node.id=1
|
|
controller.quorum.voters=1@localhost:9093
|
|
listeners=PLAINTEXT://${NODEIP}:9092,CONTROLLER://:9093
|
|
inter.broker.listener.name=PLAINTEXT
|
|
advertised.listeners=PLAINTEXT://${NODEIP}:9092
|
|
controller.listener.names=CONTROLLER
|
|
listener.security.protocol.map=CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT,SSL:SSL,SASL_PLAINTEXT:SASL_PLAINTEXT,SASL_SSL:SASL_SSL
|
|
num.network.threads=3
|
|
num.io.threads=8
|
|
socket.send.buffer.bytes=102400
|
|
socket.receive.buffer.bytes=102400
|
|
socket.request.max.bytes=104857600
|
|
log.dirs=/data/kraft-combined-logs
|
|
num.partitions=1
|
|
num.recovery.threads.per.data.dir=1
|
|
offsets.topic.replication.factor=1
|
|
transaction.state.log.replication.factor=1
|
|
transaction.state.log.min.isr=1
|
|
log.retention.hours=168
|
|
log.segment.bytes=1073741824
|
|
log.retention.check.interval.ms=300000
|
|
group.initial.rebalance.delay.ms=0
|
|
confluent.license.topic.replication.factor=1
|
|
confluent.metadata.topic.replication.factor=1
|
|
confluent.security.event.logger.exporter.kafka.topic.replicas=1
|
|
confluent.balancer.enable=true
|
|
confluent.balancer.topic.replication.factor=1
|
|
EOT
|
|
|
|
scp -i ~/.ssh/gitlab-featurebase-ci.pem -o "StrictHostKeyChecking no" server.properties ec2-user@${NODEIP}:/home/ec2-user/kafka/etc/kafka/kraft/server.properties
|
|
if (( $? != 0 ))
|
|
then
|
|
echo "kafka config file: server.properties copy failed"
|
|
exit 1
|
|
fi
|
|
rm -f server.properties
|
|
}
|
|
|
|
setupKafkaServer() {
|
|
NODEIP=$1
|
|
|
|
# install kafka
|
|
echo "setting up kafka server for: $NODEIP"
|
|
ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "sudo yum install java-1.8.0-openjdk nc -y"
|
|
ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "mkdir kafka && cd kafka && curl https://packages.confluent.io/archive/7.0/confluent-community-7.0.1.tar.gz -o kafka.tgz && tar -xvzf kafka.tgz --strip 1"
|
|
|
|
# update kafka config
|
|
writeKafkaServerConfig $NODEIP
|
|
|
|
cat << 'EOF' >> runKafka.sh
|
|
/home/ec2-user/kafka/bin/kafka-storage format --config /home/ec2-user/kafka/etc/kafka/kraft/server.properties --cluster-id $(/home/ec2-user/kafka/bin/kafka-storage random-uuid)
|
|
sudo /home/ec2-user/kafka/bin/kafka-server-start /home/ec2-user/kafka/etc/kafka/kraft/server.properties > /tmp/kafka.log &
|
|
echo "checking kafka server status"
|
|
EOF
|
|
|
|
cat runKafka.sh
|
|
# start zookeeper and kafka server
|
|
echo "starting kafka server"
|
|
scp -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ./runKafka.sh ec2-user@${NODEIP}:.
|
|
ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "bash runKafka.sh"
|
|
ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "timeout 60s nc -z ${NODEIP} 9092"
|
|
# rm ./runKafka.sh
|
|
}
|
|
|
|
writeDatadogConfigForIngest() {
|
|
BRANCH_NAME=$1
|
|
|
|
# general datadog config
|
|
cat << EOT > datadog.yaml
|
|
api_key: 6ab706d12de9cb46db25a0aabab6a004
|
|
site: datadoghq.com
|
|
tags:
|
|
- team:core
|
|
- app:${NODE_TYPE}
|
|
- branch_name:${BRANCH_NAME}
|
|
process_config:
|
|
enabled: "true"
|
|
EOT
|
|
|
|
# openmetrics config
|
|
cat << EOT > conf.yaml
|
|
# The prometheus endpoint to query from
|
|
instances:
|
|
- openmetrics_endpoint: "http://localhost:9093/metrics"
|
|
namespace: "ingest"
|
|
metrics:
|
|
- "ingester_kafka_static_.+"
|
|
- go*
|
|
EOT
|
|
}
|
|
|
|
writeDatadogConfigForFeaturebase() {
|
|
BRANCH_NAME=$1
|
|
|
|
# general datadog config
|
|
cat << EOT > datadog.yaml
|
|
api_key: 6ab706d12de9cb46db25a0aabab6a004
|
|
site: datadoghq.com
|
|
tags:
|
|
- team:core
|
|
- app:${NODE_TYPE}
|
|
- branch_name:${BRANCH_NAME}
|
|
process_config:
|
|
enabled: "true"
|
|
EOT
|
|
|
|
# openmetrics config
|
|
cat << EOT > conf.yaml
|
|
# The prometheus endpoint to query from
|
|
instances:
|
|
- prometheus_url: http://localhost:10101/metrics
|
|
namespace: "featurebase"
|
|
metrics:
|
|
- prometheus_target_interval_length_seconds: target_interval_length
|
|
- http_requests_total
|
|
- http*
|
|
- etcd*
|
|
- pilosa*
|
|
- go*
|
|
- process*
|
|
- os*
|
|
EOT
|
|
}
|
|
|
|
setupDatadog() {
|
|
NODEIP=$1
|
|
NODE_TYPE=$2
|
|
BRANCH_NAME=$3
|
|
cat << 'EOF' > runDatadog.sh
|
|
DD_AGENT_MAJOR_VERSION=7 DD_API_KEY=6ab706d12de9cb46db25a0aabab6a004 DD_SITE="datadoghq.com" bash -c "$(curl -L https://s3.amazonaws.com/dd-agent/scripts/install_script.sh)"
|
|
EOF
|
|
|
|
if [[ "$NODE_TYPE" == "featurebase" ]]
|
|
then
|
|
echo "writing datadog config for featurebase node: $NODEIP, $NODE_TYPE, $BRANCH_NAME"
|
|
writeDatadogConfigForFeaturebase $BRANCH_NAME
|
|
else
|
|
echo "writing datadog config for ingest node: $NODEIP, $NODE_TYPE, $BRANCH_NAME"
|
|
writeDatadogConfigForIngest $BRANCH_NAME
|
|
fi
|
|
|
|
echo "setting up datadog for $NODE_TYPE node: $NODEIP"
|
|
echo "copying datadog config"
|
|
scp -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ./runDatadog.sh ./datadog.yaml ./conf.yaml ec2-user@${NODEIP}:.
|
|
echo "installing datadog"
|
|
cat ./runDatadog.sh
|
|
ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "bash ./runDatadog.sh"
|
|
echo "restarting datadog and updating config"
|
|
ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "sudo systemctl stop datadog-agent"
|
|
# update configuration and openmetrics for datadog
|
|
ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "sudo mv ./datadog.yaml /etc/datadog-agent/datadog.yaml"
|
|
ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "sudo mv ./conf.yaml /etc/datadog-agent/conf.d/openmetrics.d/conf.yaml"
|
|
ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "sudo systemctl start datadog-agent"
|
|
echo "checking datadog status"
|
|
ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "sudo systemctl status datadog-agent"
|
|
ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "sudo yum install nc htop -y"
|
|
rm runDatadog.sh datadog.yaml conf.yaml
|
|
}
|
|
|
|
installDatagen() {
|
|
INGESTNODE0=$1
|
|
BRANCH_NAME=$2
|
|
# download datagen
|
|
aws s3 cp s3://molecula-artifact-storage/idk/${BRANCH_NAME}/_latest/idk-linux-arm64/datagen datagen
|
|
if (( $? != 0 ))
|
|
then
|
|
echo "datagen binary copy failed"
|
|
exit 1
|
|
fi
|
|
|
|
# make it executable
|
|
chmod +x datagen
|
|
if (( $? != 0 ))
|
|
then
|
|
echo "couldn't make datagen executable"
|
|
exit 1
|
|
fi
|
|
|
|
# copy it over to the ingest node
|
|
scp -r -i ~/.ssh/gitlab-featurebase-ci.pem ./datagen ec2-user@${INGESTNODE0}:/data
|
|
if (( $? != 0 ))
|
|
then
|
|
echo "datagen copy failed"
|
|
exit 1
|
|
fi
|
|
|
|
# setup the yum repo needed for librdkafka onto the ingest node
|
|
scp -r -i ~/.ssh/gitlab-featurebase-ci.pem ./qa/scripts/perf/delete/confluent ec2-user@${INGESTNODE0}:/data
|
|
if (( $? != 0 ))
|
|
then
|
|
echo "confluent repo setup copy failed"
|
|
exit 1
|
|
fi
|
|
|
|
echo "setting up confluent repo"
|
|
ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o "StrictHostKeyChecking no" ec2-user@${INGESTNODE0} "sudo mv /data/confluent /etc/yum.repos.d"
|
|
if (( $? != 0 ))
|
|
then
|
|
echo "setting up confluent repo failed"
|
|
exit 1
|
|
fi
|
|
|
|
|
|
echo "installing librdkafka on ingest node"
|
|
ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o "StrictHostKeyChecking no" ec2-user@${INGESTNODE0} "sudo rpm --import http://packages.confluent.io/rpm/3.1/archive.key && sudo yum clean all && sudo yum install librdkafka-devel -y"
|
|
if (( $? != 0 ))
|
|
then
|
|
echo "librdkafka install failed"
|
|
exit 1
|
|
fi
|
|
}
|
|
|
|
# copy datagen to ingest nodes
|
|
setupProducerNode(){
|
|
IP=$1
|
|
BRANCH_NAME=$2
|
|
|
|
# download producer(datagen binary) from S3
|
|
DATAGEN_BINARY=datagen
|
|
echo "Downloading Datagen binary"
|
|
aws s3 cp s3://molecula-artifact-storage/idk/${BRANCH_NAME}/_latest/idk-linux-amd64/datagen $DATAGEN_BINARY --profile $AWS_PROFILE
|
|
|
|
mountEBSVolume $IP
|
|
scp -i ~/.ssh/gitlab-featurebase-ci.pem -o "StrictHostKeyChecking no" ${DATAGEN_BINARY} ec2-user@${IP}:
|
|
ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${IP} "chmod ugo+x /home/ec2-user/${DATAGEN_BINARY}"
|
|
scp -i ~/.ssh/gitlab-featurebase-ci.pem -o "StrictHostKeyChecking no" ./qa/scripts/delete/tremor_keys.yaml ./qa/scripts/delete/runProducer.sh ec2-user@${IP}:
|
|
ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${IP} "sudo yum install jq htop tmux -y"
|
|
setupKafkaServer ${IP}
|
|
setupDatadog $IP "ingest" $BRANCH_NAME
|
|
}
|
|
|
|
setupConsumerNode(){
|
|
IP=$1
|
|
BRANCH_NAME=$2
|
|
|
|
# download consumer(kafka-static binary) from S3
|
|
KAFKA_STATIC=molecula-consumer-kafka-static
|
|
echo "Downloading Molecular Consumer Kafka Static binary"
|
|
aws s3 cp s3://molecula-artifact-storage/idk/${BRANCH_NAME}/_latest/idk-linux-amd64/molecula-consumer-kafka-static ./$KAFKA_STATIC --profile $AWS_PROFILE
|
|
|
|
mountEBSVolume $IP
|
|
|
|
scp -i ~/.ssh/gitlab-featurebase-ci.pem -o "StrictHostKeyChecking no" ./qa/scripts/delete/schema.json ./qa/scripts/delete/runConsumer.sh ec2-user@${IP}:
|
|
ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${IP} "sudo yum install librdkafka -y"
|
|
scp -i ~/.ssh/gitlab-featurebase-ci.pem -o "StrictHostKeyChecking no" ${KAFKA_STATIC} ec2-user@${IP}:
|
|
ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${IP} "chmod ugo+x /home/ec2-user/${KAFKA_STATIC}"
|
|
ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${IP} "sudo yum install jq htop tmux nc -y"
|
|
setupDatadog $IP "ingest" $BRANCH_NAME
|
|
}
|
|
|
|
# get_value [file] [var] [nodes] checks for a value stored in [nodes];
|
|
# for instance, `get_value file foo a b` checks for [a][b] in the JSON
|
|
# in file, and stores the result in $foo. it exits with a non-zero status
|
|
# if this fails.
|
|
# e.g., `get_value output.json INGESTNODE0 .ingest_ips 0 '"value"' 0` sets
|
|
# $INGESTNODE0 to the value of [.ingest_ips][0]["value"][0], or fails to
|
|
# do so and returns a non-zero status. Note the quoting to get the double
|
|
# quotes around "value".
|
|
get_value() {
|
|
file=$1
|
|
var=$2
|
|
shift 2
|
|
# assemble [a][b][c] from "a" "b" "c"
|
|
expr="["
|
|
infix=""
|
|
for word; do
|
|
expr="$expr$infix$word"
|
|
infix="]["
|
|
done
|
|
expr="$expr]"
|
|
tmp=$(jq -r "$expr" < $file)
|
|
if [ $? -ne 0 ]; then
|
|
echo >&2 "jq failed parsing input file"
|
|
return 1
|
|
fi
|
|
case $tmp in
|
|
null)
|
|
echo >&2 "reading IPs for ${node}: got null"
|
|
return 1
|
|
;;
|
|
*)
|
|
eval $var=\$tmp
|
|
return 0
|
|
;;
|
|
esac
|
|
}
|
|
|
|
# must_get_value [file] [var] [nodes] checks for the contents of [nodes], and stores
|
|
# the result in $var. if it fails, it causes the *calling script* to exit. do not
|
|
# use this if you need to handle the error more gracefully.
|
|
# e.g., `must_get_value output.json INGESTNODE0 ingest_ips 0 "value" 0` sets $INGESTNODE0
|
|
# to the value of [.ingest_ips][0]["value"][0].
|
|
must_get_value() {
|
|
if ! get_value "$@"; then
|
|
exit 1
|
|
fi
|
|
} |