From f224e64bc3fea8da5f1b41e845790432da98e153 Mon Sep 17 00:00:00 2001 From: reesporte Date: Tue, 12 Apr 2022 10:31:23 -0500 Subject: [PATCH] setup auth cluster infra also add test for sup218 Co-authored-by: rachithrr Co-authored-by: bruce-b-molecula --- .gitlab/.gitlab-ci.yml | 57 ++++++++ authn/authenticate.go | 12 +- qa/fakeidp/go.mod | 5 + qa/fakeidp/go.sum | 2 + qa/fakeidp/server.go | 65 +++++++++ qa/scripts/auth-smoke/setup.sh | 81 +++++++++++ qa/scripts/auth-smoke/teardown.sh | 7 + qa/scripts/auth-smoke/test.sh | 37 +++++ qa/scripts/auth-smoke/tests/README | 16 +++ qa/scripts/auth-smoke/tests/run-all.sh | 8 ++ qa/scripts/auth-smoke/tests/sup218-test.sh | 34 +++++ .../auth-smoke/tests/sup218_datagen.yaml | 61 +++++++++ qa/scripts/setupSmokeTest.sh | 1 + qa/scripts/setupTLS.sh | 81 +++++++++++ qa/scripts/utilCluster.sh | 129 ++++++++++++++---- qa/tf/ci/auth-smoke/main.tf | 13 ++ qa/tf/ci/auth-smoke/outputs.tf | 19 +++ qa/tf/ci/auth-smoke/provider.tf | 4 + qa/tf/ci/auth-smoke/tf.auto.tfvars | 2 + qa/tf/ci/auth-smoke/variables.tf | 15 ++ 20 files changed, 622 insertions(+), 27 deletions(-) create mode 100644 qa/fakeidp/go.mod create mode 100644 qa/fakeidp/go.sum create mode 100644 qa/fakeidp/server.go create mode 100755 qa/scripts/auth-smoke/setup.sh create mode 100755 qa/scripts/auth-smoke/teardown.sh create mode 100755 qa/scripts/auth-smoke/test.sh create mode 100644 qa/scripts/auth-smoke/tests/README create mode 100755 qa/scripts/auth-smoke/tests/run-all.sh create mode 100755 qa/scripts/auth-smoke/tests/sup218-test.sh create mode 100644 qa/scripts/auth-smoke/tests/sup218_datagen.yaml create mode 100755 qa/scripts/setupTLS.sh create mode 100644 qa/tf/ci/auth-smoke/main.tf create mode 100644 qa/tf/ci/auth-smoke/outputs.tf create mode 100644 qa/tf/ci/auth-smoke/provider.tf create mode 100644 qa/tf/ci/auth-smoke/tf.auto.tfvars create mode 100644 qa/tf/ci/auth-smoke/variables.tf diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 8919322a9..d46b72c1a 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -340,6 +340,63 @@ external lookup tests: - apt-get install -y postgresql-client - go test . -run "^TestExternalLookup" -externalLookupDSN postgresql://$POSTGRES_USER:$POSTGRES_PASSWORD@postgres/$POSTGRES_DB?sslmode=disable +smoke test auth: + stage: integration + image: registry.gitlab.com/gitlab-org/cloud-deploy/aws-base:latest + variables: + PROFILE: "service-terraform" + AWS_SSH_PRIVATE_KEY: $AWS_FBCI_SSH_KEY + AWS_ACCESS_KEY_ID: $AWS_FBCI_ACCESS_KEY_ID + AWS_SECRET_ACCESS_KEY: $AWS_FBCI_SECRET_ACCESS_KEY + TF_VAR_cluster_prefix: "" + tags: + - aws + - docker + - fbsmoke + rules: + - if: '$CI_PIPELINE_SOURCE == "push"' + before_script: + - apt-get update && apt-get install -y gnupg software-properties-common curl git + - curl -fsSL https://apt.releases.hashicorp.com/gpg | apt-key add - + - apt-add-repository "deb [arch=amd64] https://apt.releases.hashicorp.com $(lsb_release -cs) main" + - apt-get update && apt-get install terraform + - aws configure set aws_access_key_id $AWS_FBCI_ACCESS_KEY_ID --profile $PROFILE + - aws configure set aws_secret_access_key $AWS_FBCI_SECRET_ACCESS_KEY --profile $PROFILE + - aws configure set region "us-east-2" --profile $PROFILE + - aws configure set aws_profile $PROFILE + - echo $AWS_FBCI_SSH_KEY > gitlab-featurebase-ci.pem + - chmod 400 gitlab-featurebase-ci.pem + - 'which ssh-agent || ( apt-get update -y && apt-get install openssh-client -y )' + - eval $(ssh-agent -s) + - mkdir -p ~/.ssh + - echo $AWS_FBCI_SSH_KEY > /root/.ssh/gitlab-featurebase-ci.pem + - chmod 400 /root/.ssh/gitlab-featurebase-ci.pem + - echo "$AWS_FBCI_SSH_KEY" | ssh-add - + - chmod 700 /root/.ssh + - '[[ -f /.dockerenv ]] && echo -e "Host *\n\tStrictHostKeyChecking no\n\n" > ~/.ssh/config' + - apt update && apt -y install jq wget git libnss3-tools + - wget -q https://go.dev/dl/go$GOVERSION.linux-amd64.tar.gz + - tar -C /usr/local -xzf go$GOVERSION.linux-amd64.tar.gz + - export PATH=$PATH:/usr/local/go/bin + - TF_VAR_cluster_prefix="smoke-$(openssl rand -base64 12 | tr -d /=+ | cut -c -16)" + - echo "Cluster Prefix --> $TF_VAR_cluster_prefix" + # download datagen for FB-1270 repro test. TODO replace w/ locally built datagen once we merge IDK into FB + - aws s3 cp s3://molecula-artifact-storage/idk/master/_latest/idk-linux-arm64/datagen ./datagen_linux_arm64 + - chmod +x ./datagen_linux_arm64 + script: + - ./qa/scripts/auth-smoke/setup.sh + - ./qa/scripts/auth-smoke/test.sh + after_script: + - ./qa/scripts/auth-smoke/teardown.sh + needs: + - job: build for linux arm64 + artifacts: + when: always + paths: + - report.xml + reports: + junit: report.xml + smoke test: stage: integration diff --git a/authn/authenticate.go b/authn/authenticate.go index 5b8800e98..9b60ca797 100644 --- a/authn/authenticate.go +++ b/authn/authenticate.go @@ -168,10 +168,14 @@ func (a *Auth) Authenticate(ctx context.Context, bearer string) (*UserInfo, erro } userInfo := UserInfo{ - UserID: claims["oid"].(string), - UserName: claims["name"].(string), - Token: bearer, - Groups: []Group{}, + Token: bearer, + Groups: []Group{}, + } + if uid, ok := claims["oid"].(string); ok { + userInfo.UserID = uid + } + if name, ok := claims["name"].(string); ok { + userInfo.UserName = name } if userInfo.Groups, err = a.getGroups(bearer); err != nil { diff --git a/qa/fakeidp/go.mod b/qa/fakeidp/go.mod new file mode 100644 index 000000000..7d0a51cee --- /dev/null +++ b/qa/fakeidp/go.mod @@ -0,0 +1,5 @@ +module fakeidp + +go 1.17 + +require github.com/golang-jwt/jwt v3.2.2+incompatible diff --git a/qa/fakeidp/go.sum b/qa/fakeidp/go.sum new file mode 100644 index 000000000..efdb2a9a1 --- /dev/null +++ b/qa/fakeidp/go.sum @@ -0,0 +1,2 @@ +github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY= +github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= diff --git a/qa/fakeidp/server.go b/qa/fakeidp/server.go new file mode 100644 index 000000000..fb9076745 --- /dev/null +++ b/qa/fakeidp/server.go @@ -0,0 +1,65 @@ +package main + +import ( + "encoding/hex" + "log" + "net/http" + "strconv" + "strings" + "time" + + "github.com/golang-jwt/jwt" +) + +const ( + adminToken = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJuYW1lIjoiYWRtaW4ifQ.I1iCgk1VU7m6e-En4ACTHIs6V2dZpy_8j2blSSo7K3U" + readerToken = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJuYW1lIjoicmVhZGVyIn0.QcHy_W6oAYFgdBWy1CqLr55HcOyymn5zAXPJUKCvQE4" + writerToken = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJuYW1lIjoid3JpdGVyIn0.aEk-12xP9RJeXog4MHO8LhuQFEjNNG2BcWDcMzSX_HI" +) + +func groups(w http.ResponseWriter, req *http.Request) { + authToken := strings.TrimPrefix(req.Header.Get("Authorization"), "Bearer ") + status := http.StatusOK + + var val []byte + switch authToken { + case adminToken: + val = []byte(`{"value":[{"id":"group-id-admin","displayName":"group-id-admin"}]}`) + case readerToken: + val = []byte(`{"value":[{"id":"group-id-reader","displayName":"group-id-reader"}]}`) + case writerToken: + val = []byte(`{"value":[{"id":"group-id-writer","displayName":"group-id-writer"}]}`) + default: + status = http.StatusUnauthorized + } + w.WriteHeader(status) + w.Write(val) +} + +func token(w http.ResponseWriter, req *http.Request) { + tkn := jwt.New(jwt.SigningMethodHS256) + claims := tkn.Claims.(jwt.MapClaims) + claims["oid"] = "42" + claims["name"] = "valid" + expiresIn := 2 * time.Hour + claims["exp"] = strconv.Itoa(int(time.Now().Add(expiresIn).Unix())) + k, err := hex.DecodeString("DEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEF") + if err != nil { + log.Fatalf("i am not equipped to handle this!!! %v", err) + } + fresh, err := tkn.SignedString(k) + if err != nil { + log.Fatalf("i am not equipped to handle this!!! %v", err) + } + body := `{"access_token": "` + fresh + `", "refresh_token": "blah", "expires_in": "` + strconv.Itoa(int(expiresIn.Seconds())) + `"}` + w.Header().Set("Content-Type", "application/json; charset=utf-8") + w.WriteHeader(http.StatusOK) + w.Write([]byte(body)) +} + +func main() { + http.HandleFunc("/groups", groups) + http.HandleFunc("/token", token) + log.Println("FAKEIDP SERVER UP AND RUNNING") + log.Fatal(http.ListenAndServe(":12345", nil)) +} diff --git a/qa/scripts/auth-smoke/setup.sh b/qa/scripts/auth-smoke/setup.sh new file mode 100755 index 000000000..67016ea20 --- /dev/null +++ b/qa/scripts/auth-smoke/setup.sh @@ -0,0 +1,81 @@ +#!/bin/bash + +# To run script: ./setup.sh +ADMIN_TOKEN=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJuYW1lIjoiYWRtaW4ifQ.I1iCgk1VU7m6e-En4ACTHIs6V2dZpy_8j2blSSo7K3U +READER_TOKEN=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJuYW1lIjoicmVhZGVyIn0.QcHy_W6oAYFgdBWy1CqLr55HcOyymn5zAXPJUKCvQE4 +WRITER_TOKEN=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJuYW1lIjoid3JpdGVyIn0.aEk-12xP9RJeXog4MHO8LhuQFEjNNG2BcWDcMzSX_HI +export TF_IN_AUTOMATION=1 +export AUTH_ENABLED=1 + +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +source ./qa/scripts/utilCluster.sh + +pushd ./qa/tf/ci/auth-smoke +echo "Running terraform init..." +terraform init -input=false +echo "Running terraform apply..." +terraform apply -input=false -auto-approve +terraform output -json > outputs.json +popd + +# get the first ingest host +INGESTNODE0=$(cat ./qa/tf/ci/auth-smoke/outputs.json | jq -r '[.ingest_ips][0]["value"][0]') +echo "using INGESTNODE0 ${INGESTNODE0}" + +# get the first data host +DATANODE0=$(cat ./qa/tf/ci/auth-smoke/outputs.json | jq -r '[.data_node_ips][0]["value"][0]') +echo "using DATANODE0 ${DATANODE0}" + +DEPLOYED_CLUSTER_PREFIX=$(cat ./qa/tf/ci/auth-smoke/outputs.json | jq -r '[.cluster_prefix][0]["value"]') +echo "Using DEPLOYED_CLUSTER_PREFIX: ${DEPLOYED_CLUSTER_PREFIX}" + +DEPLOYED_CLUSTER_REPLICA_COUNT=$(cat ./qa/tf/ci/auth-smoke/outputs.json | jq -r '[.fb_cluster_replica_count][0]["value"]') +echo "Using DEPLOYED_CLUSTER_REPLICA_COUNT: ${DEPLOYED_CLUSTDEPLOYED_CLUSTER_REPLICA_COUNTER_PREFIX}" + +DEPLOYED_DATA_IPS=$(cat ./qa/tf/ci/auth-smoke/outputs.json | jq -r '[.data_node_ips][0]["value"][]') +echo "DEPLOYED_DATA_IPS: {" +echo "${DEPLOYED_DATA_IPS}" +echo "}" + +DEPLOYED_DATA_IPS_LEN=`echo "$DEPLOYED_DATA_IPS" | wc -l` + +DEPLOYED_INGEST_IPS=$(cat ./qa/tf/ci/auth-smoke/outputs.json | jq -r '[.ingest_ips][0]["value"][]') +echo "DEPLOYED_INGEST_IPS: {" +echo "${DEPLOYED_INGEST_IPS}" +echo "}" + +DEPLOYED_INGEST_IPS_LEN=`echo "$DEPLOYED_INGEST_IPS" | wc -l` + + +#wait until we can connect to one of the hosts +for i in {0..24} +do + ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no -o ConnectTimeout=10 ec2-user@${DATANODE0} "pwd" + if [ $? -eq 0 ] + then + echo "Cluster is up after ${i} tries." + break + fi + sleep 10 +done + +ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no -o ConnectTimeout=10 ec2-user@${DATANODE0} "pwd" +if [ $? -ne 0 ] +then + echo "Unable to connect to cluster - giving up" + exit 1 +fi + +setupClusterNodes + +# verify featurebase running +echo "Verifying featurebase cluster running..." +for i in {0..24}; do + curl -k -v https://${DATANODE0}:10101/status -H "Authorization: Bearer ${ADMIN_TOKEN}" + if [ $? -eq 0 ]; then + echo "Cluster is up after ${i} tries" + exit 0 + fi + sleep 1 +done +exit $? diff --git a/qa/scripts/auth-smoke/teardown.sh b/qa/scripts/auth-smoke/teardown.sh new file mode 100755 index 000000000..40665efd9 --- /dev/null +++ b/qa/scripts/auth-smoke/teardown.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +# To run script: ./teardown.sh + +cd qa/tf/ci/auth-smoke +export TF_IN_AUTOMATION=1 +terraform destroy -auto-approve diff --git a/qa/scripts/auth-smoke/test.sh b/qa/scripts/auth-smoke/test.sh new file mode 100755 index 000000000..7f7c0cb93 --- /dev/null +++ b/qa/scripts/auth-smoke/test.sh @@ -0,0 +1,37 @@ +#!/bin/bash + +source ./qa/scripts/utilCluster.sh + +# get the first ingest host +INGESTNODE0=$(cat ./qa/tf/ci/auth-smoke/outputs.json | jq -r '[.ingest_ips][0]["value"][0]') +echo "using INGESTNODE0 ${INGESTNODE0}" + +# get the first data host +DATANODE0=$(cat ./qa/tf/ci/auth-smoke/outputs.json | jq -r '[.data_node_ips][0]["value"][0]') +echo "using DATANODE0 ${DATANODE0}" + +HOSTS=($( cat ./qa/tf/ci/auth-smoke/outputs.json | jq -r '.data_node_ips.value' | tr -d '[],"')) + +echo "Copying tests to remote" +scp -r -i ~/.ssh/gitlab-featurebase-ci.pem ./qa/scripts/auth-smoke/tests/ ec2-user@${INGESTNODE0}:/data +scp -r -i ~/.ssh/gitlab-featurebase-ci.pem ./datagen_linux_arm64 ec2-user@${INGESTNODE0}:/data +if (( $? != 0 )) +then + echo "Copy failed" + exit 1 +fi + +# run all repros +echo "Running smoke tests..." +ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o "StrictHostKeyChecking no" ec2-user@${INGESTNODE0} "cd /data/tests; ./run-all.sh ${HOSTS[@]}" +SMOKETESTRESULT=$? + + +if (( $SMOKETESTRESULT != 0 )) +then + echo "smoke tests complete with test failures" +else + echo "smoke tests complete" +fi + +exit $SMOKETESTRESULT diff --git a/qa/scripts/auth-smoke/tests/README b/qa/scripts/auth-smoke/tests/README new file mode 100644 index 000000000..c9ebf1977 --- /dev/null +++ b/qa/scripts/auth-smoke/tests/README @@ -0,0 +1,16 @@ +adding a test case to the auth-smoke directory??? no problem!!! its a snap!!! + +just make a shell script that does the test you want and name it some thing like: + +fb42069-test.sh + +this will get picked up by run-all.sh and get run automatically!!! + +# THINGS TO NOTE +- the datanode0 ip will be passed to your script in $1 via qa/scripts/bug_repro_tests.sh + +# ENTHUSIASM +WOW + +shout out to computers for making our lives easier! :) 👍 + diff --git a/qa/scripts/auth-smoke/tests/run-all.sh b/qa/scripts/auth-smoke/tests/run-all.sh new file mode 100755 index 000000000..ac27d85ce --- /dev/null +++ b/qa/scripts/auth-smoke/tests/run-all.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +set -eou pipefail + +for file in `ls *-test.sh`; do + echo "running $file"; + ./$file "$@" +done diff --git a/qa/scripts/auth-smoke/tests/sup218-test.sh b/qa/scripts/auth-smoke/tests/sup218-test.sh new file mode 100755 index 000000000..cd75a0c0f --- /dev/null +++ b/qa/scripts/auth-smoke/tests/sup218-test.sh @@ -0,0 +1,34 @@ +#!/bin/bash + +ifErr() { + res=$? + if (( res != 0 )); then + echo "error: $1" + exit $res + fi +} + +HOSTS=($@) + +for host in ${HOSTS[@]}; do + echo $host; +done + +HOST=${HOSTS[2]} + +ADMIN_TOKEN="eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJuYW1lIjoiYWRtaW4ifQ.I1iCgk1VU7m6e-En4ACTHIs6V2dZpy_8j2blSSo7K3U" +# ingest string key data to user index +/data/datagen --source custom --custom-config /data/tests/sup218_datagen.yaml --pilosa.index=user --pilosa.hosts=https://$HOST:10101 --pilosa.batch-size=1000 --auth-token=$ADMIN_TOKEN +ifErr "running datagen on $HOST" + +# install grpcurl +wget https://github.com/fullstorydev/grpcurl/releases/download/v1.8.6/grpcurl_1.8.6_linux_arm64.tar.gz +tar -xvf grpcurl_1.8.6_linux_arm64.tar.gz +chmod +x grpcurl + +for ip in ${HOSTS[@]}; do + # then make a `select distinct test_field from user` + ./grpcurl -H 'Authorization: Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJuYW1lIjoiYWRtaW4ifQ.I1iCgk1VU7m6e-En4ACTHIs6V2dZpy_8j2blSSo7K3U' -d '{"sql": "select * from user limit 1"}' $ip:20101 pilosa.Pilosa.QuerySQL + # make sure it doesn't fail + ifErr "select * from user failed when it shouldn't have!" +done diff --git a/qa/scripts/auth-smoke/tests/sup218_datagen.yaml b/qa/scripts/auth-smoke/tests/sup218_datagen.yaml new file mode 100644 index 000000000..c9b8292c5 --- /dev/null +++ b/qa/scripts/auth-smoke/tests/sup218_datagen.yaml @@ -0,0 +1,61 @@ +fields: + - name: "a_random_string" + type: "string" # (default StringField (non-mutex)) + generator_type: "random-string" # used to generate random strings rather than pulling from known set + min_len: 8 + max_len: 12 + charset: "AB" # set of possible characters to pull from when generating random string + - name: "id" + type: "uint" + distribution: "sequential" + min: 0 + max: 1000 # 2%24 + step: 1 + repeat: false + - name: "type" + type: "int" # (default IntField) + min: 0 + max: 3 + distribution: "zipfian" + s: 1.1 + v: 5.1 + - name: "ts" + type: "timestamp" + min_date: 2006-01-02T15:04:05.001Z # RFC3339Nano + max_date: 2007-01-02T15:04:05.001Z # RFC3339Nano + distribution: "increasing" # only "increasing" is supported right now + min_step_duration: "10ns" + max_step_duration: "200ms" + - name: "slice" + type: "uint-set" # (default IDArrayField) + min: 0 + max: 35000 + distribution: "zipfian" + s: 1.1 + v: 5.1 + min_num: 1 + max_num: 50 + +# idk_params describe how data from "fields" should be ingested by IDK +idk_params: + primary_key_config: + field: "a_random_string" # if this is a single field named "id" then we'll use uint IDs, if it's empty we'll autogen ids, and if it's anything else we'll do string keys... yes this is a bit hacky, needs to be cleaned up. + # fields is keyed by names of fields from top level "fields". It is + # not required that all fields appear here, those that don't will + # use the default ingestion. + fields: + id: + - type: "ID" + type: + - type: "ID" + a_decimal_field: + - type: "Decimal" + scale: 4 + ts: + - type: "RecordTime" + layout: "2006-01-02T15:04:05Z" + epoch: 1970-01-01T00:00:00.0Z + name: "na" + slice: + - type: "IDArray" + time_quantum: "YMD" diff --git a/qa/scripts/setupSmokeTest.sh b/qa/scripts/setupSmokeTest.sh index 997974494..0eb601f26 100755 --- a/qa/scripts/setupSmokeTest.sh +++ b/qa/scripts/setupSmokeTest.sh @@ -2,6 +2,7 @@ # To run script: ./setupSmokeTest.sh export TF_IN_AUTOMATION=1 +export AUTH_ENABLED=0 SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) source $SCRIPT_DIR/utilCluster.sh diff --git a/qa/scripts/setupTLS.sh b/qa/scripts/setupTLS.sh new file mode 100755 index 000000000..e8b3f85c6 --- /dev/null +++ b/qa/scripts/setupTLS.sh @@ -0,0 +1,81 @@ +#!/bin/bash + +NODEIP=$1 + +if [[ $NODEIP = "" ]]; then + echo "usage: " + echo "./setupTLS.sh " + exit 1 +fi + +ifErr() { + res=$? + if (( res != 0 )); then + echo "error: $1" + exit $res + fi +} + +echo "installing go" +sudo yum install wget -y +ifErr "installing wget" + +sudo wget -q https://go.dev/dl/go1.17.8.linux-arm64.tar.gz +ifErr "downloading golang" + +sudo tar -C /usr/local -xzf go1.17.8.linux-arm64.tar.gz +ifErr "unpacking golang" + +echo "export PATH=$PATH:/usr/local/go/bin" >> ~/.bashrc +ifErr "adding go to path" +export PATH=$PATH:/usr/local/go/bin +ifErr "setting PATH" + +echo $PATH +ifErr "echoing path" + +go version +if (( $? != 0 )); then + echo "go not installed!" + sudo yum install golang -y + ifErr "golang not installing! >:(" +fi + +echo "setting up certs" +cd ~ + +sudo yum install nss-tools +ifErr "installing nss-tools" + +git clone https://github.com/FiloSottile/mkcert && cd mkcert +ifErr "cloning mkcert" + +go build -ldflags "-X main.Version=$(git describe --tags)" +ifErr "building mkcert" + +sudo ./mkcert -install +ifErr "installing root CA" + +sudo ./mkcert $NODEIP +ifErr "creating cert for $NODEIP" + +sudo mv $NODEIP.pem /data/cert.crt +ifErr "moving cert" + +sudo mv $NODEIP-key.pem /data/key.key +ifErr "moving key" + +sudo chown molecula /data/key.key +ifErr "chown-ing /data/key.key" + +sudo chown molecula /data/cert.crt +ifErr "chown-ing /data/cert.crt" + +echo "setting up fake IDP" +cd /etc/fakeidp + +go build . +ifErr "building fakeidp server" + +echo "starting fakeidp" +nohup ./fakeidp > /dev/null 2>&1 & diff --git a/qa/scripts/utilCluster.sh b/qa/scripts/utilCluster.sh index c0afb3a6e..9918ba3ac 100644 --- a/qa/scripts/utilCluster.sh +++ b/qa/scripts/utilCluster.sh @@ -22,6 +22,14 @@ DEPLOYED_INGEST_IPS_LEN=0 #Initial cluster string INITIAL_CLUSTER="" +ifErr() { + res=$? + if (( res != 0 )); then + echo "error: $1" + exit $res + fi +} + writeFeatureBaseNodeServiceFile() { echo "Writing featurebase.service file...index: $1, ip:$2" NODEIDX=$1 @@ -61,12 +69,12 @@ writeFeatureBaseNodeConfigFile() { echo "Writing featurebase.conf file...index: $1, ip:$2" NODEIDX=$1 NODEIP=$2 - if (( AUTH_ENABLED = 1 )); then + if [[ "$AUTH_ENABLED" = "1" ]]; then echo "writing auth enabled featurebase.conf" cat << EOT > featurebase.conf name = "p${NODEIDX}" -bind = "0.0.0.0:10101" -bind-grpc = "0.0.0.0:20101" +bind = "https://0.0.0.0:10101" +bind-grpc = "https://0.0.0.0:20101" data-dir = "/data/featurebase" log-path = "/var/log/molecula/featurebase.log" @@ -77,23 +85,47 @@ max-map-count=900000 long-query-time = "10s" [postgres] - bind = "localhost:55432" [cluster] - name = "${DEPLOYED_CLUSTER_PREFIX}" replicas = ${DEPLOYED_CLUSTER_REPLICA_COUNT} [etcd] - listen-client-address = "http://${NODEIP}:10401" listen-peer-address = "http://${NODEIP}:10301" initial-cluster = "${INITIAL_CLUSTER}" [metric] - service = "prometheus" + +[tls] + certificate = "/data/cert.crt" + key = "/data/key.key" + +[auth] + enable = true + client-id = "e9088663-eb08-41d7-8f65-efb5f54bbb71" + client-secret = "bex7Q~aiWeQ70iBXbEup-XydyQrrNc_Q5n8EW" + authorize-url = "http://localhost:12345/authorize" + redirect-base-url = "http://localhost:12345/redirect" + token-url = "http://localhost:12345/token" + group-endpoint-url = "http://localhost:12345/groups" + logout-url = "http://localhost:12345/logout" + scopes = ["openid", "https://graph.microsoft.com/.default", "offline_access"] + secret-key = "98995f0530eeba96da1d0a04311073c0abb7b6abbfb0f5f4ef3629527ff88428" + permissions = "/etc/permissions.yml" + query-log-path = "/data/featurebase/query.log" +EOT + + echo "writing the permissions file" + cat << EOT > permissions.yml +"user-groups": + "group-id-reader": + "user": "read" + "group-id-writer": + "user": "write" +admin: "group-id-admin" EOT else cat << EOT > featurebase.conf @@ -110,30 +142,22 @@ max-map-count=900000 long-query-time = "10s" [postgres] - bind = "localhost:55432" [cluster] - name = "${DEPLOYED_CLUSTER_PREFIX}" replicas = ${DEPLOYED_CLUSTER_REPLICA_COUNT} [etcd] - listen-client-address = "http://${NODEIP}:10401" listen-peer-address = "http://${NODEIP}:10301" initial-cluster = "${INITIAL_CLUSTER}" [metric] - service = "prometheus" EOT fi - #echo "featurebase.conf >>" - #cat featurebase.conf - #echo "featurebase.conf <<" - scp -i ~/.ssh/gitlab-featurebase-ci.pem -o "StrictHostKeyChecking no" featurebase.conf ec2-user@${NODEIP}: if (( $? != 0 )) then @@ -143,6 +167,47 @@ EOT rm -f featurebase.conf ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "sudo mv featurebase.conf ${CONFIG_FILE_PATH}" + + if [[ "$AUTH_ENABLED" = "1" ]]; then + scp -i ~/.ssh/gitlab-featurebase-ci.pem -o "StrictHostKeyChecking no" permissions.yml ec2-user@${NODEIP}: + ifErr "permissions.yml copy failed" + rm -f permissions.yml + + ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "sudo mv ~/permissions.yml /etc/permissions.yml" + ifErr "mv permissions.yml failed" + fi +} + +setupMkCertCA() { + # this function sets up a CA on the runner for use by all nodes in setupTLS + git clone https://github.com/FiloSottile/mkcert && cd mkcert + ifErr "cloning mkcert" + + go build -ldflags "-X main.Version=$(git describe --tags)" + ifErr "building mkcert" + + ./mkcert -install + ifErr "installing root CA" + + cp mkcert /usr/local/bin + + cd ../ + +} + +setupTLS() { + NODEIP=$1 + scp -i ~/.ssh/gitlab-featurebase-ci.pem -o "StrictHostKeyChecking no" ./qa/scripts/setupTLS.sh ec2-user@${NODEIP}: + ifErr "setupTLS.sh copy failed" + + scp -r -i ~/.ssh/gitlab-featurebase-ci.pem -o "StrictHostKeyChecking no" ./qa/fakeidp/ ec2-user@${NODEIP}: + ifErr "fakeidp copy failed" + + ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "sudo mv ~/fakeidp /etc" + ifErr "mv fakeidp failed" + + ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "sudo ./setupTLS.sh ${NODEIP}" + ifErr "error setting up TLS" } executeGeneralNodeConfigCommands() { @@ -150,7 +215,7 @@ executeGeneralNodeConfigCommands() { NODEIDX=$1 NODEIP=$2 - ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "sudo mkdir /data" + ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "sudo mkdir -p /data" ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "sudo mkfs.ext4 /dev/nvme1n1" ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "sudo mount /dev/nvme1n1 /data" @@ -171,8 +236,26 @@ executeGeneralNodeConfigCommands() { ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "chown ec2-user:ec2-user /home/ec2-user/featurebase_linux_arm64" ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "chmod ugo+x /home/ec2-user/featurebase_linux_arm64" ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "sudo mv /home/ec2-user/featurebase_linux_arm64 /usr/local/bin/featurebase" + ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "sudo yum install git -y" echo "featurebase binary copied." + + if [[ "$AUTH_ENABLED" = "1" ]]; then + caroot=$(mkcert -CAROOT) + + echo "copying root ca to ${NODEIP}" + scp -i ~/.ssh/gitlab-featurebase-ci.pem -o "StrictHostKeyChecking no" "${caroot}/rootCA.pem" ec2-user@${NODEIP}: + ifErr "copying CAROOT to ${NODEIP}" + + scp -i ~/.ssh/gitlab-featurebase-ci.pem -o "StrictHostKeyChecking no" "${caroot}/rootCA-key.pem" ec2-user@${NODEIP}: + ifErr "copying CAROOT to ${NODEIP}" + + ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "sudo mkdir -p /root/.local/share/mkcert && sudo cp ~/rootCA.pem /root/.local/share/mkcert && sudo cp ~/rootCA-key.pem /root/.local/share/mkcert" + ifErr "cp-ing rootCA.pem to /root/.local/share/mkcert on ${NODEIP}" + + echo "setting up tls certificates" + setupTLS $NODEIP + fi } executeDataStartCommands() { @@ -216,6 +299,7 @@ setupIngestNode() { ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "pip3 install -U requests" ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "pip3 install -U json" ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "sudo yum install -y jq" + installDatagen $NODEIP } setupDataNodes() { @@ -237,20 +321,15 @@ setupIngestNodes() { } generateInitialClusterString() { - if (( AUTH_ENABLED = 1 )); then - scheme=https - else - scheme=http - fi IFS=$'\n' cnt=0 for ip in $DEPLOYED_DATA_IPS do if (($cnt + 1 != $DEPLOYED_DATA_IPS_LEN)) then - INITIAL_CLUSTER="${INITIAL_CLUSTER}p${cnt}=${scheme}://$ip:10301," + INITIAL_CLUSTER="${INITIAL_CLUSTER}p${cnt}=http://$ip:10301," else - INITIAL_CLUSTER="${INITIAL_CLUSTER}p${cnt}=${scheme}://$ip:10301" + INITIAL_CLUSTER="${INITIAL_CLUSTER}p${cnt}=http://$ip:10301" fi cnt=$((cnt+1)) done @@ -263,6 +342,10 @@ setupClusterNodes() { #data nodes generateInitialClusterString + if [[ "$AUTH_ENABLED" = "1" ]]; then + setupMkCertCA + fi + setupDataNodes startDataNodes diff --git a/qa/tf/ci/auth-smoke/main.tf b/qa/tf/ci/auth-smoke/main.tf new file mode 100644 index 000000000..083e1dae4 --- /dev/null +++ b/qa/tf/ci/auth-smoke/main.tf @@ -0,0 +1,13 @@ +module "ci-cluster" { + source = "../../.modules/featurebase-cluster" + cluster_prefix = var.cluster_prefix + region = var.region + profile = var.profile + fb_data_node_type = "m6g.large" + fb_data_node_count = 3 + fb_ingest_type = "m6g.large" + vpc_id = "vpc-05a26a122f961dc2b" + vpc_cidr_block = "10.0.0.0/16" + vpc_public_subnets = ["subnet-066b4b922b54e51a2", "subnet-037b8884269a69025", "subnet-08482631514426210", ] + vpc_private_subnets = ["subnet-0319dde319380326f", "subnet-0517ca9a646d80f88", "subnet-05a7b685ed27eb1cf", ] +} diff --git a/qa/tf/ci/auth-smoke/outputs.tf b/qa/tf/ci/auth-smoke/outputs.tf new file mode 100644 index 000000000..886c6f783 --- /dev/null +++ b/qa/tf/ci/auth-smoke/outputs.tf @@ -0,0 +1,19 @@ +output "ingest_ips" { + description = "List of ingest IPs" + value = module.ci-cluster.ingest_ips +} + +output "data_node_ips" { + description = "List of data node IPs" + value = module.ci-cluster.data_node_ips +} + +output "cluster_prefix" { + description = "The cluster prefix used" + value = module.ci-cluster.cluster_prefix +} + +output "fb_cluster_replica_count" { + description = "The cluster replica count used" + value = module.ci-cluster.fb_cluster_replica_count +} diff --git a/qa/tf/ci/auth-smoke/provider.tf b/qa/tf/ci/auth-smoke/provider.tf new file mode 100644 index 000000000..c0fc95d9d --- /dev/null +++ b/qa/tf/ci/auth-smoke/provider.tf @@ -0,0 +1,4 @@ +provider "aws" { + region = var.region + profile = var.profile +} \ No newline at end of file diff --git a/qa/tf/ci/auth-smoke/tf.auto.tfvars b/qa/tf/ci/auth-smoke/tf.auto.tfvars new file mode 100644 index 000000000..ac6de62a6 --- /dev/null +++ b/qa/tf/ci/auth-smoke/tf.auto.tfvars @@ -0,0 +1,2 @@ +region = "us-east-2" +profile = "service-terraform" \ No newline at end of file diff --git a/qa/tf/ci/auth-smoke/variables.tf b/qa/tf/ci/auth-smoke/variables.tf new file mode 100644 index 000000000..a327ea4ff --- /dev/null +++ b/qa/tf/ci/auth-smoke/variables.tf @@ -0,0 +1,15 @@ +variable "region" { + description = "The AWS region in which the VPC should be built" + type = string +} + +variable "profile" { + description = "The name of the AWS profile Terraform should use for auth." + type = string +} + +variable "cluster_prefix" { + type = string + description = "This is a identifier that will be prefixed to created resources" +} +