setup auth cluster infra

also add test for sup218

Co-authored-by: rachithrr <rachith.ramaswamy@molecula.com>
Co-authored-by: bruce-b-molecula <bruce.baranowski@molecula.com>

.gitlab/.gitlab-ci.yml

@@ -340,6 +340,63 @@ external lookup tests:
     - apt-get install -y postgresql-client
     - go test . -run "^TestExternalLookup" -externalLookupDSN postgresql://$POSTGRES_USER:$POSTGRES_PASSWORD@postgres/$POSTGRES_DB?sslmode=disable
 
+smoke test auth:
+  stage: integration
+  image: registry.gitlab.com/gitlab-org/cloud-deploy/aws-base:latest
+  variables:
+    PROFILE: "service-terraform"
+    AWS_SSH_PRIVATE_KEY: $AWS_FBCI_SSH_KEY
+    AWS_ACCESS_KEY_ID: $AWS_FBCI_ACCESS_KEY_ID
+    AWS_SECRET_ACCESS_KEY: $AWS_FBCI_SECRET_ACCESS_KEY
+    TF_VAR_cluster_prefix: ""
+  tags:
+    - aws
+    - docker
+    - fbsmoke
+  rules:
+    - if: '$CI_PIPELINE_SOURCE == "push"'
+  before_script:
+    - apt-get update && apt-get install -y gnupg software-properties-common curl git
+    - curl -fsSL https://apt.releases.hashicorp.com/gpg | apt-key add -
+    - apt-add-repository "deb [arch=amd64] https://apt.releases.hashicorp.com $(lsb_release -cs) main"
+    - apt-get update && apt-get install terraform
+    - aws configure set aws_access_key_id $AWS_FBCI_ACCESS_KEY_ID --profile $PROFILE
+    - aws configure set aws_secret_access_key $AWS_FBCI_SECRET_ACCESS_KEY --profile $PROFILE
+    - aws configure set region "us-east-2" --profile $PROFILE
+    - aws configure set aws_profile $PROFILE
+    - echo $AWS_FBCI_SSH_KEY > gitlab-featurebase-ci.pem
+    - chmod 400 gitlab-featurebase-ci.pem
+    - 'which ssh-agent || ( apt-get update -y && apt-get install openssh-client -y )'
+    - eval $(ssh-agent -s)
+    - mkdir -p ~/.ssh
+    - echo $AWS_FBCI_SSH_KEY > /root/.ssh/gitlab-featurebase-ci.pem
+    - chmod 400 /root/.ssh/gitlab-featurebase-ci.pem
+    - echo "$AWS_FBCI_SSH_KEY" | ssh-add -
+    - chmod 700 /root/.ssh
+    - '[[ -f /.dockerenv ]] && echo -e "Host *\n\tStrictHostKeyChecking no\n\n" > ~/.ssh/config'
+    - apt update && apt -y install jq wget git libnss3-tools
+    - wget -q https://go.dev/dl/go$GOVERSION.linux-amd64.tar.gz
+    - tar -C /usr/local -xzf go$GOVERSION.linux-amd64.tar.gz
+    - export PATH=$PATH:/usr/local/go/bin
+    - TF_VAR_cluster_prefix="smoke-$(openssl rand -base64 12 | tr -d /=+ | cut -c -16)"
+    - echo "Cluster Prefix --> $TF_VAR_cluster_prefix"
+    # download datagen for FB-1270 repro test. TODO replace w/ locally built datagen once we merge IDK into FB
+    - aws s3 cp s3://molecula-artifact-storage/idk/master/_latest/idk-linux-arm64/datagen ./datagen_linux_arm64
+    - chmod +x ./datagen_linux_arm64
+  script:
+    - ./qa/scripts/auth-smoke/setup.sh
+    - ./qa/scripts/auth-smoke/test.sh
+  after_script:
+    - ./qa/scripts/auth-smoke/teardown.sh
+  needs:
+    - job: build for linux arm64
+  artifacts:
+    when: always
+    paths:
+      - report.xml
+    reports:
+      junit: report.xml
+
 
 smoke test:
   stage: integration

authn/authenticate.go

@@ -168,10 +168,14 @@ func (a *Auth) Authenticate(ctx context.Context, bearer string) (*UserInfo, erro
 	}
 
 	userInfo := UserInfo{
-		UserID:   claims["oid"].(string),
-		UserName: claims["name"].(string),
-		Token:    bearer,
-		Groups:   []Group{},
+		Token:  bearer,
+		Groups: []Group{},
+	}
+	if uid, ok := claims["oid"].(string); ok {
+		userInfo.UserID = uid
+	}
+	if name, ok := claims["name"].(string); ok {
+		userInfo.UserName = name
 	}
 
 	if userInfo.Groups, err = a.getGroups(bearer); err != nil {

qa/fakeidp/go.mod

@@ -0,0 +1,5 @@
+module fakeidp
+
+go 1.17
+
+require github.com/golang-jwt/jwt v3.2.2+incompatible

qa/fakeidp/go.sum

@@ -0,0 +1,2 @@
+github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY=
+github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I=

qa/fakeidp/server.go

@@ -0,0 +1,65 @@
+package main
+
+import (
+	"encoding/hex"
+	"log"
+	"net/http"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/golang-jwt/jwt"
+)
+
+const (
+	adminToken  = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJuYW1lIjoiYWRtaW4ifQ.I1iCgk1VU7m6e-En4ACTHIs6V2dZpy_8j2blSSo7K3U"
+	readerToken = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJuYW1lIjoicmVhZGVyIn0.QcHy_W6oAYFgdBWy1CqLr55HcOyymn5zAXPJUKCvQE4"
+	writerToken = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJuYW1lIjoid3JpdGVyIn0.aEk-12xP9RJeXog4MHO8LhuQFEjNNG2BcWDcMzSX_HI"
+)
+
+func groups(w http.ResponseWriter, req *http.Request) {
+	authToken := strings.TrimPrefix(req.Header.Get("Authorization"), "Bearer ")
+	status := http.StatusOK
+
+	var val []byte
+	switch authToken {
+	case adminToken:
+		val = []byte(`{"value":[{"id":"group-id-admin","displayName":"group-id-admin"}]}`)
+	case readerToken:
+		val = []byte(`{"value":[{"id":"group-id-reader","displayName":"group-id-reader"}]}`)
+	case writerToken:
+		val = []byte(`{"value":[{"id":"group-id-writer","displayName":"group-id-writer"}]}`)
+	default:
+		status = http.StatusUnauthorized
+	}
+	w.WriteHeader(status)
+	w.Write(val)
+}
+
+func token(w http.ResponseWriter, req *http.Request) {
+	tkn := jwt.New(jwt.SigningMethodHS256)
+	claims := tkn.Claims.(jwt.MapClaims)
+	claims["oid"] = "42"
+	claims["name"] = "valid"
+	expiresIn := 2 * time.Hour
+	claims["exp"] = strconv.Itoa(int(time.Now().Add(expiresIn).Unix()))
+	k, err := hex.DecodeString("DEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEF")
+	if err != nil {
+		log.Fatalf("i am not equipped to handle this!!! %v", err)
+	}
+	fresh, err := tkn.SignedString(k)
+	if err != nil {
+		log.Fatalf("i am not equipped to handle this!!! %v", err)
+	}
+	body := `{"access_token": "` + fresh + `", "refresh_token": "blah", "expires_in": "` + strconv.Itoa(int(expiresIn.Seconds())) + `"}`
+	w.Header().Set("Content-Type", "application/json; charset=utf-8")
+	w.WriteHeader(http.StatusOK)
+	w.Write([]byte(body))
+}
+
+func main() {
+	http.HandleFunc("/groups", groups)
+	http.HandleFunc("/token", token)
+	log.Println("FAKEIDP SERVER UP AND RUNNING")
+	log.Fatal(http.ListenAndServe(":12345", nil))
+}

qa/scripts/auth-smoke/setup.sh

@@ -0,0 +1,81 @@
+#!/bin/bash
+
+# To run script: ./setup.sh
+ADMIN_TOKEN=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJuYW1lIjoiYWRtaW4ifQ.I1iCgk1VU7m6e-En4ACTHIs6V2dZpy_8j2blSSo7K3U
+READER_TOKEN=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJuYW1lIjoicmVhZGVyIn0.QcHy_W6oAYFgdBWy1CqLr55HcOyymn5zAXPJUKCvQE4
+WRITER_TOKEN=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJuYW1lIjoid3JpdGVyIn0.aEk-12xP9RJeXog4MHO8LhuQFEjNNG2BcWDcMzSX_HI
+export TF_IN_AUTOMATION=1
+export AUTH_ENABLED=1
+
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+source ./qa/scripts/utilCluster.sh
+
+pushd ./qa/tf/ci/auth-smoke
+echo "Running terraform init..."
+terraform init -input=false
+echo "Running terraform apply..."
+terraform apply -input=false -auto-approve
+terraform output -json > outputs.json
+popd
+
+# get the first ingest host
+INGESTNODE0=$(cat ./qa/tf/ci/auth-smoke/outputs.json | jq -r '[.ingest_ips][0]["value"][0]')
+echo "using INGESTNODE0 ${INGESTNODE0}"
+
+# get the first data host
+DATANODE0=$(cat ./qa/tf/ci/auth-smoke/outputs.json | jq -r '[.data_node_ips][0]["value"][0]')
+echo "using DATANODE0 ${DATANODE0}"
+
+DEPLOYED_CLUSTER_PREFIX=$(cat ./qa/tf/ci/auth-smoke/outputs.json | jq -r '[.cluster_prefix][0]["value"]')
+echo "Using DEPLOYED_CLUSTER_PREFIX: ${DEPLOYED_CLUSTER_PREFIX}"
+
+DEPLOYED_CLUSTER_REPLICA_COUNT=$(cat ./qa/tf/ci/auth-smoke/outputs.json | jq -r '[.fb_cluster_replica_count][0]["value"]')
+echo "Using DEPLOYED_CLUSTER_REPLICA_COUNT: ${DEPLOYED_CLUSTDEPLOYED_CLUSTER_REPLICA_COUNTER_PREFIX}"
+
+DEPLOYED_DATA_IPS=$(cat ./qa/tf/ci/auth-smoke/outputs.json | jq -r '[.data_node_ips][0]["value"][]')
+echo "DEPLOYED_DATA_IPS: {"
+echo "${DEPLOYED_DATA_IPS}"
+echo "}"
+
+DEPLOYED_DATA_IPS_LEN=`echo "$DEPLOYED_DATA_IPS" | wc -l`
+
+DEPLOYED_INGEST_IPS=$(cat ./qa/tf/ci/auth-smoke/outputs.json | jq -r '[.ingest_ips][0]["value"][]')
+echo "DEPLOYED_INGEST_IPS: {"
+echo "${DEPLOYED_INGEST_IPS}"
+echo "}"
+
+DEPLOYED_INGEST_IPS_LEN=`echo "$DEPLOYED_INGEST_IPS" | wc -l`
+
+
+#wait until we can connect to one of the hosts
+for i in {0..24}
+do 
+    ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no -o ConnectTimeout=10 ec2-user@${DATANODE0} "pwd"
+    if [ $? -eq 0 ]
+    then
+        echo "Cluster is up after ${i} tries." 
+        break
+    fi
+    sleep 10
+done
+
+ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no -o ConnectTimeout=10 ec2-user@${DATANODE0} "pwd"
+if [ $? -ne 0 ]
+then
+    echo "Unable to connect to cluster - giving up" 
+    exit 1
+fi
+
+setupClusterNodes
+
+# verify featurebase running
+echo "Verifying featurebase cluster running..."
+for i in {0..24}; do
+    curl -k -v https://${DATANODE0}:10101/status -H "Authorization: Bearer ${ADMIN_TOKEN}"
+    if [ $? -eq 0 ]; then
+        echo "Cluster is up after ${i} tries"
+        exit 0
+    fi 
+    sleep 1
+done
+exit $?

qa/scripts/auth-smoke/teardown.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+# To run script: ./teardown.sh
+
+cd qa/tf/ci/auth-smoke
+export TF_IN_AUTOMATION=1
+terraform destroy -auto-approve

qa/scripts/auth-smoke/test.sh

@@ -0,0 +1,37 @@
+#!/bin/bash
+
+source ./qa/scripts/utilCluster.sh
+
+# get the first ingest host
+INGESTNODE0=$(cat ./qa/tf/ci/auth-smoke/outputs.json | jq -r '[.ingest_ips][0]["value"][0]')
+echo "using INGESTNODE0 ${INGESTNODE0}"
+
+# get the first data host
+DATANODE0=$(cat ./qa/tf/ci/auth-smoke/outputs.json | jq -r '[.data_node_ips][0]["value"][0]')
+echo "using DATANODE0 ${DATANODE0}"
+
+HOSTS=($( cat ./qa/tf/ci/auth-smoke/outputs.json | jq -r '.data_node_ips.value' | tr -d '[],"'))
+
+echo "Copying tests to remote"
+scp -r -i ~/.ssh/gitlab-featurebase-ci.pem ./qa/scripts/auth-smoke/tests/ ec2-user@${INGESTNODE0}:/data
+scp -r -i ~/.ssh/gitlab-featurebase-ci.pem ./datagen_linux_arm64 ec2-user@${INGESTNODE0}:/data
+if (( $? != 0 )) 
+then 
+    echo "Copy failed"
+    exit 1
+fi
+
+# run all repros
+echo "Running smoke tests..."
+ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o "StrictHostKeyChecking no" ec2-user@${INGESTNODE0} "cd /data/tests; ./run-all.sh ${HOSTS[@]}"
+SMOKETESTRESULT=$?
+
+
+if (( $SMOKETESTRESULT != 0 )) 
+then 
+    echo "smoke tests complete with test failures"
+else
+    echo "smoke tests complete"
+fi
+
+exit $SMOKETESTRESULT 

qa/scripts/auth-smoke/tests/README

@@ -0,0 +1,16 @@
+adding a test case to the auth-smoke directory??? no problem!!! its a snap!!!
+
+just make a shell script that does the test you want and name it some thing like:
+
+fb42069-test.sh
+
+this will get picked up by run-all.sh and get run automatically!!!
+
+# THINGS TO NOTE
+- the datanode0 ip will be passed to your script in $1 via qa/scripts/bug_repro_tests.sh
+
+# ENTHUSIASM
+WOW
+
+shout out to computers for making our lives easier! :) 👍
+

qa/scripts/auth-smoke/tests/run-all.sh

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+
+set -eou pipefail
+
+for file in `ls *-test.sh`; do
+    echo "running $file";
+    ./$file "$@"
+done

qa/scripts/auth-smoke/tests/sup218-test.sh

@@ -0,0 +1,34 @@
+#!/bin/bash
+
+ifErr() {
+    res=$?
+    if (( res != 0 )); then
+        echo "error: $1"
+        exit $res
+    fi
+}
+
+HOSTS=($@)
+
+for host in ${HOSTS[@]}; do
+    echo $host;
+done
+
+HOST=${HOSTS[2]}
+
+ADMIN_TOKEN="eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJuYW1lIjoiYWRtaW4ifQ.I1iCgk1VU7m6e-En4ACTHIs6V2dZpy_8j2blSSo7K3U"
+# ingest string key data to user index 
+/data/datagen --source custom --custom-config /data/tests/sup218_datagen.yaml --pilosa.index=user --pilosa.hosts=https://$HOST:10101 --pilosa.batch-size=1000 --auth-token=$ADMIN_TOKEN
+ifErr "running datagen on $HOST"
+
+# install grpcurl 
+wget https://github.com/fullstorydev/grpcurl/releases/download/v1.8.6/grpcurl_1.8.6_linux_arm64.tar.gz
+tar -xvf grpcurl_1.8.6_linux_arm64.tar.gz
+chmod +x grpcurl
+
+for ip in ${HOSTS[@]}; do
+    # then make a `select distinct test_field from user` 
+    ./grpcurl -H 'Authorization: Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJuYW1lIjoiYWRtaW4ifQ.I1iCgk1VU7m6e-En4ACTHIs6V2dZpy_8j2blSSo7K3U' -d '{"sql": "select * from user limit 1"}' $ip:20101 pilosa.Pilosa.QuerySQL
+    # make sure it doesn't fail
+    ifErr "select * from user failed when it shouldn't have!"
+done

qa/scripts/auth-smoke/tests/sup218_datagen.yaml

@@ -0,0 +1,61 @@
+fields:
+  - name: "a_random_string"
+    type: "string" # (default StringField (non-mutex))
+    generator_type: "random-string" # used to generate random strings rather than pulling from known set
+    min_len: 8
+    max_len: 12
+    charset: "AB" # set of possible characters to pull from when generating random string
+  - name: "id"
+    type: "uint"
+    distribution: "sequential"
+    min: 0
+    max: 1000 # 2%24
+    step: 1
+    repeat: false
+  - name: "type"
+    type: "int" # (default IntField)
+    min: 0
+    max: 3
+    distribution: "zipfian"
+    s: 1.1
+    v: 5.1
+  - name: "ts"
+    type: "timestamp"
+    min_date: 2006-01-02T15:04:05.001Z # RFC3339Nano
+    max_date: 2007-01-02T15:04:05.001Z # RFC3339Nano
+    distribution: "increasing" # only "increasing" is supported right now
+    min_step_duration: "10ns"
+    max_step_duration: "200ms" 
+  - name: "slice"
+    type: "uint-set" # (default IDArrayField)
+    min: 0
+    max: 35000
+    distribution: "zipfian"
+    s: 1.1
+    v: 5.1
+    min_num: 1
+    max_num: 50
+
+# idk_params describe how data from "fields" should be ingested by IDK
+idk_params:
+  primary_key_config:
+    field: "a_random_string" # if this is a single field named "id" then we'll use uint IDs, if it's empty we'll autogen ids, and if it's anything else we'll do string keys... yes this is a bit hacky, needs to be cleaned up.
+  # fields is keyed by names of fields from top level "fields". It is
+  # not required that all fields appear here, those that don't will
+  # use the default ingestion.
+  fields:
+    id:
+      - type: "ID"
+    type:
+      - type: "ID"
+    a_decimal_field:
+      - type: "Decimal"
+        scale: 4
+    ts:
+      - type: "RecordTime"
+        layout: "2006-01-02T15:04:05Z"
+        epoch: 1970-01-01T00:00:00.0Z
+        name: "na"
+    slice:
+      - type: "IDArray"
+        time_quantum: "YMD"

qa/scripts/setupSmokeTest.sh

@@ -2,6 +2,7 @@
 
 # To run script: ./setupSmokeTest.sh
 export TF_IN_AUTOMATION=1
+export AUTH_ENABLED=0
 
 SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
 source $SCRIPT_DIR/utilCluster.sh

qa/scripts/setupTLS.sh

@@ -0,0 +1,81 @@
+#!/bin/bash
+
+NODEIP=$1
+
+if [[ $NODEIP = "" ]]; then
+    echo "usage: "
+    echo "./setupTLS.sh <ip address>"
+    exit 1
+fi
+
+ifErr() {
+    res=$?
+    if (( res != 0 )); then 
+        echo "error: $1"
+        exit $res
+    fi
+}
+
+echo "installing go"
+sudo yum install wget -y
+ifErr "installing wget"
+
+sudo wget -q https://go.dev/dl/go1.17.8.linux-arm64.tar.gz
+ifErr "downloading golang"
+
+sudo tar -C /usr/local -xzf go1.17.8.linux-arm64.tar.gz
+ifErr "unpacking golang"
+
+echo "export PATH=$PATH:/usr/local/go/bin" >> ~/.bashrc
+ifErr "adding go to path"
+export PATH=$PATH:/usr/local/go/bin
+ifErr "setting PATH"
+
+echo $PATH
+ifErr "echoing path"
+
+go version
+if (( $? != 0 )); then
+    echo "go not installed!"
+    sudo yum install golang -y
+    ifErr "golang not installing! >:("
+fi
+
+echo "setting up certs"
+cd ~
+
+sudo yum install nss-tools
+ifErr "installing nss-tools"
+
+git clone https://github.com/FiloSottile/mkcert && cd mkcert
+ifErr "cloning mkcert"
+
+go build -ldflags "-X main.Version=$(git describe --tags)"
+ifErr "building mkcert"
+
+sudo ./mkcert -install
+ifErr "installing root CA"
+
+sudo ./mkcert $NODEIP
+ifErr "creating cert for $NODEIP"
+
+sudo mv $NODEIP.pem /data/cert.crt
+ifErr "moving cert"
+
+sudo mv $NODEIP-key.pem /data/key.key
+ifErr "moving key"
+
+sudo chown molecula /data/key.key
+ifErr "chown-ing /data/key.key"
+
+sudo chown molecula /data/cert.crt
+ifErr "chown-ing /data/cert.crt"
+
+echo "setting up fake IDP"
+cd /etc/fakeidp
+
+go build .
+ifErr "building fakeidp server"
+
+echo "starting fakeidp"
+nohup ./fakeidp > /dev/null 2>&1 &

qa/scripts/utilCluster.sh

@@ -22,6 +22,14 @@ DEPLOYED_INGEST_IPS_LEN=0
 #Initial cluster string
 INITIAL_CLUSTER=""
 
+ifErr() {
+    res=$?
+    if (( res != 0 )); then
+        echo "error: $1"
+        exit $res
+    fi
+}
+
 writeFeatureBaseNodeServiceFile() {
     echo "Writing featurebase.service file...index: $1, ip:$2"
     NODEIDX=$1
@@ -61,12 +69,12 @@ writeFeatureBaseNodeConfigFile() {
     echo "Writing featurebase.conf file...index: $1, ip:$2"
     NODEIDX=$1
     NODEIP=$2
-    if (( AUTH_ENABLED = 1 )); then
+    if [[ "$AUTH_ENABLED" = "1" ]]; then
         echo "writing auth enabled featurebase.conf"
         cat << EOT > featurebase.conf
 name = "p${NODEIDX}"
-bind = "0.0.0.0:10101"
-bind-grpc = "0.0.0.0:20101"
+bind = "https://0.0.0.0:10101"
+bind-grpc = "https://0.0.0.0:20101"
 
 data-dir = "/data/featurebase"
 log-path = "/var/log/molecula/featurebase.log"
@@ -77,23 +85,47 @@ max-map-count=900000
 long-query-time = "10s"
 
 [postgres]
-
     bind = "localhost:55432"
 
 [cluster]
-
     name = "${DEPLOYED_CLUSTER_PREFIX}"
     replicas = ${DEPLOYED_CLUSTER_REPLICA_COUNT}
 
 [etcd]
-
     listen-client-address = "http://${NODEIP}:10401"
     listen-peer-address = "http://${NODEIP}:10301"
     initial-cluster = "${INITIAL_CLUSTER}"
 
 [metric]
-
     service = "prometheus"
+
+[tls]
+  certificate = "/data/cert.crt"
+  key = "/data/key.key"
+
+[auth]
+    enable = true
+    client-id = "e9088663-eb08-41d7-8f65-efb5f54bbb71"
+    client-secret = "bex7Q~aiWeQ70iBXbEup-XydyQrrNc_Q5n8EW"
+    authorize-url = "http://localhost:12345/authorize"
+    redirect-base-url = "http://localhost:12345/redirect"
+    token-url = "http://localhost:12345/token"
+    group-endpoint-url = "http://localhost:12345/groups"
+    logout-url = "http://localhost:12345/logout"
+    scopes = ["openid", "https://graph.microsoft.com/.default", "offline_access"]
+    secret-key = "98995f0530eeba96da1d0a04311073c0abb7b6abbfb0f5f4ef3629527ff88428"
+    permissions = "/etc/permissions.yml"
+    query-log-path = "/data/featurebase/query.log"
+EOT
+
+    echo "writing the permissions file"
+    cat << EOT > permissions.yml
+"user-groups":
+  "group-id-reader":
+    "user": "read"
+  "group-id-writer":
+    "user": "write"
+admin: "group-id-admin"
 EOT
     else 
        cat << EOT > featurebase.conf
@@ -110,30 +142,22 @@ max-map-count=900000
 long-query-time = "10s"
 
 [postgres]
-
     bind = "localhost:55432"
 
 [cluster]
-
     name = "${DEPLOYED_CLUSTER_PREFIX}"
     replicas = ${DEPLOYED_CLUSTER_REPLICA_COUNT}
 
 [etcd]
-
     listen-client-address = "http://${NODEIP}:10401"
     listen-peer-address = "http://${NODEIP}:10301"
     initial-cluster = "${INITIAL_CLUSTER}"
 
 [metric]
-
     service = "prometheus"
 EOT
     fi
 
-    #echo "featurebase.conf >>"
-    #cat featurebase.conf
-    #echo "featurebase.conf <<"
-
     scp -i ~/.ssh/gitlab-featurebase-ci.pem -o "StrictHostKeyChecking no" featurebase.conf ec2-user@${NODEIP}:
     if (( $? != 0 )) 
     then 
@@ -143,6 +167,47 @@ EOT
     rm -f featurebase.conf
     
     ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "sudo mv featurebase.conf ${CONFIG_FILE_PATH}"
+
+    if [[ "$AUTH_ENABLED" = "1" ]]; then
+        scp -i ~/.ssh/gitlab-featurebase-ci.pem -o "StrictHostKeyChecking no" permissions.yml ec2-user@${NODEIP}:
+        ifErr "permissions.yml copy failed"
+        rm -f permissions.yml
+
+        ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "sudo mv ~/permissions.yml /etc/permissions.yml"
+        ifErr "mv permissions.yml failed"
+    fi
+}
+
+setupMkCertCA() {
+    # this function sets up a CA on the runner for use by all nodes in setupTLS
+    git clone https://github.com/FiloSottile/mkcert && cd mkcert
+    ifErr "cloning mkcert"
+
+    go build -ldflags "-X main.Version=$(git describe --tags)"
+    ifErr "building mkcert"
+
+    ./mkcert -install
+    ifErr "installing root CA"
+
+    cp mkcert /usr/local/bin
+
+    cd ../
+
+}
+
+setupTLS() {
+    NODEIP=$1
+    scp -i ~/.ssh/gitlab-featurebase-ci.pem -o "StrictHostKeyChecking no" ./qa/scripts/setupTLS.sh ec2-user@${NODEIP}:
+    ifErr "setupTLS.sh copy failed"
+
+    scp -r -i ~/.ssh/gitlab-featurebase-ci.pem -o "StrictHostKeyChecking no" ./qa/fakeidp/ ec2-user@${NODEIP}:
+    ifErr "fakeidp copy failed"
+
+    ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "sudo mv ~/fakeidp /etc"
+    ifErr "mv fakeidp failed"
+
+    ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "sudo ./setupTLS.sh ${NODEIP}"
+    ifErr "error setting up TLS"
 }
 
 executeGeneralNodeConfigCommands() {
@@ -150,7 +215,7 @@ executeGeneralNodeConfigCommands() {
     NODEIDX=$1
     NODEIP=$2
 
-    ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "sudo mkdir /data"
+    ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "sudo mkdir -p /data"
     ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "sudo mkfs.ext4 /dev/nvme1n1"
     ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "sudo mount /dev/nvme1n1 /data"
 
@@ -171,8 +236,26 @@ executeGeneralNodeConfigCommands() {
     ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "chown ec2-user:ec2-user /home/ec2-user/featurebase_linux_arm64"
     ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "chmod ugo+x /home/ec2-user/featurebase_linux_arm64"
     ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "sudo mv /home/ec2-user/featurebase_linux_arm64 /usr/local/bin/featurebase"
+    ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "sudo yum install git -y"
 
     echo "featurebase binary copied."
+
+    if [[ "$AUTH_ENABLED" = "1" ]]; then
+        caroot=$(mkcert -CAROOT)
+
+        echo "copying root ca to ${NODEIP}"
+        scp -i ~/.ssh/gitlab-featurebase-ci.pem -o "StrictHostKeyChecking no" "${caroot}/rootCA.pem" ec2-user@${NODEIP}:
+        ifErr "copying CAROOT to ${NODEIP}"
+
+        scp -i ~/.ssh/gitlab-featurebase-ci.pem -o "StrictHostKeyChecking no" "${caroot}/rootCA-key.pem" ec2-user@${NODEIP}:
+        ifErr "copying CAROOT to ${NODEIP}"
+
+        ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "sudo mkdir -p /root/.local/share/mkcert && sudo cp ~/rootCA.pem /root/.local/share/mkcert && sudo cp ~/rootCA-key.pem /root/.local/share/mkcert"
+        ifErr "cp-ing rootCA.pem to /root/.local/share/mkcert on ${NODEIP}"
+
+        echo "setting up tls certificates"
+        setupTLS $NODEIP
+    fi
 }
 
 executeDataStartCommands() {
@@ -216,6 +299,7 @@ setupIngestNode() {
     ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "pip3 install -U requests"
     ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "pip3 install -U json"
     ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${NODEIP} "sudo yum install -y jq"
+    installDatagen $NODEIP
 }
 
 setupDataNodes() {
@@ -237,20 +321,15 @@ setupIngestNodes() {
 }
 
 generateInitialClusterString() {
-    if (( AUTH_ENABLED = 1 )); then
-        scheme=https
-    else
-        scheme=http
-    fi
     IFS=$'\n'
     cnt=0
     for ip in $DEPLOYED_DATA_IPS
     do
         if (($cnt + 1 != $DEPLOYED_DATA_IPS_LEN)) 
         then
-            INITIAL_CLUSTER="${INITIAL_CLUSTER}p${cnt}=${scheme}://$ip:10301,"
+            INITIAL_CLUSTER="${INITIAL_CLUSTER}p${cnt}=http://$ip:10301,"
         else
-            INITIAL_CLUSTER="${INITIAL_CLUSTER}p${cnt}=${scheme}://$ip:10301"
+            INITIAL_CLUSTER="${INITIAL_CLUSTER}p${cnt}=http://$ip:10301"
         fi
         cnt=$((cnt+1))
     done
@@ -263,6 +342,10 @@ setupClusterNodes() {
     #data nodes
     generateInitialClusterString
 
+    if [[ "$AUTH_ENABLED" = "1" ]]; then
+        setupMkCertCA
+    fi
+
     setupDataNodes
 
     startDataNodes

qa/tf/ci/auth-smoke/main.tf

@@ -0,0 +1,13 @@
+module "ci-cluster" {
+  source              = "../../.modules/featurebase-cluster"
+  cluster_prefix      = var.cluster_prefix
+  region              = var.region
+  profile             = var.profile
+  fb_data_node_type   = "m6g.large"
+  fb_data_node_count  = 3
+  fb_ingest_type      = "m6g.large"
+  vpc_id              = "vpc-05a26a122f961dc2b"
+  vpc_cidr_block      = "10.0.0.0/16"
+  vpc_public_subnets  = ["subnet-066b4b922b54e51a2", "subnet-037b8884269a69025", "subnet-08482631514426210", ]
+  vpc_private_subnets = ["subnet-0319dde319380326f", "subnet-0517ca9a646d80f88", "subnet-05a7b685ed27eb1cf", ]
+}

qa/tf/ci/auth-smoke/outputs.tf

@@ -0,0 +1,19 @@
+output "ingest_ips" {
+    description = "List of ingest IPs"
+    value = module.ci-cluster.ingest_ips
+}
+
+output "data_node_ips" {
+  description = "List of data node IPs"
+  value = module.ci-cluster.data_node_ips
+}
+
+output "cluster_prefix" {
+  description = "The cluster prefix used"
+  value = module.ci-cluster.cluster_prefix
+}
+
+output "fb_cluster_replica_count" {
+  description = "The cluster replica count used"
+  value = module.ci-cluster.fb_cluster_replica_count
+}

qa/tf/ci/auth-smoke/provider.tf

@@ -0,0 +1,4 @@
+provider "aws" {
+  region  = var.region
+  profile = var.profile
+}

qa/tf/ci/auth-smoke/tf.auto.tfvars

@@ -0,0 +1,2 @@
+region  = "us-east-2"
+profile = "service-terraform"

qa/tf/ci/auth-smoke/variables.tf

@@ -0,0 +1,15 @@
+variable "region" {
+  description = "The AWS region in which the VPC should be built"
+  type        = string
+}
+
+variable "profile" {
+  description = "The name of the AWS profile Terraform should use for auth."
+  type        = string
+}
+
+variable "cluster_prefix" {
+  type        = string
+  description = "This is a identifier that will be prefixed to created resources"
+}
+