Merge branch 'master' into protect-endpoints

2026-09-09 14:41:02 +00:00 · 2022-01-05 17:24:41 -06:00 · 2022-01-05 17:24:41 -06:00 · 7881b0987f
commit 7881b0987f
parent fd896de270 4268804a46
32 changed files with 1812 additions and 394 deletions
--- a/.gitignore
+++ b/.gitignore
@ -18,3 +18,5 @@ pilosa
 launch.json
 .terraform.lock.hcl
 __pycache__/
+report.xml
+outputs.json
--- a/.gitlab/.gitlab-ci.yml
+++ b/.gitlab/.gitlab-ci.yml
@ -13,7 +13,6 @@ include:
 variables:
  GOVERSION: "1.16.10"

-
 stages:
  - lint
  - test
@ -27,7 +26,7 @@ golangci-lint:
  extends: .go-cache
  allow_failure: false
  rules:
-    - if: '$CI_PIPELINE_SOURCE == "push"'
+    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
  script:
    - echo "Checking for issues in new code"
    - golangci-lint run -v 
@ -38,7 +37,7 @@ build lattice:
  variables:
    CI: "false"
  rules:
-    - if: '$CI_PIPELINE_SOURCE == "push"'
+    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
  script:
    - cd lattice
    - yarn install
@ -58,7 +57,7 @@ run jest tests:
  variables:
    CI: "true"
  rules:
-    - if: '$CI_PIPELINE_SOURCE == "push"'
+    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
  script:
    - echo "Testing lattice..."
    - cd lattice
@ -73,7 +72,7 @@ run go tests:
  image: golang:$GOVERSION
  extends: .go-cache
  rules:
-    - if: '$CI_PIPELINE_SOURCE == "push"'
+    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
  script:
    - echo "Running featurebase unit tests..."
    - PKG_LIST=$(go list ./... | grep -v internal/clustertests | paste -s -d, -)
@ -87,7 +86,7 @@ run go tests future:
  image: golang:1.17.3
  extends: .go-cache
  rules:
-    - if: '$CI_PIPELINE_SOURCE == "push"'
+    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
  script:
    - echo "Running featurebase unit tests..."
    - PKG_LIST=$(go list ./... | grep -v internal/clustertests | paste -s -d, -)
@ -96,12 +95,11 @@ run go tests future:
    paths:
      - coverage.out

-
 run go tests with output:
  stage: test
  image: golang:$GOVERSION
  rules:
-    - if: '$CI_PIPELINE_SOURCE == "push"'
+    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
  script:
    - echo "Running featurebase unit tests to capture JSON output..."
    - go test -json > test-report.out
@ -115,7 +113,7 @@ upload to sonarcloud:
  variables:
    SONAR_TOKEN: $SONAR_TOKEN
  rules:
-    - if: '$CI_PIPELINE_SOURCE == "push"'
+    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
  script:
    - sonar-scanner -Dsonar.projectKey=molecula_featurebase -Dsonar.organization=molecula -Dsonar.sources=. -Dsonar.host.url=https://sonarcloud.io -Dsonar.go.coverage.reportPaths=coverage.out -Dsonar.go.tests.reportPaths=test-report.out  -Dsonar.javascript.lcov.reportPaths=lattice/coverage/lcov.info
  needs:
@ -127,7 +125,7 @@ build for linux amd64:
  stage: build
  image: golang:$GOVERSION
  rules:
-    - if: '$CI_PIPELINE_SOURCE == "push"'
+    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
  script:
    - rm -r lattice
    - tar -xvf lattice.tar.gz 
@ -142,7 +140,7 @@ build for linux arm64:
  stage: build
  image: golang:$GOVERSION
  rules:
-    - if: '$CI_PIPELINE_SOURCE == "push"'
+    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
  script:
    - rm -r lattice
    - tar -xvf lattice.tar.gz 
@ -157,7 +155,7 @@ build for darwin amd64:
  stage: build
  image: golang:$GOVERSION
  rules:
-    - if: '$CI_PIPELINE_SOURCE == "push"'
+    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
  script:
    - rm -r lattice
    - tar -xvf lattice.tar.gz 
@ -172,7 +170,7 @@ build for darwin arm64:
  stage: build
  image: golang:$GOVERSION
  rules:
-    - if: '$CI_PIPELINE_SOURCE == "push"'
+    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
  script:
    - rm -r lattice
    - tar -xvf lattice.tar.gz 
@ -187,7 +185,7 @@ package for linux amd64:
  stage: build
  image: golang:$GOVERSION
  rules:
-    - if: '$CI_PIPELINE_SOURCE == "push"'
+    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
  variables:
    GOOS: "linux"
    GOARCH: "amd64"
@ -209,7 +207,7 @@ build container fb:
  tags:
    - shell
  rules:
-    - if: '$CI_PIPELINE_SOURCE == "push"'
+    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
  before_script:    
    - echo "${DOCKER_DEPLOY_TOKEN}" | docker login -u ${DOCKER_DEPLOY_USER} --password-stdin ${CI_REGISTRY}
  script:    
@ -218,46 +216,18 @@ build container fb:
    - docker push $tag
    - echo Created docker featurebase image with tag "$tag"

-# deploy EC2 instance, configure and run featurebase 
-# diabling for now - will come back and refactor POK
-# deploy node for linux amd64:
-#  stage: integration
-#  image: registry.gitlab.com/gitlab-org/cloud-deploy/aws-base:latest
-#  variables:
-#    PROFILE: "default"
-#    AWS_SSH_PRIVATE_KEY: $AWS_FBCI_SSH_KEY 
-#  rules:
-#    - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'
-#  before_script:
-#    - aws configure set aws_access_key_id $AWS_FBCI_ACCESS_KEY_ID
-#    - aws configure set aws_secret_access_key $AWS_FBCI_SECRET_ACCESS_KEY
-#    - aws configure set region "us-east-2"
-#    - aws configure set aws_profile $PROFILE
-#    - echo $AWS_FBCI_SSH_KEY > gitlab-featurebase-ci.pem
-#    - chmod 400 gitlab-featurebase-ci.pem
-#    - 'which ssh-agent || ( apt-get update -y && apt-get install openssh-client -y )'
-#    - eval `ssh-agent -s`
-#    - mkdir -p ~/.ssh 
-#    - echo "$AWS_FBCI_SSH_KEY" | ssh-add -
-#    - chmod 700 /root/.ssh
-#    - '[[ -f /.dockerenv ]] && echo -e "Host *\n\tStrictHostKeyChecking no\n\n" > ~/.ssh/config'
-#    - apt update && apt -y install jq
-#  script:
-#    - ./qa/scripts/deployNode.sh $PROFILE
-#  needs:
-#    - job: build for linux amd64
-
-gauntlet:
-  stage: gauntlet
-  timeout: 4h
+smoke test:
+  stage: integration
  image: registry.gitlab.com/gitlab-org/cloud-deploy/aws-base:latest
  variables:
-    PROFILE: "default"
+    PROFILE: "service-terraform"
    AWS_SSH_PRIVATE_KEY: $AWS_FBCI_SSH_KEY
    AWS_ACCESS_KEY_ID: $AWS_FBCI_ACCESS_KEY_ID
    AWS_SECRET_ACCESS_KEY: $AWS_FBCI_SECRET_ACCESS_KEY
+    TF_VAR_cluster_prefix: ""
+    TF_VAR_branch: ""
  rules:
-    - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && ($CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web")'
+    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
  before_script:
    - apt-get update && apt-get install -y gnupg software-properties-common curl git
    - curl -fsSL https://apt.releases.hashicorp.com/gpg | apt-key add -
@ -281,6 +251,65 @@ gauntlet:
    - wget https://go.dev/dl/go1.17.5.linux-amd64.tar.gz
    - tar -C /usr/local -xzf go1.17.5.linux-amd64.tar.gz
    - export PATH=$PATH:/usr/local/go/bin
+    - TF_VAR_cluster_prefix="smoke-$(openssl rand -base64 12 | tr -d /=+ | cut -c -16)"
+    - echo "Cluster Prefix --> $TF_VAR_cluster_prefix"
+    - TF_VAR_branch=$CI_COMMIT_BRANCH
+    - echo "Branch --> $TF_VAR_branch"
+  script:
+    - ./qa/scripts/setupSmokeTest.sh
+    - ./qa/scripts/testSmokeTest.sh
+  after_script:
+    - ./qa/scripts/teardownSmokeTest.sh
+  needs:
+    - job: build for linux arm64
+  artifacts:
+    when: always
+    paths:
+      - report.xml
+    reports:
+      junit: report.xml
+
+gauntlet:
+  stage: gauntlet
+  timeout: 4h
+  image: registry.gitlab.com/gitlab-org/cloud-deploy/aws-base:latest
+  variables:
+    PROFILE: "default"
+    AWS_SSH_PRIVATE_KEY: $AWS_FBCI_SSH_KEY
+    AWS_ACCESS_KEY_ID: $AWS_FBCI_ACCESS_KEY_ID
+    AWS_SECRET_ACCESS_KEY: $AWS_FBCI_SECRET_ACCESS_KEY
+    TF_VAR_cluster_prefix: ""
+    TF_VAR_branch: ""
+  rules:
+    - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'
+    - if: '$CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
+  before_script:
+    - apt-get update && apt-get install -y gnupg software-properties-common curl git
+    - curl -fsSL https://apt.releases.hashicorp.com/gpg | apt-key add -
+    - apt-add-repository "deb [arch=amd64] https://apt.releases.hashicorp.com $(lsb_release -cs) main"
+    - apt-get update && apt-get install terraform
+    - aws configure set aws_access_key_id $AWS_FBCI_ACCESS_KEY_ID
+    - aws configure set aws_secret_access_key $AWS_FBCI_SECRET_ACCESS_KEY
+    - aws configure set region "us-east-2"
+    - aws configure set aws_profile $PROFILE
+    - echo $AWS_FBCI_SSH_KEY > gitlab-featurebase-ci.pem
+    - chmod 400 gitlab-featurebase-ci.pem
+    - 'which ssh-agent || ( apt-get update -y && apt-get install openssh-client -y )'
+    - eval $(ssh-agent -s)
+    - mkdir -p ~/.ssh
+    - echo $AWS_FBCI_SSH_KEY > /root/.ssh/gitlab-featurebase-ci.pem
+    - chmod 400 /root/.ssh/gitlab-featurebase-ci.pem
+    - echo "$AWS_FBCI_SSH_KEY" | ssh-add -
+    - chmod 700 /root/.ssh
+    - '[[ -f /.dockerenv ]] && echo -e "Host *\n\tStrictHostKeyChecking no\n\n" > ~/.ssh/config'
+    - apt update && apt -y install jq wget
+    - wget https://go.dev/dl/go1.17.5.linux-amd64.tar.gz
+    - tar -C /usr/local -xzf go1.17.5.linux-amd64.tar.gz
+    - export PATH=$PATH:/usr/local/go/bin
+    - TF_VAR_cluster_prefix="smoke-$(openssl rand -base64 12 | tr -d /=+ | cut -c -16)"
+    - echo "Cluster Prefix --> $TF_VAR_cluster_prefix"
+    - TF_VAR_branch=$CI_COMMIT_BRANCH
+    - echo "Branch --> $TF_VAR_branch"
  script:
    - ./qa/scripts/setupSamsungGauntlet.sh
    - ./qa/scripts/testSamsungGauntlet.sh
--- a/qa/scripts/cloud-init.sh
+++ b/qa/scripts/cloud-init.sh
@ -1,26 +0,0 @@
-#!/bin/bash -ex
-# generate log 
-exec > >(tee /var/log/user-data.log|logger -t user-data -s 2>/dev/console) 2>&1
-
-# Install packages 
-yum update -y
-yum install postgresql -y
-
-# Configure host system  
-echo 'cat /proc/sys/fs/file-max'
-sysctl -w fs.file-max=262144
-sysctl -p
-echo 'cat /proc/sys/fs/file-max'
-
-# yum install golang -y # latest verion in ec2 is 1.15.14 
-# install go 1.16.9 manually
-curl -O https://dl.google.com/go/go1.16.10.linux-amd64.tar.gz
-tar xvf go1.16.10.linux-amd64.tar.gz
-chown -R root:root ./go
-mv go /usr/local
-echo "export PATH=/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/home/ec2-user/.local/bin:/home/ec2-user/bin:/usr/local/go/bin" | tee -a /etc/profile > /dev/null
-source /etc/profile
-
-# install aws session manager pluggin
-curl "https://s3.amazonaws.com/session-manager-downloads/plugin/latest/linux_64bit/session-manager-plugin.rpm" -o "session-manager-plugin.rpm"
-yum install -y session-manager-plugin.rpm
--- a/qa/scripts/config.json
+++ b/qa/scripts/config.json
--- a/qa/scripts/configureFeatureBase.json
+++ b/qa/scripts/configureFeatureBase.json
@ -1,20 +0,0 @@
-{
-    "Parameters": {
-        "commands": [
-            "#!/bin/bash",
-            "mv /home/ec2-user/featurebase_linux_amd64 /usr/local/bin/featurebase",
-            "mv /home/ec2-user/featurebase.conf /etc/",
-            "mv /home/ec2-user/featurebase.service /etc/systemd/system/",
-            "adduser molecula",
-            "sudo mkdir /var/log/molecula",
-            "sudo chown molecula /var/log/molecula",
-            "sudo mkdir -p /opt/molecula/featurebase",
-            "sudo chown molecula /opt/molecula/featurebase",
-            "systemctl daemon-reload",
-            "sudo systemctl start featurebase",
-            "sudo systemctl enable featurebase",
-            "sudo systemctl status featurebase",
-            "curl localhost:10101"
-        ]
-    }
-}
--- a/qa/scripts/deployNode.sh
+++ b/qa/scripts/deployNode.sh
@ -1,98 +0,0 @@
-#!/bin/bash
-
-# To run script: ./deployNode.sh $PROFILE
-
-# default to the VPC initially created
-VPC=${VPC:-vpc-0582f594d7d2ca2d4}
-
-function deploy_node() {
-    # get AMI, security group and subnet ID 
-    AMI=$(aws ssm get-parameters --names /aws/service/ami-amazon-linux-latest/amzn2-ami-hvm-x86_64-ebs --query 'Parameters[0].[Value]' --output text --profile $PROFILE)
-    if [[ $? > 0 ]]; then 
-        echo "aws session manager failed to find AMI"
-        exit 1
-    fi
-
-    SECURITY_GROUP=$(aws ec2 describe-security-groups --filters "Name=vpc-id,Values=$VPC" Name=group-name,Values=default --query 'SecurityGroups[*].[GroupId]' --output text --profile $PROFILE)
-    if [[ $? > 0 ]]; then 
-        echo "aws session manager failed to find security group"
-        exit 1
-    fi
-    
-    SUBNET_ID=$(aws ec2 describe-subnets --filters "Name=vpc-id,Values=$VPC" 'Name=availability-zone,Values=us-east-2a' --query 'Subnets[0].SubnetId' --output text --profile $PROFILE)
-    if [[ $? > 0 ]]; then 
-        echo "aws session manager failed to find subnet ID"
-        exit 1
-    fi
-
-    # launch EC2 instance and get instance ID
-    aws ec2 run-instances --image-id $AMI --instance-type $INSTANCE --security-group-ids $SECURITY_GROUP --subnet-id $SUBNET_ID --key-name gitlab-featurebase-dev --tag-specifications 'ResourceType=instance,Tags=[{Key=Name,Value=linux-amd64-node}]' --profile $PROFILE --user-data file://./qa/scripts/cloud-init.sh --iam-instance-profile Name=featurebase-dev-ssm > config.json
-    if [[ $? > 0 ]]; then 
-        echo "aws run-instances failed to launch a new EC2 instance"
-        exit 1
-    fi
-
-    INSTANCE_ID=$(jq '.Instances | .[0] |.InstanceId' config.json | tr -d '"') 
-    echo "aws run-instances succeeded in launching a new EC2 instance with instance ID: " $INSTANCE_ID
-}
-
-function initialize_featurebase() {
-    # get IP for node 
-    for i in {0..24}
-    do 
-        IP=$(aws ec2 describe-instances --instance-ids $INSTANCE_ID --filters 'Name=instance-state-name, Values=running' --query 'Reservations[*].Instances[*].PublicIpAddress' --output text --profile $PROFILE)
-        if [ -n "$IP" ]; then
-            echo "Public IP for EC2 instance found: " $IP 
-            break
-        fi
-
-        if [[ $? > 0 ]]; then 
-            echo "aws cli describe-instances command failed to find public IP"
-            terminate_node
-            exit 1
-        fi
-
-        sleep 5
-    done
-    
-    sleep 60 # to allow enough time for node to be ready for use
-
-    # copy featurebase binary and files to ec2 instance
-    scp  -o StrictHostKeyChecking=no -i gitlab-featurebase-dev.pem featurebase_linux_amd64 ./qa/scripts/featurebase.conf ./qa/scripts/featurebase.service ec2-user@$IP:.
-    if [[ $? > 0 ]]; then 
-        echo "scp of featurebase binary, service and config files to EC2 instance failed"
-        terminate_node
-        exit 1
-    fi
-
-    # execute script to configure featurebase on the EC2 node 
-    aws ssm send-command --document-name "AWS-RunShellScript" --instance-ids $INSTANCE_ID --cli-input-json file://./qa/scripts/configureFeatureBase.json --profile $PROFILE --region $REGION
-    if [[ $? > 0 ]]; then 
-        echo "aws cli session manager send-command failed"
-        terminate_node
-        exit 1
-    fi
-}
-
-function terminate_node() {
-    aws ec2 terminate-instances --instance-ids $INSTANCE_ID --profile $PROFILE 
-}
-
-# Pass variables to shell script
-PROFILE=$1
-shift 
-
-# set some variables 
-INSTANCE="t3a.large"
-REGION="us-east-2"
-
-# get AMI, security group and subnet for EC2 instance,
-# launch instance, save instance Id and run cloud-init to set up node env
-deploy_node
-
-# Get IP for instance, scp featurebase binary, config and service files;
-# set up featurebase config in node 
-initialize_featurebase
-
-# terminate node 
-terminate_node
--- a/qa/scripts/deploySingleNodeCluster.sh
+++ b/qa/scripts/deploySingleNodeCluster.sh
@ -1,28 +0,0 @@
-#!/bin/bash
-
-# To run script: ./deploySingleNodeCluster.sh
-# requires TF_VAR_gitlab_token env var to be set
-
-echo “$(pwd)”
-
-pushd ./qa/tf/ci/singlenode
-export TF_IN_AUTOMATION=1
-terraform init -input=false
-terraform apply -input=false -auto-approve
-popd
-
-# configure Featurebase
-
-# step 1a: get IPs of the cluster
-
-
-
-# step 1b: get IPs of the ingest nodes
-
-# step 2: write a featurebase.conf file
-
-# step 3: write featurebase.service
-
-# step 4: start featurebase
-
-# step 5: verify featurebase running
--- a/qa/scripts/featurebase.conf
+++ b/qa/scripts/featurebase.conf
@ -1,30 +0,0 @@
-name = "pilosa1"
-bind = "0.0.0.0:10101"
-bind-grpc = "0.0.0.0:20101"
-
-data-dir = "/opt/molecula/featurebase"
-log-path = "/var/log/molecula/featurebase.log"
-
-max-file-count=900000
-max-map-count=900000
-
-long-query-time = "10s"
-
-[postgres]
-
-    bind = "localhost:55432"
-
-[cluster]
-
-    name = "cluster1"
-    replicas = 1
-
-[etcd]
-
-    listen-client-address = "http://localhost:10401"
-    listen-peer-address = "http://localhost:10301"
-    initial-cluster = "pilosa1=http://localhost:10301"
-
-[metric]
-
-    service = "prometheus"
--- a/qa/scripts/featurebase.service
+++ b/qa/scripts/featurebase.service
@ -1,13 +0,0 @@
-# Not Ansible managed
-
-[Unit]
-Description="Service for FeatureBase"
-
-[Service]
-RestartSec=30
-Restart=on-failure
-EnvironmentFile=
-User=molecula
-ExecStart=/usr/local/bin/featurebase server -c /etc/featurebase.conf
-
-[Install]
--- a/qa/scripts/ingestWorkload.sh
+++ b/qa/scripts/ingestWorkload.sh
@ -1,84 +0,0 @@
-#!/usr/bin/env bash
-
-# path for featurebase binary
-FEATUREBASE_PATH=/usr/local/bin
-
-# path for directory with csv directory files for all fields to be ingested 
-CSV_DIR_PATH=/data
-
-
-# To run:
-# ./ingestWorkload.sh {Local host & port for featurebase} {initialize flag}
-
-function delete_field {
-    if (($INITIALIZE == 0));
-    then 
-        curl -XDELETE $HOST/index/$INDEX/field/$FIELD
-    fi
-}
-
-# Script to replicate samsung workload of deleting and re-ingesting fields every night 
-# outline delete and re-ingest workload 
-function ingest_int_field { 
-    delete_field
-    curl -XPOST $HOST/index/$INDEX/field/$FIELD -d '{"options": {"type": "int", "min": 0, "max":'$MAX'}}'
-    $FEATUREBASE_PATH/featurebase import --host $HOST -i $INDEX -f $FIELD $CSV_FILE 
-}
-
-function ingest_time_field {
-    delete_field
-    curl -XPOST $HOST/index/$INDEX/field/$FIELD -d '{"options": {"keys": true, "type": "time", "timeQuantum": "YMD"}}'
-    $FEATUREBASE_PATH/featurebase import --host $HOST -i $INDEX -f $FIELD $CSV_FILE 
-}
-
-function ingest_set_field {
-    delete_field
-    curl -XPOST $HOST/index/$INDEX/field/$FIELD -d '{"options": {"keys": true}}'
-    $FEATUREBASE_PATH/featurebase import --host $HOST -i $INDEX -f $FIELD $CSV_FILE 
-}
-
-# featurebase host & port
-HOST=$1
-shift
-
-# intialize flag - 0:disabled, 1:enabled - creates the index and fields for testing
-INITIALIZE=$1
-shift 
-
-# get a list of csv files in the directory 
-CSV_FILES=`ls $CSV_DIR_PATH/*.csv`
-
-# assign index name 
-INDEX="samsung"
-if (($INITIALIZE == 1));
-then 
-    curl -XPOST $HOST/index/$INDEX
-fi
-
-# perform delete and re-ingest for all fields 
-for CSV_FILE in ${CSV_FILES[@]}
-    do 
-        # get field name from csv file path 
-        FIELD="$(basename $CSV_FILE .csv)"
-        if [[ "$FIELD" == *"age"* ]];
-        then 
-            MAX=100 
-            ingest_int_field
-        elif [[ "$FIELD" == *"identifier"* ]];
-        then 
-            MAX=$((2**63 - 1)) # compute max value for 64bit
-            ingest_int_field
-        elif [[ "$FIELD" == *"ip"* ]];
-        then 
-            MAX=$((2**31 - 1)) # compute max value for 32bit
-            ingest_int_field
-        elif [[ "$FIELD" == *"time"* ]];
-        then 
-            ingest_time_field
-        else
-            ingest_set_field
-        fi
-    done
-
-
-
--- a/qa/scripts/perf.sh
+++ b/qa/scripts/perf.sh
@ -1,3 +0,0 @@
-#!/bin/bash
-echo >&2 "performance testing"
-time ./simulacraData
--- a/qa/scripts/runSamsungGauntlet.sh
+++ b/qa/scripts/runSamsungGauntlet.sh
@ -1,8 +1,5 @@
 #!/bin/bash

-
-#openssl rand -base64 32 | tr -d /=+ | cut -c -16
-
 ./setupSamsungGauntlet.sh
 ./testSamsungGauntlet.sh
 ./teardownSamsungGauntlet.sh
--- a/qa/scripts/runSmokeTest.sh
+++ b/qa/scripts/runSmokeTest.sh
@ -0,0 +1,5 @@
+#!/bin/bash
+
+./setupSmokeTest.sh
+./testSmokeTest.sh
+./teardownSmokeTest.sh
--- a/qa/scripts/setupSamsungGauntlet.sh
+++ b/qa/scripts/setupSamsungGauntlet.sh
@ -9,14 +9,14 @@ echo "Running terraform init..."
 terraform init -input=false
 echo "Running terraform apply..."
 terraform apply -input=false -auto-approve
-terraform output -json > samsung-gauntlet.json
+terraform output -json > outputs.json
 popd

 # get the bastion host
-BASTION=$(cat ./qa/tf/gauntlet/samsung/samsung-gauntlet.json | jq -r '[.ingest_ips][0]["value"][0]')
+BASTION=$(cat ./qa/tf/gauntlet/samsung/outputs.json | jq -r '[.ingest_ips][0]["value"][0]')
 echo "using bastion ${BASTION}"

-NODE=$(cat ./qa/tf/gauntlet/samsung/samsung-gauntlet.json | jq -r '[.data_node_ips][0]["value"][0]')
+NODE=$(cat ./qa/tf/gauntlet/samsung/outputs.json | jq -r '[.data_node_ips][0]["value"][0]')
 echo "using node ${NODE}"

 # remember that the nodes will take at least 2 mins to be up and going and finish cloud-init
--- a/qa/scripts/setupSmokeTest.sh
+++ b/qa/scripts/setupSmokeTest.sh
@ -0,0 +1,38 @@
+#!/bin/bash
+
+# To run script: ./setupSmokeTest.sh
+# requires TF_VAR_gitlab_token env var to be set
+
+pushd ./qa/tf/ci/smoketest
+export TF_IN_AUTOMATION=1
+echo "Running terraform init..."
+terraform init -input=false
+echo "Running terraform apply..."
+terraform apply -input=false -auto-approve
+terraform output -json > outputs.json
+popd
+
+# get the bastion host
+BASTION=$(cat ./qa/tf/ci/smoketest/outputs.json | jq -r '[.ingest_ips][0]["value"][0]')
+echo "using bastion ${BASTION}"
+
+NODE=$(cat ./qa/tf/ci/smoketest/outputs.json | jq -r '[.data_node_ips][0]["value"][0]')
+echo "using node ${NODE}"
+
+# remember that the nodes will take at least 2 mins to be up and going and finish cloud-init
+echo "Waiting for cluster to become available..."
+# jaffee - I do wanna do a loop here, but I give up, and am running home to sleep... -POK
+sleep 150
+
+# verify featurebase running
+echo "Verifying featurebase cluster running..."
+ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${BASTION} "curl -s http://${NODE}:10101/status" 
+if (( $? != 0 )) 
+then 
+    echo "Featurebase cluster not running"
+    exit 1
+fi
+
+echo "Cluster running."
+
+
--- a/qa/scripts/teardownSmokeTest.sh
+++ b/qa/scripts/teardownSmokeTest.sh
@ -0,0 +1,8 @@
+#!/bin/bash
+
+# To run script: ./teardownSmokeTest.sh
+# requires TF_VAR_gitlab_token env var to be set
+
+cd qa/tf/ci/smoketest
+export TF_IN_AUTOMATION=1
+terraform destroy -auto-approve
--- a/qa/scripts/testSamsungGauntlet.sh
+++ b/qa/scripts/testSamsungGauntlet.sh
@ -1,10 +1,10 @@
 #!/bin/bash

 # get the bastion host
-BASTION=$(cat ./qa/tf/gauntlet/samsung/samsung-gauntlet.json | jq -r '[.ingest_ips][0]["value"][0]')
+BASTION=$(cat ./qa/tf/gauntlet/samsung/outputs.json | jq -r '[.ingest_ips][0]["value"][0]')
 echo "using bastion ${BASTION}"

-NODE=$(cat ./qa/tf/gauntlet/samsung/samsung-gauntlet.json | jq -r '[.data_node_ips][0]["value"][0]')
+NODE=$(cat ./qa/tf/gauntlet/samsung/outputs.json | jq -r '[.data_node_ips][0]["value"][0]')
 echo "using node ${NODE}"

 # generate csv files
--- a/qa/scripts/testSmokeTest.sh
+++ b/qa/scripts/testSmokeTest.sh
@ -0,0 +1,35 @@
+#!/bin/bash
+
+# get the bastion host
+BASTION=$(cat ./qa/tf/ci/smoketest/outputs.json | jq -r '[.ingest_ips][0]["value"][0]')
+echo "using bastion ${BASTION}"
+
+NODE=$(cat ./qa/tf/ci/smoketest/outputs.json | jq -r '[.data_node_ips][0]["value"][0]')
+echo "using node ${NODE}"
+
+echo "Copying tests to remote"
+scp -r -i ~/.ssh/gitlab-featurebase-ci.pem ./qa/testcases/smoketest/*.py ec2-user@${BASTION}:/data
+if (( $? != 0 )) 
+then 
+    echo "Copy failed"
+    exit 1
+fi
+
+# run smoke test
+echo "Running smoke test..."
+ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o "StrictHostKeyChecking no" ec2-user@${BASTION} " pushd /data; pytest --junitxml=report.xml; popd" 
+if (( $? != 0 )) 
+then 
+    echo "Unable to run smoke test"
+    exit 1
+fi
+
+echo "Copying test report to local"
+scp -r -i ~/.ssh/gitlab-featurebase-ci.pem ec2-user@${BASTION}:/data/report.xml report.xml
+if (( $? != 0 )) 
+then 
+    echo "Copy failed"
+    exit 1
+fi
+
+echo "Smoke test complete"
--- a/qa/testcases/smoketest/test_smoke.py
+++ b/qa/testcases/smoketest/test_smoke.py
@ -0,0 +1,7 @@
+# content of test_smoke.py
+def inc(x):
+    return x + 1
+
+
+def test_answer():
+    assert inc(3) == 5
--- a/qa/tf/.modules/featurebase-cluster/main.tf
+++ b/qa/tf/.modules/featurebase-cluster/main.tf
@ -46,7 +46,7 @@ resource "aws_instance" "fb_cluster_nodes" {
    Role   = "cluster_node"
  }

-  user_data = base64encode(templatefile("${path.module}/setup_cluster_node.sh.tpl", { gitlab_token = var.gitlab_token, cluster_prefix = var.cluster_prefix, node_count = var.fb_data_node_count, fb_cluster_replica_count = var.fb_cluster_replica_count, region = var.region }))
+  user_data = base64encode(templatefile("${path.module}/setup_cluster_node.sh.tpl", { gitlab_token = var.gitlab_token, branch = var.branch, cluster_prefix = var.cluster_prefix, node_count = var.fb_data_node_count, fb_cluster_replica_count = var.fb_cluster_replica_count, region = var.region }))
 }

 resource "aws_instance" "fb_ingest" {
@ -79,7 +79,7 @@ resource "aws_instance" "fb_ingest" {
    Role   = "ingest_node"
  }

-  user_data = base64encode(templatefile("${path.module}/setup_ingest_node.sh.tpl", { gitlab_token = var.gitlab_token, cluster_prefix = var.cluster_prefix, node_count = var.fb_ingest_node_count, this_node = count.index, region = var.region }))
+  user_data = base64encode(templatefile("${path.module}/setup_ingest_node.sh.tpl", { gitlab_token = var.gitlab_token, branch = var.branch, cluster_prefix = var.cluster_prefix, node_count = var.fb_ingest_node_count, this_node = count.index, region = var.region }))
 }

 resource "aws_key_pair" "gitlab-featurebase-ci" {
--- a/qa/tf/.modules/featurebase-cluster/setup_cluster_node.sh.tpl
+++ b/qa/tf/.modules/featurebase-cluster/setup_cluster_node.sh.tpl
@ -164,7 +164,7 @@ write_featurebase_service_file

 #get the featurebase binary and put in in the right spot
 echo "Getting featurebase binary..."
-curl --header "PRIVATE-TOKEN: ${gitlab_token}" -o "/home/ec2-user/featurebase_linux_arm64" https://gitlab.com/api/v4/projects/molecula%2Ffeaturebase/jobs/artifacts/master/raw/featurebase_linux_arm64?job=build%20for%20linux%20arm64
+curl --fail --header "PRIVATE-TOKEN: ${gitlab_token}" -o "/home/ec2-user/featurebase_linux_arm64" https://gitlab.com/api/v4/projects/molecula%2Ffeaturebase/jobs/artifacts/${branch}/raw/featurebase_linux_arm64?job=build%20for%20linux%20arm64
 chown ec2-user:ec2-user "/home/ec2-user/featurebase_linux_arm64"
 chmod ugo+x "/home/ec2-user/featurebase_linux_arm64"

--- a/qa/tf/.modules/featurebase-cluster/setup_ingest_node.sh.tpl
+++ b/qa/tf/.modules/featurebase-cluster/setup_ingest_node.sh.tpl
@ -48,7 +48,7 @@ get_aws_instance_id
 wait_on_all_ingest_ips

 echo "Getting featurebase binary..."
-curl --header "PRIVATE-TOKEN: ${gitlab_token}" -o "/home/ec2-user/featurebase_linux_arm64" https://gitlab.com/api/v4/projects/molecula%2Ffeaturebase/jobs/artifacts/master/raw/featurebase_linux_arm64?job=build%20for%20linux%20arm64
+curl --fail --header "PRIVATE-TOKEN: ${gitlab_token}" -o "/home/ec2-user/featurebase_linux_arm64" https://gitlab.com/api/v4/projects/molecula%2Ffeaturebase/jobs/artifacts/${branch}/raw/featurebase_linux_arm64?job=build%20for%20linux%20arm64
 chown ec2-user:ec2-user "/home/ec2-user/featurebase_linux_arm64"
 chmod ugo+x "/home/ec2-user/featurebase_linux_arm64"

@ -62,5 +62,11 @@ sudo mount /dev/nvme1n1 /data

 sudo chown -R ec2-user /data

+echo "Installing pytest"
+pip3 install -U pytest
+pip3 install -U requests
+
+echo "Done."
+


--- a/qa/tf/.modules/featurebase-cluster/variables.tf
+++ b/qa/tf/.modules/featurebase-cluster/variables.tf
@ -91,3 +91,8 @@ variable "gitlab_token" {
  description = "Gitlab API token"
  type        = string
 }
+
+variable "branch" {
+  description = "The branch we are on"
+  type        = string
+}
--- a/qa/tf/ci/singlenode/main.tf
+++ b/qa/tf/ci/singlenode/main.tf
@ -1,10 +1,11 @@

 module "ci-cluster" {
    source = "../../.modules/featurebase-cluster"
-    cluster_prefix = "ci-single-node"
+    cluster_prefix = var.cluster_prefix
    region = var.region
    profile = var.profile
    fb_data_node_type = "m6g.large"
    fb_data_node_count = 1
    gitlab_token = var.gitlab_token
+    branch = var.branch 
 }
--- a/qa/tf/ci/singlenode/outputs.tf
+++ b/qa/tf/ci/singlenode/outputs.tf
--- a/qa/tf/ci/singlenode/provider.tf
+++ b/qa/tf/ci/singlenode/provider.tf
--- a/qa/tf/ci/smoketest/terraform.tfstate.backup
+++ b/qa/tf/ci/smoketest/terraform.tfstate.backup
--- a/qa/tf/ci/singlenode/tf.auto.tfvars
+++ b/qa/tf/ci/singlenode/tf.auto.tfvars
--- a/qa/tf/ci/singlenode/variables.tf
+++ b/qa/tf/ci/singlenode/variables.tf
@ -11,4 +11,14 @@ variable "profile" {
 variable "gitlab_token" {
  description = "The API token for taking to Gitlab API - expected to come from an env variable."
  type        = string
+}
+
+variable "cluster_prefix" {
+  type        = string
+  description = "This is a identifier that will be prefixed to created resources"
+}
+
+variable "branch" {
+  type        = string
+  description = "The branch we are on"
 }
--- a/qa/tf/gauntlet/samsung/main.tf
+++ b/qa/tf/gauntlet/samsung/main.tf
@ -10,4 +10,5 @@ module "samsung-cluster" {
    fb_ingest_disk_iops = 10000
    fb_ingest_node_count = 1
    gitlab_token = var.gitlab_token
+    branch = var.branch
 }
--- a/qa/tf/gauntlet/samsung/samsung-gauntlet.json
+++ b/qa/tf/gauntlet/samsung/samsung-gauntlet.json
@ -1,30 +0,0 @@
-{
-  "data_node_ips": {
-    "sensitive": false,
-    "type": [
-      "tuple",
-      [
-        "string",
-        "string",
-        "string"
-      ]
-    ],
-    "value": [
-      "10.0.1.144",
-      "10.0.2.108",
-      "10.0.3.178"
-    ]
-  },
-  "ingest_ips": {
-    "sensitive": false,
-    "type": [
-      "tuple",
-      [
-        "string"
-      ]
-    ],
-    "value": [
-      "3.145.104.76"
-    ]
-  }
-}
--- a/qa/tf/gauntlet/samsung/variables.tf
+++ b/qa/tf/gauntlet/samsung/variables.tf
@ -11,4 +11,14 @@ variable "profile" {
 variable "gitlab_token" {
  description = "The API token for taking to Gitlab API - expected to come from an env variable."
  type        = string
+}
+
+variable "cluster_prefix" {
+  type        = string
+  description = "This is a identifier that will be prefixed to created resources"
+}
+
+variable "branch" {
+  type        = string
+  description = "The branch we are on"
 }