Merge branch 'master' into 54mir/authentication

2026-09-10 15:01:03 +00:00 · 2022-01-03 17:55:58 -06:00 · 2022-01-03 17:55:58 -06:00 · 16600a219c
commit 16600a219c
parent d5dcdcd40a 58a70863eb
56 changed files with 1390 additions and 84 deletions
--- a/.gitignore
+++ b/.gitignore
@ -13,3 +13,8 @@ pilosa
 *.dot
 .idea/
 .*.swp
+.terraform/
+*.tfstate
+launch.json
+.terraform.lock.hcl
+__pycache__/
--- a/.gitlab/.gitlab-ci.yml
+++ b/.gitlab/.gitlab-ci.yml
@ -19,6 +19,7 @@ stages:
  - test
  - build
  - integration
+  - gauntlet

 golangci-lint:
  image: golangci/golangci-lint:v1.39.0
@ -244,3 +245,45 @@ deploy node for linux amd64:
    - ./qa/scripts/deployNode.sh $PROFILE
  needs:
    - job: build for linux amd64
+
+gauntlet:
+  stage: gauntlet
+  timeout: 4h
+  image: registry.gitlab.com/gitlab-org/cloud-deploy/aws-base:latest
+  variables:
+    PROFILE: "default"
+    AWS_SSH_PRIVATE_KEY: $AWS_FBCI_SSH_KEY
+    AWS_ACCESS_KEY_ID: $AWS_FBCI_ACCESS_KEY_ID
+    AWS_SECRET_ACCESS_KEY: $AWS_FBCI_SECRET_ACCESS_KEY
+  # TODO: For now, run always
+  rules:
+    - if: '$CI_PIPELINE_SOURCE == "schedule" && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'
+  before_script:
+    - apt-get update && apt-get install -y gnupg software-properties-common curl git
+    - curl -fsSL https://apt.releases.hashicorp.com/gpg | apt-key add -
+    - apt-add-repository "deb [arch=amd64] https://apt.releases.hashicorp.com $(lsb_release -cs) main"
+    - apt-get update && apt-get install terraform
+    - aws configure set aws_access_key_id $AWS_FBCI_ACCESS_KEY_ID
+    - aws configure set aws_secret_access_key $AWS_FBCI_SECRET_ACCESS_KEY
+    - aws configure set region "us-east-2"
+    - aws configure set aws_profile $PROFILE
+    - echo $AWS_FBCI_SSH_KEY > gitlab-featurebase-ci.pem
+    - chmod 400 gitlab-featurebase-ci.pem
+    - 'which ssh-agent || ( apt-get update -y && apt-get install openssh-client -y )'
+    - eval $(ssh-agent -s)
+    - mkdir -p ~/.ssh
+    - echo $AWS_FBCI_SSH_KEY > /root/.ssh/gitlab-featurebase-ci.pem
+    - chmod 400 /root/.ssh/gitlab-featurebase-ci.pem
+    - echo "$AWS_FBCI_SSH_KEY" | ssh-add -
+    - chmod 700 /root/.ssh
+    - '[[ -f /.dockerenv ]] && echo -e "Host *\n\tStrictHostKeyChecking no\n\n" > ~/.ssh/config'
+    - apt update && apt -y install jq wget
+    - wget https://go.dev/dl/go1.17.5.linux-amd64.tar.gz
+    - tar -C /usr/local -xzf go1.17.5.linux-amd64.tar.gz
+    - export PATH=$PATH:/usr/local/go/bin
+  script:
+    - ./qa/scripts/setupSamsungGauntlet.sh
+    - ./qa/scripts/testSamsungGauntlet.sh
+  after_script:
+    - ./qa/scripts/teardownSamsungGauntlet.sh
+  needs: ["build for linux arm64"]
--- a/ctl/rbf_check.go
+++ b/ctl/rbf_check.go
@ -26,7 +26,7 @@ func NewRBFCheckCommand(stdin io.Reader, stdout, stderr io.Writer) *RBFCheckComm
 	}
 }

-// Run executes the export.
+// Run executes a consistency check of an RBF database.
 func (cmd *RBFCheckCommand) Run(ctx context.Context) error {
 	// Open database.
 	db := rbf.NewDB(cmd.Path, nil)
@ -37,7 +37,15 @@ func (cmd *RBFCheckCommand) Run(ctx context.Context) error {

 	// Run check on the database.
 	if err := db.Check(); err != nil {
-		return err
+		switch err := err.(type) {
+		case rbf.ErrorList:
+			for i := range err {
+				fmt.Fprintln(cmd.Stdout, err[i])
+			}
+		default:
+			fmt.Fprintln(cmd.Stdout, err)
+		}
+		return fmt.Errorf("check failed")
 	}

 	// If successful, print a success message.
--- a/ctl/rbf_check_test.go
+++ b/ctl/rbf_check_test.go
@ -0,0 +1,33 @@
+// Copyright 2021 Molecula Corp. All rights reserved.
+package ctl
+
+import (
+	"bytes"
+	"context"
+	"path/filepath"
+	"testing"
+)
+
+func TestRBFCheckCommand_Run(t *testing.T) {
+	t.Run("OK", func(t *testing.T) {
+		var stdout, stderr bytes.Buffer
+		cmd := NewRBFCheckCommand(bytes.NewReader(nil), &stdout, &stderr)
+		cmd.Path = filepath.Join("testdata", "rbf-check", "ok")
+		if err := cmd.Run(context.Background()); err != nil {
+			t.Fatal(err)
+		} else if got, want := stdout.String(), `ok`+"\n"; got != want {
+			t.Fatalf("got:\n%s\n\nwant:\n%s", got, want)
+		}
+	})
+
+	t.Run("ErrInvalidPageType", func(t *testing.T) {
+		var stdout, stderr bytes.Buffer
+		cmd := NewRBFCheckCommand(bytes.NewReader(nil), &stdout, &stderr)
+		cmd.Path = filepath.Join("testdata", "rbf-check", "err-invalid-page-type")
+		if err := cmd.Run(context.Background()); err == nil || err.Error() != `check failed` {
+			t.Fatal(err)
+		} else if got, want := stdout.String(), `page not in-use & not free: pgno=4`+"\n"; got != want {
+			t.Fatalf("got:\n%s\n\nwant:\n%s", got, want)
+		}
+	})
+}
--- a/ctl/rbf_pages.go
+++ b/ctl/rbf_pages.go
@ -49,7 +49,16 @@ func (cmd *RBFPagesCommand) Run(ctx context.Context) error {
 	// Iterate over each page and grab info.
 	infos, err := tx.PageInfos()
 	if err != nil {
-		return err
+		fmt.Fprintln(cmd.Stdout, "ERRORS:")
+		switch err := err.(type) {
+		case rbf.ErrorList:
+			for i := range err {
+				fmt.Fprintln(cmd.Stdout, err[i])
+			}
+		default:
+			fmt.Fprintln(cmd.Stdout, err)
+		}
+		fmt.Fprintln(cmd.Stdout, "")
 	}

 	// Write header.
--- a/ctl/rbf_pages_test.go
+++ b/ctl/rbf_pages_test.go
@ -0,0 +1,52 @@
+// Copyright 2021 Molecula Corp. All rights reserved.
+package ctl
+
+import (
+	"bytes"
+	"context"
+	"path/filepath"
+	"testing"
+)
+
+func TestRBFPagesCommand_Run(t *testing.T) {
+	t.Run("OK", func(t *testing.T) {
+		want := `
+ID       TYPE       EXTRA
+======== ========== ====================
+0        meta       pageN=4,walid=4,rootrec=1,freelist=2
+1        rootrec    next=0
+2        leaf       flags=x2,celln=0
+3        leaf       flags=x2,celln=1
+`[1:]
+
+		var stdout, stderr bytes.Buffer
+		cmd := NewRBFPagesCommand(bytes.NewReader(nil), &stdout, &stderr)
+		cmd.Path = filepath.Join("testdata", "rbf-pages", "ok")
+		if err := cmd.Run(context.Background()); err != nil {
+			t.Fatal(err)
+		} else if got := stdout.String(); got != want {
+			t.Fatalf("got:\n%s\n\nwant:\n%s", got, want)
+		}
+	})
+
+	t.Run("ErrInvalidPageType", func(t *testing.T) {
+		want := `
+ID       TYPE       EXTRA
+======== ========== ====================
+0        meta       pageN=5,walid=4,rootrec=1,freelist=2
+1        rootrec    next=0
+2        leaf       flags=x2,celln=0
+3        leaf       flags=x2,celln=1
+4        unknown [<nil>]
+`[1:]
+
+		var stdout, stderr bytes.Buffer
+		cmd := NewRBFPagesCommand(bytes.NewReader(nil), &stdout, &stderr)
+		cmd.Path = filepath.Join("testdata", "rbf-pages", "err-invalid-page-type")
+		if err := cmd.Run(context.Background()); err != nil {
+			t.Fatal(err)
+		} else if got := stdout.String(); got != want {
+			t.Fatalf("got:\n%s\n\nwant:\n%s", got, want)
+		}
+	})
+}
--- a/ctl/testdata/rbf-check/err-invalid-page-type/data
+++ b/ctl/testdata/rbf-check/err-invalid-page-type/data
--- a/ctl/testdata/rbf-check/err-invalid-page-type/wal
+++ b/ctl/testdata/rbf-check/err-invalid-page-type/wal
--- a/ctl/testdata/rbf-check/ok/data
+++ b/ctl/testdata/rbf-check/ok/data
--- a/ctl/testdata/rbf-check/ok/wal
+++ b/ctl/testdata/rbf-check/ok/wal
--- a/ctl/testdata/rbf-pages/err-invalid-page-type/data
+++ b/ctl/testdata/rbf-pages/err-invalid-page-type/data
--- a/ctl/testdata/rbf-pages/err-invalid-page-type/wal
+++ b/ctl/testdata/rbf-pages/err-invalid-page-type/wal
--- a/ctl/testdata/rbf-pages/ok/data
+++ b/ctl/testdata/rbf-pages/ok/data
--- a/ctl/testdata/rbf-pages/ok/wal
+++ b/ctl/testdata/rbf-pages/ok/wal
--- a/qa/scripts/config.json
+++ b/qa/scripts/config.json
--- a/qa/scripts/deployNode.sh
+++ b/qa/scripts/deployNode.sh
@ -2,6 +2,9 @@

 # To run script: ./deployNode.sh $PROFILE

+# default to the VPC initially created
+VPC=${VPC:-vpc-0582f594d7d2ca2d4}
+
 function deploy_node() {
    # get AMI, security group and subnet ID 
    AMI=$(aws ssm get-parameters --names /aws/service/ami-amazon-linux-latest/amzn2-ami-hvm-x86_64-ebs --query 'Parameters[0].[Value]' --output text --profile $PROFILE)
@ -10,13 +13,13 @@ function deploy_node() {
        exit 1
    fi

-    SECURITY_GROUP=$(aws ec2 describe-security-groups --filters Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978 Name=group-name,Values=default --query 'SecurityGroups[*].[GroupId]' --output text --profile $PROFILE)
+    SECURITY_GROUP=$(aws ec2 describe-security-groups --filters "Name=vpc-id,Values=$VPC" Name=group-name,Values=default --query 'SecurityGroups[*].[GroupId]' --output text --profile $PROFILE)
    if [[ $? > 0 ]]; then 
        echo "aws session manager failed to find security group"
        exit 1
    fi
    
-    SUBNET_ID=$(aws ec2 describe-subnets --filters 'Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978' 'Name=availability-zone,Values=us-east-2a' --query 'Subnets[0].SubnetId' --output text --profile $PROFILE)
+    SUBNET_ID=$(aws ec2 describe-subnets --filters "Name=vpc-id,Values=$VPC" 'Name=availability-zone,Values=us-east-2a' --query 'Subnets[0].SubnetId' --output text --profile $PROFILE)
    if [[ $? > 0 ]]; then 
        echo "aws session manager failed to find subnet ID"
        exit 1
--- a/qa/scripts/deploySingleNodeCluster.sh
+++ b/qa/scripts/deploySingleNodeCluster.sh
@ -0,0 +1,28 @@
+#!/bin/bash
+
+# To run script: ./deploySingleNodeCluster.sh
+# requires TF_VAR_gitlab_token env var to be set
+
+echo “$(pwd)”
+
+pushd ./qa/tf/ci/singlenode
+export TF_IN_AUTOMATION=1
+terraform init -input=false
+terraform apply -input=false -auto-approve
+popd
+
+# configure Featurebase
+
+# step 1a: get IPs of the cluster
+
+
+
+# step 1b: get IPs of the ingest nodes
+
+# step 2: write a featurebase.conf file
+
+# step 3: write featurebase.service
+
+# step 4: start featurebase
+
+# step 5: verify featurebase running
--- a/qa/scripts/ingestWorkload.sh
+++ b/qa/scripts/ingestWorkload.sh
@ -1,7 +1,14 @@
 #!/usr/bin/env bash

+# path for featurebase binary
+FEATUREBASE_PATH=/usr/local/bin
+
+# path for directory with csv directory files for all fields to be ingested 
+CSV_DIR_PATH=/data
+
+
 # To run:
-# ./ingestWorkload.sh {Path for featurebase binary} {Local host & port for featurebase} {Path for directory with csv files} {initialize flag}
+# ./ingestWorkload.sh {Local host & port for featurebase} {initialize flag}

 function delete_field {
    if (($INITIALIZE == 0));
@ -30,18 +37,10 @@ function ingest_set_field {
    $FEATUREBASE_PATH/featurebase import --host $HOST -i $INDEX -f $FIELD $CSV_FILE 
 }

-# path for featurebase binary
-FEATUREBASE_PATH=$1
-shift
-
 # featurebase host & port
 HOST=$1
 shift

-# path for directory with csv directory files for all fields to be ingested 
-CSV_DIR_PATH=$1
-shift 
-
 # intialize flag - 0:disabled, 1:enabled - creates the index and fields for testing
 INITIALIZE=$1
 shift 
--- a/qa/scripts/perf.sh
+++ b/qa/scripts/perf.sh
@ -0,0 +1,3 @@
+#!/bin/bash
+echo >&2 "performance testing"
+time ./simulacraData
--- a/qa/scripts/runSamsungGauntlet.sh
+++ b/qa/scripts/runSamsungGauntlet.sh
@ -0,0 +1,8 @@
+#!/bin/bash
+
+
+#openssl rand -base64 32 | tr -d /=+ | cut -c -16
+
+./setupSamsungGauntlet.sh
+./testSamsungGauntlet.sh
+./teardownSamsungGauntlet.sh
--- a/qa/scripts/setupSamsungGauntlet.sh
+++ b/qa/scripts/setupSamsungGauntlet.sh
@ -0,0 +1,41 @@
+#!/bin/bash
+
+# To run script: ./setupSamsungGauntlet.sh
+# requires TF_VAR_gitlab_token env var to be set
+
+pushd ./qa/tf/gauntlet/samsung
+export TF_IN_AUTOMATION=1
+echo "Running terraform init..."
+terraform init -input=false
+echo "Running terraform apply..."
+terraform apply -input=false -auto-approve
+terraform output -json > samsung-gauntlet.json
+popd
+
+# get the bastion host
+BASTION=$(cat ./qa/tf/gauntlet/samsung/samsung-gauntlet.json | jq -r '[.ingest_ips][0]["value"][0]')
+echo "using bastion ${BASTION}"
+
+NODE=$(cat ./qa/tf/gauntlet/samsung/samsung-gauntlet.json | jq -r '[.data_node_ips][0]["value"][0]')
+echo "using node ${NODE}"
+
+# remember that the nodes will take at least 2 mins to be up and going and finish cloud-init
+#while true
+#do
+#  nc -G 2 -w 1 $BASTION 22
+#  if [ $? -eq 0 ]
+#  then
+#    break
+#  fi
+#done
+sleep 150
+
+# verify featurebase running
+ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o "StrictHostKeyChecking no" ec2-user@${BASTION} "curl -s http://${NODE}:10101/status" 
+if (( $? != 0 )) 
+then 
+    echo "Featurebase cluster not running"
+    exit 1
+fi
+
+
--- a/qa/scripts/teardownSamsungGauntlet.sh
+++ b/qa/scripts/teardownSamsungGauntlet.sh
@ -0,0 +1,8 @@
+#!/bin/bash
+
+# To run script: ./teardownSamsungGauntlet.sh
+# requires TF_VAR_gitlab_token env var to be set
+
+cd qa/tf/gauntlet/samsung
+export TF_IN_AUTOMATION=1
+terraform destroy -auto-approve
--- a/qa/scripts/testSamsungGauntlet.sh
+++ b/qa/scripts/testSamsungGauntlet.sh
@ -0,0 +1,48 @@
+#!/bin/bash
+
+# get the bastion host
+BASTION=$(cat ./qa/tf/gauntlet/samsung/samsung-gauntlet.json | jq -r '[.ingest_ips][0]["value"][0]')
+echo "using bastion ${BASTION}"
+
+NODE=$(cat ./qa/tf/gauntlet/samsung/samsung-gauntlet.json | jq -r '[.data_node_ips][0]["value"][0]')
+echo "using node ${NODE}"
+
+# generate csv files
+GOOS=linux GOARCH=arm64 go build ./qa/simulacraData/...
+scp -i ~/.ssh/gitlab-featurebase-ci.pem simulacraData ec2-user@${BASTION}:/data
+if (( $? != 0 )) 
+then 
+    echo "Copy failed"
+    exit 1
+fi
+
+ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem ec2-user@${BASTION} "cd /data && /data/simulacraData" 
+if (( $? != 0 )) 
+then 
+    echo "Making big files failed"
+    exit 1
+fi
+
+# ingest these files the way that samsung does it
+scp -i ~/.ssh/gitlab-featurebase-ci.pem ./qa/scripts/testSamsungPayload.sh ec2-user@${BASTION}:
+if (( $? != 0 )) 
+then 
+    echo "Copy ingest script failed"
+    exit 1
+fi
+
+ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem ec2-user@${BASTION} "./testSamsungPayload.sh http://${NODE}:10101 1" 
+if (( $? != 0 )) 
+then 
+    echo "Running 1 testSamsungPayload.sh failed"
+    exit 1
+fi
+
+ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem ec2-user@${BASTION} "./testSamsungPayload.sh http://${NODE}:10101 0" 
+if (( $? != 0 )) 
+then 
+    echo "Running 0 testSamsungPayload.sh failed"
+    exit 1
+fi
+
+# query workload that runs
--- a/qa/scripts/testSamsungPayload.sh
+++ b/qa/scripts/testSamsungPayload.sh
@ -0,0 +1,84 @@
+#!/usr/bin/env bash
+
+# path for featurebase binary
+FEATUREBASE_PATH=/usr/local/bin
+
+# path for directory with csv directory files for all fields to be ingested 
+CSV_DIR_PATH=/data
+
+
+# To run:
+# ./testSamsungPayload.sh {Local host & port for featurebase} {initialize flag}
+
+function delete_field {
+    if (($INITIALIZE == 0));
+    then 
+        curl -XDELETE $HOST/index/$INDEX/field/$FIELD
+    fi
+}
+
+# Script to replicate samsung workload of deleting and re-ingesting fields every night 
+# outline delete and re-ingest workload 
+function ingest_int_field { 
+    delete_field
+    curl -XPOST $HOST/index/$INDEX/field/$FIELD -d '{"options": {"type": "int", "min": 0, "max":'$MAX'}}'
+    $FEATUREBASE_PATH/featurebase import --host $HOST -i $INDEX -f $FIELD $CSV_FILE 
+}
+
+function ingest_time_field {
+    delete_field
+    curl -XPOST $HOST/index/$INDEX/field/$FIELD -d '{"options": {"keys": true, "type": "time", "timeQuantum": "YMD"}}'
+    $FEATUREBASE_PATH/featurebase import --host $HOST -i $INDEX -f $FIELD $CSV_FILE 
+}
+
+function ingest_set_field {
+    delete_field
+    curl -XPOST $HOST/index/$INDEX/field/$FIELD -d '{"options": {"keys": true}}'
+    $FEATUREBASE_PATH/featurebase import --host $HOST -i $INDEX -f $FIELD $CSV_FILE 
+}
+
+# featurebase host & port
+HOST=$1
+shift
+
+# intialize flag - 0:disabled, 1:enabled - creates the index and fields for testing
+INITIALIZE=$1
+shift 
+
+# get a list of csv files in the directory 
+CSV_FILES=`ls $CSV_DIR_PATH/*.csv`
+
+# assign index name 
+INDEX="samsung"
+if (($INITIALIZE == 1));
+then 
+    curl -XPOST $HOST/index/$INDEX
+fi
+
+# perform delete and re-ingest for all fields 
+for CSV_FILE in ${CSV_FILES[@]}
+    do 
+        # get field name from csv file path 
+        FIELD="$(basename $CSV_FILE .csv)"
+        if [[ "$FIELD" == *"age"* ]];
+        then 
+            MAX=100 
+            ingest_int_field
+        elif [[ "$FIELD" == *"identifier"* ]];
+        then 
+            MAX=$((2**63 - 1)) # compute max value for 64bit
+            ingest_int_field
+        elif [[ "$FIELD" == *"ip"* ]];
+        then 
+            MAX=$((2**31 - 1)) # compute max value for 32bit
+            ingest_int_field
+        elif [[ "$FIELD" == *"time"* ]];
+        then 
+            ingest_time_field
+        else
+            ingest_set_field
+        fi
+    done
+
+
+
--- a/qa/tf/.modules/featurebase-cluster/README.md
+++ b/qa/tf/.modules/featurebase-cluster/README.md
@ -0,0 +1,37 @@
+# Summary
+
+This module provisions a VPC, subnets, instances, keys, and security groups needed for a basic featurebase cluster running in AWS. It is meant to be used as a module. For example:
+
+```hcl
+module "featurebase" {
+  source "/path/to/module/"
+  cluster_prefix = "sprockets"
+  azs = ["us-east-1a", "us-east-1b", "us-east-1c"]
+}
+```
+
+The path to the module is wherever the `featurebase-cloud` directory is. So if you have put it in `/var/opt/terraform/modules/featurebase-cloud` then calling the module would look like:
+
+```hcl
+module "featurebase" {
+  source "/var/opt/terraform/modules/featurebase-cloud"
+  cluster_prefix = "sprockets"
+}
+```
+
+Much more is configurable; for a complete list, look in `variables.tf`. Reasonable defaults have been set.
+
+## AWS Access
+
+Please make sure you have set up your AWS access in either environment variables, or in the credentials file.
+
+Some useful links for this are:
+
+AWS Environment Variables <https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-envvars.html>
+
+## State
+
+State is currently kept locally, for as this is intended for PoCs. It can be stored in a remote s3 or GCS bucket if desired.
+
+
+ 
--- a/qa/tf/.modules/featurebase-cluster/main.tf
+++ b/qa/tf/.modules/featurebase-cluster/main.tf
@ -0,0 +1,229 @@
+data "aws_ami" "amazon_linux_2" {
+  most_recent = true
+  owners      = ["amazon"]
+  filter {
+    name   = "name"
+    values = ["amzn2-ami-hvm-*"]
+  }
+
+  filter {
+    name   = "virtualization-type"
+    values = ["hvm"]
+  }
+
+  filter {
+    name   = "architecture"
+    values = ["arm64"]
+  }
+}
+
+resource "aws_instance" "fb_cluster_nodes" {
+  count                  = var.fb_data_node_count
+  ami                    = data.aws_ami.amazon_linux_2.id
+  instance_type          = var.fb_data_node_type
+  key_name               = aws_key_pair.gitlab-featurebase-ci.key_name
+  vpc_security_group_ids = [aws_security_group.featurebase.id]
+  monitoring             = true
+  subnet_id              = var.subnet != "" ? var.subnet : module.vpc.private_subnets[count.index % length(module.vpc.private_subnets)]
+  availability_zone      = var.zone != "" ? var.zone : var.azs[count.index % length(var.azs)]
+  iam_instance_profile   = "${aws_iam_instance_profile.fb_cluster_node_profile.name}"
+
+  root_block_device {
+    volume_type = "gp3"
+    volume_size = 20
+  }
+
+  ebs_block_device {
+    device_name = "/dev/sdb"
+    volume_type = var.fb_data_disk_type
+    volume_size = var.fb_data_disk_size_gb
+    iops        = var.fb_data_disk_iops
+  }
+
+  tags = {
+    Prefix = "${var.cluster_prefix}"
+    Name   = "${var.cluster_prefix}-featurebase-cluster-${count.index}"
+    Role   = "cluster_node"
+  }
+
+  user_data = base64encode(templatefile("${path.module}/setup_cluster_node.sh.tpl", { gitlab_token = var.gitlab_token, cluster_prefix = var.cluster_prefix, node_count = var.fb_data_node_count, fb_cluster_replica_count = var.fb_cluster_replica_count, region = var.region }))
+}
+
+resource "aws_instance" "fb_ingest" {
+  count                       = var.fb_ingest_node_count
+  ami                         = data.aws_ami.amazon_linux_2.id
+  key_name                    = aws_key_pair.gitlab-featurebase-ci.key_name
+  vpc_security_group_ids      = [aws_security_group.ingest.id]
+  instance_type               = var.fb_ingest_type
+  associate_public_ip_address = true
+  monitoring                  = true
+  subnet_id                   = var.subnet != "" ? var.subnet : module.vpc.public_subnets[count.index % length(module.vpc.public_subnets)]
+  availability_zone           = var.zone != "" ? var.zone : var.azs[count.index % length(var.azs)]
+  iam_instance_profile        = "${aws_iam_instance_profile.fb_cluster_node_profile.name}"
+
+  root_block_device {
+    volume_type = "gp3"
+    volume_size = 20
+  }
+
+  ebs_block_device {
+    device_name = "/dev/sdb"
+    volume_type = var.fb_ingest_disk_type
+    volume_size = var.fb_ingest_disk_size_gb
+    iops        = var.fb_ingest_disk_iops
+  }
+
+  tags = {
+    Prefix = "${var.cluster_prefix}"
+    Name   = "${var.cluster_prefix}-featurebase-ingest-${count.index}"
+    Role   = "ingest_node"
+  }
+
+  user_data = base64encode(templatefile("${path.module}/setup_ingest_node.sh.tpl", { gitlab_token = var.gitlab_token, cluster_prefix = var.cluster_prefix, node_count = var.fb_ingest_node_count, this_node = count.index, region = var.region }))
+}
+
+resource "aws_key_pair" "gitlab-featurebase-ci" {
+  key_name   = "gitlab-featurebase-ci"
+  public_key = "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC91hhpVHNonAG7ku2ugpxEskf9KHeyHJPQJT26OHrMUw7R+T5A8TjqSzTau07sXQ/E9SO3ebV8SJ5PqeaQOnQB8VEvVNK0DjQH7ppvNg1Rfs42FZT9ttzTMvOjsSbK3vZTHXdoKQEdC9NxBwSkFIRGQojK1HUOq9xGrw31fA1OjSwlpLcbx7yyg18lcqW6UOptnVR8U9Yy9qQ5jZF1HtkQ6L9J+gv4o1UyNAUK2bopeGiXpBc3PQ/CFaFT2h/aqLBP66qAHsHVyAFD3PIRtplC5EHa8jXDgLacEls0uF7Q3kRPxvzcuo4g4VkOn1rDy9qH3vd2hT3aKVnM73FIDUiL"
+}
+
+resource "aws_security_group" "featurebase" {
+  name        = "allow_featurebase"
+  description = "Allow featurebase inbound traffic"
+  vpc_id      = module.vpc.vpc_id
+
+  ingress {
+    description = "TLS from Internal"
+    from_port   = 10101
+    to_port     = 10101
+    protocol    = "tcp"
+    cidr_blocks = [module.vpc.vpc_cidr_block]
+  }
+
+  ingress {
+
+    description = "GRPC from Internal"
+    from_port   = 20101
+    to_port     = 20101
+    protocol    = "tcp"
+    cidr_blocks = [module.vpc.vpc_cidr_block]
+  }
+
+  ingress {
+    description = "PostgreSQL from Internal"
+    from_port   = 55432
+    to_port     = 55432
+    protocol    = "tcp"
+    cidr_blocks = [module.vpc.vpc_cidr_block]
+  }
+
+  ingress {
+    description = "etcd from internal"
+    from_port   = 10301
+    to_port     = 10301
+    protocol    = "tcp"
+    cidr_blocks = [module.vpc.vpc_cidr_block]
+  }
+
+  ingress {
+    description = "etcd from internal 2"
+    from_port   = 10401
+    to_port     = 10401
+    protocol    = "tcp"
+    cidr_blocks = [module.vpc.vpc_cidr_block]
+  }
+
+  ingress {
+    description      = "SSH"
+    from_port        = 22
+    to_port          = 22
+    protocol         = "tcp"
+    cidr_blocks      = ["0.0.0.0/0"]
+    ipv6_cidr_blocks = ["::/0"]
+  }
+
+  egress {
+    from_port        = 0
+    to_port          = 0
+    protocol         = "-1"
+    cidr_blocks      = ["0.0.0.0/0"]
+    ipv6_cidr_blocks = ["::/0"]
+  }
+
+  tags = {
+    Name = "allow_featurebase"
+  }
+}
+
+resource "aws_security_group" "ingest" {
+  name        = "allow_ingest"
+  description = "Allow ingest inbound traffic"
+  vpc_id      = module.vpc.vpc_id
+
+  ingress {
+    from_port        = 10101
+    to_port          = 10101
+    protocol         = "tcp"
+    cidr_blocks      = ["0.0.0.0/0"]
+    ipv6_cidr_blocks = ["::/0"]
+  }
+
+  ingress {
+    description      = "SSH"
+    from_port        = 22
+    to_port          = 22
+    protocol         = "tcp"
+    cidr_blocks      = ["0.0.0.0/0"]
+    ipv6_cidr_blocks = ["::/0"]
+  }
+
+  egress {
+    from_port        = 0
+    to_port          = 0
+    protocol         = "-1"
+    cidr_blocks      = ["0.0.0.0/0"]
+    ipv6_cidr_blocks = ["::/0"]
+  }
+
+  tags = {
+    Name = "allow_ingest"
+  }
+}
+
+resource "aws_iam_instance_profile" "fb_cluster_node_profile" {
+  name = "fb_cluster_node_profile"
+  role = aws_iam_role.fb_cluster_node_role.name
+}
+
+resource "aws_iam_role" "fb_cluster_node_role" {
+  name = "fb_cluster_node"
+
+  assume_role_policy = jsonencode({
+    Version = "2012-10-17"
+    Statement = [
+      {
+        Action = "sts:AssumeRole"
+        Effect = "Allow"
+        Sid    = ""
+        Principal = {
+          Service = "ec2.amazonaws.com"
+        }
+      },
+    ]
+  })
+
+  inline_policy {
+    name = "ec2_read_all"
+    policy = jsonencode({
+      Version = "2012-10-17"
+      Statement = [
+        {
+          Action   = ["ec2:Describe*"]
+          Effect   = "Allow"
+          Resource = "*"
+        },
+      ]
+    })
+  }
+
+}
--- a/qa/tf/.modules/featurebase-cluster/outputs.tf
+++ b/qa/tf/.modules/featurebase-cluster/outputs.tf
@ -0,0 +1,7 @@
+output "ingest_ips" {
+    value = aws_instance.fb_ingest.*.public_ip
+}
+
+output "data_node_ips" {
+    value = aws_instance.fb_cluster_nodes.*.private_ip
+}
--- a/qa/tf/.modules/featurebase-cluster/provider.tf
+++ b/qa/tf/.modules/featurebase-cluster/provider.tf
@ -0,0 +1,11 @@
+terraform {
+  required_version = ">= 0.13.1"
+
+  required_providers {
+    aws = {
+      source  = "hashicorp/aws"
+      version = ">= 3.38.0"
+    }
+  }
+}
+
--- a/qa/tf/.modules/featurebase-cluster/setup_cluster_node.sh.tpl
+++ b/qa/tf/.modules/featurebase-cluster/setup_cluster_node.sh.tpl
@ -0,0 +1,188 @@
+#!/bin/bash
+
+#path to the featurebase.conf file
+CONFIG_FILE_PATH="/etc/featurebase.conf"
+#path to the featurebase.service file
+SERVICE_FILE_PATH="/etc/systemd/system/featurebase.service"
+
+AWS_INSTANCE_ID=""
+#IP of this node
+PRIVATE_IP=""
+PRIVATE_IP_INDEX=-1
+#IPs of the cluster
+CLUSTER_IPS=""
+
+get_aws_instance_id() {
+    echo "Getting AWS instance ID..."
+    while true
+    do
+        curl -s http://169.254.169.254/latest/meta-data/instance-id > /dev/null
+        if [ $? -eq 0 ]
+        then
+            break
+        fi
+    done
+    AWS_INSTANCE_ID=`curl http://169.254.169.254/latest/meta-data/instance-id`
+    echo "AWS instance ID is: $${AWS_INSTANCE_ID}"
+}
+
+wait_on_all_cluster_ips() {
+    echo "Waiting on all cluster IPs..."
+    # get IP for node 
+    IPS=$(aws ec2 describe-instances --filters "Name=instance-state-name, Values=running" "Name=tag:Role, Values=cluster_node" "Name=tag:Prefix, Values=${cluster_prefix}" --query 'Reservations[*].Instances[*].PrivateIpAddress' --output text --region ${region})
+    IP_LENGTH=`echo "$IPS" | wc -l`
+
+    for i in {0..24}
+    do 
+        echo "Comparing $${IP_LENGTH} with ${node_count}"
+        if [ $IP_LENGTH == "${node_count}" ]; then
+            echo "Cluster is up after $${i} tries." 
+            break
+        fi
+        sleep 10s
+    done
+
+    if [ $IP_LENGTH != "${node_count}" ]; then
+        echo "Timed out waiting for cluster to be available $${IP_LENGTH} actual nodes compared with ${node_count} desire nodes."
+        exit 1
+    fi
+}
+
+get_private_ip() {
+    echo "Getting private IP address..."
+    PRIVATE_IP=$(aws ec2 describe-instances --filters "Name=instance-state-name, Values=running" "Name=instance-id,Values=$${AWS_INSTANCE_ID}" --query 'Reservations[*].Instances[*].PrivateIpAddress' --output text --region ${region})
+    echo "Private IP is $${PRIVATE_IP}"
+}
+
+get_cluster_ips() {
+    echo "Getting cluster IPs..."
+    # get IP for node 
+    IPS=$(aws ec2 describe-instances --filters "Name=instance-state-name, Values=running" "Name=tag:Role, Values=cluster_node" "Name=tag:Prefix, Values=${cluster_prefix}" --query 'Reservations[*].Instances[*].PrivateIpAddress' --output text --region ${region})
+    IP_LENGTH=`echo "$IPS" | wc -l`
+
+    IFS=$'\n'
+    cnt=0
+    for ip in $IPS
+    do
+        echo $cnt $ip
+        if (($cnt + 1 != $IP_LENGTH)) 
+        then
+            CLUSTER_IPS="$${CLUSTER_IPS}p$${cnt}=http://$ip:10301,"
+        else
+            CLUSTER_IPS="$${CLUSTER_IPS}p$${cnt}=http://$ip:10301"
+        fi
+        echo "comparing $ip to $PRIVATE_IP"
+        if [ "$ip" = "$PRIVATE_IP" ]; then
+            PRIVATE_IP_INDEX=$cnt
+        fi
+        cnt=$((cnt+1))
+    done
+
+    echo "CLUSTER_IPS are: $${CLUSTER_IPS}"
+}
+
+write_featurebase_config_file() {
+    echo "Writing featurebase.conf file..."
+    cat << EOT > $${CONFIG_FILE_PATH}
+name = "p$${PRIVATE_IP_INDEX}"
+bind = "0.0.0.0:10101"
+bind-grpc = "0.0.0.0:20101"
+
+data-dir = "/data/featurebase"
+log-path = "/var/log/molecula/featurebase.log"
+
+max-file-count=900000
+max-map-count=900000
+
+long-query-time = "10s"
+
+[postgres]
+
+    bind = "localhost:55432"
+
+[cluster]
+
+    name = "${cluster_prefix}"
+    replicas = ${fb_cluster_replica_count}
+
+[etcd]
+
+    listen-client-address = "http://$${PRIVATE_IP}:10401"
+    listen-peer-address = "http://$${PRIVATE_IP}:10301"
+    initial-cluster = "$${CLUSTER_IPS}"
+
+[metric]
+
+    service = "prometheus"
+EOT
+
+    echo "featurebase.conf written to $${CONFIG_FILE_PATH}."
+}
+
+write_featurebase_service_file() {
+    echo "Writing featurebase.service file..."
+    cat << EOT > $${SERVICE_FILE_PATH}
+# Not Ansible managed
+
+[Unit]
+Description="Service for FeatureBase"
+
+[Service]
+RestartSec=30
+Restart=on-failure
+EnvironmentFile=
+User=molecula
+ExecStart=/usr/local/bin/featurebase server -c /etc/featurebase.conf
+
+[Install]
+EOT
+
+    echo "featurebase.service written to $${SERVICE_FILE_PATH}."
+    
+}
+
+#get the instance id
+get_aws_instance_id
+
+#copy the script so we can look at it later if needed
+sudo cp /var/lib/cloud/instances/$${AWS_INSTANCE_ID}/user-data.txt /home/ec2-user/setup_cluster_node.sh
+
+#wait for the count of nodes to equal requested nodes
+wait_on_all_cluster_ips
+
+#get private ip
+get_private_ip
+
+#generate cluster ips
+get_cluster_ips
+
+#write the featurebase config file
+write_featurebase_config_file
+
+#write the featurebase service file
+write_featurebase_service_file
+
+#get the featurebase binary and put in in the right spot
+echo "Getting featurebase binary..."
+curl --header "PRIVATE-TOKEN: ${gitlab_token}" -o "/home/ec2-user/featurebase_linux_arm64" https://gitlab.com/api/v4/projects/molecula%2Ffeaturebase/jobs/artifacts/master/raw/featurebase_linux_arm64?job=build%20for%20linux%20arm64
+chown ec2-user:ec2-user "/home/ec2-user/featurebase_linux_arm64"
+chmod ugo+x "/home/ec2-user/featurebase_linux_arm64"
+
+mv /home/ec2-user/featurebase_linux_arm64 /usr/local/bin/featurebase
+echo "featurebase binary copied."
+
+sudo mkdir /data
+sudo mkfs.ext4 /dev/nvme1n1
+sudo mount /dev/nvme1n1 /data
+
+adduser molecula
+sudo mkdir /var/log/molecula
+sudo chown molecula /var/log/molecula
+sudo mkdir -p /data/featurebase
+sudo chown molecula /data/featurebase
+sudo systemctl daemon-reload
+sudo systemctl start featurebase
+sudo systemctl enable featurebase
+sudo systemctl status featurebase
+
+echo "Done!"
--- a/qa/tf/.modules/featurebase-cluster/setup_ingest_node.sh.tpl
+++ b/qa/tf/.modules/featurebase-cluster/setup_ingest_node.sh.tpl
@ -0,0 +1,66 @@
+#!/bin/bash
+
+AWS_INSTANCE_ID=""
+
+get_aws_instance_id() {
+    echo "Getting AWS instance ID..."
+    while true
+    do
+        curl -s http://169.254.169.254/latest/meta-data/instance-id > /dev/null
+        if [ $? -eq 0 ]
+        then
+            break
+        fi
+    done
+    AWS_INSTANCE_ID=`curl http://169.254.169.254/latest/meta-data/instance-id`
+    echo "AWS instance ID is: $${AWS_INSTANCE_ID}"
+}
+
+wait_on_all_ingest_ips() {
+    echo "Waiting on all cluster IPs..."
+    # get IP for node 
+    IPS=$(aws ec2 describe-instances --filters "Name=instance-state-name, Values=running" "Name=tag:Role, Values=ingest_node" "Name=tag:Prefix, Values=${cluster_prefix}" --query 'Reservations[*].Instances[*].PrivateIpAddress' --output text --region ${region})
+    IP_LENGTH=`echo "$IPS" | wc -l`
+
+    for i in {0..24}
+    do 
+        echo "Comparing $${IP_LENGTH} with ${node_count}"
+        if [ $IP_LENGTH == "${node_count}" ]; then
+            echo "Cluster is up after $${i} tries." 
+            break
+        fi
+        sleep 10s
+    done
+
+    if [ $IP_LENGTH != "${node_count}" ]; then
+        echo "Timed out waiting for cluster to be available $${IP_LENGTH} actual nodes compared with ${node_count} desire nodes."
+        exit 1
+    fi
+}
+
+#copy the script so we can look at it later if needed
+sudo cp /var/lib/cloud/instances/$${AWS_INSTANCE_ID}/user-data.txt ~/setup_ingest_node.sh
+
+#get the instance id
+get_aws_instance_id
+
+#wait for the count of nodes to equal requested nodes
+wait_on_all_ingest_ips
+
+echo "Getting featurebase binary..."
+curl --header "PRIVATE-TOKEN: ${gitlab_token}" -o "/home/ec2-user/featurebase_linux_arm64" https://gitlab.com/api/v4/projects/molecula%2Ffeaturebase/jobs/artifacts/master/raw/featurebase_linux_arm64?job=build%20for%20linux%20arm64
+chown ec2-user:ec2-user "/home/ec2-user/featurebase_linux_arm64"
+chmod ugo+x "/home/ec2-user/featurebase_linux_arm64"
+
+mv /home/ec2-user/featurebase_linux_arm64 /usr/local/bin/featurebase
+echo "featurebase binary copied."
+
+
+sudo mkdir /data
+sudo mkfs.ext4 /dev/nvme1n1
+sudo mount /dev/nvme1n1 /data
+
+sudo chown -R ec2-user /data
+
+
+
--- a/qa/tf/.modules/featurebase-cluster/variables.tf
+++ b/qa/tf/.modules/featurebase-cluster/variables.tf
@ -0,0 +1,93 @@
+variable "cluster_prefix" {
+  type        = string
+  description = "This is a identifier that will be prefixed to created resources"
+}
+
+variable "fb_ingest_type" {
+  type    = string
+  default = "c6g.2xlarge"
+}
+
+variable "fb_ingest_node_count" {
+  type    = number
+  default = 1
+}
+
+variable "fb_data_node_type" {
+  type    = string
+  default = "c6g.16xlarge"
+}
+
+variable "fb_data_node_count" {
+  type    = number
+  default = 3
+}
+
+variable "fb_cluster_replica_count" {
+  type    = number
+  default = 1
+}
+
+variable "subnet" {
+  default = ""
+}
+
+variable "zone" {
+  default = ""
+}
+
+variable "fb_data_disk_type" {
+  default = "gp3"
+}
+variable "fb_data_disk_iops" {
+  default = 1000
+}
+
+variable "fb_data_disk_size_gb" {
+  default = 100
+}
+
+variable "fb_ingest_disk_type" {
+  default = "gp3"
+}
+variable "fb_ingest_disk_iops" {
+  default = 1000
+}
+
+variable "fb_ingest_disk_size_gb" {
+  default = 100
+}
+
+variable "azs" {
+  type    = list(any)
+  default = ["us-east-2a", "us-east-2b", "us-east-2c"]
+}
+
+variable "private_subnets" {
+  type    = list(any)
+  default = ["10.0.1.0/24", "10.0.2.0/24", "10.0.3.0/24"]
+}
+
+variable "public_subnets" {
+  type    = list(any)
+  default = ["10.0.101.0/24", "10.0.102.0/24", "10.0.103.0/24"]
+}
+
+variable "vpc_cidr" {
+  default = "10.0.0.0/16"
+}
+
+variable "region" {
+  description = "Region to create AWS resources in"
+  type        = string
+}
+
+variable "profile" {
+  description = "Profile to use to authenticate with AWS"
+  type        = string
+}
+
+variable "gitlab_token" {
+  description = "Gitlab API token"
+  type        = string
+}
--- a/qa/tf/.modules/featurebase-cluster/vpc.tf
+++ b/qa/tf/.modules/featurebase-cluster/vpc.tf
@ -0,0 +1,16 @@
+module "vpc" {
+  source = "terraform-aws-modules/vpc/aws"
+
+  name = "${var.cluster_prefix}"
+  cidr = var.vpc_cidr
+  azs             = var.azs
+  private_subnets = var.private_subnets
+  public_subnets  = var.public_subnets
+
+  enable_nat_gateway = true
+  enable_vpn_gateway = false
+
+  tags = {
+    Name = "${var.cluster_prefix}"
+  }
+}
--- a/qa/tf/README.md
+++ b/qa/tf/README.md
@ -0,0 +1,16 @@
+# Deploy testing environments with this one weird trick!
+
+This directory contains Terraform to deploy test environments both ad-hoc and as part of CI/CD pipelines.
+
+The .modules contains the guts of the operation, the things you probably want are in the other directories, each with a README.
+
+## How to Terraform
+
+With terraform installed (`brew install terraform` if not)...
+
+You can do `terraform plan` -> `terraform apply` to spin up a cluster, `terraform destroy` to tear one down.
+
+## Other prerequisites:
+Please read these carefully.
+
+
--- a/qa/tf/ci/singlenode/main.tf
+++ b/qa/tf/ci/singlenode/main.tf
@ -0,0 +1,10 @@
+
+module "ci-cluster" {
+    source = "../../.modules/featurebase-cluster"
+    cluster_prefix = "ci-single-node"
+    region = var.region
+    profile = var.profile
+    fb_data_node_type = "m6g.large"
+    fb_data_node_count = 1
+    gitlab_token = var.gitlab_token
+}
--- a/qa/tf/ci/singlenode/outputs.tf
+++ b/qa/tf/ci/singlenode/outputs.tf
@ -0,0 +1,9 @@
+output "ingest_ips" {
+    description = "List of ingest IPs"
+    value = module.ci-cluster.ingest_ips
+}
+
+output "data_node_ips" {
+  description = "List of data node IPs"
+  value = module.ci-cluster.data_node_ips
+}
--- a/qa/tf/ci/singlenode/provider.tf
+++ b/qa/tf/ci/singlenode/provider.tf
@ -0,0 +1,4 @@
+provider "aws" {
+  region  = var.region
+  profile = var.profile
+}
--- a/qa/tf/ci/singlenode/tf.auto.tfvars
+++ b/qa/tf/ci/singlenode/tf.auto.tfvars
@ -0,0 +1,2 @@
+region  = "us-east-2"
+profile = "service-terraform"
--- a/qa/tf/ci/singlenode/variables.tf
+++ b/qa/tf/ci/singlenode/variables.tf
@ -0,0 +1,14 @@
+variable "region" {
+  description = "The AWS region in which the VPC should be built"
+  type        = string
+}
+
+variable "profile" {
+  description = "The name of the AWS profile Terraform should use for auth."
+  type        = string
+}
+
+variable "gitlab_token" {
+  description = "The API token for taking to Gitlab API - expected to come from an env variable."
+  type        = string
+}
--- a/qa/tf/gauntlet/samsung/README.md
+++ b/qa/tf/gauntlet/samsung/README.md
@ -0,0 +1,35 @@
+With terraform installed (`brew install terraform` if not)...
+
+You can do `terraform plan` -> `terraform apply` to spin up a cluster, `terraform destroy` to tear one down.
+
+## Other prerequisites:
+Please read these carefully.
+
+Be in the `tf` directory (e.g., when you try to run a `terraform` command, the output of `pwd` should be `.../featurebase/qa/tf`)
+
+Currently, the path to the terraform module is using a local reference, i.e., in `main.tf`, the source line is assuming that you have `molecula-terraform` project installed locally, such that the `molecular-terraform` project and `featurebase` have the same parent directory (e.g., `...A/featurebase/qa/tf` and `...A/molecular-terraform/aws/.modules/featurebase-cluster` should both be valid paths).
+
+In addition, you must currently have a local copy of the `fb901` branch for the `molecular-terraform` project (located in the previously specified directory).
+
+Last thing, there is a key that is currently in 1Password (in the `Shared` vault, called `gitlab-featurebase-ci AWS key`) that must be in `~/.ssh/`, `chmod 400`, named `gitlab-featurebase-ci.pem`. You need this key to SSH to these instances. Assuming an `~/.ssh/config` like the following (append to the top of yours)
+```
+Host test_*
+  User ec2-user
+  IdentityFile ~/.ssh/gitlab-featurebase-ci.pem
+Host test_ingest
+  HostName 3.143.237.165
+Host test_node
+  HostName 10.0.1.142
+  ProxyJump test_ingest
+```
+except with the `test_ingest`'s `HostName` being the public, `ingest_ips` output from `terraform output` and `test_node`'s `HostName` being one of the private, `data_node_ips` output from `terraform output`. (Hopefully the rationale to use the ssh config to do the jumping like this makes sense; you can do `ssh test_ingest` or `ssh test_node` with minimal further fiddling.)
+
+OR specify cert to us directly thus:
+
+`ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem ec2-user@ip_address`
+
+-A is used to ensure key forwarding.
+
+### TODOs
+* We need a `user-data.sh` script which sets up/installs featurebase (possibly installs go, most likely pulls the artifacts from GitLab; sets up featurebase on both the node and data workers).
+* Logs get sent to DataDog?
--- a/qa/tf/gauntlet/samsung/main.tf
+++ b/qa/tf/gauntlet/samsung/main.tf
@ -0,0 +1,13 @@
+module "samsung-cluster" {
+    source = "../../.modules/featurebase-cluster"
+    cluster_prefix = "samsung-gauntlet"
+    region = var.region
+    profile = var.profile
+    fb_data_node_type = "m6g.xlarge"
+    fb_data_disk_iops = 10000
+    fb_data_node_count = 3
+    fb_ingest_type = "m6g.large"
+    fb_ingest_disk_iops = 10000
+    fb_ingest_node_count = 1
+    gitlab_token = var.gitlab_token
+}
--- a/qa/tf/gauntlet/samsung/outputs.tf
+++ b/qa/tf/gauntlet/samsung/outputs.tf
@ -0,0 +1,9 @@
+output "ingest_ips" {
+    description = "List of ingest IPs"
+    value = module.samsung-cluster.ingest_ips
+}
+
+output "data_node_ips" {
+  description = "List of data node IPs"
+  value = module.samsung-cluster.data_node_ips
+}
--- a/qa/tf/gauntlet/samsung/provider.tf
+++ b/qa/tf/gauntlet/samsung/provider.tf
@ -0,0 +1,4 @@
+provider "aws" {
+  region  = var.region
+  profile = var.profile
+}
--- a/qa/tf/gauntlet/samsung/samsung-gauntlet.json
+++ b/qa/tf/gauntlet/samsung/samsung-gauntlet.json
@ -0,0 +1,30 @@
+{
+  "data_node_ips": {
+    "sensitive": false,
+    "type": [
+      "tuple",
+      [
+        "string",
+        "string",
+        "string"
+      ]
+    ],
+    "value": [
+      "10.0.1.144",
+      "10.0.2.108",
+      "10.0.3.178"
+    ]
+  },
+  "ingest_ips": {
+    "sensitive": false,
+    "type": [
+      "tuple",
+      [
+        "string"
+      ]
+    ],
+    "value": [
+      "3.145.104.76"
+    ]
+  }
+}
--- a/qa/tf/gauntlet/samsung/tf.auto.tfvars
+++ b/qa/tf/gauntlet/samsung/tf.auto.tfvars
@ -0,0 +1,2 @@
+region  = "us-east-2"
+profile = "service-terraform"
--- a/qa/tf/gauntlet/samsung/variables.tf
+++ b/qa/tf/gauntlet/samsung/variables.tf
@ -0,0 +1,14 @@
+variable "region" {
+  description = "The AWS region in which the VPC should be built"
+  type        = string
+}
+
+variable "profile" {
+  description = "The name of the AWS profile Terraform should use for auth."
+  type        = string
+}
+
+variable "gitlab_token" {
+  description = "The API token for taking to Gitlab API - expected to come from an env variable."
+  type        = string
+}
--- a/rbf/cursor.go
+++ b/rbf/cursor.go
@ -932,6 +932,10 @@ func (c *Cursor) First() error {
 		case PageTypeBranch:
 			elem.index = 0

+			if n := readCellN(buf); elem.index >= n { // branch cell index must less than cell count
+				return fmt.Errorf("branch cell index out of range: pgno=%d i=%d n=%d", elem.pgno, elem.index, n)
+			}
+
 			// Read cell pgno into the next stack level.
 			cell := readBranchCell(buf, elem.index)

--- a/rbf/rbf.go
+++ b/rbf/rbf.go
@ -799,3 +799,46 @@ func (m *Metric) Inc(d time.Duration) {
 		fmt.Printf("metric:%10s avg=%dns\n", m.name, int(m.d)/m.n)
 	}
 }
+
+// ErrorList represents a list of errors.
+type ErrorList []error
+
+// Err returns the list if it contains errors. Otherwise returns nil.
+func (a ErrorList) Err() error {
+	if len(a) > 0 {
+		return a
+	}
+	return nil
+}
+
+func (a ErrorList) Error() string {
+	switch len(a) {
+	case 0:
+		return "no errors"
+	case 1:
+		return a[0].Error()
+	}
+	return fmt.Sprintf("%s (and %d more errors)", a[0], len(a)-1)
+}
+
+func (a ErrorList) FullError() string {
+	if len(a) == 0 {
+		return ""
+	}
+
+	var buf bytes.Buffer
+	for _, err := range a {
+		fmt.Fprintln(&buf, err)
+	}
+	return buf.String()
+}
+
+// Append appends an error to the list. If err is an ErrorList then all errors are appended.
+func (a *ErrorList) Append(err error) {
+	switch err := err.(type) {
+	case ErrorList:
+		*a = append(*a, err...)
+	default:
+		*a = append(*a, err)
+	}
+}
--- a/rbf/rbf/testdata/check/bad-freelist/data
+++ b/rbf/rbf/testdata/check/bad-freelist/data
--- a/rbf/rbf/testdata/check/bad-freelist/wal
+++ b/rbf/rbf/testdata/check/bad-freelist/wal
--- a/rbf/rbf_test.go
+++ b/rbf/rbf_test.go
@ -54,17 +54,30 @@ func NewDB(tb testing.TB, cfg ...*rbfcfg.Config) *rbf.DB {
 	if err != nil {
 		panic(err)
 	}
+	return NewDBAt(tb, path, cfg...)
+}

+// NewDBAt returns a new instance of DB with a given path.
+func NewDBAt(tb testing.TB, path string, cfg ...*rbfcfg.Config) *rbf.DB {
 	var cfg0 *rbfcfg.Config
 	if len(cfg) > 0 {
 		cfg0 = cfg[0]
 	}
-	db := rbf.NewDB(path, cfg0)
-	return db
+	return rbf.NewDB(path, cfg0)
 }

 // MustOpenDB returns a db opened on a temporary file. On error, fail test.
 func MustOpenDB(tb testing.TB, cfg ...*rbfcfg.Config) *rbf.DB {
+	tb.Helper()
+	path, err := testhook.TempDir(tb, "rbfdb")
+	if err != nil {
+		panic(err)
+	}
+	return MustOpenDBAt(tb, path, cfg...)
+}
+
+// MustOpenDBAt returns a db opened on an existing file. On error, fail test.
+func MustOpenDBAt(tb testing.TB, path string, cfg ...*rbfcfg.Config) *rbf.DB {
 	tb.Helper()
 	if len(cfg) == 0 || cfg[0] == nil {
 		newconf := rbfcfg.NewDefaultConfig()
@ -73,7 +86,7 @@ func MustOpenDB(tb testing.TB, cfg ...*rbfcfg.Config) *rbf.DB {
 	} else if cfg[0].Logger == nil {
 		cfg[0].Logger = logger.NewLogfLogger(tb)
 	}
-	db := NewDB(tb, cfg...)
+	db := NewDBAt(tb, path, cfg...)
 	if err := db.Open(); err != nil {
 		tb.Fatal(err)
 	}
--- a/rbf/testdata/check/bad-bitmap/data
+++ b/rbf/testdata/check/bad-bitmap/data
--- a/rbf/testdata/check/bad-bitmap/wal
+++ b/rbf/testdata/check/bad-bitmap/wal
--- a/rbf/testdata/check/bad-freelist/data
+++ b/rbf/testdata/check/bad-freelist/data
--- a/rbf/testdata/check/bad-freelist/wal
+++ b/rbf/testdata/check/bad-freelist/wal
--- a/rbf/tx.go
+++ b/rbf/tx.go
@ -738,10 +738,11 @@ func (tx *Tx) Check() error {
 		return ErrTxClosed
 	}

+	var errorList ErrorList
 	if err := tx.checkPageAllocations(); err != nil {
-		return fmt.Errorf("page allocations: %w", err)
+		errorList.Append(err)
 	}
-	return nil
+	return errorList.Err()
 }

 func (tx *Tx) checkPage(pgno, parent, typ uint32) error {
@ -767,14 +768,15 @@ func (tx *Tx) checkBranchPage(pgno, parent, typ uint32) error {

 // checkPageAllocations ensures that all pages are either in-use or on the freelist.
 func (tx *Tx) checkPageAllocations() error {
+	var errorList ErrorList
 	freePageSet, err := tx.freePageSet()
 	if err != nil {
-		return err
+		errorList.Append(err)
 	}

 	inusePageSet, err := tx.inusePageSet()
 	if err != nil {
-		return err
+		errorList.Append(err)
 	}

 	// Iterate over all pages and ensure they are either in-use or free.
@ -785,26 +787,23 @@ func (tx *Tx) checkPageAllocations() error {
 		_, isFree := freePageSet[pgno]

 		if isInuse && isFree {
-			return fmt.Errorf("page in-use & free: pgno=%d", pgno)
-		} else if !isInuse && !isFree {
-			page, _, err := tx.readPage(pgno)
-			if err != nil {
-				return err
-			}
-			flags := readFlags(page)
-			if flags == PageTypeBranch || flags == PageTypeLeaf {
-				return fmt.Errorf("page not in-use & not free: pgno=%d", pgno)
-			}
-			//assuming its a bitmap so its ok TODO ben?
-			return nil
+			errorList.Append(fmt.Errorf("page in-use & free: pgno=%d", pgno))
+			continue
+		}
+
+		if !isInuse && !isFree {
+			errorList.Append(fmt.Errorf("page not in-use & not free: pgno=%d", pgno))
+			continue
 		}
 	}

-	return nil
+	return errorList.Err()
 }

 // freePageSet returns the set of pages in the freelist.
 func (tx *Tx) freePageSet() (map[uint32]struct{}, error) {
+	var errorList ErrorList
+
 	m := make(map[uint32]struct{})
 	c := Cursor{tx: tx}
 	c.stack.elems[0] = stackElem{pgno: readMetaFreelistPageNo(tx.meta[:])}
@ -816,18 +815,20 @@ func (tx *Tx) freePageSet() (map[uint32]struct{}, error) {

 	for {
 		if err := c.Next(); err == io.EOF {
-			return m, nil
+			return m, errorList.Err()
 		} else if err != nil {
-			return m, err
+			errorList.Append(err)
+			return m, errorList.Err()
 		}

 		elem := &c.stack.elems[c.stack.top]
 		leafPage, _, err := c.tx.readPage(elem.pgno)
 		if err != nil {
-			return nil, err
+			errorList.Append(fmt.Errorf("cannot read free page: pgno=%d err=%w", elem.pgno, err))
+			continue
 		}
-		cell := readLeafCell(leafPage, elem.index)

+		cell := readLeafCell(leafPage, elem.index)
 		for _, v := range cell.Values(tx) {
 			pgno := uint32((cell.Key << 16) | uint64(v))
 			m[pgno] = struct{}{}
@ -837,6 +838,7 @@ func (tx *Tx) freePageSet() (map[uint32]struct{}, error) {

 // inusePageSet returns the set of pages in use by the root records or b-trees.
 func (tx *Tx) inusePageSet() (map[uint32]struct{}, error) {
+	var errorList ErrorList
 	m := make(map[uint32]struct{})
 	m[0] = struct{}{} // meta page

@ -846,15 +848,24 @@ func (tx *Tx) inusePageSet() (map[uint32]struct{}, error) {

 		page, _, err := tx.readPage(pgno)
 		if err != nil {
-			return nil, err
+			errorList.Append(err)
+			break
 		}
 		pgno = WalkRootRecordPages(page)
 	}

 	// Traverse freelist and mark pages as in-use.
-	if err := tx.walkTree(readMetaFreelistPageNo(tx.meta[:]), 0, func(pgno, parent, typ uint32) error {
+	if err := tx.walkTree(readMetaFreelistPageNo(tx.meta[:]), 0, func(pgno, parent, typ uint32, err error) error {
+		if err != nil {
+			errorList.Append(err)
+			return nil
+		}
+
 		m[pgno] = struct{}{}
-		return tx.checkPage(pgno, parent, typ)
+		if err := tx.checkPage(pgno, parent, typ); err != nil {
+			errorList.Append(err)
+		}
+		return nil
 	}); err != nil {
 		return m, err
 	}
@ -862,22 +873,28 @@ func (tx *Tx) inusePageSet() (map[uint32]struct{}, error) {
 	// Traverse every b-tree and mark pages as in-use.
 	records, err := tx.RootRecords()
 	if err != nil {
-		return m, err
-	}
+		errorList.Append(err)
+	} else {
+		for itr := records.Iterator(); !itr.Done(); {
+			_, pgno := itr.Next()

-	for itr := records.Iterator(); !itr.Done(); {
-		_, pgno := itr.Next()
+			if err := tx.walkTree(pgno.(uint32), 0, func(pgno, parent, typ uint32, err error) error {
+				if err != nil {
+					errorList.Append(err)
+				}

-		if err := tx.walkTree(pgno.(uint32), 0, func(pgno, parent, typ uint32) error {
-			m[pgno] = struct{}{}
-
-			return tx.checkPage(pgno, parent, typ)
-		}); err != nil {
-			return m, err
+				m[pgno] = struct{}{}
+				if err := tx.checkPage(pgno, parent, typ); err != nil {
+					errorList.Append(err)
+				}
+				return nil
+			}); err != nil {
+				return m, err
+			}
 		}
 	}

-	return m, nil
+	return m, errorList.Err()
 }

 // GetSizeBytesWithPrefix returns the size of bitmaps with a given key prefix.
@ -897,9 +914,9 @@ func (tx *Tx) GetSizeBytesWithPrefix(prefix string) (n uint64, err error) {
 		}

 		// Traverse the bitmap's b-tree and count the bytes for each page.
-		if err := tx.walkTree(pgno.(uint32), 0, func(pgno, parent, typ uint32) error {
+		if err := tx.walkTree(pgno.(uint32), 0, func(pgno, parent, typ uint32, err error) error {
 			n += PageSize
-			return nil
+			return err
 		}); err != nil {
 			return 0, err
 		}
@ -908,21 +925,19 @@ func (tx *Tx) GetSizeBytesWithPrefix(prefix string) (n uint64, err error) {
 }

 // walkTree recursively iterates over a page and all its children.
-func (tx *Tx) walkTree(pgno, parent uint32, fn func(pgno, parent, typ uint32) error) error {
+func (tx *Tx) walkTree(pgno, parent uint32, fn func(pgno, parent, typ uint32, err error) error) error {
 	// Read page and iterate over children.
 	page, _, err := tx.readPage(pgno)
 	if err != nil {
-		return err
+		return fn(pgno, parent, 0, fmt.Errorf("cannot read page: pgno=%d parent=%d err=%s", pgno, parent, err))
 	}

-	// Execute callback.
-	typ := readFlags(page)
-	if err := fn(pgno, parent, typ); err != nil {
-		return err
-	}
-
-	switch typ {
+	switch typ := readFlags(page); typ {
 	case PageTypeBranch:
+		if err := fn(pgno, parent, typ, nil); err != nil {
+			return err
+		}
+
 		for i, n := 0, readCellN(page); i < n; i++ {
 			cell := readBranchCell(page, i)
 			if err := tx.walkTree(cell.ChildPgno, pgno, fn); err != nil {
@ -930,18 +945,24 @@ func (tx *Tx) walkTree(pgno, parent uint32, fn func(pgno, parent, typ uint32) er
 			}
 		}
 		return nil
+
 	case PageTypeLeaf:
+		if err := fn(pgno, parent, typ, nil); err != nil {
+			return err
+		}
+
 		// Execute callback only for bitmap pages pointed to by this leaf.
 		for i, n := 0, readCellN(page); i < n; i++ {
 			if cell := readLeafCell(page, i); cell.Type == ContainerTypeBitmapPtr {
-				if err := fn(toPgno(cell.Data), pgno, PageTypeBitmap); err != nil {
+				if err := fn(toPgno(cell.Data), pgno, PageTypeBitmap, nil); err != nil {
 					return err
 				}
 			}
 		}
 		return nil
+
 	default:
-		return fmt.Errorf("rbf.Tx.forEachTreePage(): invalid page type: pgno=%d type=%d", pgno, typ)
+		return fn(pgno, parent, typ, fmt.Errorf("invalid page type: pgno=%d parent=%d type=%d", pgno, parent, typ))
 	}
 }

@ -1906,6 +1927,7 @@ func (tx *Tx) Pages(pgnos []uint32) ([]Page, error) {

 // PageInfos returns meta data about all pages in the database.
 func (tx *Tx) PageInfos() ([]PageInfo, error) {
+	var errorList ErrorList
 	infos := make([]PageInfo, tx.PageN())

 	// Read meta page info.
@ -1919,7 +1941,8 @@ func (tx *Tx) PageInfos() ([]PageInfo, error) {
 	for pgno := metaInfo.RootRecordPageNo; pgno != 0; {
 		info, err := tx.rootRecordPageInfo(pgno)
 		if err != nil {
-			return nil, err
+			errorList.Append(err)
+			break
 		}
 		infos[pgno] = info
 		pgno = info.Next
@ -1927,33 +1950,34 @@ func (tx *Tx) PageInfos() ([]PageInfo, error) {

 	// Traverse freelist and mark pages as in-use.
 	if err := tx.walkPageInfo(infos, metaInfo.FreelistPageNo, "freelist"); err != nil {
-		return nil, err
+		errorList.Append(err)
 	}

 	// Traverse every b-tree and mark pages as in-use.
 	records, err := tx.RootRecords()
 	if err != nil {
-		return nil, err
-	}
+		errorList.Append(err)
+	} else {
+		for itr := records.Iterator(); !itr.Done(); {
+			name, pgno := itr.Next()

-	for itr := records.Iterator(); !itr.Done(); {
-		name, pgno := itr.Next()
-
-		if err := tx.walkPageInfo(infos, pgno.(uint32), name.(string)); err != nil {
-			return nil, err
+			if err := tx.walkPageInfo(infos, pgno.(uint32), name.(string)); err != nil {
+				errorList.Append(err)
+			}
 		}
 	}

 	// Build page info objects for each free page.
 	freePageSet, err := tx.freePageSet()
 	if err != nil {
-		return nil, err
-	}
-	for pgno := range freePageSet {
-		infos[pgno] = &FreePageInfo{Pgno: pgno}
+		errorList.Append(err)
+	} else {
+		for pgno := range freePageSet {
+			infos[pgno] = &FreePageInfo{Pgno: pgno}
+		}
 	}

-	return infos, nil
+	return infos, errorList.Err()
 }

 // metaPageInfo returns page metadata for the meta page.
@ -1987,10 +2011,18 @@ func (tx *Tx) rootRecordPageInfo(pgno uint32) (*RootRecordPageInfo, error) {
 }

 func (tx *Tx) walkPageInfo(infos []PageInfo, root uint32, name string) error {
-	return tx.walkTree(root, 0, func(pgno, parent, typ uint32) error {
+	var errorList ErrorList
+
+	if err := tx.walkTree(root, 0, func(pgno, parent, typ uint32, err error) error {
+		if err != nil {
+			errorList.Append(err)
+			return nil
+		}
+
 		buf, _, err := tx.readPage(pgno)
 		if err != nil {
-			return err
+			errorList.Append(fmt.Errorf("cannot read page: pgno=%d parent=%d typ=%d err=%d", pgno, parent, typ, err))
+			return nil
 		}

 		switch typ {
@ -2016,12 +2048,14 @@ func (tx *Tx) walkPageInfo(infos []PageInfo, root uint32, name string) error {
 				Parent: parent,
 				Tree:   name,
 			}
-		default:
-			vprint.PanicOn(fmt.Sprintf("unexpected page type %d for page %d", typ, pgno))
 		}

 		return nil
-	})
+	}); err != nil {
+		errorList.Append(err)
+	}
+
+	return errorList.Err()
 }

 // PageData returns the raw page data for a single page.
--- a/rbf/tx_test.go
+++ b/rbf/tx_test.go
@ -6,6 +6,7 @@ import (
 	"fmt"
 	"math/rand"
 	"os"
+	"path/filepath"
 	"strings"
 	"sync"
 	"testing"
@ -870,6 +871,36 @@ func TestTx_Check(t *testing.T) {
 			t.Fatalf("unexpected error: %#v", err)
 		}
 	})
+
+	t.Run("ErrBadFreelist", func(t *testing.T) {
+		t.Parallel()
+
+		db := MustOpenDBAt(t, filepath.Join("testdata", "check", "bad-freelist"))
+		defer db.Close()
+		tx := MustBegin(t, db, false)
+		defer tx.Rollback()
+
+		if err, ok := tx.Check().(rbf.ErrorList); !ok {
+			t.Fatal("expected error list")
+		} else if s := err.FullError(); !strings.Contains(s, `branch cell index out of range: pgno=2 i=0 n=0`) {
+			t.Fatalf("unexpected error:\n%s", s)
+		}
+	})
+
+	t.Run("ErrBadBitmap", func(t *testing.T) {
+		t.Parallel()
+
+		db := MustOpenDBAt(t, filepath.Join("testdata", "check", "bad-bitmap"))
+		defer db.Close()
+		tx := MustBegin(t, db, false)
+		defer tx.Rollback()
+
+		if err, ok := tx.Check().(rbf.ErrorList); !ok {
+			t.Fatal("expected error list")
+		} else if s := err.FullError(); !strings.Contains(s, `cannot read page: pgno=65537 parent=3 err=rbf: page read out of bounds: pgno=65537 max=3`) {
+			t.Fatalf("unexpected error:\n%s", s)
+		}
+	})
 }

 func mustReadPage(tb testing.TB, path string, pgno uint32) []byte {