Merge pull request #1823 from molecula/fb901

Fb901

.gitignore

@@ -13,3 +13,8 @@ pilosa
 *.dot
 .idea/
 .*.swp
+.terraform/
+*.tfstate
+launch.json
+.terraform.lock.hcl
+__pycache__/

.gitlab/.gitlab-ci.yml

@@ -19,6 +19,7 @@ stages:
   - test
   - build
   - integration
+  - gauntlet
 
 golangci-lint:
   image: golangci/golangci-lint:v1.39.0
@@ -244,3 +245,45 @@ deploy node for linux amd64:
     - ./qa/scripts/deployNode.sh $PROFILE
   needs:
     - job: build for linux amd64
+
+gauntlet:
+  stage: gauntlet
+  timeout: 4h
+  image: registry.gitlab.com/gitlab-org/cloud-deploy/aws-base:latest
+  variables:
+    PROFILE: "default"
+    AWS_SSH_PRIVATE_KEY: $AWS_FBCI_SSH_KEY
+    AWS_ACCESS_KEY_ID: $AWS_FBCI_ACCESS_KEY_ID
+    AWS_SECRET_ACCESS_KEY: $AWS_FBCI_SECRET_ACCESS_KEY
+  # TODO: For now, run always
+  rules:
+    - if: '$CI_PIPELINE_SOURCE == "schedule" && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'
+  before_script:
+    - apt-get update && apt-get install -y gnupg software-properties-common curl git
+    - curl -fsSL https://apt.releases.hashicorp.com/gpg | apt-key add -
+    - apt-add-repository "deb [arch=amd64] https://apt.releases.hashicorp.com $(lsb_release -cs) main"
+    - apt-get update && apt-get install terraform
+    - aws configure set aws_access_key_id $AWS_FBCI_ACCESS_KEY_ID
+    - aws configure set aws_secret_access_key $AWS_FBCI_SECRET_ACCESS_KEY
+    - aws configure set region "us-east-2"
+    - aws configure set aws_profile $PROFILE
+    - echo $AWS_FBCI_SSH_KEY > gitlab-featurebase-ci.pem
+    - chmod 400 gitlab-featurebase-ci.pem
+    - 'which ssh-agent || ( apt-get update -y && apt-get install openssh-client -y )'
+    - eval $(ssh-agent -s)
+    - mkdir -p ~/.ssh
+    - echo $AWS_FBCI_SSH_KEY > /root/.ssh/gitlab-featurebase-ci.pem
+    - chmod 400 /root/.ssh/gitlab-featurebase-ci.pem
+    - echo "$AWS_FBCI_SSH_KEY" | ssh-add -
+    - chmod 700 /root/.ssh
+    - '[[ -f /.dockerenv ]] && echo -e "Host *\n\tStrictHostKeyChecking no\n\n" > ~/.ssh/config'
+    - apt update && apt -y install jq wget
+    - wget https://go.dev/dl/go1.17.5.linux-amd64.tar.gz
+    - tar -C /usr/local -xzf go1.17.5.linux-amd64.tar.gz
+    - export PATH=$PATH:/usr/local/go/bin
+  script:
+    - ./qa/scripts/setupSamsungGauntlet.sh
+    - ./qa/scripts/testSamsungGauntlet.sh
+  after_script:
+    - ./qa/scripts/teardownSamsungGauntlet.sh
+  needs: ["build for linux arm64"]

qa/scripts/deployNode.sh

@@ -2,6 +2,9 @@
 
 # To run script: ./deployNode.sh $PROFILE
 
+# default to the VPC initially created
+VPC=${VPC:-vpc-0582f594d7d2ca2d4}
+
 function deploy_node() {
     # get AMI, security group and subnet ID 
     AMI=$(aws ssm get-parameters --names /aws/service/ami-amazon-linux-latest/amzn2-ami-hvm-x86_64-ebs --query 'Parameters[0].[Value]' --output text --profile $PROFILE)
@@ -10,13 +13,13 @@ function deploy_node() {
         exit 1
     fi
 
-    SECURITY_GROUP=$(aws ec2 describe-security-groups --filters Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978 Name=group-name,Values=default --query 'SecurityGroups[*].[GroupId]' --output text --profile $PROFILE)
+    SECURITY_GROUP=$(aws ec2 describe-security-groups --filters "Name=vpc-id,Values=$VPC" Name=group-name,Values=default --query 'SecurityGroups[*].[GroupId]' --output text --profile $PROFILE)
     if [[ $? > 0 ]]; then 
         echo "aws session manager failed to find security group"
         exit 1
     fi
     
-    SUBNET_ID=$(aws ec2 describe-subnets --filters 'Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978' 'Name=availability-zone,Values=us-east-2a' --query 'Subnets[0].SubnetId' --output text --profile $PROFILE)
+    SUBNET_ID=$(aws ec2 describe-subnets --filters "Name=vpc-id,Values=$VPC" 'Name=availability-zone,Values=us-east-2a' --query 'Subnets[0].SubnetId' --output text --profile $PROFILE)
     if [[ $? > 0 ]]; then 
         echo "aws session manager failed to find subnet ID"
         exit 1

qa/scripts/deploySingleNodeCluster.sh

@@ -0,0 +1,28 @@
+#!/bin/bash
+
+# To run script: ./deploySingleNodeCluster.sh
+# requires TF_VAR_gitlab_token env var to be set
+
+echo “$(pwd)”
+
+pushd ./qa/tf/ci/singlenode
+export TF_IN_AUTOMATION=1
+terraform init -input=false
+terraform apply -input=false -auto-approve
+popd
+
+# configure Featurebase
+
+# step 1a: get IPs of the cluster
+
+
+
+# step 1b: get IPs of the ingest nodes
+
+# step 2: write a featurebase.conf file
+
+# step 3: write featurebase.service
+
+# step 4: start featurebase
+
+# step 5: verify featurebase running

qa/scripts/ingestWorkload.sh

@@ -1,7 +1,14 @@
 #!/usr/bin/env bash
 
+# path for featurebase binary
+FEATUREBASE_PATH=/usr/local/bin
+
+# path for directory with csv directory files for all fields to be ingested 
+CSV_DIR_PATH=/data
+
+
 # To run:
-# ./ingestWorkload.sh {Path for featurebase binary} {Local host & port for featurebase} {Path for directory with csv files} {initialize flag}
+# ./ingestWorkload.sh {Local host & port for featurebase} {initialize flag}
 
 function delete_field {
     if (($INITIALIZE == 0));
@@ -30,18 +37,10 @@ function ingest_set_field {
     $FEATUREBASE_PATH/featurebase import --host $HOST -i $INDEX -f $FIELD $CSV_FILE 
 }
 
-# path for featurebase binary
-FEATUREBASE_PATH=$1
-shift
-
 # featurebase host & port
 HOST=$1
 shift
 
-# path for directory with csv directory files for all fields to be ingested 
-CSV_DIR_PATH=$1
-shift 
-
 # intialize flag - 0:disabled, 1:enabled - creates the index and fields for testing
 INITIALIZE=$1
 shift 

qa/scripts/perf.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+echo >&2 "performance testing"
+time ./simulacraData

qa/scripts/runSamsungGauntlet.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+
+#openssl rand -base64 32 | tr -d /=+ | cut -c -16
+
+./setupSamsungGauntlet.sh
+./testSamsungGauntlet.sh
+./teardownSamsungGauntlet.sh

qa/scripts/setupSamsungGauntlet.sh

@@ -0,0 +1,41 @@
+#!/bin/bash
+
+# To run script: ./setupSamsungGauntlet.sh
+# requires TF_VAR_gitlab_token env var to be set
+
+pushd ./qa/tf/gauntlet/samsung
+export TF_IN_AUTOMATION=1
+echo "Running terraform init..."
+terraform init -input=false
+echo "Running terraform apply..."
+terraform apply -input=false -auto-approve
+terraform output -json > samsung-gauntlet.json
+popd
+
+# get the bastion host
+BASTION=$(cat ./qa/tf/gauntlet/samsung/samsung-gauntlet.json | jq -r '[.ingest_ips][0]["value"][0]')
+echo "using bastion ${BASTION}"
+
+NODE=$(cat ./qa/tf/gauntlet/samsung/samsung-gauntlet.json | jq -r '[.data_node_ips][0]["value"][0]')
+echo "using node ${NODE}"
+
+# remember that the nodes will take at least 2 mins to be up and going and finish cloud-init
+#while true
+#do
+#  nc -G 2 -w 1 $BASTION 22
+#  if [ $? -eq 0 ]
+#  then
+#    break
+#  fi
+#done
+sleep 150
+
+# verify featurebase running
+ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o "StrictHostKeyChecking no" ec2-user@${BASTION} "curl -s http://${NODE}:10101/status" 
+if (( $? != 0 )) 
+then 
+    echo "Featurebase cluster not running"
+    exit 1
+fi
+
+

qa/scripts/teardownSamsungGauntlet.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+# To run script: ./teardownSamsungGauntlet.sh
+# requires TF_VAR_gitlab_token env var to be set
+
+cd qa/tf/gauntlet/samsung
+export TF_IN_AUTOMATION=1
+terraform destroy -auto-approve

qa/scripts/testSamsungGauntlet.sh

@@ -0,0 +1,48 @@
+#!/bin/bash
+
+# get the bastion host
+BASTION=$(cat ./qa/tf/gauntlet/samsung/samsung-gauntlet.json | jq -r '[.ingest_ips][0]["value"][0]')
+echo "using bastion ${BASTION}"
+
+NODE=$(cat ./qa/tf/gauntlet/samsung/samsung-gauntlet.json | jq -r '[.data_node_ips][0]["value"][0]')
+echo "using node ${NODE}"
+
+# generate csv files
+GOOS=linux GOARCH=arm64 go build ./qa/simulacraData/...
+scp -i ~/.ssh/gitlab-featurebase-ci.pem simulacraData ec2-user@${BASTION}:/data
+if (( $? != 0 )) 
+then 
+    echo "Copy failed"
+    exit 1
+fi
+
+ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem ec2-user@${BASTION} "cd /data && /data/simulacraData" 
+if (( $? != 0 )) 
+then 
+    echo "Making big files failed"
+    exit 1
+fi
+
+# ingest these files the way that samsung does it
+scp -i ~/.ssh/gitlab-featurebase-ci.pem ./qa/scripts/testSamsungPayload.sh ec2-user@${BASTION}:
+if (( $? != 0 )) 
+then 
+    echo "Copy ingest script failed"
+    exit 1
+fi
+
+ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem ec2-user@${BASTION} "./testSamsungPayload.sh http://${NODE}:10101 1" 
+if (( $? != 0 )) 
+then 
+    echo "Running 1 testSamsungPayload.sh failed"
+    exit 1
+fi
+
+ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem ec2-user@${BASTION} "./testSamsungPayload.sh http://${NODE}:10101 0" 
+if (( $? != 0 )) 
+then 
+    echo "Running 0 testSamsungPayload.sh failed"
+    exit 1
+fi
+
+# query workload that runs

qa/scripts/testSamsungPayload.sh

@@ -0,0 +1,84 @@
+#!/usr/bin/env bash
+
+# path for featurebase binary
+FEATUREBASE_PATH=/usr/local/bin
+
+# path for directory with csv directory files for all fields to be ingested 
+CSV_DIR_PATH=/data
+
+
+# To run:
+# ./testSamsungPayload.sh {Local host & port for featurebase} {initialize flag}
+
+function delete_field {
+    if (($INITIALIZE == 0));
+    then 
+        curl -XDELETE $HOST/index/$INDEX/field/$FIELD
+    fi
+}
+
+# Script to replicate samsung workload of deleting and re-ingesting fields every night 
+# outline delete and re-ingest workload 
+function ingest_int_field { 
+    delete_field
+    curl -XPOST $HOST/index/$INDEX/field/$FIELD -d '{"options": {"type": "int", "min": 0, "max":'$MAX'}}'
+    $FEATUREBASE_PATH/featurebase import --host $HOST -i $INDEX -f $FIELD $CSV_FILE 
+}
+
+function ingest_time_field {
+    delete_field
+    curl -XPOST $HOST/index/$INDEX/field/$FIELD -d '{"options": {"keys": true, "type": "time", "timeQuantum": "YMD"}}'
+    $FEATUREBASE_PATH/featurebase import --host $HOST -i $INDEX -f $FIELD $CSV_FILE 
+}
+
+function ingest_set_field {
+    delete_field
+    curl -XPOST $HOST/index/$INDEX/field/$FIELD -d '{"options": {"keys": true}}'
+    $FEATUREBASE_PATH/featurebase import --host $HOST -i $INDEX -f $FIELD $CSV_FILE 
+}
+
+# featurebase host & port
+HOST=$1
+shift
+
+# intialize flag - 0:disabled, 1:enabled - creates the index and fields for testing
+INITIALIZE=$1
+shift 
+
+# get a list of csv files in the directory 
+CSV_FILES=`ls $CSV_DIR_PATH/*.csv`
+
+# assign index name 
+INDEX="samsung"
+if (($INITIALIZE == 1));
+then 
+    curl -XPOST $HOST/index/$INDEX
+fi
+
+# perform delete and re-ingest for all fields 
+for CSV_FILE in ${CSV_FILES[@]}
+    do 
+        # get field name from csv file path 
+        FIELD="$(basename $CSV_FILE .csv)"
+        if [[ "$FIELD" == *"age"* ]];
+        then 
+            MAX=100 
+            ingest_int_field
+        elif [[ "$FIELD" == *"identifier"* ]];
+        then 
+            MAX=$((2**63 - 1)) # compute max value for 64bit
+            ingest_int_field
+        elif [[ "$FIELD" == *"ip"* ]];
+        then 
+            MAX=$((2**31 - 1)) # compute max value for 32bit
+            ingest_int_field
+        elif [[ "$FIELD" == *"time"* ]];
+        then 
+            ingest_time_field
+        else
+            ingest_set_field
+        fi
+    done
+
+
+

qa/tf/.modules/featurebase-cluster/README.md

@@ -0,0 +1,37 @@
+# Summary
+
+This module provisions a VPC, subnets, instances, keys, and security groups needed for a basic featurebase cluster running in AWS. It is meant to be used as a module. For example:
+
+```hcl
+module "featurebase" {
+  source "/path/to/module/"
+  cluster_prefix = "sprockets"
+  azs = ["us-east-1a", "us-east-1b", "us-east-1c"]
+}
+```
+
+The path to the module is wherever the `featurebase-cloud` directory is. So if you have put it in `/var/opt/terraform/modules/featurebase-cloud` then calling the module would look like:
+
+```hcl
+module "featurebase" {
+  source "/var/opt/terraform/modules/featurebase-cloud"
+  cluster_prefix = "sprockets"
+}
+```
+
+Much more is configurable; for a complete list, look in `variables.tf`. Reasonable defaults have been set.
+
+## AWS Access
+
+Please make sure you have set up your AWS access in either environment variables, or in the credentials file.
+
+Some useful links for this are:
+
+AWS Environment Variables <https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-envvars.html>
+
+## State
+
+State is currently kept locally, for as this is intended for PoCs. It can be stored in a remote s3 or GCS bucket if desired.
+
+
+ 

qa/tf/.modules/featurebase-cluster/main.tf

@@ -0,0 +1,229 @@
+data "aws_ami" "amazon_linux_2" {
+  most_recent = true
+  owners      = ["amazon"]
+  filter {
+    name   = "name"
+    values = ["amzn2-ami-hvm-*"]
+  }
+
+  filter {
+    name   = "virtualization-type"
+    values = ["hvm"]
+  }
+
+  filter {
+    name   = "architecture"
+    values = ["arm64"]
+  }
+}
+
+resource "aws_instance" "fb_cluster_nodes" {
+  count                  = var.fb_data_node_count
+  ami                    = data.aws_ami.amazon_linux_2.id
+  instance_type          = var.fb_data_node_type
+  key_name               = aws_key_pair.gitlab-featurebase-ci.key_name
+  vpc_security_group_ids = [aws_security_group.featurebase.id]
+  monitoring             = true
+  subnet_id              = var.subnet != "" ? var.subnet : module.vpc.private_subnets[count.index % length(module.vpc.private_subnets)]
+  availability_zone      = var.zone != "" ? var.zone : var.azs[count.index % length(var.azs)]
+  iam_instance_profile   = "${aws_iam_instance_profile.fb_cluster_node_profile.name}"
+
+  root_block_device {
+    volume_type = "gp3"
+    volume_size = 20
+  }
+
+  ebs_block_device {
+    device_name = "/dev/sdb"
+    volume_type = var.fb_data_disk_type
+    volume_size = var.fb_data_disk_size_gb
+    iops        = var.fb_data_disk_iops
+  }
+
+  tags = {
+    Prefix = "${var.cluster_prefix}"
+    Name   = "${var.cluster_prefix}-featurebase-cluster-${count.index}"
+    Role   = "cluster_node"
+  }
+
+  user_data = base64encode(templatefile("${path.module}/setup_cluster_node.sh.tpl", { gitlab_token = var.gitlab_token, cluster_prefix = var.cluster_prefix, node_count = var.fb_data_node_count, fb_cluster_replica_count = var.fb_cluster_replica_count, region = var.region }))
+}
+
+resource "aws_instance" "fb_ingest" {
+  count                       = var.fb_ingest_node_count
+  ami                         = data.aws_ami.amazon_linux_2.id
+  key_name                    = aws_key_pair.gitlab-featurebase-ci.key_name
+  vpc_security_group_ids      = [aws_security_group.ingest.id]
+  instance_type               = var.fb_ingest_type
+  associate_public_ip_address = true
+  monitoring                  = true
+  subnet_id                   = var.subnet != "" ? var.subnet : module.vpc.public_subnets[count.index % length(module.vpc.public_subnets)]
+  availability_zone           = var.zone != "" ? var.zone : var.azs[count.index % length(var.azs)]
+  iam_instance_profile        = "${aws_iam_instance_profile.fb_cluster_node_profile.name}"
+
+  root_block_device {
+    volume_type = "gp3"
+    volume_size = 20
+  }
+
+  ebs_block_device {
+    device_name = "/dev/sdb"
+    volume_type = var.fb_ingest_disk_type
+    volume_size = var.fb_ingest_disk_size_gb
+    iops        = var.fb_ingest_disk_iops
+  }
+
+  tags = {
+    Prefix = "${var.cluster_prefix}"
+    Name   = "${var.cluster_prefix}-featurebase-ingest-${count.index}"
+    Role   = "ingest_node"
+  }
+
+  user_data = base64encode(templatefile("${path.module}/setup_ingest_node.sh.tpl", { gitlab_token = var.gitlab_token, cluster_prefix = var.cluster_prefix, node_count = var.fb_ingest_node_count, this_node = count.index, region = var.region }))
+}
+
+resource "aws_key_pair" "gitlab-featurebase-ci" {
+  key_name   = "gitlab-featurebase-ci"
+  public_key = "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC91hhpVHNonAG7ku2ugpxEskf9KHeyHJPQJT26OHrMUw7R+T5A8TjqSzTau07sXQ/E9SO3ebV8SJ5PqeaQOnQB8VEvVNK0DjQH7ppvNg1Rfs42FZT9ttzTMvOjsSbK3vZTHXdoKQEdC9NxBwSkFIRGQojK1HUOq9xGrw31fA1OjSwlpLcbx7yyg18lcqW6UOptnVR8U9Yy9qQ5jZF1HtkQ6L9J+gv4o1UyNAUK2bopeGiXpBc3PQ/CFaFT2h/aqLBP66qAHsHVyAFD3PIRtplC5EHa8jXDgLacEls0uF7Q3kRPxvzcuo4g4VkOn1rDy9qH3vd2hT3aKVnM73FIDUiL"
+}
+
+resource "aws_security_group" "featurebase" {
+  name        = "allow_featurebase"
+  description = "Allow featurebase inbound traffic"
+  vpc_id      = module.vpc.vpc_id
+
+  ingress {
+    description = "TLS from Internal"
+    from_port   = 10101
+    to_port     = 10101
+    protocol    = "tcp"
+    cidr_blocks = [module.vpc.vpc_cidr_block]
+  }
+
+  ingress {
+
+    description = "GRPC from Internal"
+    from_port   = 20101
+    to_port     = 20101
+    protocol    = "tcp"
+    cidr_blocks = [module.vpc.vpc_cidr_block]
+  }
+
+  ingress {
+    description = "PostgreSQL from Internal"
+    from_port   = 55432
+    to_port     = 55432
+    protocol    = "tcp"
+    cidr_blocks = [module.vpc.vpc_cidr_block]
+  }
+
+  ingress {
+    description = "etcd from internal"
+    from_port   = 10301
+    to_port     = 10301
+    protocol    = "tcp"
+    cidr_blocks = [module.vpc.vpc_cidr_block]
+  }
+
+  ingress {
+    description = "etcd from internal 2"
+    from_port   = 10401
+    to_port     = 10401
+    protocol    = "tcp"
+    cidr_blocks = [module.vpc.vpc_cidr_block]
+  }
+
+  ingress {
+    description      = "SSH"
+    from_port        = 22
+    to_port          = 22
+    protocol         = "tcp"
+    cidr_blocks      = ["0.0.0.0/0"]
+    ipv6_cidr_blocks = ["::/0"]
+  }
+
+  egress {
+    from_port        = 0
+    to_port          = 0
+    protocol         = "-1"
+    cidr_blocks      = ["0.0.0.0/0"]
+    ipv6_cidr_blocks = ["::/0"]
+  }
+
+  tags = {
+    Name = "allow_featurebase"
+  }
+}
+
+resource "aws_security_group" "ingest" {
+  name        = "allow_ingest"
+  description = "Allow ingest inbound traffic"
+  vpc_id      = module.vpc.vpc_id
+
+  ingress {
+    from_port        = 10101
+    to_port          = 10101
+    protocol         = "tcp"
+    cidr_blocks      = ["0.0.0.0/0"]
+    ipv6_cidr_blocks = ["::/0"]
+  }
+
+  ingress {
+    description      = "SSH"
+    from_port        = 22
+    to_port          = 22
+    protocol         = "tcp"
+    cidr_blocks      = ["0.0.0.0/0"]
+    ipv6_cidr_blocks = ["::/0"]
+  }
+
+  egress {
+    from_port        = 0
+    to_port          = 0
+    protocol         = "-1"
+    cidr_blocks      = ["0.0.0.0/0"]
+    ipv6_cidr_blocks = ["::/0"]
+  }
+
+  tags = {
+    Name = "allow_ingest"
+  }
+}
+
+resource "aws_iam_instance_profile" "fb_cluster_node_profile" {
+  name = "fb_cluster_node_profile"
+  role = aws_iam_role.fb_cluster_node_role.name
+}
+
+resource "aws_iam_role" "fb_cluster_node_role" {
+  name = "fb_cluster_node"
+
+  assume_role_policy = jsonencode({
+    Version = "2012-10-17"
+    Statement = [
+      {
+        Action = "sts:AssumeRole"
+        Effect = "Allow"
+        Sid    = ""
+        Principal = {
+          Service = "ec2.amazonaws.com"
+        }
+      },
+    ]
+  })
+
+  inline_policy {
+    name = "ec2_read_all"
+    policy = jsonencode({
+      Version = "2012-10-17"
+      Statement = [
+        {
+          Action   = ["ec2:Describe*"]
+          Effect   = "Allow"
+          Resource = "*"
+        },
+      ]
+    })
+  }
+
+}

qa/tf/.modules/featurebase-cluster/outputs.tf

@@ -0,0 +1,7 @@
+output "ingest_ips" {
+    value = aws_instance.fb_ingest.*.public_ip
+}
+
+output "data_node_ips" {
+    value = aws_instance.fb_cluster_nodes.*.private_ip
+}

qa/tf/.modules/featurebase-cluster/provider.tf

@@ -0,0 +1,11 @@
+terraform {
+  required_version = ">= 0.13.1"
+
+  required_providers {
+    aws = {
+      source  = "hashicorp/aws"
+      version = ">= 3.38.0"
+    }
+  }
+}
+

qa/tf/.modules/featurebase-cluster/setup_cluster_node.sh.tpl

@@ -0,0 +1,188 @@
+#!/bin/bash
+
+#path to the featurebase.conf file
+CONFIG_FILE_PATH="/etc/featurebase.conf"
+#path to the featurebase.service file
+SERVICE_FILE_PATH="/etc/systemd/system/featurebase.service"
+
+AWS_INSTANCE_ID=""
+#IP of this node
+PRIVATE_IP=""
+PRIVATE_IP_INDEX=-1
+#IPs of the cluster
+CLUSTER_IPS=""
+
+get_aws_instance_id() {
+    echo "Getting AWS instance ID..."
+    while true
+    do
+        curl -s http://169.254.169.254/latest/meta-data/instance-id > /dev/null
+        if [ $? -eq 0 ]
+        then
+            break
+        fi
+    done
+    AWS_INSTANCE_ID=`curl http://169.254.169.254/latest/meta-data/instance-id`
+    echo "AWS instance ID is: $${AWS_INSTANCE_ID}"
+}
+
+wait_on_all_cluster_ips() {
+    echo "Waiting on all cluster IPs..."
+    # get IP for node 
+    IPS=$(aws ec2 describe-instances --filters "Name=instance-state-name, Values=running" "Name=tag:Role, Values=cluster_node" "Name=tag:Prefix, Values=${cluster_prefix}" --query 'Reservations[*].Instances[*].PrivateIpAddress' --output text --region ${region})
+    IP_LENGTH=`echo "$IPS" | wc -l`
+
+    for i in {0..24}
+    do 
+        echo "Comparing $${IP_LENGTH} with ${node_count}"
+        if [ $IP_LENGTH == "${node_count}" ]; then
+            echo "Cluster is up after $${i} tries." 
+            break
+        fi
+        sleep 10s
+    done
+
+    if [ $IP_LENGTH != "${node_count}" ]; then
+        echo "Timed out waiting for cluster to be available $${IP_LENGTH} actual nodes compared with ${node_count} desire nodes."
+        exit 1
+    fi
+}
+
+get_private_ip() {
+    echo "Getting private IP address..."
+    PRIVATE_IP=$(aws ec2 describe-instances --filters "Name=instance-state-name, Values=running" "Name=instance-id,Values=$${AWS_INSTANCE_ID}" --query 'Reservations[*].Instances[*].PrivateIpAddress' --output text --region ${region})
+    echo "Private IP is $${PRIVATE_IP}"
+}
+
+get_cluster_ips() {
+    echo "Getting cluster IPs..."
+    # get IP for node 
+    IPS=$(aws ec2 describe-instances --filters "Name=instance-state-name, Values=running" "Name=tag:Role, Values=cluster_node" "Name=tag:Prefix, Values=${cluster_prefix}" --query 'Reservations[*].Instances[*].PrivateIpAddress' --output text --region ${region})
+    IP_LENGTH=`echo "$IPS" | wc -l`
+
+    IFS=$'\n'
+    cnt=0
+    for ip in $IPS
+    do
+        echo $cnt $ip
+        if (($cnt + 1 != $IP_LENGTH)) 
+        then
+            CLUSTER_IPS="$${CLUSTER_IPS}p$${cnt}=http://$ip:10301,"
+        else
+            CLUSTER_IPS="$${CLUSTER_IPS}p$${cnt}=http://$ip:10301"
+        fi
+        echo "comparing $ip to $PRIVATE_IP"
+        if [ "$ip" = "$PRIVATE_IP" ]; then
+            PRIVATE_IP_INDEX=$cnt
+        fi
+        cnt=$((cnt+1))
+    done
+
+    echo "CLUSTER_IPS are: $${CLUSTER_IPS}"
+}
+
+write_featurebase_config_file() {
+    echo "Writing featurebase.conf file..."
+    cat << EOT > $${CONFIG_FILE_PATH}
+name = "p$${PRIVATE_IP_INDEX}"
+bind = "0.0.0.0:10101"
+bind-grpc = "0.0.0.0:20101"
+
+data-dir = "/data/featurebase"
+log-path = "/var/log/molecula/featurebase.log"
+
+max-file-count=900000
+max-map-count=900000
+
+long-query-time = "10s"
+
+[postgres]
+
+    bind = "localhost:55432"
+
+[cluster]
+
+    name = "${cluster_prefix}"
+    replicas = ${fb_cluster_replica_count}
+
+[etcd]
+
+    listen-client-address = "http://$${PRIVATE_IP}:10401"
+    listen-peer-address = "http://$${PRIVATE_IP}:10301"
+    initial-cluster = "$${CLUSTER_IPS}"
+
+[metric]
+
+    service = "prometheus"
+EOT
+
+    echo "featurebase.conf written to $${CONFIG_FILE_PATH}."
+}
+
+write_featurebase_service_file() {
+    echo "Writing featurebase.service file..."
+    cat << EOT > $${SERVICE_FILE_PATH}
+# Not Ansible managed
+
+[Unit]
+Description="Service for FeatureBase"
+
+[Service]
+RestartSec=30
+Restart=on-failure
+EnvironmentFile=
+User=molecula
+ExecStart=/usr/local/bin/featurebase server -c /etc/featurebase.conf
+
+[Install]
+EOT
+
+    echo "featurebase.service written to $${SERVICE_FILE_PATH}."
+    
+}
+
+#get the instance id
+get_aws_instance_id
+
+#copy the script so we can look at it later if needed
+sudo cp /var/lib/cloud/instances/$${AWS_INSTANCE_ID}/user-data.txt /home/ec2-user/setup_cluster_node.sh
+
+#wait for the count of nodes to equal requested nodes
+wait_on_all_cluster_ips
+
+#get private ip
+get_private_ip
+
+#generate cluster ips
+get_cluster_ips
+
+#write the featurebase config file
+write_featurebase_config_file
+
+#write the featurebase service file
+write_featurebase_service_file
+
+#get the featurebase binary and put in in the right spot
+echo "Getting featurebase binary..."
+curl --header "PRIVATE-TOKEN: ${gitlab_token}" -o "/home/ec2-user/featurebase_linux_arm64" https://gitlab.com/api/v4/projects/molecula%2Ffeaturebase/jobs/artifacts/master/raw/featurebase_linux_arm64?job=build%20for%20linux%20arm64
+chown ec2-user:ec2-user "/home/ec2-user/featurebase_linux_arm64"
+chmod ugo+x "/home/ec2-user/featurebase_linux_arm64"
+
+mv /home/ec2-user/featurebase_linux_arm64 /usr/local/bin/featurebase
+echo "featurebase binary copied."
+
+sudo mkdir /data
+sudo mkfs.ext4 /dev/nvme1n1
+sudo mount /dev/nvme1n1 /data
+
+adduser molecula
+sudo mkdir /var/log/molecula
+sudo chown molecula /var/log/molecula
+sudo mkdir -p /data/featurebase
+sudo chown molecula /data/featurebase
+sudo systemctl daemon-reload
+sudo systemctl start featurebase
+sudo systemctl enable featurebase
+sudo systemctl status featurebase
+
+echo "Done!"

qa/tf/.modules/featurebase-cluster/setup_ingest_node.sh.tpl

@@ -0,0 +1,66 @@
+#!/bin/bash
+
+AWS_INSTANCE_ID=""
+
+get_aws_instance_id() {
+    echo "Getting AWS instance ID..."
+    while true
+    do
+        curl -s http://169.254.169.254/latest/meta-data/instance-id > /dev/null
+        if [ $? -eq 0 ]
+        then
+            break
+        fi
+    done
+    AWS_INSTANCE_ID=`curl http://169.254.169.254/latest/meta-data/instance-id`
+    echo "AWS instance ID is: $${AWS_INSTANCE_ID}"
+}
+
+wait_on_all_ingest_ips() {
+    echo "Waiting on all cluster IPs..."
+    # get IP for node 
+    IPS=$(aws ec2 describe-instances --filters "Name=instance-state-name, Values=running" "Name=tag:Role, Values=ingest_node" "Name=tag:Prefix, Values=${cluster_prefix}" --query 'Reservations[*].Instances[*].PrivateIpAddress' --output text --region ${region})
+    IP_LENGTH=`echo "$IPS" | wc -l`
+
+    for i in {0..24}
+    do 
+        echo "Comparing $${IP_LENGTH} with ${node_count}"
+        if [ $IP_LENGTH == "${node_count}" ]; then
+            echo "Cluster is up after $${i} tries." 
+            break
+        fi
+        sleep 10s
+    done
+
+    if [ $IP_LENGTH != "${node_count}" ]; then
+        echo "Timed out waiting for cluster to be available $${IP_LENGTH} actual nodes compared with ${node_count} desire nodes."
+        exit 1
+    fi
+}
+
+#copy the script so we can look at it later if needed
+sudo cp /var/lib/cloud/instances/$${AWS_INSTANCE_ID}/user-data.txt ~/setup_ingest_node.sh
+
+#get the instance id
+get_aws_instance_id
+
+#wait for the count of nodes to equal requested nodes
+wait_on_all_ingest_ips
+
+echo "Getting featurebase binary..."
+curl --header "PRIVATE-TOKEN: ${gitlab_token}" -o "/home/ec2-user/featurebase_linux_arm64" https://gitlab.com/api/v4/projects/molecula%2Ffeaturebase/jobs/artifacts/master/raw/featurebase_linux_arm64?job=build%20for%20linux%20arm64
+chown ec2-user:ec2-user "/home/ec2-user/featurebase_linux_arm64"
+chmod ugo+x "/home/ec2-user/featurebase_linux_arm64"
+
+mv /home/ec2-user/featurebase_linux_arm64 /usr/local/bin/featurebase
+echo "featurebase binary copied."
+
+
+sudo mkdir /data
+sudo mkfs.ext4 /dev/nvme1n1
+sudo mount /dev/nvme1n1 /data
+
+sudo chown -R ec2-user /data
+
+
+

qa/tf/.modules/featurebase-cluster/variables.tf

@@ -0,0 +1,93 @@
+variable "cluster_prefix" {
+  type        = string
+  description = "This is a identifier that will be prefixed to created resources"
+}
+
+variable "fb_ingest_type" {
+  type    = string
+  default = "c6g.2xlarge"
+}
+
+variable "fb_ingest_node_count" {
+  type    = number
+  default = 1
+}
+
+variable "fb_data_node_type" {
+  type    = string
+  default = "c6g.16xlarge"
+}
+
+variable "fb_data_node_count" {
+  type    = number
+  default = 3
+}
+
+variable "fb_cluster_replica_count" {
+  type    = number
+  default = 1
+}
+
+variable "subnet" {
+  default = ""
+}
+
+variable "zone" {
+  default = ""
+}
+
+variable "fb_data_disk_type" {
+  default = "gp3"
+}
+variable "fb_data_disk_iops" {
+  default = 1000
+}
+
+variable "fb_data_disk_size_gb" {
+  default = 100
+}
+
+variable "fb_ingest_disk_type" {
+  default = "gp3"
+}
+variable "fb_ingest_disk_iops" {
+  default = 1000
+}
+
+variable "fb_ingest_disk_size_gb" {
+  default = 100
+}
+
+variable "azs" {
+  type    = list(any)
+  default = ["us-east-2a", "us-east-2b", "us-east-2c"]
+}
+
+variable "private_subnets" {
+  type    = list(any)
+  default = ["10.0.1.0/24", "10.0.2.0/24", "10.0.3.0/24"]
+}
+
+variable "public_subnets" {
+  type    = list(any)
+  default = ["10.0.101.0/24", "10.0.102.0/24", "10.0.103.0/24"]
+}
+
+variable "vpc_cidr" {
+  default = "10.0.0.0/16"
+}
+
+variable "region" {
+  description = "Region to create AWS resources in"
+  type        = string
+}
+
+variable "profile" {
+  description = "Profile to use to authenticate with AWS"
+  type        = string
+}
+
+variable "gitlab_token" {
+  description = "Gitlab API token"
+  type        = string
+}

qa/tf/.modules/featurebase-cluster/vpc.tf

@@ -0,0 +1,16 @@
+module "vpc" {
+  source = "terraform-aws-modules/vpc/aws"
+
+  name = "${var.cluster_prefix}"
+  cidr = var.vpc_cidr
+  azs             = var.azs
+  private_subnets = var.private_subnets
+  public_subnets  = var.public_subnets
+
+  enable_nat_gateway = true
+  enable_vpn_gateway = false
+
+  tags = {
+    Name = "${var.cluster_prefix}"
+  }
+}

qa/tf/README.md

@@ -0,0 +1,16 @@
+# Deploy testing environments with this one weird trick!
+
+This directory contains Terraform to deploy test environments both ad-hoc and as part of CI/CD pipelines.
+
+The .modules contains the guts of the operation, the things you probably want are in the other directories, each with a README.
+
+## How to Terraform
+
+With terraform installed (`brew install terraform` if not)...
+
+You can do `terraform plan` -> `terraform apply` to spin up a cluster, `terraform destroy` to tear one down.
+
+## Other prerequisites:
+Please read these carefully.
+
+

qa/tf/ci/singlenode/main.tf

@@ -0,0 +1,10 @@
+
+module "ci-cluster" {
+    source = "../../.modules/featurebase-cluster"
+    cluster_prefix = "ci-single-node"
+    region = var.region
+    profile = var.profile
+    fb_data_node_type = "m6g.large"
+    fb_data_node_count = 1
+    gitlab_token = var.gitlab_token
+}

qa/tf/ci/singlenode/outputs.tf

@@ -0,0 +1,9 @@
+output "ingest_ips" {
+    description = "List of ingest IPs"
+    value = module.ci-cluster.ingest_ips
+}
+
+output "data_node_ips" {
+  description = "List of data node IPs"
+  value = module.ci-cluster.data_node_ips
+}

qa/tf/ci/singlenode/provider.tf

@@ -0,0 +1,4 @@
+provider "aws" {
+  region  = var.region
+  profile = var.profile
+}

qa/tf/ci/singlenode/tf.auto.tfvars

@@ -0,0 +1,2 @@
+region  = "us-east-2"
+profile = "service-terraform"

qa/tf/ci/singlenode/variables.tf

@@ -0,0 +1,14 @@
+variable "region" {
+  description = "The AWS region in which the VPC should be built"
+  type        = string
+}
+
+variable "profile" {
+  description = "The name of the AWS profile Terraform should use for auth."
+  type        = string
+}
+
+variable "gitlab_token" {
+  description = "The API token for taking to Gitlab API - expected to come from an env variable."
+  type        = string
+}

qa/tf/gauntlet/samsung/README.md

@@ -0,0 +1,35 @@
+With terraform installed (`brew install terraform` if not)...
+
+You can do `terraform plan` -> `terraform apply` to spin up a cluster, `terraform destroy` to tear one down.
+
+## Other prerequisites:
+Please read these carefully.
+
+Be in the `tf` directory (e.g., when you try to run a `terraform` command, the output of `pwd` should be `.../featurebase/qa/tf`)
+
+Currently, the path to the terraform module is using a local reference, i.e., in `main.tf`, the source line is assuming that you have `molecula-terraform` project installed locally, such that the `molecular-terraform` project and `featurebase` have the same parent directory (e.g., `...A/featurebase/qa/tf` and `...A/molecular-terraform/aws/.modules/featurebase-cluster` should both be valid paths).
+
+In addition, you must currently have a local copy of the `fb901` branch for the `molecular-terraform` project (located in the previously specified directory).
+
+Last thing, there is a key that is currently in 1Password (in the `Shared` vault, called `gitlab-featurebase-ci AWS key`) that must be in `~/.ssh/`, `chmod 400`, named `gitlab-featurebase-ci.pem`. You need this key to SSH to these instances. Assuming an `~/.ssh/config` like the following (append to the top of yours)
+```
+Host test_*
+  User ec2-user
+  IdentityFile ~/.ssh/gitlab-featurebase-ci.pem
+Host test_ingest
+  HostName 3.143.237.165
+Host test_node
+  HostName 10.0.1.142
+  ProxyJump test_ingest
+```
+except with the `test_ingest`'s `HostName` being the public, `ingest_ips` output from `terraform output` and `test_node`'s `HostName` being one of the private, `data_node_ips` output from `terraform output`. (Hopefully the rationale to use the ssh config to do the jumping like this makes sense; you can do `ssh test_ingest` or `ssh test_node` with minimal further fiddling.)
+
+OR specify cert to us directly thus:
+
+`ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem ec2-user@ip_address`
+
+-A is used to ensure key forwarding.
+
+### TODOs
+* We need a `user-data.sh` script which sets up/installs featurebase (possibly installs go, most likely pulls the artifacts from GitLab; sets up featurebase on both the node and data workers).
+* Logs get sent to DataDog?

qa/tf/gauntlet/samsung/main.tf

@@ -0,0 +1,13 @@
+module "samsung-cluster" {
+    source = "../../.modules/featurebase-cluster"
+    cluster_prefix = "samsung-gauntlet"
+    region = var.region
+    profile = var.profile
+    fb_data_node_type = "m6g.xlarge"
+    fb_data_disk_iops = 10000
+    fb_data_node_count = 3
+    fb_ingest_type = "m6g.large"
+    fb_ingest_disk_iops = 10000
+    fb_ingest_node_count = 1
+    gitlab_token = var.gitlab_token
+}

qa/tf/gauntlet/samsung/outputs.tf

@@ -0,0 +1,9 @@
+output "ingest_ips" {
+    description = "List of ingest IPs"
+    value = module.samsung-cluster.ingest_ips
+}
+
+output "data_node_ips" {
+  description = "List of data node IPs"
+  value = module.samsung-cluster.data_node_ips
+}

qa/tf/gauntlet/samsung/provider.tf

@@ -0,0 +1,4 @@
+provider "aws" {
+  region  = var.region
+  profile = var.profile
+}

qa/tf/gauntlet/samsung/samsung-gauntlet.json

@@ -0,0 +1,30 @@
+{
+  "data_node_ips": {
+    "sensitive": false,
+    "type": [
+      "tuple",
+      [
+        "string",
+        "string",
+        "string"
+      ]
+    ],
+    "value": [
+      "10.0.1.144",
+      "10.0.2.108",
+      "10.0.3.178"
+    ]
+  },
+  "ingest_ips": {
+    "sensitive": false,
+    "type": [
+      "tuple",
+      [
+        "string"
+      ]
+    ],
+    "value": [
+      "3.145.104.76"
+    ]
+  }
+}

qa/tf/gauntlet/samsung/tf.auto.tfvars

@@ -0,0 +1,2 @@
+region  = "us-east-2"
+profile = "service-terraform"

qa/tf/gauntlet/samsung/variables.tf

@@ -0,0 +1,14 @@
+variable "region" {
+  description = "The AWS region in which the VPC should be built"
+  type        = string
+}
+
+variable "profile" {
+  description = "The name of the AWS profile Terraform should use for auth."
+  type        = string
+}
+
+variable "gitlab_token" {
+  description = "The API token for taking to Gitlab API - expected to come from an env variable."
+  type        = string
+}