diff --git a/.gitignore b/.gitignore index 2082bd5c0..d7356ece0 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,8 @@ pilosa *.dot .idea/ .*.swp +.terraform/ +*.tfstate +launch.json +.terraform.lock.hcl +__pycache__/ diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 60d1a309d..5ceba4d2d 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -19,6 +19,7 @@ stages: - test - build - integration + - gauntlet golangci-lint: image: golangci/golangci-lint:v1.39.0 @@ -244,3 +245,45 @@ deploy node for linux amd64: - ./qa/scripts/deployNode.sh $PROFILE needs: - job: build for linux amd64 + +gauntlet: + stage: gauntlet + timeout: 4h + image: registry.gitlab.com/gitlab-org/cloud-deploy/aws-base:latest + variables: + PROFILE: "default" + AWS_SSH_PRIVATE_KEY: $AWS_FBCI_SSH_KEY + AWS_ACCESS_KEY_ID: $AWS_FBCI_ACCESS_KEY_ID + AWS_SECRET_ACCESS_KEY: $AWS_FBCI_SECRET_ACCESS_KEY + # TODO: For now, run always + rules: + - if: '$CI_PIPELINE_SOURCE == "schedule" && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH' + before_script: + - apt-get update && apt-get install -y gnupg software-properties-common curl git + - curl -fsSL https://apt.releases.hashicorp.com/gpg | apt-key add - + - apt-add-repository "deb [arch=amd64] https://apt.releases.hashicorp.com $(lsb_release -cs) main" + - apt-get update && apt-get install terraform + - aws configure set aws_access_key_id $AWS_FBCI_ACCESS_KEY_ID + - aws configure set aws_secret_access_key $AWS_FBCI_SECRET_ACCESS_KEY + - aws configure set region "us-east-2" + - aws configure set aws_profile $PROFILE + - echo $AWS_FBCI_SSH_KEY > gitlab-featurebase-ci.pem + - chmod 400 gitlab-featurebase-ci.pem + - 'which ssh-agent || ( apt-get update -y && apt-get install openssh-client -y )' + - eval $(ssh-agent -s) + - mkdir -p ~/.ssh + - echo $AWS_FBCI_SSH_KEY > /root/.ssh/gitlab-featurebase-ci.pem + - chmod 400 /root/.ssh/gitlab-featurebase-ci.pem + - echo "$AWS_FBCI_SSH_KEY" | ssh-add - + - chmod 700 /root/.ssh + - '[[ -f /.dockerenv ]] && echo -e "Host *\n\tStrictHostKeyChecking no\n\n" > ~/.ssh/config' + - apt update && apt -y install jq wget + - wget https://go.dev/dl/go1.17.5.linux-amd64.tar.gz + - tar -C /usr/local -xzf go1.17.5.linux-amd64.tar.gz + - export PATH=$PATH:/usr/local/go/bin + script: + - ./qa/scripts/setupSamsungGauntlet.sh + - ./qa/scripts/testSamsungGauntlet.sh + after_script: + - ./qa/scripts/teardownSamsungGauntlet.sh + needs: ["build for linux arm64"] \ No newline at end of file diff --git a/ctl/rbf_check.go b/ctl/rbf_check.go index 00594b861..f3d40912d 100644 --- a/ctl/rbf_check.go +++ b/ctl/rbf_check.go @@ -26,7 +26,7 @@ func NewRBFCheckCommand(stdin io.Reader, stdout, stderr io.Writer) *RBFCheckComm } } -// Run executes the export. +// Run executes a consistency check of an RBF database. func (cmd *RBFCheckCommand) Run(ctx context.Context) error { // Open database. db := rbf.NewDB(cmd.Path, nil) @@ -37,7 +37,15 @@ func (cmd *RBFCheckCommand) Run(ctx context.Context) error { // Run check on the database. if err := db.Check(); err != nil { - return err + switch err := err.(type) { + case rbf.ErrorList: + for i := range err { + fmt.Fprintln(cmd.Stdout, err[i]) + } + default: + fmt.Fprintln(cmd.Stdout, err) + } + return fmt.Errorf("check failed") } // If successful, print a success message. diff --git a/ctl/rbf_check_test.go b/ctl/rbf_check_test.go new file mode 100644 index 000000000..c11cd30dc --- /dev/null +++ b/ctl/rbf_check_test.go @@ -0,0 +1,33 @@ +// Copyright 2021 Molecula Corp. All rights reserved. +package ctl + +import ( + "bytes" + "context" + "path/filepath" + "testing" +) + +func TestRBFCheckCommand_Run(t *testing.T) { + t.Run("OK", func(t *testing.T) { + var stdout, stderr bytes.Buffer + cmd := NewRBFCheckCommand(bytes.NewReader(nil), &stdout, &stderr) + cmd.Path = filepath.Join("testdata", "rbf-check", "ok") + if err := cmd.Run(context.Background()); err != nil { + t.Fatal(err) + } else if got, want := stdout.String(), `ok`+"\n"; got != want { + t.Fatalf("got:\n%s\n\nwant:\n%s", got, want) + } + }) + + t.Run("ErrInvalidPageType", func(t *testing.T) { + var stdout, stderr bytes.Buffer + cmd := NewRBFCheckCommand(bytes.NewReader(nil), &stdout, &stderr) + cmd.Path = filepath.Join("testdata", "rbf-check", "err-invalid-page-type") + if err := cmd.Run(context.Background()); err == nil || err.Error() != `check failed` { + t.Fatal(err) + } else if got, want := stdout.String(), `page not in-use & not free: pgno=4`+"\n"; got != want { + t.Fatalf("got:\n%s\n\nwant:\n%s", got, want) + } + }) +} diff --git a/ctl/rbf_pages.go b/ctl/rbf_pages.go index ca3484f31..b774175ae 100644 --- a/ctl/rbf_pages.go +++ b/ctl/rbf_pages.go @@ -49,7 +49,16 @@ func (cmd *RBFPagesCommand) Run(ctx context.Context) error { // Iterate over each page and grab info. infos, err := tx.PageInfos() if err != nil { - return err + fmt.Fprintln(cmd.Stdout, "ERRORS:") + switch err := err.(type) { + case rbf.ErrorList: + for i := range err { + fmt.Fprintln(cmd.Stdout, err[i]) + } + default: + fmt.Fprintln(cmd.Stdout, err) + } + fmt.Fprintln(cmd.Stdout, "") } // Write header. diff --git a/ctl/rbf_pages_test.go b/ctl/rbf_pages_test.go new file mode 100644 index 000000000..73c395e1b --- /dev/null +++ b/ctl/rbf_pages_test.go @@ -0,0 +1,52 @@ +// Copyright 2021 Molecula Corp. All rights reserved. +package ctl + +import ( + "bytes" + "context" + "path/filepath" + "testing" +) + +func TestRBFPagesCommand_Run(t *testing.T) { + t.Run("OK", func(t *testing.T) { + want := ` +ID TYPE EXTRA +======== ========== ==================== +0 meta pageN=4,walid=4,rootrec=1,freelist=2 +1 rootrec next=0 +2 leaf flags=x2,celln=0 +3 leaf flags=x2,celln=1 +`[1:] + + var stdout, stderr bytes.Buffer + cmd := NewRBFPagesCommand(bytes.NewReader(nil), &stdout, &stderr) + cmd.Path = filepath.Join("testdata", "rbf-pages", "ok") + if err := cmd.Run(context.Background()); err != nil { + t.Fatal(err) + } else if got := stdout.String(); got != want { + t.Fatalf("got:\n%s\n\nwant:\n%s", got, want) + } + }) + + t.Run("ErrInvalidPageType", func(t *testing.T) { + want := ` +ID TYPE EXTRA +======== ========== ==================== +0 meta pageN=5,walid=4,rootrec=1,freelist=2 +1 rootrec next=0 +2 leaf flags=x2,celln=0 +3 leaf flags=x2,celln=1 +4 unknown [] +`[1:] + + var stdout, stderr bytes.Buffer + cmd := NewRBFPagesCommand(bytes.NewReader(nil), &stdout, &stderr) + cmd.Path = filepath.Join("testdata", "rbf-pages", "err-invalid-page-type") + if err := cmd.Run(context.Background()); err != nil { + t.Fatal(err) + } else if got := stdout.String(); got != want { + t.Fatalf("got:\n%s\n\nwant:\n%s", got, want) + } + }) +} diff --git a/ctl/testdata/rbf-check/err-invalid-page-type/data b/ctl/testdata/rbf-check/err-invalid-page-type/data new file mode 100644 index 000000000..f088c25c9 Binary files /dev/null and b/ctl/testdata/rbf-check/err-invalid-page-type/data differ diff --git a/ctl/testdata/rbf-check/err-invalid-page-type/wal b/ctl/testdata/rbf-check/err-invalid-page-type/wal new file mode 100644 index 000000000..e69de29bb diff --git a/ctl/testdata/rbf-check/ok/data b/ctl/testdata/rbf-check/ok/data new file mode 100644 index 000000000..e4c3b621e Binary files /dev/null and b/ctl/testdata/rbf-check/ok/data differ diff --git a/ctl/testdata/rbf-check/ok/wal b/ctl/testdata/rbf-check/ok/wal new file mode 100644 index 000000000..e69de29bb diff --git a/ctl/testdata/rbf-pages/err-invalid-page-type/data b/ctl/testdata/rbf-pages/err-invalid-page-type/data new file mode 100644 index 000000000..f088c25c9 Binary files /dev/null and b/ctl/testdata/rbf-pages/err-invalid-page-type/data differ diff --git a/ctl/testdata/rbf-pages/err-invalid-page-type/wal b/ctl/testdata/rbf-pages/err-invalid-page-type/wal new file mode 100644 index 000000000..e69de29bb diff --git a/ctl/testdata/rbf-pages/ok/data b/ctl/testdata/rbf-pages/ok/data new file mode 100644 index 000000000..e4c3b621e Binary files /dev/null and b/ctl/testdata/rbf-pages/ok/data differ diff --git a/ctl/testdata/rbf-pages/ok/wal b/ctl/testdata/rbf-pages/ok/wal new file mode 100644 index 000000000..e69de29bb diff --git a/qa/scripts/config.json b/qa/scripts/config.json new file mode 100644 index 000000000..e69de29bb diff --git a/qa/scripts/deployNode.sh b/qa/scripts/deployNode.sh index b928a31e7..7b5f53796 100755 --- a/qa/scripts/deployNode.sh +++ b/qa/scripts/deployNode.sh @@ -2,6 +2,9 @@ # To run script: ./deployNode.sh $PROFILE +# default to the VPC initially created +VPC=${VPC:-vpc-0582f594d7d2ca2d4} + function deploy_node() { # get AMI, security group and subnet ID AMI=$(aws ssm get-parameters --names /aws/service/ami-amazon-linux-latest/amzn2-ami-hvm-x86_64-ebs --query 'Parameters[0].[Value]' --output text --profile $PROFILE) @@ -10,13 +13,13 @@ function deploy_node() { exit 1 fi - SECURITY_GROUP=$(aws ec2 describe-security-groups --filters Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978 Name=group-name,Values=default --query 'SecurityGroups[*].[GroupId]' --output text --profile $PROFILE) + SECURITY_GROUP=$(aws ec2 describe-security-groups --filters "Name=vpc-id,Values=$VPC" Name=group-name,Values=default --query 'SecurityGroups[*].[GroupId]' --output text --profile $PROFILE) if [[ $? > 0 ]]; then echo "aws session manager failed to find security group" exit 1 fi - SUBNET_ID=$(aws ec2 describe-subnets --filters 'Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978' 'Name=availability-zone,Values=us-east-2a' --query 'Subnets[0].SubnetId' --output text --profile $PROFILE) + SUBNET_ID=$(aws ec2 describe-subnets --filters "Name=vpc-id,Values=$VPC" 'Name=availability-zone,Values=us-east-2a' --query 'Subnets[0].SubnetId' --output text --profile $PROFILE) if [[ $? > 0 ]]; then echo "aws session manager failed to find subnet ID" exit 1 diff --git a/qa/scripts/deploySingleNodeCluster.sh b/qa/scripts/deploySingleNodeCluster.sh new file mode 100644 index 000000000..d4d1240ec --- /dev/null +++ b/qa/scripts/deploySingleNodeCluster.sh @@ -0,0 +1,28 @@ +#!/bin/bash + +# To run script: ./deploySingleNodeCluster.sh +# requires TF_VAR_gitlab_token env var to be set + +echo “$(pwd)” + +pushd ./qa/tf/ci/singlenode +export TF_IN_AUTOMATION=1 +terraform init -input=false +terraform apply -input=false -auto-approve +popd + +# configure Featurebase + +# step 1a: get IPs of the cluster + + + +# step 1b: get IPs of the ingest nodes + +# step 2: write a featurebase.conf file + +# step 3: write featurebase.service + +# step 4: start featurebase + +# step 5: verify featurebase running \ No newline at end of file diff --git a/qa/scripts/ingestWorkload.sh b/qa/scripts/ingestWorkload.sh index 23d11dd32..6b87fb762 100755 --- a/qa/scripts/ingestWorkload.sh +++ b/qa/scripts/ingestWorkload.sh @@ -1,7 +1,14 @@ #!/usr/bin/env bash +# path for featurebase binary +FEATUREBASE_PATH=/usr/local/bin + +# path for directory with csv directory files for all fields to be ingested +CSV_DIR_PATH=/data + + # To run: -# ./ingestWorkload.sh {Path for featurebase binary} {Local host & port for featurebase} {Path for directory with csv files} {initialize flag} +# ./ingestWorkload.sh {Local host & port for featurebase} {initialize flag} function delete_field { if (($INITIALIZE == 0)); @@ -30,18 +37,10 @@ function ingest_set_field { $FEATUREBASE_PATH/featurebase import --host $HOST -i $INDEX -f $FIELD $CSV_FILE } -# path for featurebase binary -FEATUREBASE_PATH=$1 -shift - # featurebase host & port HOST=$1 shift -# path for directory with csv directory files for all fields to be ingested -CSV_DIR_PATH=$1 -shift - # intialize flag - 0:disabled, 1:enabled - creates the index and fields for testing INITIALIZE=$1 shift diff --git a/qa/scripts/perf.sh b/qa/scripts/perf.sh new file mode 100644 index 000000000..41a734d59 --- /dev/null +++ b/qa/scripts/perf.sh @@ -0,0 +1,3 @@ +#!/bin/bash +echo >&2 "performance testing" +time ./simulacraData diff --git a/qa/scripts/runSamsungGauntlet.sh b/qa/scripts/runSamsungGauntlet.sh new file mode 100644 index 000000000..d99a03baa --- /dev/null +++ b/qa/scripts/runSamsungGauntlet.sh @@ -0,0 +1,8 @@ +#!/bin/bash + + +#openssl rand -base64 32 | tr -d /=+ | cut -c -16 + +./setupSamsungGauntlet.sh +./testSamsungGauntlet.sh +./teardownSamsungGauntlet.sh diff --git a/qa/scripts/setupSamsungGauntlet.sh b/qa/scripts/setupSamsungGauntlet.sh new file mode 100755 index 000000000..6a867c18d --- /dev/null +++ b/qa/scripts/setupSamsungGauntlet.sh @@ -0,0 +1,41 @@ +#!/bin/bash + +# To run script: ./setupSamsungGauntlet.sh +# requires TF_VAR_gitlab_token env var to be set + +pushd ./qa/tf/gauntlet/samsung +export TF_IN_AUTOMATION=1 +echo "Running terraform init..." +terraform init -input=false +echo "Running terraform apply..." +terraform apply -input=false -auto-approve +terraform output -json > samsung-gauntlet.json +popd + +# get the bastion host +BASTION=$(cat ./qa/tf/gauntlet/samsung/samsung-gauntlet.json | jq -r '[.ingest_ips][0]["value"][0]') +echo "using bastion ${BASTION}" + +NODE=$(cat ./qa/tf/gauntlet/samsung/samsung-gauntlet.json | jq -r '[.data_node_ips][0]["value"][0]') +echo "using node ${NODE}" + +# remember that the nodes will take at least 2 mins to be up and going and finish cloud-init +#while true +#do +# nc -G 2 -w 1 $BASTION 22 +# if [ $? -eq 0 ] +# then +# break +# fi +#done +sleep 150 + +# verify featurebase running +ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o "StrictHostKeyChecking no" ec2-user@${BASTION} "curl -s http://${NODE}:10101/status" +if (( $? != 0 )) +then + echo "Featurebase cluster not running" + exit 1 +fi + + diff --git a/qa/scripts/teardownSamsungGauntlet.sh b/qa/scripts/teardownSamsungGauntlet.sh new file mode 100755 index 000000000..ae614d99b --- /dev/null +++ b/qa/scripts/teardownSamsungGauntlet.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +# To run script: ./teardownSamsungGauntlet.sh +# requires TF_VAR_gitlab_token env var to be set + +cd qa/tf/gauntlet/samsung +export TF_IN_AUTOMATION=1 +terraform destroy -auto-approve diff --git a/qa/scripts/testSamsungGauntlet.sh b/qa/scripts/testSamsungGauntlet.sh new file mode 100755 index 000000000..ff8d13cb7 --- /dev/null +++ b/qa/scripts/testSamsungGauntlet.sh @@ -0,0 +1,48 @@ +#!/bin/bash + +# get the bastion host +BASTION=$(cat ./qa/tf/gauntlet/samsung/samsung-gauntlet.json | jq -r '[.ingest_ips][0]["value"][0]') +echo "using bastion ${BASTION}" + +NODE=$(cat ./qa/tf/gauntlet/samsung/samsung-gauntlet.json | jq -r '[.data_node_ips][0]["value"][0]') +echo "using node ${NODE}" + +# generate csv files +GOOS=linux GOARCH=arm64 go build ./qa/simulacraData/... +scp -i ~/.ssh/gitlab-featurebase-ci.pem simulacraData ec2-user@${BASTION}:/data +if (( $? != 0 )) +then + echo "Copy failed" + exit 1 +fi + +ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem ec2-user@${BASTION} "cd /data && /data/simulacraData" +if (( $? != 0 )) +then + echo "Making big files failed" + exit 1 +fi + +# ingest these files the way that samsung does it +scp -i ~/.ssh/gitlab-featurebase-ci.pem ./qa/scripts/testSamsungPayload.sh ec2-user@${BASTION}: +if (( $? != 0 )) +then + echo "Copy ingest script failed" + exit 1 +fi + +ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem ec2-user@${BASTION} "./testSamsungPayload.sh http://${NODE}:10101 1" +if (( $? != 0 )) +then + echo "Running 1 testSamsungPayload.sh failed" + exit 1 +fi + +ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem ec2-user@${BASTION} "./testSamsungPayload.sh http://${NODE}:10101 0" +if (( $? != 0 )) +then + echo "Running 0 testSamsungPayload.sh failed" + exit 1 +fi + +# query workload that runs \ No newline at end of file diff --git a/qa/scripts/testSamsungPayload.sh b/qa/scripts/testSamsungPayload.sh new file mode 100755 index 000000000..8c36c63dc --- /dev/null +++ b/qa/scripts/testSamsungPayload.sh @@ -0,0 +1,84 @@ +#!/usr/bin/env bash + +# path for featurebase binary +FEATUREBASE_PATH=/usr/local/bin + +# path for directory with csv directory files for all fields to be ingested +CSV_DIR_PATH=/data + + +# To run: +# ./testSamsungPayload.sh {Local host & port for featurebase} {initialize flag} + +function delete_field { + if (($INITIALIZE == 0)); + then + curl -XDELETE $HOST/index/$INDEX/field/$FIELD + fi +} + +# Script to replicate samsung workload of deleting and re-ingesting fields every night +# outline delete and re-ingest workload +function ingest_int_field { + delete_field + curl -XPOST $HOST/index/$INDEX/field/$FIELD -d '{"options": {"type": "int", "min": 0, "max":'$MAX'}}' + $FEATUREBASE_PATH/featurebase import --host $HOST -i $INDEX -f $FIELD $CSV_FILE +} + +function ingest_time_field { + delete_field + curl -XPOST $HOST/index/$INDEX/field/$FIELD -d '{"options": {"keys": true, "type": "time", "timeQuantum": "YMD"}}' + $FEATUREBASE_PATH/featurebase import --host $HOST -i $INDEX -f $FIELD $CSV_FILE +} + +function ingest_set_field { + delete_field + curl -XPOST $HOST/index/$INDEX/field/$FIELD -d '{"options": {"keys": true}}' + $FEATUREBASE_PATH/featurebase import --host $HOST -i $INDEX -f $FIELD $CSV_FILE +} + +# featurebase host & port +HOST=$1 +shift + +# intialize flag - 0:disabled, 1:enabled - creates the index and fields for testing +INITIALIZE=$1 +shift + +# get a list of csv files in the directory +CSV_FILES=`ls $CSV_DIR_PATH/*.csv` + +# assign index name +INDEX="samsung" +if (($INITIALIZE == 1)); +then + curl -XPOST $HOST/index/$INDEX +fi + +# perform delete and re-ingest for all fields +for CSV_FILE in ${CSV_FILES[@]} + do + # get field name from csv file path + FIELD="$(basename $CSV_FILE .csv)" + if [[ "$FIELD" == *"age"* ]]; + then + MAX=100 + ingest_int_field + elif [[ "$FIELD" == *"identifier"* ]]; + then + MAX=$((2**63 - 1)) # compute max value for 64bit + ingest_int_field + elif [[ "$FIELD" == *"ip"* ]]; + then + MAX=$((2**31 - 1)) # compute max value for 32bit + ingest_int_field + elif [[ "$FIELD" == *"time"* ]]; + then + ingest_time_field + else + ingest_set_field + fi + done + + + diff --git a/qa/tf/.modules/featurebase-cluster/README.md b/qa/tf/.modules/featurebase-cluster/README.md new file mode 100644 index 000000000..d1525adba --- /dev/null +++ b/qa/tf/.modules/featurebase-cluster/README.md @@ -0,0 +1,37 @@ +# Summary + +This module provisions a VPC, subnets, instances, keys, and security groups needed for a basic featurebase cluster running in AWS. It is meant to be used as a module. For example: + +```hcl +module "featurebase" { + source "/path/to/module/" + cluster_prefix = "sprockets" + azs = ["us-east-1a", "us-east-1b", "us-east-1c"] +} +``` + +The path to the module is wherever the `featurebase-cloud` directory is. So if you have put it in `/var/opt/terraform/modules/featurebase-cloud` then calling the module would look like: + +```hcl +module "featurebase" { + source "/var/opt/terraform/modules/featurebase-cloud" + cluster_prefix = "sprockets" +} +``` + +Much more is configurable; for a complete list, look in `variables.tf`. Reasonable defaults have been set. + +## AWS Access + +Please make sure you have set up your AWS access in either environment variables, or in the credentials file. + +Some useful links for this are: + +AWS Environment Variables + +## State + +State is currently kept locally, for as this is intended for PoCs. It can be stored in a remote s3 or GCS bucket if desired. + + + \ No newline at end of file diff --git a/qa/tf/.modules/featurebase-cluster/main.tf b/qa/tf/.modules/featurebase-cluster/main.tf new file mode 100644 index 000000000..f8c612e49 --- /dev/null +++ b/qa/tf/.modules/featurebase-cluster/main.tf @@ -0,0 +1,229 @@ +data "aws_ami" "amazon_linux_2" { + most_recent = true + owners = ["amazon"] + filter { + name = "name" + values = ["amzn2-ami-hvm-*"] + } + + filter { + name = "virtualization-type" + values = ["hvm"] + } + + filter { + name = "architecture" + values = ["arm64"] + } +} + +resource "aws_instance" "fb_cluster_nodes" { + count = var.fb_data_node_count + ami = data.aws_ami.amazon_linux_2.id + instance_type = var.fb_data_node_type + key_name = aws_key_pair.gitlab-featurebase-ci.key_name + vpc_security_group_ids = [aws_security_group.featurebase.id] + monitoring = true + subnet_id = var.subnet != "" ? var.subnet : module.vpc.private_subnets[count.index % length(module.vpc.private_subnets)] + availability_zone = var.zone != "" ? var.zone : var.azs[count.index % length(var.azs)] + iam_instance_profile = "${aws_iam_instance_profile.fb_cluster_node_profile.name}" + + root_block_device { + volume_type = "gp3" + volume_size = 20 + } + + ebs_block_device { + device_name = "/dev/sdb" + volume_type = var.fb_data_disk_type + volume_size = var.fb_data_disk_size_gb + iops = var.fb_data_disk_iops + } + + tags = { + Prefix = "${var.cluster_prefix}" + Name = "${var.cluster_prefix}-featurebase-cluster-${count.index}" + Role = "cluster_node" + } + + user_data = base64encode(templatefile("${path.module}/setup_cluster_node.sh.tpl", { gitlab_token = var.gitlab_token, cluster_prefix = var.cluster_prefix, node_count = var.fb_data_node_count, fb_cluster_replica_count = var.fb_cluster_replica_count, region = var.region })) +} + +resource "aws_instance" "fb_ingest" { + count = var.fb_ingest_node_count + ami = data.aws_ami.amazon_linux_2.id + key_name = aws_key_pair.gitlab-featurebase-ci.key_name + vpc_security_group_ids = [aws_security_group.ingest.id] + instance_type = var.fb_ingest_type + associate_public_ip_address = true + monitoring = true + subnet_id = var.subnet != "" ? var.subnet : module.vpc.public_subnets[count.index % length(module.vpc.public_subnets)] + availability_zone = var.zone != "" ? var.zone : var.azs[count.index % length(var.azs)] + iam_instance_profile = "${aws_iam_instance_profile.fb_cluster_node_profile.name}" + + root_block_device { + volume_type = "gp3" + volume_size = 20 + } + + ebs_block_device { + device_name = "/dev/sdb" + volume_type = var.fb_ingest_disk_type + volume_size = var.fb_ingest_disk_size_gb + iops = var.fb_ingest_disk_iops + } + + tags = { + Prefix = "${var.cluster_prefix}" + Name = "${var.cluster_prefix}-featurebase-ingest-${count.index}" + Role = "ingest_node" + } + + user_data = base64encode(templatefile("${path.module}/setup_ingest_node.sh.tpl", { gitlab_token = var.gitlab_token, cluster_prefix = var.cluster_prefix, node_count = var.fb_ingest_node_count, this_node = count.index, region = var.region })) +} + +resource "aws_key_pair" "gitlab-featurebase-ci" { + key_name = "gitlab-featurebase-ci" + public_key = "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC91hhpVHNonAG7ku2ugpxEskf9KHeyHJPQJT26OHrMUw7R+T5A8TjqSzTau07sXQ/E9SO3ebV8SJ5PqeaQOnQB8VEvVNK0DjQH7ppvNg1Rfs42FZT9ttzTMvOjsSbK3vZTHXdoKQEdC9NxBwSkFIRGQojK1HUOq9xGrw31fA1OjSwlpLcbx7yyg18lcqW6UOptnVR8U9Yy9qQ5jZF1HtkQ6L9J+gv4o1UyNAUK2bopeGiXpBc3PQ/CFaFT2h/aqLBP66qAHsHVyAFD3PIRtplC5EHa8jXDgLacEls0uF7Q3kRPxvzcuo4g4VkOn1rDy9qH3vd2hT3aKVnM73FIDUiL" +} + +resource "aws_security_group" "featurebase" { + name = "allow_featurebase" + description = "Allow featurebase inbound traffic" + vpc_id = module.vpc.vpc_id + + ingress { + description = "TLS from Internal" + from_port = 10101 + to_port = 10101 + protocol = "tcp" + cidr_blocks = [module.vpc.vpc_cidr_block] + } + + ingress { + + description = "GRPC from Internal" + from_port = 20101 + to_port = 20101 + protocol = "tcp" + cidr_blocks = [module.vpc.vpc_cidr_block] + } + + ingress { + description = "PostgreSQL from Internal" + from_port = 55432 + to_port = 55432 + protocol = "tcp" + cidr_blocks = [module.vpc.vpc_cidr_block] + } + + ingress { + description = "etcd from internal" + from_port = 10301 + to_port = 10301 + protocol = "tcp" + cidr_blocks = [module.vpc.vpc_cidr_block] + } + + ingress { + description = "etcd from internal 2" + from_port = 10401 + to_port = 10401 + protocol = "tcp" + cidr_blocks = [module.vpc.vpc_cidr_block] + } + + ingress { + description = "SSH" + from_port = 22 + to_port = 22 + protocol = "tcp" + cidr_blocks = ["0.0.0.0/0"] + ipv6_cidr_blocks = ["::/0"] + } + + egress { + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] + ipv6_cidr_blocks = ["::/0"] + } + + tags = { + Name = "allow_featurebase" + } +} + +resource "aws_security_group" "ingest" { + name = "allow_ingest" + description = "Allow ingest inbound traffic" + vpc_id = module.vpc.vpc_id + + ingress { + from_port = 10101 + to_port = 10101 + protocol = "tcp" + cidr_blocks = ["0.0.0.0/0"] + ipv6_cidr_blocks = ["::/0"] + } + + ingress { + description = "SSH" + from_port = 22 + to_port = 22 + protocol = "tcp" + cidr_blocks = ["0.0.0.0/0"] + ipv6_cidr_blocks = ["::/0"] + } + + egress { + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] + ipv6_cidr_blocks = ["::/0"] + } + + tags = { + Name = "allow_ingest" + } +} + +resource "aws_iam_instance_profile" "fb_cluster_node_profile" { + name = "fb_cluster_node_profile" + role = aws_iam_role.fb_cluster_node_role.name +} + +resource "aws_iam_role" "fb_cluster_node_role" { + name = "fb_cluster_node" + + assume_role_policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Action = "sts:AssumeRole" + Effect = "Allow" + Sid = "" + Principal = { + Service = "ec2.amazonaws.com" + } + }, + ] + }) + + inline_policy { + name = "ec2_read_all" + policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Action = ["ec2:Describe*"] + Effect = "Allow" + Resource = "*" + }, + ] + }) + } + +} \ No newline at end of file diff --git a/qa/tf/.modules/featurebase-cluster/outputs.tf b/qa/tf/.modules/featurebase-cluster/outputs.tf new file mode 100644 index 000000000..e52e7344c --- /dev/null +++ b/qa/tf/.modules/featurebase-cluster/outputs.tf @@ -0,0 +1,7 @@ +output "ingest_ips" { + value = aws_instance.fb_ingest.*.public_ip +} + +output "data_node_ips" { + value = aws_instance.fb_cluster_nodes.*.private_ip +} \ No newline at end of file diff --git a/qa/tf/.modules/featurebase-cluster/provider.tf b/qa/tf/.modules/featurebase-cluster/provider.tf new file mode 100644 index 000000000..cf53a4ed7 --- /dev/null +++ b/qa/tf/.modules/featurebase-cluster/provider.tf @@ -0,0 +1,11 @@ +terraform { + required_version = ">= 0.13.1" + + required_providers { + aws = { + source = "hashicorp/aws" + version = ">= 3.38.0" + } + } +} + diff --git a/qa/tf/.modules/featurebase-cluster/setup_cluster_node.sh.tpl b/qa/tf/.modules/featurebase-cluster/setup_cluster_node.sh.tpl new file mode 100644 index 000000000..05d55d973 --- /dev/null +++ b/qa/tf/.modules/featurebase-cluster/setup_cluster_node.sh.tpl @@ -0,0 +1,188 @@ +#!/bin/bash + +#path to the featurebase.conf file +CONFIG_FILE_PATH="/etc/featurebase.conf" +#path to the featurebase.service file +SERVICE_FILE_PATH="/etc/systemd/system/featurebase.service" + +AWS_INSTANCE_ID="" +#IP of this node +PRIVATE_IP="" +PRIVATE_IP_INDEX=-1 +#IPs of the cluster +CLUSTER_IPS="" + +get_aws_instance_id() { + echo "Getting AWS instance ID..." + while true + do + curl -s http://169.254.169.254/latest/meta-data/instance-id > /dev/null + if [ $? -eq 0 ] + then + break + fi + done + AWS_INSTANCE_ID=`curl http://169.254.169.254/latest/meta-data/instance-id` + echo "AWS instance ID is: $${AWS_INSTANCE_ID}" +} + +wait_on_all_cluster_ips() { + echo "Waiting on all cluster IPs..." + # get IP for node + IPS=$(aws ec2 describe-instances --filters "Name=instance-state-name, Values=running" "Name=tag:Role, Values=cluster_node" "Name=tag:Prefix, Values=${cluster_prefix}" --query 'Reservations[*].Instances[*].PrivateIpAddress' --output text --region ${region}) + IP_LENGTH=`echo "$IPS" | wc -l` + + for i in {0..24} + do + echo "Comparing $${IP_LENGTH} with ${node_count}" + if [ $IP_LENGTH == "${node_count}" ]; then + echo "Cluster is up after $${i} tries." + break + fi + sleep 10s + done + + if [ $IP_LENGTH != "${node_count}" ]; then + echo "Timed out waiting for cluster to be available $${IP_LENGTH} actual nodes compared with ${node_count} desire nodes." + exit 1 + fi +} + +get_private_ip() { + echo "Getting private IP address..." + PRIVATE_IP=$(aws ec2 describe-instances --filters "Name=instance-state-name, Values=running" "Name=instance-id,Values=$${AWS_INSTANCE_ID}" --query 'Reservations[*].Instances[*].PrivateIpAddress' --output text --region ${region}) + echo "Private IP is $${PRIVATE_IP}" +} + +get_cluster_ips() { + echo "Getting cluster IPs..." + # get IP for node + IPS=$(aws ec2 describe-instances --filters "Name=instance-state-name, Values=running" "Name=tag:Role, Values=cluster_node" "Name=tag:Prefix, Values=${cluster_prefix}" --query 'Reservations[*].Instances[*].PrivateIpAddress' --output text --region ${region}) + IP_LENGTH=`echo "$IPS" | wc -l` + + IFS=$'\n' + cnt=0 + for ip in $IPS + do + echo $cnt $ip + if (($cnt + 1 != $IP_LENGTH)) + then + CLUSTER_IPS="$${CLUSTER_IPS}p$${cnt}=http://$ip:10301," + else + CLUSTER_IPS="$${CLUSTER_IPS}p$${cnt}=http://$ip:10301" + fi + echo "comparing $ip to $PRIVATE_IP" + if [ "$ip" = "$PRIVATE_IP" ]; then + PRIVATE_IP_INDEX=$cnt + fi + cnt=$((cnt+1)) + done + + echo "CLUSTER_IPS are: $${CLUSTER_IPS}" +} + +write_featurebase_config_file() { + echo "Writing featurebase.conf file..." + cat << EOT > $${CONFIG_FILE_PATH} +name = "p$${PRIVATE_IP_INDEX}" +bind = "0.0.0.0:10101" +bind-grpc = "0.0.0.0:20101" + +data-dir = "/data/featurebase" +log-path = "/var/log/molecula/featurebase.log" + +max-file-count=900000 +max-map-count=900000 + +long-query-time = "10s" + +[postgres] + + bind = "localhost:55432" + +[cluster] + + name = "${cluster_prefix}" + replicas = ${fb_cluster_replica_count} + +[etcd] + + listen-client-address = "http://$${PRIVATE_IP}:10401" + listen-peer-address = "http://$${PRIVATE_IP}:10301" + initial-cluster = "$${CLUSTER_IPS}" + +[metric] + + service = "prometheus" +EOT + + echo "featurebase.conf written to $${CONFIG_FILE_PATH}." +} + +write_featurebase_service_file() { + echo "Writing featurebase.service file..." + cat << EOT > $${SERVICE_FILE_PATH} +# Not Ansible managed + +[Unit] +Description="Service for FeatureBase" + +[Service] +RestartSec=30 +Restart=on-failure +EnvironmentFile= +User=molecula +ExecStart=/usr/local/bin/featurebase server -c /etc/featurebase.conf + +[Install] +EOT + + echo "featurebase.service written to $${SERVICE_FILE_PATH}." + +} + +#get the instance id +get_aws_instance_id + +#copy the script so we can look at it later if needed +sudo cp /var/lib/cloud/instances/$${AWS_INSTANCE_ID}/user-data.txt /home/ec2-user/setup_cluster_node.sh + +#wait for the count of nodes to equal requested nodes +wait_on_all_cluster_ips + +#get private ip +get_private_ip + +#generate cluster ips +get_cluster_ips + +#write the featurebase config file +write_featurebase_config_file + +#write the featurebase service file +write_featurebase_service_file + +#get the featurebase binary and put in in the right spot +echo "Getting featurebase binary..." +curl --header "PRIVATE-TOKEN: ${gitlab_token}" -o "/home/ec2-user/featurebase_linux_arm64" https://gitlab.com/api/v4/projects/molecula%2Ffeaturebase/jobs/artifacts/master/raw/featurebase_linux_arm64?job=build%20for%20linux%20arm64 +chown ec2-user:ec2-user "/home/ec2-user/featurebase_linux_arm64" +chmod ugo+x "/home/ec2-user/featurebase_linux_arm64" + +mv /home/ec2-user/featurebase_linux_arm64 /usr/local/bin/featurebase +echo "featurebase binary copied." + +sudo mkdir /data +sudo mkfs.ext4 /dev/nvme1n1 +sudo mount /dev/nvme1n1 /data + +adduser molecula +sudo mkdir /var/log/molecula +sudo chown molecula /var/log/molecula +sudo mkdir -p /data/featurebase +sudo chown molecula /data/featurebase +sudo systemctl daemon-reload +sudo systemctl start featurebase +sudo systemctl enable featurebase +sudo systemctl status featurebase + +echo "Done!" \ No newline at end of file diff --git a/qa/tf/.modules/featurebase-cluster/setup_ingest_node.sh.tpl b/qa/tf/.modules/featurebase-cluster/setup_ingest_node.sh.tpl new file mode 100644 index 000000000..1e93149b1 --- /dev/null +++ b/qa/tf/.modules/featurebase-cluster/setup_ingest_node.sh.tpl @@ -0,0 +1,66 @@ +#!/bin/bash + +AWS_INSTANCE_ID="" + +get_aws_instance_id() { + echo "Getting AWS instance ID..." + while true + do + curl -s http://169.254.169.254/latest/meta-data/instance-id > /dev/null + if [ $? -eq 0 ] + then + break + fi + done + AWS_INSTANCE_ID=`curl http://169.254.169.254/latest/meta-data/instance-id` + echo "AWS instance ID is: $${AWS_INSTANCE_ID}" +} + +wait_on_all_ingest_ips() { + echo "Waiting on all cluster IPs..." + # get IP for node + IPS=$(aws ec2 describe-instances --filters "Name=instance-state-name, Values=running" "Name=tag:Role, Values=ingest_node" "Name=tag:Prefix, Values=${cluster_prefix}" --query 'Reservations[*].Instances[*].PrivateIpAddress' --output text --region ${region}) + IP_LENGTH=`echo "$IPS" | wc -l` + + for i in {0..24} + do + echo "Comparing $${IP_LENGTH} with ${node_count}" + if [ $IP_LENGTH == "${node_count}" ]; then + echo "Cluster is up after $${i} tries." + break + fi + sleep 10s + done + + if [ $IP_LENGTH != "${node_count}" ]; then + echo "Timed out waiting for cluster to be available $${IP_LENGTH} actual nodes compared with ${node_count} desire nodes." + exit 1 + fi +} + +#copy the script so we can look at it later if needed +sudo cp /var/lib/cloud/instances/$${AWS_INSTANCE_ID}/user-data.txt ~/setup_ingest_node.sh + +#get the instance id +get_aws_instance_id + +#wait for the count of nodes to equal requested nodes +wait_on_all_ingest_ips + +echo "Getting featurebase binary..." +curl --header "PRIVATE-TOKEN: ${gitlab_token}" -o "/home/ec2-user/featurebase_linux_arm64" https://gitlab.com/api/v4/projects/molecula%2Ffeaturebase/jobs/artifacts/master/raw/featurebase_linux_arm64?job=build%20for%20linux%20arm64 +chown ec2-user:ec2-user "/home/ec2-user/featurebase_linux_arm64" +chmod ugo+x "/home/ec2-user/featurebase_linux_arm64" + +mv /home/ec2-user/featurebase_linux_arm64 /usr/local/bin/featurebase +echo "featurebase binary copied." + + +sudo mkdir /data +sudo mkfs.ext4 /dev/nvme1n1 +sudo mount /dev/nvme1n1 /data + +sudo chown -R ec2-user /data + + + diff --git a/qa/tf/.modules/featurebase-cluster/variables.tf b/qa/tf/.modules/featurebase-cluster/variables.tf new file mode 100644 index 000000000..dd0da90c7 --- /dev/null +++ b/qa/tf/.modules/featurebase-cluster/variables.tf @@ -0,0 +1,93 @@ +variable "cluster_prefix" { + type = string + description = "This is a identifier that will be prefixed to created resources" +} + +variable "fb_ingest_type" { + type = string + default = "c6g.2xlarge" +} + +variable "fb_ingest_node_count" { + type = number + default = 1 +} + +variable "fb_data_node_type" { + type = string + default = "c6g.16xlarge" +} + +variable "fb_data_node_count" { + type = number + default = 3 +} + +variable "fb_cluster_replica_count" { + type = number + default = 1 +} + +variable "subnet" { + default = "" +} + +variable "zone" { + default = "" +} + +variable "fb_data_disk_type" { + default = "gp3" +} +variable "fb_data_disk_iops" { + default = 1000 +} + +variable "fb_data_disk_size_gb" { + default = 100 +} + +variable "fb_ingest_disk_type" { + default = "gp3" +} +variable "fb_ingest_disk_iops" { + default = 1000 +} + +variable "fb_ingest_disk_size_gb" { + default = 100 +} + +variable "azs" { + type = list(any) + default = ["us-east-2a", "us-east-2b", "us-east-2c"] +} + +variable "private_subnets" { + type = list(any) + default = ["10.0.1.0/24", "10.0.2.0/24", "10.0.3.0/24"] +} + +variable "public_subnets" { + type = list(any) + default = ["10.0.101.0/24", "10.0.102.0/24", "10.0.103.0/24"] +} + +variable "vpc_cidr" { + default = "10.0.0.0/16" +} + +variable "region" { + description = "Region to create AWS resources in" + type = string +} + +variable "profile" { + description = "Profile to use to authenticate with AWS" + type = string +} + +variable "gitlab_token" { + description = "Gitlab API token" + type = string +} diff --git a/qa/tf/.modules/featurebase-cluster/vpc.tf b/qa/tf/.modules/featurebase-cluster/vpc.tf new file mode 100644 index 000000000..4a09c9275 --- /dev/null +++ b/qa/tf/.modules/featurebase-cluster/vpc.tf @@ -0,0 +1,16 @@ +module "vpc" { + source = "terraform-aws-modules/vpc/aws" + + name = "${var.cluster_prefix}" + cidr = var.vpc_cidr + azs = var.azs + private_subnets = var.private_subnets + public_subnets = var.public_subnets + + enable_nat_gateway = true + enable_vpn_gateway = false + + tags = { + Name = "${var.cluster_prefix}" + } +} \ No newline at end of file diff --git a/qa/tf/README.md b/qa/tf/README.md new file mode 100644 index 000000000..7117ebf3c --- /dev/null +++ b/qa/tf/README.md @@ -0,0 +1,16 @@ +# Deploy testing environments with this one weird trick! + +This directory contains Terraform to deploy test environments both ad-hoc and as part of CI/CD pipelines. + +The .modules contains the guts of the operation, the things you probably want are in the other directories, each with a README. + +## How to Terraform + +With terraform installed (`brew install terraform` if not)... + +You can do `terraform plan` -> `terraform apply` to spin up a cluster, `terraform destroy` to tear one down. + +## Other prerequisites: +Please read these carefully. + + diff --git a/qa/tf/ci/singlenode/main.tf b/qa/tf/ci/singlenode/main.tf new file mode 100644 index 000000000..d50f41182 --- /dev/null +++ b/qa/tf/ci/singlenode/main.tf @@ -0,0 +1,10 @@ + +module "ci-cluster" { + source = "../../.modules/featurebase-cluster" + cluster_prefix = "ci-single-node" + region = var.region + profile = var.profile + fb_data_node_type = "m6g.large" + fb_data_node_count = 1 + gitlab_token = var.gitlab_token +} diff --git a/qa/tf/ci/singlenode/outputs.tf b/qa/tf/ci/singlenode/outputs.tf new file mode 100644 index 000000000..adcc96dc9 --- /dev/null +++ b/qa/tf/ci/singlenode/outputs.tf @@ -0,0 +1,9 @@ +output "ingest_ips" { + description = "List of ingest IPs" + value = module.ci-cluster.ingest_ips +} + +output "data_node_ips" { + description = "List of data node IPs" + value = module.ci-cluster.data_node_ips +} \ No newline at end of file diff --git a/qa/tf/ci/singlenode/provider.tf b/qa/tf/ci/singlenode/provider.tf new file mode 100644 index 000000000..c0fc95d9d --- /dev/null +++ b/qa/tf/ci/singlenode/provider.tf @@ -0,0 +1,4 @@ +provider "aws" { + region = var.region + profile = var.profile +} \ No newline at end of file diff --git a/qa/tf/ci/singlenode/tf.auto.tfvars b/qa/tf/ci/singlenode/tf.auto.tfvars new file mode 100644 index 000000000..ac6de62a6 --- /dev/null +++ b/qa/tf/ci/singlenode/tf.auto.tfvars @@ -0,0 +1,2 @@ +region = "us-east-2" +profile = "service-terraform" \ No newline at end of file diff --git a/qa/tf/ci/singlenode/variables.tf b/qa/tf/ci/singlenode/variables.tf new file mode 100644 index 000000000..e87a8f517 --- /dev/null +++ b/qa/tf/ci/singlenode/variables.tf @@ -0,0 +1,14 @@ +variable "region" { + description = "The AWS region in which the VPC should be built" + type = string +} + +variable "profile" { + description = "The name of the AWS profile Terraform should use for auth." + type = string +} + +variable "gitlab_token" { + description = "The API token for taking to Gitlab API - expected to come from an env variable." + type = string +} \ No newline at end of file diff --git a/qa/tf/gauntlet/samsung/README.md b/qa/tf/gauntlet/samsung/README.md new file mode 100644 index 000000000..e7b633516 --- /dev/null +++ b/qa/tf/gauntlet/samsung/README.md @@ -0,0 +1,35 @@ +With terraform installed (`brew install terraform` if not)... + +You can do `terraform plan` -> `terraform apply` to spin up a cluster, `terraform destroy` to tear one down. + +## Other prerequisites: +Please read these carefully. + +Be in the `tf` directory (e.g., when you try to run a `terraform` command, the output of `pwd` should be `.../featurebase/qa/tf`) + +Currently, the path to the terraform module is using a local reference, i.e., in `main.tf`, the source line is assuming that you have `molecula-terraform` project installed locally, such that the `molecular-terraform` project and `featurebase` have the same parent directory (e.g., `...A/featurebase/qa/tf` and `...A/molecular-terraform/aws/.modules/featurebase-cluster` should both be valid paths). + +In addition, you must currently have a local copy of the `fb901` branch for the `molecular-terraform` project (located in the previously specified directory). + +Last thing, there is a key that is currently in 1Password (in the `Shared` vault, called `gitlab-featurebase-ci AWS key`) that must be in `~/.ssh/`, `chmod 400`, named `gitlab-featurebase-ci.pem`. You need this key to SSH to these instances. Assuming an `~/.ssh/config` like the following (append to the top of yours) +``` +Host test_* + User ec2-user + IdentityFile ~/.ssh/gitlab-featurebase-ci.pem +Host test_ingest + HostName 3.143.237.165 +Host test_node + HostName 10.0.1.142 + ProxyJump test_ingest +``` +except with the `test_ingest`'s `HostName` being the public, `ingest_ips` output from `terraform output` and `test_node`'s `HostName` being one of the private, `data_node_ips` output from `terraform output`. (Hopefully the rationale to use the ssh config to do the jumping like this makes sense; you can do `ssh test_ingest` or `ssh test_node` with minimal further fiddling.) + +OR specify cert to us directly thus: + +`ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem ec2-user@ip_address` + +-A is used to ensure key forwarding. + +### TODOs +* We need a `user-data.sh` script which sets up/installs featurebase (possibly installs go, most likely pulls the artifacts from GitLab; sets up featurebase on both the node and data workers). +* Logs get sent to DataDog? diff --git a/qa/tf/gauntlet/samsung/main.tf b/qa/tf/gauntlet/samsung/main.tf new file mode 100644 index 000000000..926e54cc3 --- /dev/null +++ b/qa/tf/gauntlet/samsung/main.tf @@ -0,0 +1,13 @@ +module "samsung-cluster" { + source = "../../.modules/featurebase-cluster" + cluster_prefix = "samsung-gauntlet" + region = var.region + profile = var.profile + fb_data_node_type = "m6g.xlarge" + fb_data_disk_iops = 10000 + fb_data_node_count = 3 + fb_ingest_type = "m6g.large" + fb_ingest_disk_iops = 10000 + fb_ingest_node_count = 1 + gitlab_token = var.gitlab_token +} diff --git a/qa/tf/gauntlet/samsung/outputs.tf b/qa/tf/gauntlet/samsung/outputs.tf new file mode 100644 index 000000000..0c405bed0 --- /dev/null +++ b/qa/tf/gauntlet/samsung/outputs.tf @@ -0,0 +1,9 @@ +output "ingest_ips" { + description = "List of ingest IPs" + value = module.samsung-cluster.ingest_ips +} + +output "data_node_ips" { + description = "List of data node IPs" + value = module.samsung-cluster.data_node_ips +} \ No newline at end of file diff --git a/qa/tf/gauntlet/samsung/provider.tf b/qa/tf/gauntlet/samsung/provider.tf new file mode 100644 index 000000000..c0fc95d9d --- /dev/null +++ b/qa/tf/gauntlet/samsung/provider.tf @@ -0,0 +1,4 @@ +provider "aws" { + region = var.region + profile = var.profile +} \ No newline at end of file diff --git a/qa/tf/gauntlet/samsung/samsung-gauntlet.json b/qa/tf/gauntlet/samsung/samsung-gauntlet.json new file mode 100644 index 000000000..8760c7322 --- /dev/null +++ b/qa/tf/gauntlet/samsung/samsung-gauntlet.json @@ -0,0 +1,30 @@ +{ + "data_node_ips": { + "sensitive": false, + "type": [ + "tuple", + [ + "string", + "string", + "string" + ] + ], + "value": [ + "10.0.1.144", + "10.0.2.108", + "10.0.3.178" + ] + }, + "ingest_ips": { + "sensitive": false, + "type": [ + "tuple", + [ + "string" + ] + ], + "value": [ + "3.145.104.76" + ] + } +} diff --git a/qa/tf/gauntlet/samsung/tf.auto.tfvars b/qa/tf/gauntlet/samsung/tf.auto.tfvars new file mode 100644 index 000000000..ac6de62a6 --- /dev/null +++ b/qa/tf/gauntlet/samsung/tf.auto.tfvars @@ -0,0 +1,2 @@ +region = "us-east-2" +profile = "service-terraform" \ No newline at end of file diff --git a/qa/tf/gauntlet/samsung/variables.tf b/qa/tf/gauntlet/samsung/variables.tf new file mode 100644 index 000000000..e87a8f517 --- /dev/null +++ b/qa/tf/gauntlet/samsung/variables.tf @@ -0,0 +1,14 @@ +variable "region" { + description = "The AWS region in which the VPC should be built" + type = string +} + +variable "profile" { + description = "The name of the AWS profile Terraform should use for auth." + type = string +} + +variable "gitlab_token" { + description = "The API token for taking to Gitlab API - expected to come from an env variable." + type = string +} \ No newline at end of file diff --git a/rbf/cursor.go b/rbf/cursor.go index eda6bec1d..2419bc5f8 100644 --- a/rbf/cursor.go +++ b/rbf/cursor.go @@ -932,6 +932,10 @@ func (c *Cursor) First() error { case PageTypeBranch: elem.index = 0 + if n := readCellN(buf); elem.index >= n { // branch cell index must less than cell count + return fmt.Errorf("branch cell index out of range: pgno=%d i=%d n=%d", elem.pgno, elem.index, n) + } + // Read cell pgno into the next stack level. cell := readBranchCell(buf, elem.index) diff --git a/rbf/rbf.go b/rbf/rbf.go index 74a5135eb..4a66ba309 100644 --- a/rbf/rbf.go +++ b/rbf/rbf.go @@ -799,3 +799,46 @@ func (m *Metric) Inc(d time.Duration) { fmt.Printf("metric:%10s avg=%dns\n", m.name, int(m.d)/m.n) } } + +// ErrorList represents a list of errors. +type ErrorList []error + +// Err returns the list if it contains errors. Otherwise returns nil. +func (a ErrorList) Err() error { + if len(a) > 0 { + return a + } + return nil +} + +func (a ErrorList) Error() string { + switch len(a) { + case 0: + return "no errors" + case 1: + return a[0].Error() + } + return fmt.Sprintf("%s (and %d more errors)", a[0], len(a)-1) +} + +func (a ErrorList) FullError() string { + if len(a) == 0 { + return "" + } + + var buf bytes.Buffer + for _, err := range a { + fmt.Fprintln(&buf, err) + } + return buf.String() +} + +// Append appends an error to the list. If err is an ErrorList then all errors are appended. +func (a *ErrorList) Append(err error) { + switch err := err.(type) { + case ErrorList: + *a = append(*a, err...) + default: + *a = append(*a, err) + } +} diff --git a/rbf/rbf/testdata/check/bad-freelist/data b/rbf/rbf/testdata/check/bad-freelist/data new file mode 100644 index 000000000..8b03c7b02 Binary files /dev/null and b/rbf/rbf/testdata/check/bad-freelist/data differ diff --git a/rbf/rbf/testdata/check/bad-freelist/wal b/rbf/rbf/testdata/check/bad-freelist/wal new file mode 100644 index 000000000..e69de29bb diff --git a/rbf/rbf_test.go b/rbf/rbf_test.go index 4071a1a95..17605a6fc 100644 --- a/rbf/rbf_test.go +++ b/rbf/rbf_test.go @@ -54,17 +54,30 @@ func NewDB(tb testing.TB, cfg ...*rbfcfg.Config) *rbf.DB { if err != nil { panic(err) } + return NewDBAt(tb, path, cfg...) +} +// NewDBAt returns a new instance of DB with a given path. +func NewDBAt(tb testing.TB, path string, cfg ...*rbfcfg.Config) *rbf.DB { var cfg0 *rbfcfg.Config if len(cfg) > 0 { cfg0 = cfg[0] } - db := rbf.NewDB(path, cfg0) - return db + return rbf.NewDB(path, cfg0) } // MustOpenDB returns a db opened on a temporary file. On error, fail test. func MustOpenDB(tb testing.TB, cfg ...*rbfcfg.Config) *rbf.DB { + tb.Helper() + path, err := testhook.TempDir(tb, "rbfdb") + if err != nil { + panic(err) + } + return MustOpenDBAt(tb, path, cfg...) +} + +// MustOpenDBAt returns a db opened on an existing file. On error, fail test. +func MustOpenDBAt(tb testing.TB, path string, cfg ...*rbfcfg.Config) *rbf.DB { tb.Helper() if len(cfg) == 0 || cfg[0] == nil { newconf := rbfcfg.NewDefaultConfig() @@ -73,7 +86,7 @@ func MustOpenDB(tb testing.TB, cfg ...*rbfcfg.Config) *rbf.DB { } else if cfg[0].Logger == nil { cfg[0].Logger = logger.NewLogfLogger(tb) } - db := NewDB(tb, cfg...) + db := NewDBAt(tb, path, cfg...) if err := db.Open(); err != nil { tb.Fatal(err) } diff --git a/rbf/testdata/check/bad-bitmap/data b/rbf/testdata/check/bad-bitmap/data new file mode 100644 index 000000000..6cc32c0a1 Binary files /dev/null and b/rbf/testdata/check/bad-bitmap/data differ diff --git a/rbf/testdata/check/bad-bitmap/wal b/rbf/testdata/check/bad-bitmap/wal new file mode 100644 index 000000000..e69de29bb diff --git a/rbf/testdata/check/bad-freelist/data b/rbf/testdata/check/bad-freelist/data new file mode 100644 index 000000000..a762ee9db Binary files /dev/null and b/rbf/testdata/check/bad-freelist/data differ diff --git a/rbf/testdata/check/bad-freelist/wal b/rbf/testdata/check/bad-freelist/wal new file mode 100644 index 000000000..e69de29bb diff --git a/rbf/tx.go b/rbf/tx.go index 5d731fe4f..c45380469 100644 --- a/rbf/tx.go +++ b/rbf/tx.go @@ -738,10 +738,11 @@ func (tx *Tx) Check() error { return ErrTxClosed } + var errorList ErrorList if err := tx.checkPageAllocations(); err != nil { - return fmt.Errorf("page allocations: %w", err) + errorList.Append(err) } - return nil + return errorList.Err() } func (tx *Tx) checkPage(pgno, parent, typ uint32) error { @@ -767,14 +768,15 @@ func (tx *Tx) checkBranchPage(pgno, parent, typ uint32) error { // checkPageAllocations ensures that all pages are either in-use or on the freelist. func (tx *Tx) checkPageAllocations() error { + var errorList ErrorList freePageSet, err := tx.freePageSet() if err != nil { - return err + errorList.Append(err) } inusePageSet, err := tx.inusePageSet() if err != nil { - return err + errorList.Append(err) } // Iterate over all pages and ensure they are either in-use or free. @@ -785,26 +787,23 @@ func (tx *Tx) checkPageAllocations() error { _, isFree := freePageSet[pgno] if isInuse && isFree { - return fmt.Errorf("page in-use & free: pgno=%d", pgno) - } else if !isInuse && !isFree { - page, _, err := tx.readPage(pgno) - if err != nil { - return err - } - flags := readFlags(page) - if flags == PageTypeBranch || flags == PageTypeLeaf { - return fmt.Errorf("page not in-use & not free: pgno=%d", pgno) - } - //assuming its a bitmap so its ok TODO ben? - return nil + errorList.Append(fmt.Errorf("page in-use & free: pgno=%d", pgno)) + continue + } + + if !isInuse && !isFree { + errorList.Append(fmt.Errorf("page not in-use & not free: pgno=%d", pgno)) + continue } } - return nil + return errorList.Err() } // freePageSet returns the set of pages in the freelist. func (tx *Tx) freePageSet() (map[uint32]struct{}, error) { + var errorList ErrorList + m := make(map[uint32]struct{}) c := Cursor{tx: tx} c.stack.elems[0] = stackElem{pgno: readMetaFreelistPageNo(tx.meta[:])} @@ -816,18 +815,20 @@ func (tx *Tx) freePageSet() (map[uint32]struct{}, error) { for { if err := c.Next(); err == io.EOF { - return m, nil + return m, errorList.Err() } else if err != nil { - return m, err + errorList.Append(err) + return m, errorList.Err() } elem := &c.stack.elems[c.stack.top] leafPage, _, err := c.tx.readPage(elem.pgno) if err != nil { - return nil, err + errorList.Append(fmt.Errorf("cannot read free page: pgno=%d err=%w", elem.pgno, err)) + continue } - cell := readLeafCell(leafPage, elem.index) + cell := readLeafCell(leafPage, elem.index) for _, v := range cell.Values(tx) { pgno := uint32((cell.Key << 16) | uint64(v)) m[pgno] = struct{}{} @@ -837,6 +838,7 @@ func (tx *Tx) freePageSet() (map[uint32]struct{}, error) { // inusePageSet returns the set of pages in use by the root records or b-trees. func (tx *Tx) inusePageSet() (map[uint32]struct{}, error) { + var errorList ErrorList m := make(map[uint32]struct{}) m[0] = struct{}{} // meta page @@ -846,15 +848,24 @@ func (tx *Tx) inusePageSet() (map[uint32]struct{}, error) { page, _, err := tx.readPage(pgno) if err != nil { - return nil, err + errorList.Append(err) + break } pgno = WalkRootRecordPages(page) } // Traverse freelist and mark pages as in-use. - if err := tx.walkTree(readMetaFreelistPageNo(tx.meta[:]), 0, func(pgno, parent, typ uint32) error { + if err := tx.walkTree(readMetaFreelistPageNo(tx.meta[:]), 0, func(pgno, parent, typ uint32, err error) error { + if err != nil { + errorList.Append(err) + return nil + } + m[pgno] = struct{}{} - return tx.checkPage(pgno, parent, typ) + if err := tx.checkPage(pgno, parent, typ); err != nil { + errorList.Append(err) + } + return nil }); err != nil { return m, err } @@ -862,22 +873,28 @@ func (tx *Tx) inusePageSet() (map[uint32]struct{}, error) { // Traverse every b-tree and mark pages as in-use. records, err := tx.RootRecords() if err != nil { - return m, err - } + errorList.Append(err) + } else { + for itr := records.Iterator(); !itr.Done(); { + _, pgno := itr.Next() - for itr := records.Iterator(); !itr.Done(); { - _, pgno := itr.Next() + if err := tx.walkTree(pgno.(uint32), 0, func(pgno, parent, typ uint32, err error) error { + if err != nil { + errorList.Append(err) + } - if err := tx.walkTree(pgno.(uint32), 0, func(pgno, parent, typ uint32) error { - m[pgno] = struct{}{} - - return tx.checkPage(pgno, parent, typ) - }); err != nil { - return m, err + m[pgno] = struct{}{} + if err := tx.checkPage(pgno, parent, typ); err != nil { + errorList.Append(err) + } + return nil + }); err != nil { + return m, err + } } } - return m, nil + return m, errorList.Err() } // GetSizeBytesWithPrefix returns the size of bitmaps with a given key prefix. @@ -897,9 +914,9 @@ func (tx *Tx) GetSizeBytesWithPrefix(prefix string) (n uint64, err error) { } // Traverse the bitmap's b-tree and count the bytes for each page. - if err := tx.walkTree(pgno.(uint32), 0, func(pgno, parent, typ uint32) error { + if err := tx.walkTree(pgno.(uint32), 0, func(pgno, parent, typ uint32, err error) error { n += PageSize - return nil + return err }); err != nil { return 0, err } @@ -908,21 +925,19 @@ func (tx *Tx) GetSizeBytesWithPrefix(prefix string) (n uint64, err error) { } // walkTree recursively iterates over a page and all its children. -func (tx *Tx) walkTree(pgno, parent uint32, fn func(pgno, parent, typ uint32) error) error { +func (tx *Tx) walkTree(pgno, parent uint32, fn func(pgno, parent, typ uint32, err error) error) error { // Read page and iterate over children. page, _, err := tx.readPage(pgno) if err != nil { - return err + return fn(pgno, parent, 0, fmt.Errorf("cannot read page: pgno=%d parent=%d err=%s", pgno, parent, err)) } - // Execute callback. - typ := readFlags(page) - if err := fn(pgno, parent, typ); err != nil { - return err - } - - switch typ { + switch typ := readFlags(page); typ { case PageTypeBranch: + if err := fn(pgno, parent, typ, nil); err != nil { + return err + } + for i, n := 0, readCellN(page); i < n; i++ { cell := readBranchCell(page, i) if err := tx.walkTree(cell.ChildPgno, pgno, fn); err != nil { @@ -930,18 +945,24 @@ func (tx *Tx) walkTree(pgno, parent uint32, fn func(pgno, parent, typ uint32) er } } return nil + case PageTypeLeaf: + if err := fn(pgno, parent, typ, nil); err != nil { + return err + } + // Execute callback only for bitmap pages pointed to by this leaf. for i, n := 0, readCellN(page); i < n; i++ { if cell := readLeafCell(page, i); cell.Type == ContainerTypeBitmapPtr { - if err := fn(toPgno(cell.Data), pgno, PageTypeBitmap); err != nil { + if err := fn(toPgno(cell.Data), pgno, PageTypeBitmap, nil); err != nil { return err } } } return nil + default: - return fmt.Errorf("rbf.Tx.forEachTreePage(): invalid page type: pgno=%d type=%d", pgno, typ) + return fn(pgno, parent, typ, fmt.Errorf("invalid page type: pgno=%d parent=%d type=%d", pgno, parent, typ)) } } @@ -1906,6 +1927,7 @@ func (tx *Tx) Pages(pgnos []uint32) ([]Page, error) { // PageInfos returns meta data about all pages in the database. func (tx *Tx) PageInfos() ([]PageInfo, error) { + var errorList ErrorList infos := make([]PageInfo, tx.PageN()) // Read meta page info. @@ -1919,7 +1941,8 @@ func (tx *Tx) PageInfos() ([]PageInfo, error) { for pgno := metaInfo.RootRecordPageNo; pgno != 0; { info, err := tx.rootRecordPageInfo(pgno) if err != nil { - return nil, err + errorList.Append(err) + break } infos[pgno] = info pgno = info.Next @@ -1927,33 +1950,34 @@ func (tx *Tx) PageInfos() ([]PageInfo, error) { // Traverse freelist and mark pages as in-use. if err := tx.walkPageInfo(infos, metaInfo.FreelistPageNo, "freelist"); err != nil { - return nil, err + errorList.Append(err) } // Traverse every b-tree and mark pages as in-use. records, err := tx.RootRecords() if err != nil { - return nil, err - } + errorList.Append(err) + } else { + for itr := records.Iterator(); !itr.Done(); { + name, pgno := itr.Next() - for itr := records.Iterator(); !itr.Done(); { - name, pgno := itr.Next() - - if err := tx.walkPageInfo(infos, pgno.(uint32), name.(string)); err != nil { - return nil, err + if err := tx.walkPageInfo(infos, pgno.(uint32), name.(string)); err != nil { + errorList.Append(err) + } } } // Build page info objects for each free page. freePageSet, err := tx.freePageSet() if err != nil { - return nil, err - } - for pgno := range freePageSet { - infos[pgno] = &FreePageInfo{Pgno: pgno} + errorList.Append(err) + } else { + for pgno := range freePageSet { + infos[pgno] = &FreePageInfo{Pgno: pgno} + } } - return infos, nil + return infos, errorList.Err() } // metaPageInfo returns page metadata for the meta page. @@ -1987,10 +2011,18 @@ func (tx *Tx) rootRecordPageInfo(pgno uint32) (*RootRecordPageInfo, error) { } func (tx *Tx) walkPageInfo(infos []PageInfo, root uint32, name string) error { - return tx.walkTree(root, 0, func(pgno, parent, typ uint32) error { + var errorList ErrorList + + if err := tx.walkTree(root, 0, func(pgno, parent, typ uint32, err error) error { + if err != nil { + errorList.Append(err) + return nil + } + buf, _, err := tx.readPage(pgno) if err != nil { - return err + errorList.Append(fmt.Errorf("cannot read page: pgno=%d parent=%d typ=%d err=%d", pgno, parent, typ, err)) + return nil } switch typ { @@ -2016,12 +2048,14 @@ func (tx *Tx) walkPageInfo(infos []PageInfo, root uint32, name string) error { Parent: parent, Tree: name, } - default: - vprint.PanicOn(fmt.Sprintf("unexpected page type %d for page %d", typ, pgno)) } return nil - }) + }); err != nil { + errorList.Append(err) + } + + return errorList.Err() } // PageData returns the raw page data for a single page. diff --git a/rbf/tx_test.go b/rbf/tx_test.go index 5be3a50ae..7a726c2ed 100644 --- a/rbf/tx_test.go +++ b/rbf/tx_test.go @@ -6,6 +6,7 @@ import ( "fmt" "math/rand" "os" + "path/filepath" "strings" "sync" "testing" @@ -870,6 +871,36 @@ func TestTx_Check(t *testing.T) { t.Fatalf("unexpected error: %#v", err) } }) + + t.Run("ErrBadFreelist", func(t *testing.T) { + t.Parallel() + + db := MustOpenDBAt(t, filepath.Join("testdata", "check", "bad-freelist")) + defer db.Close() + tx := MustBegin(t, db, false) + defer tx.Rollback() + + if err, ok := tx.Check().(rbf.ErrorList); !ok { + t.Fatal("expected error list") + } else if s := err.FullError(); !strings.Contains(s, `branch cell index out of range: pgno=2 i=0 n=0`) { + t.Fatalf("unexpected error:\n%s", s) + } + }) + + t.Run("ErrBadBitmap", func(t *testing.T) { + t.Parallel() + + db := MustOpenDBAt(t, filepath.Join("testdata", "check", "bad-bitmap")) + defer db.Close() + tx := MustBegin(t, db, false) + defer tx.Rollback() + + if err, ok := tx.Check().(rbf.ErrorList); !ok { + t.Fatal("expected error list") + } else if s := err.FullError(); !strings.Contains(s, `cannot read page: pgno=65537 parent=3 err=rbf: page read out of bounds: pgno=65537 max=3`) { + t.Fatalf("unexpected error:\n%s", s) + } + }) } func mustReadPage(tb testing.TB, path string, pgno uint32) []byte {