From aa8057e9e4a62f32a78de44f461fd497b5e671e1 Mon Sep 17 00:00:00 2001
From: Bruce Baranowski <92940816+bruce-b-molecula@users.noreply.github.com>
Date: Tue, 31 May 2022 17:25:09 -0400
Subject: [PATCH] CICD - Spot instances (#2087)

* spot instance test

* update outputs.tf to provide spot instances

* uptate outputs.tf data_node_ips

* propogate spot instance request tags to the instances
---
 qa/tf/.modules/featurebase-cluster/main.tf    | 81 ++++++++++++++++++-
 qa/tf/.modules/featurebase-cluster/outputs.tf |  4 +-
 .../.modules/featurebase-cluster/variables.tf |  4 +
 qa/tf/ci/auth-smoke/main.tf                   |  1 +
 qa/tf/ci/deletetest/main.tf                   |  1 +
 qa/tf/ci/smoketest/main.tf                    |  1 +
 qa/tf/gauntlet/backuprestore/main.tf          |  1 +
 qa/tf/gauntlet/samsung/main.tf                |  1 +
 qa/tf/perf/able/main.tf                       |  1 +
 qa/tf/perf/delete/main.tf                     |  1 +
 10 files changed, 92 insertions(+), 4 deletions(-)

diff --git a/qa/tf/.modules/featurebase-cluster/main.tf b/qa/tf/.modules/featurebase-cluster/main.tf
index 348df359e..141190037 100644
--- a/qa/tf/.modules/featurebase-cluster/main.tf
+++ b/qa/tf/.modules/featurebase-cluster/main.tf
@@ -18,7 +18,7 @@ data "aws_ami" "amazon_linux_2" {
 }
 
 resource "aws_instance" "fb_cluster_nodes" {
-  count                  = var.fb_data_node_count
+  count                  = var.use_spot_instances ? 0 : var.fb_data_node_count
   ami                    = data.aws_ami.amazon_linux_2.id
   instance_type          = var.fb_data_node_type
   key_name               = aws_key_pair.gitlab-featurebase-ci.key_name
@@ -53,7 +53,7 @@ resource "aws_instance" "fb_cluster_nodes" {
 }
 
 resource "aws_instance" "fb_ingest" {
-  count                       = var.fb_ingest_node_count
+  count                       = var.use_spot_instances ? 0 : var.fb_ingest_node_count
   ami                         = data.aws_ami.amazon_linux_2.id
   key_name                    = aws_key_pair.gitlab-featurebase-ci.key_name
   vpc_security_group_ids      = [aws_security_group.ingest.id]
@@ -83,6 +83,83 @@ resource "aws_instance" "fb_ingest" {
     Role   = "ingest_node"
   }
 
+}
+resource "aws_spot_instance_request" "fb_cluster_nodes" {
+  wait_for_fulfillment   = true
+  count                  = var.use_spot_instances ? var.fb_data_node_count : 0
+  ami                    = data.aws_ami.amazon_linux_2.id
+  instance_type          = var.fb_data_node_type
+  key_name               = aws_key_pair.gitlab-featurebase-ci.key_name
+  vpc_security_group_ids = [aws_security_group.featurebase.id]
+  monitoring             = true
+  subnet_id              = var.subnet != "" ? var.subnet : var.vpc_private_subnets[count.index % length(var.vpc_private_subnets)]
+  availability_zone      = var.zone != "" ? var.zone : var.azs[count.index % length(var.azs)]
+  iam_instance_profile   = "${aws_iam_instance_profile.fb_cluster_node_profile.name}"
+  user_data = var.user_data != "" ? file("${var.user_data}") : file("${path.module}/cloud-init.sh") 
+  
+  root_block_device {
+    volume_type = "gp3"
+    volume_size = 20
+  }
+
+  dynamic "ebs_block_device" {
+    for_each = var.ebs_volumes
+    content {
+      device_name = "/dev/sdb"
+      volume_type = var.fb_data_disk_type
+      volume_size = var.fb_data_disk_size_gb
+      iops        = var.fb_data_disk_iops
+      encrypted   = true 
+    }
+  }
+
+  tags = {
+    Prefix = "${var.cluster_prefix}"
+    Name   = "${var.cluster_prefix}-featurebase-cluster-${count.index}"
+    Role   = "cluster_node"
+  }
+
+  provisioner "local-exec" {
+    command = "aws ec2 create-tags --profile ${var.profile} --resources ${self.spot_instance_id} --tags Key=Prefix,Value='${var.cluster_prefix}' Key=Name,Value='${var.cluster_prefix}-featurebase-cluster-${count.index}' Key=Role,Value=cluster_node --region ${var.region}"
+  }
+
+}
+
+resource "aws_spot_instance_request" "fb_ingest" {
+  wait_for_fulfillment        = true
+  count                       = var.use_spot_instances ? var.fb_ingest_node_count : 0
+  ami                         = data.aws_ami.amazon_linux_2.id
+  key_name                    = aws_key_pair.gitlab-featurebase-ci.key_name
+  vpc_security_group_ids      = [aws_security_group.ingest.id]
+  instance_type               = var.fb_ingest_type
+  associate_public_ip_address = true
+  monitoring                  = true
+  subnet_id                   = var.subnet != "" ? var.subnet : var.vpc_public_subnets[count.index % length(var.vpc_public_subnets)]
+  availability_zone           = var.zone != "" ? var.zone : var.azs[count.index % length(var.azs)]
+  iam_instance_profile        = "${aws_iam_instance_profile.fb_cluster_node_profile.name}"
+
+  root_block_device {
+    volume_type = "gp3"
+    volume_size = 20
+  }
+
+  ebs_block_device {
+    device_name = "/dev/sdb"
+    volume_type = var.fb_ingest_disk_type
+    volume_size = var.fb_ingest_disk_size_gb
+    iops        = var.fb_ingest_disk_iops
+    encrypted   = true
+  }
+
+  tags = {
+    Prefix = "${var.cluster_prefix}"
+    Name   = "${var.cluster_prefix}-featurebase-ingest-${count.index}"
+    Role   = "ingest_node"
+  }
+  provisioner "local-exec" {
+    command = "aws ec2 create-tags --profile ${var.profile} --resources ${self.spot_instance_id} --tags Key=Prefix,Value='${var.cluster_prefix}' Key=Name,Value='${var.cluster_prefix}-featurebase-ingest-${count.index}' Key=Role,Value=ingest_node --region ${var.region}"
+  }
+
 }
 
 resource "aws_key_pair" "gitlab-featurebase-ci" {
diff --git a/qa/tf/.modules/featurebase-cluster/outputs.tf b/qa/tf/.modules/featurebase-cluster/outputs.tf
index 9f55082ac..070159697 100644
--- a/qa/tf/.modules/featurebase-cluster/outputs.tf
+++ b/qa/tf/.modules/featurebase-cluster/outputs.tf
@@ -1,9 +1,9 @@
 output "ingest_ips" {
-    value = aws_instance.fb_ingest.*.private_ip
+    value = var.use_spot_instances ? aws_spot_instance_request.fb_ingest.*.private_ip : aws_instance.fb_ingest.*.private_ip
 }
 
 output "data_node_ips" {
-    value = aws_instance.fb_cluster_nodes.*.private_ip
+    value = var.use_spot_instances ? aws_spot_instance_request.fb_cluster_nodes.*.private_ip : aws_instance.fb_cluster_nodes.*.private_ip 
 }
 
 output "cluster_prefix" {
diff --git a/qa/tf/.modules/featurebase-cluster/variables.tf b/qa/tf/.modules/featurebase-cluster/variables.tf
index 1e256be3d..8cc9533f4 100644
--- a/qa/tf/.modules/featurebase-cluster/variables.tf
+++ b/qa/tf/.modules/featurebase-cluster/variables.tf
@@ -123,3 +123,7 @@ variable "ebs_volumes" {
   default = ["/dev/sdb"]
 }
 
+variable "use_spot_instances" {
+  type = bool
+  default = false
+}
diff --git a/qa/tf/ci/auth-smoke/main.tf b/qa/tf/ci/auth-smoke/main.tf
index a974129a3..cb8041fae 100644
--- a/qa/tf/ci/auth-smoke/main.tf
+++ b/qa/tf/ci/auth-smoke/main.tf
@@ -11,4 +11,5 @@ module "ci-cluster" {
   vpc_public_subnets  = ["subnet-066b4b922b54e51a2", "subnet-037b8884269a69025", "subnet-08482631514426210", ]
   vpc_private_subnets = ["subnet-0319dde319380326f", "subnet-0517ca9a646d80f88", "subnet-05a7b685ed27eb1cf", ]
   user_data = "../../.modules/featurebase-cluster/cloud-init.sh"
+  use_spot_instances = true
 }
diff --git a/qa/tf/ci/deletetest/main.tf b/qa/tf/ci/deletetest/main.tf
index 1b2f98a31..3d6bd2c77 100644
--- a/qa/tf/ci/deletetest/main.tf
+++ b/qa/tf/ci/deletetest/main.tf
@@ -15,4 +15,5 @@ module "ci-cluster" {
     vpc_public_subnets = ["subnet-066b4b922b54e51a2","subnet-037b8884269a69025","subnet-08482631514426210",]
     vpc_private_subnets = ["subnet-0319dde319380326f","subnet-0517ca9a646d80f88","subnet-05a7b685ed27eb1cf",]
     user_data = "./delete_cloud_init.sh"
+    use_spot_instances = true
 }
\ No newline at end of file
diff --git a/qa/tf/ci/smoketest/main.tf b/qa/tf/ci/smoketest/main.tf
index 9c26db185..19ef9ccca 100644
--- a/qa/tf/ci/smoketest/main.tf
+++ b/qa/tf/ci/smoketest/main.tf
@@ -12,4 +12,5 @@ module "ci-cluster" {
     vpc_public_subnets = ["subnet-066b4b922b54e51a2","subnet-037b8884269a69025","subnet-08482631514426210",]
     vpc_private_subnets = ["subnet-0319dde319380326f","subnet-0517ca9a646d80f88","subnet-05a7b685ed27eb1cf",]
     user_data = "../../.modules/featurebase-cluster/cloud-init.sh"
+    use_spot_instances = true
 }
diff --git a/qa/tf/gauntlet/backuprestore/main.tf b/qa/tf/gauntlet/backuprestore/main.tf
index e613e4dbb..d09415b94 100644
--- a/qa/tf/gauntlet/backuprestore/main.tf
+++ b/qa/tf/gauntlet/backuprestore/main.tf
@@ -15,4 +15,5 @@ module "samsung-cluster" {
   vpc_public_subnets       = ["subnet-066b4b922b54e51a2", "subnet-037b8884269a69025", "subnet-08482631514426210", ]
   vpc_private_subnets      = ["subnet-0319dde319380326f", "subnet-0517ca9a646d80f88", "subnet-05a7b685ed27eb1cf", ]
   user_data = "../../.modules/featurebase-cluster/cloud-init.sh"
+  use_spot_instances = true
 }
diff --git a/qa/tf/gauntlet/samsung/main.tf b/qa/tf/gauntlet/samsung/main.tf
index 710fcdbd6..a4e63aece 100644
--- a/qa/tf/gauntlet/samsung/main.tf
+++ b/qa/tf/gauntlet/samsung/main.tf
@@ -14,4 +14,5 @@ module "samsung-cluster" {
     vpc_public_subnets = ["subnet-066b4b922b54e51a2","subnet-037b8884269a69025","subnet-08482631514426210",]
     vpc_private_subnets = ["subnet-0319dde319380326f","subnet-0517ca9a646d80f88","subnet-05a7b685ed27eb1cf",]
     user_data = "../../.modules/featurebase-cluster/cloud-init.sh"
+    use_spot_instances = true
 }
diff --git a/qa/tf/perf/able/main.tf b/qa/tf/perf/able/main.tf
index ff1d12446..a86e6dc4f 100644
--- a/qa/tf/perf/able/main.tf
+++ b/qa/tf/perf/able/main.tf
@@ -15,4 +15,5 @@ module "able-cluster" {
     vpc_public_subnets = ["subnet-066b4b922b54e51a2","subnet-037b8884269a69025","subnet-08482631514426210",]
     vpc_private_subnets = ["subnet-0319dde319380326f","subnet-0517ca9a646d80f88","subnet-05a7b685ed27eb1cf",]
     user_data = "../../.modules/featurebase-cluster/cloud-init.sh"
+    use_spot_instances = true
 }
diff --git a/qa/tf/perf/delete/main.tf b/qa/tf/perf/delete/main.tf
index bb772315e..f03041d6f 100644
--- a/qa/tf/perf/delete/main.tf
+++ b/qa/tf/perf/delete/main.tf
@@ -15,4 +15,5 @@ module "delete-cluster" {
   vpc_public_subnets     = ["subnet-066b4b922b54e51a2", "subnet-037b8884269a69025", "subnet-08482631514426210", ]
   vpc_private_subnets    = ["subnet-0319dde319380326f", "subnet-0517ca9a646d80f88", "subnet-05a7b685ed27eb1cf", ]
   user_data = "../../.modules/featurebase-cluster/cloud-init.sh"
+  use_spot_instances = true
 }