scriptoriumadmin/featurebase
1
0
Fork 0
mirror of https://github.com/featurebasedb/featurebase.git synced 2026-09-11 23:31:03 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

CICD - Spot instances (#2087)

Browse source 
* spot instance test

* update outputs.tf to provide spot instances

* uptate outputs.tf data_node_ips

* propogate spot instance request tags to the instances
This commit is contained in:
Bruce Baranowski 2022-05-31 17:25:09 -04:00 • committed by GitHub
parent c0d56e13f2
commit aa8057e9e4
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
10 changed files with 92 additions and 4 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

81
qa/tf/.modules/featurebase-cluster/main.tf
View file

@ -18,7 +18,7 @@ data "aws_ami" "amazon_linux_2" {
}
resource "aws_instance" "fb_cluster_nodes" {
count = var.fb_data_node_count
count = var.use_spot_instances ? 0 : var.fb_data_node_count
ami = data.aws_ami.amazon_linux_2.id
instance_type = var.fb_data_node_type
key_name = aws_key_pair.gitlab-featurebase-ci.key_name
@ -53,7 +53,7 @@ resource "aws_instance" "fb_cluster_nodes" {
}
resource "aws_instance" "fb_ingest" {
count = var.fb_ingest_node_count
count = var.use_spot_instances ? 0 : var.fb_ingest_node_count
ami = data.aws_ami.amazon_linux_2.id
key_name = aws_key_pair.gitlab-featurebase-ci.key_name
vpc_security_group_ids = [aws_security_group.ingest.id]
@ -83,6 +83,83 @@ resource "aws_instance" "fb_ingest" {
Role = "ingest_node"
}
}
resource "aws_spot_instance_request" "fb_cluster_nodes" {
wait_for_fulfillment = true
count = var.use_spot_instances ? var.fb_data_node_count : 0
ami = data.aws_ami.amazon_linux_2.id
instance_type = var.fb_data_node_type
key_name = aws_key_pair.gitlab-featurebase-ci.key_name
vpc_security_group_ids = [aws_security_group.featurebase.id]
monitoring = true
subnet_id = var.subnet != "" ? var.subnet : var.vpc_private_subnets[count.index % length(var.vpc_private_subnets)]
availability_zone = var.zone != "" ? var.zone : var.azs[count.index % length(var.azs)]
iam_instance_profile = "${aws_iam_instance_profile.fb_cluster_node_profile.name}"
user_data = var.user_data != "" ? file("${var.user_data}") : file("${path.module}/cloud-init.sh")
root_block_device {
volume_type = "gp3"
volume_size = 20
}
dynamic "ebs_block_device" {
for_each = var.ebs_volumes
content {
device_name = "/dev/sdb"
volume_type = var.fb_data_disk_type
volume_size = var.fb_data_disk_size_gb
iops = var.fb_data_disk_iops
encrypted = true
}
}
tags = {
Prefix = "${var.cluster_prefix}"
Name = "${var.cluster_prefix}-featurebase-cluster-${count.index}"
Role = "cluster_node"
}
provisioner "local-exec" {
command = "aws ec2 create-tags --profile ${var.profile} --resources ${self.spot_instance_id} --tags Key=Prefix,Value='${var.cluster_prefix}' Key=Name,Value='${var.cluster_prefix}-featurebase-cluster-${count.index}' Key=Role,Value=cluster_node --region ${var.region}"
}
}
resource "aws_spot_instance_request" "fb_ingest" {
wait_for_fulfillment = true
count = var.use_spot_instances ? var.fb_ingest_node_count : 0
ami = data.aws_ami.amazon_linux_2.id
key_name = aws_key_pair.gitlab-featurebase-ci.key_name
vpc_security_group_ids = [aws_security_group.ingest.id]
instance_type = var.fb_ingest_type
associate_public_ip_address = true
monitoring = true
subnet_id = var.subnet != "" ? var.subnet : var.vpc_public_subnets[count.index % length(var.vpc_public_subnets)]
availability_zone = var.zone != "" ? var.zone : var.azs[count.index % length(var.azs)]
iam_instance_profile = "${aws_iam_instance_profile.fb_cluster_node_profile.name}"
root_block_device {
volume_type = "gp3"
volume_size = 20
}
ebs_block_device {
device_name = "/dev/sdb"
volume_type = var.fb_ingest_disk_type
volume_size = var.fb_ingest_disk_size_gb
iops = var.fb_ingest_disk_iops
encrypted = true
}
tags = {
Prefix = "${var.cluster_prefix}"
Name = "${var.cluster_prefix}-featurebase-ingest-${count.index}"
Role = "ingest_node"
}
provisioner "local-exec" {
command = "aws ec2 create-tags --profile ${var.profile} --resources ${self.spot_instance_id} --tags Key=Prefix,Value='${var.cluster_prefix}' Key=Name,Value='${var.cluster_prefix}-featurebase-ingest-${count.index}' Key=Role,Value=ingest_node --region ${var.region}"
}
}
resource "aws_key_pair" "gitlab-featurebase-ci" {

4
qa/tf/.modules/featurebase-cluster/outputs.tf
View file

@ -1,9 +1,9 @@
output "ingest_ips" {
value = aws_instance.fb_ingest.*.private_ip
value = var.use_spot_instances ? aws_spot_instance_request.fb_ingest.*.private_ip : aws_instance.fb_ingest.*.private_ip
}
output "data_node_ips" {
value = aws_instance.fb_cluster_nodes.*.private_ip
value = var.use_spot_instances ? aws_spot_instance_request.fb_cluster_nodes.*.private_ip : aws_instance.fb_cluster_nodes.*.private_ip
}
output "cluster_prefix" {

4
qa/tf/.modules/featurebase-cluster/variables.tf
View file

@ -123,3 +123,7 @@ variable "ebs_volumes" {
default = ["/dev/sdb"]
}
variable "use_spot_instances" {
type = bool
default = false
}

1
qa/tf/ci/auth-smoke/main.tf
View file

@ -11,4 +11,5 @@ module "ci-cluster" {
vpc_public_subnets = ["subnet-066b4b922b54e51a2", "subnet-037b8884269a69025", "subnet-08482631514426210", ]
vpc_private_subnets = ["subnet-0319dde319380326f", "subnet-0517ca9a646d80f88", "subnet-05a7b685ed27eb1cf", ]
user_data = "../../.modules/featurebase-cluster/cloud-init.sh"
use_spot_instances = true
}

1
qa/tf/ci/deletetest/main.tf
View file

@ -15,4 +15,5 @@ module "ci-cluster" {
vpc_public_subnets = ["subnet-066b4b922b54e51a2","subnet-037b8884269a69025","subnet-08482631514426210",]
vpc_private_subnets = ["subnet-0319dde319380326f","subnet-0517ca9a646d80f88","subnet-05a7b685ed27eb1cf",]
user_data = "./delete_cloud_init.sh"
use_spot_instances = true
}

1
qa/tf/ci/smoketest/main.tf
View file

@ -12,4 +12,5 @@ module "ci-cluster" {
vpc_public_subnets = ["subnet-066b4b922b54e51a2","subnet-037b8884269a69025","subnet-08482631514426210",]
vpc_private_subnets = ["subnet-0319dde319380326f","subnet-0517ca9a646d80f88","subnet-05a7b685ed27eb1cf",]
user_data = "../../.modules/featurebase-cluster/cloud-init.sh"
use_spot_instances = true
}

1
qa/tf/gauntlet/backuprestore/main.tf
View file

@ -15,4 +15,5 @@ module "samsung-cluster" {
vpc_public_subnets = ["subnet-066b4b922b54e51a2", "subnet-037b8884269a69025", "subnet-08482631514426210", ]
vpc_private_subnets = ["subnet-0319dde319380326f", "subnet-0517ca9a646d80f88", "subnet-05a7b685ed27eb1cf", ]
user_data = "../../.modules/featurebase-cluster/cloud-init.sh"
use_spot_instances = true
}

1
qa/tf/gauntlet/samsung/main.tf
View file

@ -14,4 +14,5 @@ module "samsung-cluster" {
vpc_public_subnets = ["subnet-066b4b922b54e51a2","subnet-037b8884269a69025","subnet-08482631514426210",]
vpc_private_subnets = ["subnet-0319dde319380326f","subnet-0517ca9a646d80f88","subnet-05a7b685ed27eb1cf",]
user_data = "../../.modules/featurebase-cluster/cloud-init.sh"
use_spot_instances = true
}

1
qa/tf/perf/able/main.tf
View file

@ -15,4 +15,5 @@ module "able-cluster" {
vpc_public_subnets = ["subnet-066b4b922b54e51a2","subnet-037b8884269a69025","subnet-08482631514426210",]
vpc_private_subnets = ["subnet-0319dde319380326f","subnet-0517ca9a646d80f88","subnet-05a7b685ed27eb1cf",]
user_data = "../../.modules/featurebase-cluster/cloud-init.sh"
use_spot_instances = true
}

1
qa/tf/perf/delete/main.tf
View file

@ -15,4 +15,5 @@ module "delete-cluster" {
vpc_public_subnets = ["subnet-066b4b922b54e51a2", "subnet-037b8884269a69025", "subnet-08482631514426210", ]
vpc_private_subnets = ["subnet-0319dde319380326f", "subnet-0517ca9a646d80f88", "subnet-05a7b685ed27eb1cf", ]
user_data = "../../.modules/featurebase-cluster/cloud-init.sh"
use_spot_instances = true
}