mirror of
https://github.com/featurebasedb/featurebase.git
synced 2026-08-28 10:54:59 +00:00
Make spot instance selection dynamic
The idea behind this is to give AWS more information about what instances we can let it actually instantitate, rather than have it be one fixed instance type. e.g., in this case, we are okay with any Graviton instance with at least 2 vCPU and 8 GiB memory. The easist way to do that is to instead use a launch template, with ec2_fleets or spot fleets. I took out the part where we even support on-demand instances. This can be readded later if it is necessary.
This commit is contained in:
Garrison Davis
seebs
parent
49d0e0fbc8
commit
3bfb65eb58
3 changed files with 170 additions and 149 deletions
|
|
@ -17,159 +17,165 @@ data "aws_ami" "amazon_linux_2" {
|
|||
}
|
||||
}
|
||||
|
||||
resource "aws_instance" "fb_cluster_nodes" {
|
||||
count = var.use_spot_instances ? 0 : var.fb_data_node_count
|
||||
ami = data.aws_ami.amazon_linux_2.id
|
||||
instance_type = var.fb_data_node_type
|
||||
key_name = aws_key_pair.gitlab-featurebase-ci.key_name
|
||||
vpc_security_group_ids = [aws_security_group.featurebase.id]
|
||||
monitoring = true
|
||||
subnet_id = var.subnet != "" ? var.subnet : var.vpc_private_subnets[count.index % length(var.vpc_private_subnets)]
|
||||
availability_zone = var.zone != "" ? var.zone : var.azs[count.index % length(var.azs)]
|
||||
iam_instance_profile = "${aws_iam_instance_profile.fb_cluster_node_profile.name}"
|
||||
user_data = var.user_data != "" ? file("${var.user_data}") : file("${path.module}/cloud-init.sh")
|
||||
|
||||
root_block_device {
|
||||
volume_type = "gp3"
|
||||
volume_size = 20
|
||||
}
|
||||
resource "aws_spot_fleet_request" "fb_data_nodes" {
|
||||
target_capacity = var.fb_data_node_count // TODO
|
||||
iam_fleet_role = "arn:aws:iam::977373308795:role/aws-ec2-spot-fleet-tagging-role" // TODO
|
||||
|
||||
dynamic "ebs_block_device" {
|
||||
for_each = var.ebs_volumes
|
||||
content {
|
||||
device_name = "/dev/sdb"
|
||||
volume_type = var.fb_data_disk_type
|
||||
volume_size = var.fb_data_disk_size_gb
|
||||
iops = var.fb_data_disk_iops
|
||||
encrypted = true
|
||||
fleet_type = "request"
|
||||
wait_for_fulfillment = true
|
||||
|
||||
terminate_instances_with_expiration = true
|
||||
launch_template_config {
|
||||
launch_template_specification {
|
||||
id = aws_launch_template.fb_data_node.id
|
||||
version = aws_launch_template.fb_data_node.latest_version
|
||||
}
|
||||
overrides {
|
||||
subnet_id = var.vpc_private_subnets[0] // TODO
|
||||
}
|
||||
}
|
||||
|
||||
tags = {
|
||||
Prefix = "${var.cluster_prefix}"
|
||||
Name = "${var.cluster_prefix}-featurebase-cluster-${count.index}"
|
||||
Role = "cluster_node"
|
||||
}
|
||||
}
|
||||
|
||||
resource "aws_instance" "fb_ingest" {
|
||||
count = var.use_spot_instances ? 0 : var.fb_ingest_node_count
|
||||
ami = data.aws_ami.amazon_linux_2.id
|
||||
key_name = aws_key_pair.gitlab-featurebase-ci.key_name
|
||||
vpc_security_group_ids = [aws_security_group.ingest.id]
|
||||
instance_type = var.fb_ingest_type
|
||||
associate_public_ip_address = true
|
||||
monitoring = true
|
||||
subnet_id = var.subnet != "" ? var.subnet : var.vpc_public_subnets[count.index % length(var.vpc_public_subnets)]
|
||||
availability_zone = var.zone != "" ? var.zone : var.azs[count.index % length(var.azs)]
|
||||
iam_instance_profile = "${aws_iam_instance_profile.fb_cluster_node_profile.name}"
|
||||
resource "aws_spot_fleet_request" "fb_ingest_nodes" {
|
||||
target_capacity = var.fb_ingest_node_count
|
||||
iam_fleet_role = "arn:aws:iam::977373308795:role/aws-ec2-spot-fleet-tagging-role" // TODO
|
||||
|
||||
root_block_device {
|
||||
volume_type = "gp3"
|
||||
volume_size = 20
|
||||
}
|
||||
fleet_type = "request"
|
||||
wait_for_fulfillment = true
|
||||
|
||||
ebs_block_device {
|
||||
device_name = "/dev/sdb"
|
||||
volume_type = var.fb_ingest_disk_type
|
||||
volume_size = var.fb_ingest_disk_size_gb
|
||||
iops = var.fb_ingest_disk_iops
|
||||
encrypted = true
|
||||
}
|
||||
|
||||
tags = {
|
||||
Prefix = "${var.cluster_prefix}"
|
||||
Name = "${var.cluster_prefix}-featurebase-ingest-${count.index}"
|
||||
Role = "ingest_node"
|
||||
}
|
||||
|
||||
}
|
||||
resource "aws_spot_instance_request" "fb_cluster_nodes" {
|
||||
wait_for_fulfillment = true
|
||||
spot_type = "one-time"
|
||||
count = var.use_spot_instances ? var.fb_data_node_count : 0
|
||||
ami = data.aws_ami.amazon_linux_2.id
|
||||
instance_type = var.fb_data_node_type
|
||||
key_name = aws_key_pair.gitlab-featurebase-ci.key_name
|
||||
vpc_security_group_ids = [aws_security_group.featurebase.id]
|
||||
monitoring = true
|
||||
subnet_id = var.subnet != "" ? var.subnet : var.vpc_private_subnets[count.index % length(var.vpc_private_subnets)]
|
||||
availability_zone = var.zone != "" ? var.zone : var.azs[count.index % length(var.azs)]
|
||||
iam_instance_profile = "${aws_iam_instance_profile.fb_cluster_node_profile.name}"
|
||||
user_data = var.user_data != "" ? file("${var.user_data}") : file("${path.module}/cloud-init.sh")
|
||||
|
||||
root_block_device {
|
||||
volume_type = "gp3"
|
||||
volume_size = 20
|
||||
}
|
||||
|
||||
dynamic "ebs_block_device" {
|
||||
for_each = var.ebs_volumes
|
||||
content {
|
||||
device_name = "/dev/sdb"
|
||||
volume_type = var.fb_data_disk_type
|
||||
volume_size = var.fb_data_disk_size_gb
|
||||
iops = var.fb_data_disk_iops
|
||||
encrypted = true
|
||||
terminate_instances_with_expiration = true
|
||||
launch_template_config {
|
||||
launch_template_specification {
|
||||
id = aws_launch_template.fb_ingest_node.id
|
||||
version = aws_launch_template.fb_ingest_node.latest_version
|
||||
}
|
||||
overrides {
|
||||
subnet_id = var.vpc_public_subnets[0] // TODO
|
||||
}
|
||||
}
|
||||
|
||||
tags = {
|
||||
Prefix = "${var.cluster_prefix}"
|
||||
Name = "${var.cluster_prefix}-featurebase-cluster-${count.index}"
|
||||
Role = "cluster_node"
|
||||
}
|
||||
|
||||
provisioner "local-exec" {
|
||||
command = "aws ec2 create-tags --profile ${var.profile} --resources ${self.spot_instance_id} --tags Key=Prefix,Value='${var.cluster_prefix}' Key=Name,Value='${var.cluster_prefix}-featurebase-cluster-${count.index}' Key=Role,Value=cluster_node --region ${var.region}"
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
resource "aws_spot_instance_request" "fb_ingest" {
|
||||
wait_for_fulfillment = true
|
||||
spot_type = "one-time"
|
||||
count = var.use_spot_instances ? var.fb_ingest_node_count : 0
|
||||
ami = data.aws_ami.amazon_linux_2.id
|
||||
key_name = aws_key_pair.gitlab-featurebase-ci.key_name
|
||||
vpc_security_group_ids = [aws_security_group.ingest.id]
|
||||
instance_type = var.fb_ingest_type
|
||||
associate_public_ip_address = true
|
||||
monitoring = true
|
||||
subnet_id = var.subnet != "" ? var.subnet : var.vpc_public_subnets[count.index % length(var.vpc_public_subnets)]
|
||||
availability_zone = var.zone != "" ? var.zone : var.azs[count.index % length(var.azs)]
|
||||
iam_instance_profile = "${aws_iam_instance_profile.fb_cluster_node_profile.name}"
|
||||
|
||||
root_block_device {
|
||||
volume_type = "gp3"
|
||||
volume_size = 20
|
||||
resource "aws_launch_template" "fb_data_node" {
|
||||
name = "${var.cluster_prefix}-fb-data-node-launch-template"
|
||||
image_id = data.aws_ami.amazon_linux_2.id
|
||||
key_name = aws_key_pair.gitlab-featurebase-ci.key_name
|
||||
user_data = var.user_data != "" ? filebase64(var.user_data) : filebase64("${path.module}/cloud-init.sh")
|
||||
block_device_mappings {
|
||||
device_name = data.aws_ami.amazon_linux_2.root_device_name
|
||||
ebs {
|
||||
encrypted = true
|
||||
volume_size = 20 // GiB
|
||||
volume_type = "gp3"
|
||||
}
|
||||
}
|
||||
|
||||
ebs_block_device {
|
||||
block_device_mappings {
|
||||
device_name = "/dev/sdb"
|
||||
volume_type = var.fb_ingest_disk_type
|
||||
volume_size = var.fb_ingest_disk_size_gb
|
||||
iops = var.fb_ingest_disk_iops
|
||||
encrypted = true
|
||||
ebs {
|
||||
encrypted = true
|
||||
iops = var.fb_data_disk_iops
|
||||
volume_size = var.fb_data_disk_size_gb
|
||||
volume_type = var.fb_data_disk_type
|
||||
}
|
||||
}
|
||||
instance_requirements {
|
||||
memory_mib {
|
||||
min = 8192 // MiB
|
||||
}
|
||||
vcpu_count {
|
||||
min = 2
|
||||
}
|
||||
instance_generations = ["current"]
|
||||
}
|
||||
monitoring {
|
||||
enabled = true
|
||||
}
|
||||
iam_instance_profile {
|
||||
name = aws_iam_instance_profile.fb_cluster_node_profile.name
|
||||
}
|
||||
network_interfaces {
|
||||
device_index = 0
|
||||
subnet_id = var.subnet != "" ? var.subnet : var.vpc_private_subnets[0] // TODO don't always just use the first subnet.
|
||||
security_groups = [aws_security_group.featurebase.id]
|
||||
}
|
||||
instance_market_options {
|
||||
spot_options {
|
||||
spot_instance_type = "one-time"
|
||||
}
|
||||
}
|
||||
tag_specifications {
|
||||
resource_type = "instance"
|
||||
tags = {
|
||||
Name = "${var.cluster_prefix}-featurebase-cluster"
|
||||
Prefix = var.cluster_prefix
|
||||
Role = "cluster_node"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
tags = {
|
||||
Prefix = "${var.cluster_prefix}"
|
||||
Name = "${var.cluster_prefix}-featurebase-ingest-${count.index}"
|
||||
Role = "ingest_node"
|
||||
resource "aws_launch_template" "fb_ingest_node" {
|
||||
name = "${var.cluster_prefix}-fb-ingest-node-launch-template"
|
||||
image_id = data.aws_ami.amazon_linux_2.id
|
||||
key_name = aws_key_pair.gitlab-featurebase-ci.key_name
|
||||
user_data = var.user_data != "" ? filebase64(var.user_data) : filebase64("${path.module}/cloud-init.sh")
|
||||
block_device_mappings {
|
||||
device_name = data.aws_ami.amazon_linux_2.root_device_name
|
||||
ebs {
|
||||
encrypted = true
|
||||
volume_size = 20 // GiB
|
||||
volume_type = "gp3"
|
||||
}
|
||||
}
|
||||
provisioner "local-exec" {
|
||||
command = "aws ec2 create-tags --profile ${var.profile} --resources ${self.spot_instance_id} --tags Key=Prefix,Value='${var.cluster_prefix}' Key=Name,Value='${var.cluster_prefix}-featurebase-ingest-${count.index}' Key=Role,Value=ingest_node --region ${var.region}"
|
||||
block_device_mappings {
|
||||
device_name = "/dev/sdb"
|
||||
ebs {
|
||||
encrypted = true
|
||||
iops = var.fb_ingest_disk_iops
|
||||
volume_size = var.fb_ingest_disk_size_gb
|
||||
volume_type = var.fb_ingest_disk_type
|
||||
}
|
||||
}
|
||||
instance_requirements {
|
||||
memory_mib {
|
||||
min = 8192 // MiB
|
||||
}
|
||||
vcpu_count {
|
||||
min = 2
|
||||
}
|
||||
instance_generations = ["current"]
|
||||
}
|
||||
monitoring {
|
||||
enabled = true
|
||||
}
|
||||
iam_instance_profile {
|
||||
name = aws_iam_instance_profile.fb_cluster_node_profile.name
|
||||
}
|
||||
network_interfaces {
|
||||
associate_public_ip_address = true
|
||||
device_index = 0
|
||||
subnet_id = var.subnet != "" ? var.subnet : var.vpc_public_subnets[0] // TODO don't always just use the first subnet.
|
||||
security_groups = [aws_security_group.featurebase.id]
|
||||
}
|
||||
instance_market_options {
|
||||
spot_options {
|
||||
spot_instance_type = "one-time"
|
||||
}
|
||||
}
|
||||
tag_specifications {
|
||||
resource_type = "instance"
|
||||
tags = {
|
||||
Name = "${var.cluster_prefix}-featurebase-ingest"
|
||||
Prefix = var.cluster_prefix
|
||||
Role = "ingest_node"
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
resource "aws_key_pair" "gitlab-featurebase-ci" {
|
||||
key_name = "${var.cluster_prefix}-gitlab-ci"
|
||||
public_key = "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC91hhpVHNonAG7ku2ugpxEskf9KHeyHJPQJT26OHrMUw7R+T5A8TjqSzTau07sXQ/E9SO3ebV8SJ5PqeaQOnQB8VEvVNK0DjQH7ppvNg1Rfs42FZT9ttzTMvOjsSbK3vZTHXdoKQEdC9NxBwSkFIRGQojK1HUOq9xGrw31fA1OjSwlpLcbx7yyg18lcqW6UOptnVR8U9Yy9qQ5jZF1HtkQ6L9J+gv4o1UyNAUK2bopeGiXpBc3PQ/CFaFT2h/aqLBP66qAHsHVyAFD3PIRtplC5EHa8jXDgLacEls0uF7Q3kRPxvzcuo4g4VkOn1rDy9qH3vd2hT3aKVnM73FIDUiL"
|
||||
|
||||
|
||||
tags = {
|
||||
Prefix = "${var.cluster_prefix}"
|
||||
Prefix = var.cluster_prefix
|
||||
Name = "${var.cluster_prefix}-gitlab-featurebase-ci"
|
||||
Role = "ssh_keypair"
|
||||
}
|
||||
|
|
@ -246,7 +252,7 @@ resource "aws_security_group" "featurebase" {
|
|||
}
|
||||
|
||||
tags = {
|
||||
Prefix = "${var.cluster_prefix}"
|
||||
Prefix = var.cluster_prefix
|
||||
Name = "${var.cluster_prefix}-allow_featurebase"
|
||||
Role = "allow_featurebase"
|
||||
}
|
||||
|
|
@ -280,7 +286,7 @@ resource "aws_security_group" "ingest" {
|
|||
protocol = "tcp"
|
||||
cidr_blocks = ["10.0.0.0/8", "172.31.0.0/16"]
|
||||
}
|
||||
|
||||
|
||||
ingress {
|
||||
description = "SSH"
|
||||
from_port = 22
|
||||
|
|
@ -299,7 +305,7 @@ resource "aws_security_group" "ingest" {
|
|||
}
|
||||
|
||||
tags = {
|
||||
Prefix = "${var.cluster_prefix}"
|
||||
Prefix = var.cluster_prefix
|
||||
Name = "${var.cluster_prefix}-allow_ingest"
|
||||
Role = "allow_ingest"
|
||||
}
|
||||
|
|
@ -310,7 +316,7 @@ resource "aws_iam_instance_profile" "fb_cluster_node_profile" {
|
|||
role = aws_iam_role.fb_cluster_node_role.name
|
||||
|
||||
tags = {
|
||||
Prefix = "${var.cluster_prefix}"
|
||||
Prefix = var.cluster_prefix
|
||||
Name = "${var.cluster_prefix}-fb_cluster_node_profile"
|
||||
Role = "fb_cluster_node_profile"
|
||||
}
|
||||
|
|
@ -353,24 +359,24 @@ resource "aws_iam_role" "fb_cluster_node_role" {
|
|||
Version = "2012-10-17"
|
||||
Statement = [
|
||||
{
|
||||
Sid = "VisualEditor0",
|
||||
Effect = "Allow",
|
||||
Action = ["s3:PutObject", "s3:GetObject"],
|
||||
Resource = "arn:aws:s3:::molecula-perf-storage/*"
|
||||
Sid = "VisualEditor0",
|
||||
Effect = "Allow",
|
||||
Action = ["s3:PutObject", "s3:GetObject"],
|
||||
Resource = "arn:aws:s3:::molecula-perf-storage/*"
|
||||
},
|
||||
{
|
||||
Sid = "VisualEditor1",
|
||||
Effect = "Allow",
|
||||
Action = "s3:PutObject",
|
||||
Resource = "arn:aws:s3:::molecula-artifact-storage/*"
|
||||
Sid = "VisualEditor1",
|
||||
Effect = "Allow",
|
||||
Action = "s3:PutObject",
|
||||
Resource = "arn:aws:s3:::molecula-artifact-storage/*"
|
||||
}
|
||||
]
|
||||
})
|
||||
}
|
||||
|
||||
tags = {
|
||||
Prefix = "${var.cluster_prefix}"
|
||||
Prefix = var.cluster_prefix
|
||||
Name = "${var.cluster_prefix}-fb_cluster_node_role"
|
||||
Role = "fb_cluster_node_role"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,15 +1,30 @@
|
|||
output "ingest_ips" {
|
||||
value = var.use_spot_instances ? aws_spot_instance_request.fb_ingest.*.private_ip : aws_instance.fb_ingest.*.private_ip
|
||||
value = data.aws_instances.ingest_nodes.private_ips
|
||||
}
|
||||
|
||||
output "data_node_ips" {
|
||||
value = var.use_spot_instances ? aws_spot_instance_request.fb_cluster_nodes.*.private_ip : aws_instance.fb_cluster_nodes.*.private_ip
|
||||
value = data.aws_instances.data_nodes.private_ips
|
||||
}
|
||||
|
||||
output "cluster_prefix" {
|
||||
value = var.cluster_prefix
|
||||
value = var.cluster_prefix
|
||||
}
|
||||
|
||||
output "fb_cluster_replica_count" {
|
||||
value = var.fb_cluster_replica_count
|
||||
value = var.fb_cluster_replica_count
|
||||
}
|
||||
|
||||
// Provide the ingest and data node information for module outputs
|
||||
data "aws_instances" "ingest_nodes" {
|
||||
instance_tags = {
|
||||
Prefix = var.cluster_prefix
|
||||
Role = "ingest_node"
|
||||
}
|
||||
}
|
||||
|
||||
data "aws_instances" "data_nodes" {
|
||||
instance_tags = {
|
||||
Prefix = var.cluster_prefix
|
||||
Role = "cluster_node"
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ terraform {
|
|||
required_providers {
|
||||
aws = {
|
||||
source = "hashicorp/aws"
|
||||
version = ">= 3.38.0"
|
||||
version = ">= 4.38.0"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue