From 3bfb65eb583132a7839b335f898d7bba003e7e2c Mon Sep 17 00:00:00 2001 From: Garrison Davis Date: Mon, 7 Nov 2022 16:04:09 -0700 Subject: [PATCH] Make spot instance selection dynamic The idea behind this is to give AWS more information about what instances we can let it actually instantitate, rather than have it be one fixed instance type. e.g., in this case, we are okay with any Graviton instance with at least 2 vCPU and 8 GiB memory. The easist way to do that is to instead use a launch template, with ec2_fleets or spot fleets. I took out the part where we even support on-demand instances. This can be readded later if it is necessary. --- qa/tf/.modules/featurebase-cluster/main.tf | 294 +++++++++--------- qa/tf/.modules/featurebase-cluster/outputs.tf | 23 +- .../.modules/featurebase-cluster/provider.tf | 2 +- 3 files changed, 170 insertions(+), 149 deletions(-) diff --git a/qa/tf/.modules/featurebase-cluster/main.tf b/qa/tf/.modules/featurebase-cluster/main.tf index 8aafa4be7..c7398f6d3 100644 --- a/qa/tf/.modules/featurebase-cluster/main.tf +++ b/qa/tf/.modules/featurebase-cluster/main.tf @@ -17,159 +17,165 @@ data "aws_ami" "amazon_linux_2" { } } -resource "aws_instance" "fb_cluster_nodes" { - count = var.use_spot_instances ? 0 : var.fb_data_node_count - ami = data.aws_ami.amazon_linux_2.id - instance_type = var.fb_data_node_type - key_name = aws_key_pair.gitlab-featurebase-ci.key_name - vpc_security_group_ids = [aws_security_group.featurebase.id] - monitoring = true - subnet_id = var.subnet != "" ? var.subnet : var.vpc_private_subnets[count.index % length(var.vpc_private_subnets)] - availability_zone = var.zone != "" ? var.zone : var.azs[count.index % length(var.azs)] - iam_instance_profile = "${aws_iam_instance_profile.fb_cluster_node_profile.name}" - user_data = var.user_data != "" ? file("${var.user_data}") : file("${path.module}/cloud-init.sh") - - root_block_device { - volume_type = "gp3" - volume_size = 20 - } +resource "aws_spot_fleet_request" "fb_data_nodes" { + target_capacity = var.fb_data_node_count // TODO + iam_fleet_role = "arn:aws:iam::977373308795:role/aws-ec2-spot-fleet-tagging-role" // TODO - dynamic "ebs_block_device" { - for_each = var.ebs_volumes - content { - device_name = "/dev/sdb" - volume_type = var.fb_data_disk_type - volume_size = var.fb_data_disk_size_gb - iops = var.fb_data_disk_iops - encrypted = true + fleet_type = "request" + wait_for_fulfillment = true + + terminate_instances_with_expiration = true + launch_template_config { + launch_template_specification { + id = aws_launch_template.fb_data_node.id + version = aws_launch_template.fb_data_node.latest_version + } + overrides { + subnet_id = var.vpc_private_subnets[0] // TODO } } - - tags = { - Prefix = "${var.cluster_prefix}" - Name = "${var.cluster_prefix}-featurebase-cluster-${count.index}" - Role = "cluster_node" - } } -resource "aws_instance" "fb_ingest" { - count = var.use_spot_instances ? 0 : var.fb_ingest_node_count - ami = data.aws_ami.amazon_linux_2.id - key_name = aws_key_pair.gitlab-featurebase-ci.key_name - vpc_security_group_ids = [aws_security_group.ingest.id] - instance_type = var.fb_ingest_type - associate_public_ip_address = true - monitoring = true - subnet_id = var.subnet != "" ? var.subnet : var.vpc_public_subnets[count.index % length(var.vpc_public_subnets)] - availability_zone = var.zone != "" ? var.zone : var.azs[count.index % length(var.azs)] - iam_instance_profile = "${aws_iam_instance_profile.fb_cluster_node_profile.name}" +resource "aws_spot_fleet_request" "fb_ingest_nodes" { + target_capacity = var.fb_ingest_node_count + iam_fleet_role = "arn:aws:iam::977373308795:role/aws-ec2-spot-fleet-tagging-role" // TODO - root_block_device { - volume_type = "gp3" - volume_size = 20 - } + fleet_type = "request" + wait_for_fulfillment = true - ebs_block_device { - device_name = "/dev/sdb" - volume_type = var.fb_ingest_disk_type - volume_size = var.fb_ingest_disk_size_gb - iops = var.fb_ingest_disk_iops - encrypted = true - } - - tags = { - Prefix = "${var.cluster_prefix}" - Name = "${var.cluster_prefix}-featurebase-ingest-${count.index}" - Role = "ingest_node" - } - -} -resource "aws_spot_instance_request" "fb_cluster_nodes" { - wait_for_fulfillment = true - spot_type = "one-time" - count = var.use_spot_instances ? var.fb_data_node_count : 0 - ami = data.aws_ami.amazon_linux_2.id - instance_type = var.fb_data_node_type - key_name = aws_key_pair.gitlab-featurebase-ci.key_name - vpc_security_group_ids = [aws_security_group.featurebase.id] - monitoring = true - subnet_id = var.subnet != "" ? var.subnet : var.vpc_private_subnets[count.index % length(var.vpc_private_subnets)] - availability_zone = var.zone != "" ? var.zone : var.azs[count.index % length(var.azs)] - iam_instance_profile = "${aws_iam_instance_profile.fb_cluster_node_profile.name}" - user_data = var.user_data != "" ? file("${var.user_data}") : file("${path.module}/cloud-init.sh") - - root_block_device { - volume_type = "gp3" - volume_size = 20 - } - - dynamic "ebs_block_device" { - for_each = var.ebs_volumes - content { - device_name = "/dev/sdb" - volume_type = var.fb_data_disk_type - volume_size = var.fb_data_disk_size_gb - iops = var.fb_data_disk_iops - encrypted = true + terminate_instances_with_expiration = true + launch_template_config { + launch_template_specification { + id = aws_launch_template.fb_ingest_node.id + version = aws_launch_template.fb_ingest_node.latest_version + } + overrides { + subnet_id = var.vpc_public_subnets[0] // TODO } } - - tags = { - Prefix = "${var.cluster_prefix}" - Name = "${var.cluster_prefix}-featurebase-cluster-${count.index}" - Role = "cluster_node" - } - - provisioner "local-exec" { - command = "aws ec2 create-tags --profile ${var.profile} --resources ${self.spot_instance_id} --tags Key=Prefix,Value='${var.cluster_prefix}' Key=Name,Value='${var.cluster_prefix}-featurebase-cluster-${count.index}' Key=Role,Value=cluster_node --region ${var.region}" - } - } -resource "aws_spot_instance_request" "fb_ingest" { - wait_for_fulfillment = true - spot_type = "one-time" - count = var.use_spot_instances ? var.fb_ingest_node_count : 0 - ami = data.aws_ami.amazon_linux_2.id - key_name = aws_key_pair.gitlab-featurebase-ci.key_name - vpc_security_group_ids = [aws_security_group.ingest.id] - instance_type = var.fb_ingest_type - associate_public_ip_address = true - monitoring = true - subnet_id = var.subnet != "" ? var.subnet : var.vpc_public_subnets[count.index % length(var.vpc_public_subnets)] - availability_zone = var.zone != "" ? var.zone : var.azs[count.index % length(var.azs)] - iam_instance_profile = "${aws_iam_instance_profile.fb_cluster_node_profile.name}" - - root_block_device { - volume_type = "gp3" - volume_size = 20 +resource "aws_launch_template" "fb_data_node" { + name = "${var.cluster_prefix}-fb-data-node-launch-template" + image_id = data.aws_ami.amazon_linux_2.id + key_name = aws_key_pair.gitlab-featurebase-ci.key_name + user_data = var.user_data != "" ? filebase64(var.user_data) : filebase64("${path.module}/cloud-init.sh") + block_device_mappings { + device_name = data.aws_ami.amazon_linux_2.root_device_name + ebs { + encrypted = true + volume_size = 20 // GiB + volume_type = "gp3" + } } - - ebs_block_device { + block_device_mappings { device_name = "/dev/sdb" - volume_type = var.fb_ingest_disk_type - volume_size = var.fb_ingest_disk_size_gb - iops = var.fb_ingest_disk_iops - encrypted = true + ebs { + encrypted = true + iops = var.fb_data_disk_iops + volume_size = var.fb_data_disk_size_gb + volume_type = var.fb_data_disk_type + } } + instance_requirements { + memory_mib { + min = 8192 // MiB + } + vcpu_count { + min = 2 + } + instance_generations = ["current"] + } + monitoring { + enabled = true + } + iam_instance_profile { + name = aws_iam_instance_profile.fb_cluster_node_profile.name + } + network_interfaces { + device_index = 0 + subnet_id = var.subnet != "" ? var.subnet : var.vpc_private_subnets[0] // TODO don't always just use the first subnet. + security_groups = [aws_security_group.featurebase.id] + } + instance_market_options { + spot_options { + spot_instance_type = "one-time" + } + } + tag_specifications { + resource_type = "instance" + tags = { + Name = "${var.cluster_prefix}-featurebase-cluster" + Prefix = var.cluster_prefix + Role = "cluster_node" + } + } +} - tags = { - Prefix = "${var.cluster_prefix}" - Name = "${var.cluster_prefix}-featurebase-ingest-${count.index}" - Role = "ingest_node" +resource "aws_launch_template" "fb_ingest_node" { + name = "${var.cluster_prefix}-fb-ingest-node-launch-template" + image_id = data.aws_ami.amazon_linux_2.id + key_name = aws_key_pair.gitlab-featurebase-ci.key_name + user_data = var.user_data != "" ? filebase64(var.user_data) : filebase64("${path.module}/cloud-init.sh") + block_device_mappings { + device_name = data.aws_ami.amazon_linux_2.root_device_name + ebs { + encrypted = true + volume_size = 20 // GiB + volume_type = "gp3" + } } - provisioner "local-exec" { - command = "aws ec2 create-tags --profile ${var.profile} --resources ${self.spot_instance_id} --tags Key=Prefix,Value='${var.cluster_prefix}' Key=Name,Value='${var.cluster_prefix}-featurebase-ingest-${count.index}' Key=Role,Value=ingest_node --region ${var.region}" + block_device_mappings { + device_name = "/dev/sdb" + ebs { + encrypted = true + iops = var.fb_ingest_disk_iops + volume_size = var.fb_ingest_disk_size_gb + volume_type = var.fb_ingest_disk_type + } + } + instance_requirements { + memory_mib { + min = 8192 // MiB + } + vcpu_count { + min = 2 + } + instance_generations = ["current"] + } + monitoring { + enabled = true + } + iam_instance_profile { + name = aws_iam_instance_profile.fb_cluster_node_profile.name + } + network_interfaces { + associate_public_ip_address = true + device_index = 0 + subnet_id = var.subnet != "" ? var.subnet : var.vpc_public_subnets[0] // TODO don't always just use the first subnet. + security_groups = [aws_security_group.featurebase.id] + } + instance_market_options { + spot_options { + spot_instance_type = "one-time" + } + } + tag_specifications { + resource_type = "instance" + tags = { + Name = "${var.cluster_prefix}-featurebase-ingest" + Prefix = var.cluster_prefix + Role = "ingest_node" + } } - } resource "aws_key_pair" "gitlab-featurebase-ci" { key_name = "${var.cluster_prefix}-gitlab-ci" public_key = "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC91hhpVHNonAG7ku2ugpxEskf9KHeyHJPQJT26OHrMUw7R+T5A8TjqSzTau07sXQ/E9SO3ebV8SJ5PqeaQOnQB8VEvVNK0DjQH7ppvNg1Rfs42FZT9ttzTMvOjsSbK3vZTHXdoKQEdC9NxBwSkFIRGQojK1HUOq9xGrw31fA1OjSwlpLcbx7yyg18lcqW6UOptnVR8U9Yy9qQ5jZF1HtkQ6L9J+gv4o1UyNAUK2bopeGiXpBc3PQ/CFaFT2h/aqLBP66qAHsHVyAFD3PIRtplC5EHa8jXDgLacEls0uF7Q3kRPxvzcuo4g4VkOn1rDy9qH3vd2hT3aKVnM73FIDUiL" - + tags = { - Prefix = "${var.cluster_prefix}" + Prefix = var.cluster_prefix Name = "${var.cluster_prefix}-gitlab-featurebase-ci" Role = "ssh_keypair" } @@ -246,7 +252,7 @@ resource "aws_security_group" "featurebase" { } tags = { - Prefix = "${var.cluster_prefix}" + Prefix = var.cluster_prefix Name = "${var.cluster_prefix}-allow_featurebase" Role = "allow_featurebase" } @@ -280,7 +286,7 @@ resource "aws_security_group" "ingest" { protocol = "tcp" cidr_blocks = ["10.0.0.0/8", "172.31.0.0/16"] } - + ingress { description = "SSH" from_port = 22 @@ -299,7 +305,7 @@ resource "aws_security_group" "ingest" { } tags = { - Prefix = "${var.cluster_prefix}" + Prefix = var.cluster_prefix Name = "${var.cluster_prefix}-allow_ingest" Role = "allow_ingest" } @@ -310,7 +316,7 @@ resource "aws_iam_instance_profile" "fb_cluster_node_profile" { role = aws_iam_role.fb_cluster_node_role.name tags = { - Prefix = "${var.cluster_prefix}" + Prefix = var.cluster_prefix Name = "${var.cluster_prefix}-fb_cluster_node_profile" Role = "fb_cluster_node_profile" } @@ -353,24 +359,24 @@ resource "aws_iam_role" "fb_cluster_node_role" { Version = "2012-10-17" Statement = [ { - Sid = "VisualEditor0", - Effect = "Allow", - Action = ["s3:PutObject", "s3:GetObject"], - Resource = "arn:aws:s3:::molecula-perf-storage/*" + Sid = "VisualEditor0", + Effect = "Allow", + Action = ["s3:PutObject", "s3:GetObject"], + Resource = "arn:aws:s3:::molecula-perf-storage/*" }, { - Sid = "VisualEditor1", - Effect = "Allow", - Action = "s3:PutObject", - Resource = "arn:aws:s3:::molecula-artifact-storage/*" + Sid = "VisualEditor1", + Effect = "Allow", + Action = "s3:PutObject", + Resource = "arn:aws:s3:::molecula-artifact-storage/*" } ] }) } tags = { - Prefix = "${var.cluster_prefix}" + Prefix = var.cluster_prefix Name = "${var.cluster_prefix}-fb_cluster_node_role" Role = "fb_cluster_node_role" } -} \ No newline at end of file +} diff --git a/qa/tf/.modules/featurebase-cluster/outputs.tf b/qa/tf/.modules/featurebase-cluster/outputs.tf index 070159697..fd9905c13 100644 --- a/qa/tf/.modules/featurebase-cluster/outputs.tf +++ b/qa/tf/.modules/featurebase-cluster/outputs.tf @@ -1,15 +1,30 @@ output "ingest_ips" { - value = var.use_spot_instances ? aws_spot_instance_request.fb_ingest.*.private_ip : aws_instance.fb_ingest.*.private_ip + value = data.aws_instances.ingest_nodes.private_ips } output "data_node_ips" { - value = var.use_spot_instances ? aws_spot_instance_request.fb_cluster_nodes.*.private_ip : aws_instance.fb_cluster_nodes.*.private_ip + value = data.aws_instances.data_nodes.private_ips } output "cluster_prefix" { - value = var.cluster_prefix + value = var.cluster_prefix } output "fb_cluster_replica_count" { - value = var.fb_cluster_replica_count + value = var.fb_cluster_replica_count +} + +// Provide the ingest and data node information for module outputs +data "aws_instances" "ingest_nodes" { + instance_tags = { + Prefix = var.cluster_prefix + Role = "ingest_node" + } +} + +data "aws_instances" "data_nodes" { + instance_tags = { + Prefix = var.cluster_prefix + Role = "cluster_node" + } } diff --git a/qa/tf/.modules/featurebase-cluster/provider.tf b/qa/tf/.modules/featurebase-cluster/provider.tf index cf53a4ed7..118d882c6 100644 --- a/qa/tf/.modules/featurebase-cluster/provider.tf +++ b/qa/tf/.modules/featurebase-cluster/provider.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 3.38.0" + version = ">= 4.38.0" } } }