diff --git a/terraform/litellm/gcp/README.md b/terraform/litellm/gcp/README.md index 4b2f576adc1..85363074086 100644 --- a/terraform/litellm/gcp/README.md +++ b/terraform/litellm/gcp/README.md @@ -16,8 +16,10 @@ Deploys the componentized LiteLLM proxy on GCP: - **Cloud Run v2** services for `gateway` (port 4000), `backend` (port 4001), and `ui` (port 3000), all using a shared runtime service account - **Cloud Run Job** (`litellm-migrations`) that runs `prisma migrate deploy` from the dedicated `ghcr.io/berriai/litellm-migrations` image -- **External global HTTP(S) load balancer** with serverless NEGs and a URL - map mirroring the helm-chart ingress path routing: +- **HTTP(S) load balancer** with serverless NEGs and a URL map mirroring the + helm-chart ingress path routing. `load_balancing_scheme` controls mode: + - `EXTERNAL_MANAGED`: global external LB + - `INTERNAL_MANAGED`: global internal managed LB (private IP) - LLM data-plane prefixes → `gateway` - UI asset paths → `ui` - Everything else → `backend` @@ -440,6 +442,9 @@ vpcaccess, compute, servicenetworking, storage, artifactregistry). ## TLS +TLS and `lb_domains` apply only when `load_balancing_scheme = "EXTERNAL_MANAGED"`. +For `INTERNAL_MANAGED`, set `lb_domains = []` (or leave it unset) and `allow_plaintext_lb = true`. + `terraform plan` refuses to provision an HTTP-only LB by default — TLS is the supported posture. Two paths: diff --git a/terraform/litellm/gcp/examples/default/main.tf b/terraform/litellm/gcp/examples/default/main.tf index 782d4ee4b65..23cde567b8e 100644 --- a/terraform/litellm/gcp/examples/default/main.tf +++ b/terraform/litellm/gcp/examples/default/main.tf @@ -43,6 +43,7 @@ module "litellm" { image_registry = var.image_registry image_tag = var.image_tag + load_balancing_scheme = var.load_balancing_scheme lb_domains = var.lb_domains allow_plaintext_lb = var.allow_plaintext_lb cloudsql_deletion_protection = var.cloudsql_deletion_protection diff --git a/terraform/litellm/gcp/examples/default/terraform.tfvars.example b/terraform/litellm/gcp/examples/default/terraform.tfvars.example index ec6206734e2..553fb1aab85 100644 --- a/terraform/litellm/gcp/examples/default/terraform.tfvars.example +++ b/terraform/litellm/gcp/examples/default/terraform.tfvars.example @@ -27,7 +27,9 @@ env = "stage" # TLS: provide DNS names already pointing at the LB IP for a Google-managed # cert. Without one, plan fails unless allow_plaintext_lb = true is set -# explicitly (trial/dev only). +# explicitly (trial/dev only). INTERNAL_MANAGED is private-IP only and +# should keep lb_domains empty. +# load_balancing_scheme = "EXTERNAL_MANAGED" # or "INTERNAL_MANAGED" # lb_domains = ["proxy.example.com"] # allow_plaintext_lb = true diff --git a/terraform/litellm/gcp/examples/default/variables.tf b/terraform/litellm/gcp/examples/default/variables.tf index 08b78346df3..cf66b58f20a 100644 --- a/terraform/litellm/gcp/examples/default/variables.tf +++ b/terraform/litellm/gcp/examples/default/variables.tf @@ -90,11 +90,22 @@ variable "image_tag" { # TLS — provide DNS names for a managed cert, or opt into HTTP-only for dev. variable "lb_domains" { - description = "DNS names (already pointing at lb_ip) for a Google-managed cert. Empty → no TLS." + description = "DNS names (already pointing at lb_ip) for a Google-managed cert in EXTERNAL_MANAGED mode. Empty → no TLS." type = list(string) default = [] } +variable "load_balancing_scheme" { + description = "Load balancer scheme. Allowed values: EXTERNAL_MANAGED or INTERNAL_MANAGED." + type = string + default = "EXTERNAL_MANAGED" + + validation { + condition = contains(["EXTERNAL_MANAGED", "INTERNAL_MANAGED"], var.load_balancing_scheme) + error_message = "load_balancing_scheme must be EXTERNAL_MANAGED or INTERNAL_MANAGED." + } +} + variable "allow_plaintext_lb" { description = "Opt into HTTP-only LB (trial/dev only)." type = bool diff --git a/terraform/litellm/gcp/load_balancer.tf b/terraform/litellm/gcp/load_balancer.tf index 57e8af8210f..86bf977ffeb 100644 --- a/terraform/litellm/gcp/load_balancer.tf +++ b/terraform/litellm/gcp/load_balancer.tf @@ -1,5 +1,7 @@ -# External global HTTP(S) load balancer fronting all three Cloud Run -# services. URL map mirrors the helm-chart ingress path routing: +# HTTP(S) load balancer fronting all three Cloud Run services. +# EXTERNAL_MANAGED uses the global external path, INTERNAL_MANAGED uses +# the global (cross-region) internal managed path. URL map mirrors the helm-chart +# ingress path routing: # - LLM data-plane paths → gateway # - UI asset paths → ui # - Everything else → backend (management API: /key/*, /user/*, …) @@ -11,10 +13,12 @@ locals { tls_enabled = length(var.lb_domains) > 0 + is_external = var.load_balancing_scheme == "EXTERNAL_MANAGED" + is_internal = var.load_balancing_scheme == "INTERNAL_MANAGED" } resource "google_compute_global_address" "lb" { - count = var.create_runtime ? 1 : 0 + count = var.create_runtime && local.is_external ? 1 : 0 name = "${local.name}-lb-ip" labels = local.labels @@ -57,13 +61,14 @@ resource "google_compute_region_network_endpoint_group" "ui" { } } -# Backend services wrap each NEG. +# Backend services wrap each NEG. The selected load_balancing_scheme controls +# whether these serve EXTERNAL_MANAGED or INTERNAL_MANAGED. resource "google_compute_backend_service" "gateway" { count = var.create_runtime ? 1 : 0 name = "${local.name}-gateway-bs" protocol = "HTTP" - load_balancing_scheme = "EXTERNAL_MANAGED" + load_balancing_scheme = var.load_balancing_scheme backend { group = google_compute_region_network_endpoint_group.gateway[0].id @@ -75,7 +80,7 @@ resource "google_compute_backend_service" "backend" { name = "${local.name}-backend-bs" protocol = "HTTP" - load_balancing_scheme = "EXTERNAL_MANAGED" + load_balancing_scheme = var.load_balancing_scheme backend { group = google_compute_region_network_endpoint_group.backend[0].id @@ -87,7 +92,7 @@ resource "google_compute_backend_service" "ui" { name = "${local.name}-ui-bs" protocol = "HTTP" - load_balancing_scheme = "EXTERNAL_MANAGED" + load_balancing_scheme = var.load_balancing_scheme backend { group = google_compute_region_network_endpoint_group.ui[0].id @@ -134,7 +139,7 @@ resource "google_compute_url_map" "this" { # target proxy when TLS is enabled; otherwise the regular path-routing # URL map is attached to the HTTP proxy and everything stays plaintext. resource "google_compute_url_map" "https_redirect" { - count = var.create_runtime && local.tls_enabled ? 1 : 0 + count = var.create_runtime && local.tls_enabled && local.is_external ? 1 : 0 name = "${local.name}-redirect" default_url_redirect { @@ -153,6 +158,11 @@ resource "google_compute_target_http_proxy" "this" { # Default-deny on the HTTP-only path: TLS is the supported posture. # Operators must either supply DNS names or explicitly opt in. lifecycle { + precondition { + condition = var.load_balancing_scheme != "INTERNAL_MANAGED" || length(var.lb_domains) == 0 + error_message = "INTERNAL_MANAGED does not support lb_domains/TLS in this module. Set load_balancing_scheme = \"EXTERNAL_MANAGED\" to use lb_domains, or leave lb_domains empty for INTERNAL_MANAGED." + } + precondition { condition = local.tls_enabled || var.allow_plaintext_lb error_message = "LB has no HTTPS forwarding rule. Either set `lb_domains` to a list of DNS names you want a Google-managed cert for, or set `allow_plaintext_lb = true` to opt into HTTP-only (trial / dev only)." @@ -161,17 +171,31 @@ resource "google_compute_target_http_proxy" "this" { } resource "google_compute_global_forwarding_rule" "http" { - count = var.create_runtime ? 1 : 0 + count = var.create_runtime && local.is_external ? 1 : 0 name = "${local.name}-http" ip_protocol = "TCP" port_range = "80" - load_balancing_scheme = "EXTERNAL_MANAGED" + load_balancing_scheme = var.load_balancing_scheme ip_address = google_compute_global_address.lb[0].address target = google_compute_target_http_proxy.this[0].id labels = local.labels } +resource "google_compute_global_forwarding_rule" "http_internal" { + count = var.create_runtime && local.is_internal ? 1 : 0 + name = "${local.name}-http" + network = google_compute_network.this[0].id + subnetwork = google_compute_subnetwork.this[0].id + ip_protocol = "TCP" + port_range = "80" + load_balancing_scheme = var.load_balancing_scheme + target = google_compute_target_http_proxy.this[0].id + labels = local.labels + + depends_on = [google_compute_subnetwork.managed_proxy] +} + # ---------- HTTPS (gated on var.lb_domains) ---------- # # Google-managed certs require each listed domain to resolve to lb_ip @@ -181,7 +205,7 @@ resource "google_compute_global_forwarding_rule" "http" { # transitions to ACTIVE. resource "google_compute_managed_ssl_certificate" "this" { - count = var.create_runtime && local.tls_enabled ? 1 : 0 + count = var.create_runtime && local.tls_enabled && local.is_external ? 1 : 0 # A managed cert's `domains` is immutable, so changing var.lb_domains # forces replacement, and the cert is referenced by the HTTPS target @@ -201,18 +225,18 @@ resource "google_compute_managed_ssl_certificate" "this" { } resource "google_compute_target_https_proxy" "this" { - count = var.create_runtime && local.tls_enabled ? 1 : 0 + count = var.create_runtime && local.tls_enabled && local.is_external ? 1 : 0 name = "${local.name}-https" url_map = google_compute_url_map.this[0].id ssl_certificates = [google_compute_managed_ssl_certificate.this[0].id] } resource "google_compute_global_forwarding_rule" "https" { - count = var.create_runtime && local.tls_enabled ? 1 : 0 + count = var.create_runtime && local.tls_enabled && local.is_external ? 1 : 0 name = "${local.name}-https" ip_protocol = "TCP" port_range = "443" - load_balancing_scheme = "EXTERNAL_MANAGED" + load_balancing_scheme = var.load_balancing_scheme ip_address = google_compute_global_address.lb[0].address target = google_compute_target_https_proxy.this[0].id labels = local.labels diff --git a/terraform/litellm/gcp/network.tf b/terraform/litellm/gcp/network.tf index 47c7bf94a2b..cae593a3676 100644 --- a/terraform/litellm/gcp/network.tf +++ b/terraform/litellm/gcp/network.tf @@ -16,6 +16,17 @@ resource "google_compute_subnetwork" "this" { private_ip_google_access = true } +resource "google_compute_subnetwork" "managed_proxy" { + count = local.is_internal ? 1 : 0 + + name = "${local.name}-${var.region}-managed-proxy" + region = var.region + network = google_compute_network.this[0].id + ip_cidr_range = var.lb_proxy_only_subnet_cidr + purpose = "GLOBAL_MANAGED_PROXY" + role = "ACTIVE" +} + # Private Services Access (PSA) range for Cloud SQL + Memorystore. Both # managed services peer with the VPC over the connection below using # addresses from this range. diff --git a/terraform/litellm/gcp/outputs.tf b/terraform/litellm/gcp/outputs.tf index 2a4742f42cf..cd015594c85 100644 --- a/terraform/litellm/gcp/outputs.tf +++ b/terraform/litellm/gcp/outputs.tf @@ -1,11 +1,21 @@ output "lb_ip" { - description = "Global anycast IP of the external HTTPS load balancer. Null when create_runtime is false." - value = var.create_runtime ? one(google_compute_global_address.lb[*].address) : null + description = "Load balancer IP. Global anycast for EXTERNAL_MANAGED, global private IP for INTERNAL_MANAGED. Null when create_runtime is false." + value = var.create_runtime ? ( + var.load_balancing_scheme == "EXTERNAL_MANAGED" ? google_compute_global_address.lb[0].address : google_compute_global_forwarding_rule.http_internal[0].ip_address + ) : ( + null + ) } output "lb_url" { - description = "Proxy URL, or null when create_runtime is false. Switches scheme based on whether lb_domains is set." - value = var.create_runtime ? (local.tls_enabled ? "https://${var.lb_domains[0]}" : "http://${one(google_compute_global_address.lb[*].address)}") : null + description = "Proxy URL, or null when create_runtime is false. Switches scheme based on whether lb_domains is set; when TLS is enabled the URL points at the first listed domain (since managed certs are tied to the hostname, not the anycast IP). The dashboard is served at /, the API at /v1/*." + value = var.create_runtime ? (local.tls_enabled ? ( + "https://${var.lb_domains[0]}" + ) : ( + format("http://%s", var.load_balancing_scheme == "EXTERNAL_MANAGED" ? google_compute_global_address.lb[0].address : google_compute_global_forwarding_rule.http_internal[0].ip_address) + )) : ( + null + ) } output "gateway_service_url" { diff --git a/terraform/litellm/gcp/variables.tf b/terraform/litellm/gcp/variables.tf index 412f919ab89..857b34af1e1 100644 --- a/terraform/litellm/gcp/variables.tf +++ b/terraform/litellm/gcp/variables.tf @@ -119,6 +119,12 @@ variable "vpc_connector_cidr" { default = "10.41.0.0/28" } +variable "lb_proxy_only_subnet_cidr" { + description = "CIDR for the regional proxy-only subnet used by INTERNAL_MANAGED load balancing." + type = string + default = "10.42.0.0/23" +} + # ---------- Component images ---------- # # Cloud Run only pulls from Artifact Registry, [region.]gcr.io, or @@ -380,6 +386,21 @@ variable "lb_domains" { default = [] } +variable "load_balancing_scheme" { + description = <<-EOT + Load balancer scheme for the GCP HTTP(S) load balancer resources. + Allowed values match the provider/API values directly: + `EXTERNAL_MANAGED` (public) and `INTERNAL_MANAGED` (private). + EOT + type = string + default = "EXTERNAL_MANAGED" + + validation { + condition = contains(["EXTERNAL_MANAGED", "INTERNAL_MANAGED"], var.load_balancing_scheme) + error_message = "load_balancing_scheme must be EXTERNAL_MANAGED or INTERNAL_MANAGED." + } +} + variable "allow_plaintext_lb" { description = <<-EOT Opt into HTTP-only mode on the load balancer (port 80, no TLS).