Merge 5434dd8e1c into 30f33a949b

terraform/litellm/gcp/README.md

@@ -16,8 +16,10 @@ Deploys the componentized LiteLLM proxy on GCP:
 - **Cloud Run v2** services for `gateway` (port 4000), `backend` (port 4001),
   and `ui` (port 3000), all using a shared runtime service account
 - **Cloud Run Job** (`litellm-migrations`) that runs `prisma migrate deploy` from the dedicated `ghcr.io/berriai/litellm-migrations` image
-- **External global HTTP(S) load balancer** with serverless NEGs and a URL
-  map mirroring the helm-chart ingress path routing:
+- **HTTP(S) load balancer** with serverless NEGs and a URL map mirroring the
+  helm-chart ingress path routing. `load_balancing_scheme` controls mode:
+  - `EXTERNAL_MANAGED`: global external LB
+  - `INTERNAL_MANAGED`: global internal managed LB (private IP)
   - LLM data-plane prefixes → `gateway`
   - UI asset paths → `ui`
   - Everything else → `backend`
@@ -440,10 +442,15 @@ vpcaccess, compute, servicenetworking, storage, artifactregistry).
 
 ## TLS
 
+TLS is supported for both LB schemes, with different certificate inputs:
+
+- `EXTERNAL_MANAGED`: set `lb_domains`; this module creates a Google-managed cert
+- `INTERNAL_MANAGED`: set both `lb_domains` and `certificate_manager_certificates`; this module uses your pre-existing Certificate Manager certificates
+
 `terraform plan` refuses to provision an HTTP-only LB by default — TLS
 is the supported posture. Two paths:
 
-**Production / staging — set `lb_domains`:**
+**Production / staging (INTERNAL_MANAGED) — set `lb_domains`:**
 
 1. `terraform apply` once with `allow_plaintext_lb = true` (intentional
    chicken-and-egg escape hatch) to provision the LB and read the anycast
@@ -459,10 +466,22 @@ managed cert sits in `PROVISIONING` for ~15-60 min on first apply until
 DNS propagation completes — `gcloud compute ssl-certificates describe
 <tenant>-litellm-<env>-cert` shows the state.
 
+**Production / staging (INTERNAL_MANAGED) — set both hostnames and cert refs:**
+
+Set:
+
+- `lb_domains = ["proxy.internal.example.com"]`
+- `certificate_manager_certificates = ["projects/<project>/locations/global/certificates/<name>"]`
+
+Result: a 443 internal forwarding rule and HTTPS target proxy using the
+provided certificates. If exactly one of the two variables is set,
+`terraform plan` fails with a clear precondition error.
+
 **Trial / dev — explicitly opt into HTTP-only:**
 
-Set `allow_plaintext_lb = true` and leave `lb_domains = []`. Without the
-flag, plan fails with a clear error pointing at the precondition.
+Set `allow_plaintext_lb = true`, leave `lb_domains = []`, and leave
+`certificate_manager_certificates = []`. Without the flag, plan fails
+with a clear error pointing at the precondition.
 Intended for short-lived trial / dev stacks only.
 
 ## Using as a module

terraform/litellm/gcp/examples/default/TUTORIAL.md

@@ -81,7 +81,12 @@ gcloud secrets versions access latest \
 
 ## Going to TLS
 
-If you picked `allow_plaintext_lb=true` to bootstrap but want HTTPS for real, point a DNS A record at the LB IP, then re-run terraform with `lb_domains` set and `allow_plaintext_lb` removed:
+If you picked `allow_plaintext_lb=true` to bootstrap but want HTTPS for real:
+
+- EXTERNAL_MANAGED: point a DNS A record at the LB IP and set `lb_domains`
+- INTERNAL_MANAGED: set both `lb_domains` and `certificate_manager_certificates`
+
+Then re-run terraform with `allow_plaintext_lb` removed:
 
 ```bash
 terraform apply \

terraform/litellm/gcp/examples/default/deploystack.json

@@ -34,7 +34,7 @@
     },
     {
       "name": "allow_plaintext_lb",
-      "description": "Skip TLS on the load balancer (HTTP-only). Set true for trial/dev. For production, leave false and add lb_domains to terraform.tfvars after the first apply",
+      "description": "Skip TLS on the load balancer (HTTP-only). Set true for trial/dev. For EXTERNAL_MANAGED production, leave false and add lb_domains after bootstrap. For INTERNAL_MANAGED TLS, set both lb_domains and certificate_manager_certificates",
       "default": "true",
       "options": ["true", "false"]
     }

terraform/litellm/gcp/examples/default/main.tf

@@ -43,10 +43,12 @@ module "litellm" {
   image_registry = var.image_registry
   image_tag      = var.image_tag
 
-  lb_domains                   = var.lb_domains
-  allow_plaintext_lb           = var.allow_plaintext_lb
-  cloudsql_deletion_protection = var.cloudsql_deletion_protection
-  gcs_force_destroy            = var.gcs_force_destroy
+  load_balancing_scheme            = var.load_balancing_scheme
+  lb_domains                       = var.lb_domains
+  certificate_manager_certificates = var.certificate_manager_certificates
+  allow_plaintext_lb               = var.allow_plaintext_lb
+  cloudsql_deletion_protection     = var.cloudsql_deletion_protection
+  gcs_force_destroy                = var.gcs_force_destroy
 
   proxy_config          = var.proxy_config
   gateway_extra_env     = var.gateway_extra_env

terraform/litellm/gcp/examples/default/terraform.tfvars.example

@@ -27,8 +27,11 @@ env    = "stage"
 
 # TLS: provide DNS names already pointing at the LB IP for a Google-managed
 # cert. Without one, plan fails unless allow_plaintext_lb = true is set
-# explicitly (trial/dev only).
-# lb_domains         = ["proxy.example.com"]
+# explicitly (trial/dev only). INTERNAL_MANAGED TLS requires both
+# lb_domains and certificate_manager_certificates.
+# load_balancing_scheme = "EXTERNAL_MANAGED" # or "INTERNAL_MANAGED"
+# lb_domains = ["proxy.example.com"]
+# certificate_manager_certificates = ["projects/<project>/locations/global/certificates/<name>"]
 # allow_plaintext_lb = true
 
 # Storage and database retention. Defaults are safe — destroy preserves

terraform/litellm/gcp/examples/default/variables.tf

@@ -90,11 +90,28 @@ variable "image_tag" {
 
 # TLS — provide DNS names for a managed cert, or opt into HTTP-only for dev.
 variable "lb_domains" {
-  description = "DNS names (already pointing at lb_ip) for a Google-managed cert. Empty → no TLS."
+  description = "TLS hostnames. EXTERNAL_MANAGED creates a Google-managed cert; INTERNAL_MANAGED requires this plus certificate_manager_certificates. Empty -> no TLS."
   type        = list(string)
   default     = []
 }
 
+variable "certificate_manager_certificates" {
+  description = "Pre-existing Certificate Manager certificate self_links for INTERNAL_MANAGED TLS."
+  type        = list(string)
+  default     = []
+}
+
+variable "load_balancing_scheme" {
+  description = "Load balancer scheme. Allowed values: EXTERNAL_MANAGED or INTERNAL_MANAGED."
+  type        = string
+  default     = "EXTERNAL_MANAGED"
+
+  validation {
+    condition     = contains(["EXTERNAL_MANAGED", "INTERNAL_MANAGED"], var.load_balancing_scheme)
+    error_message = "load_balancing_scheme must be EXTERNAL_MANAGED or INTERNAL_MANAGED."
+  }
+}
+
 variable "allow_plaintext_lb" {
   description = "Opt into HTTP-only LB (trial/dev only)."
   type        = bool

terraform/litellm/gcp/load_balancer.tf

@@ -1,5 +1,7 @@
-# External global HTTP(S) load balancer fronting all three Cloud Run
-# services. URL map mirrors the helm-chart ingress path routing:
+# HTTP(S) load balancer fronting all three Cloud Run services.
+# EXTERNAL_MANAGED uses the global external path, INTERNAL_MANAGED uses
+# the global (cross-region) internal managed path. URL map mirrors the helm-chart
+# ingress path routing:
 #   - LLM data-plane paths → gateway
 #   - UI asset paths → ui
 #   - Everything else → backend (management API: /key/*, /user/*, …)
@@ -10,11 +12,16 @@
 # is rewritten to redirect HTTP→HTTPS via a redirect-only URL map.
 
 locals {
-  tls_enabled = length(var.lb_domains) > 0
+  is_external = var.load_balancing_scheme == "EXTERNAL_MANAGED"
+  is_internal = var.load_balancing_scheme == "INTERNAL_MANAGED"
+
+  external_tls_enabled = local.is_external && length(var.lb_domains) > 0
+  internal_tls_enabled = local.is_internal && length(var.lb_domains) > 0 && length(var.certificate_manager_certificates) > 0
+  tls_enabled          = local.external_tls_enabled || local.internal_tls_enabled
 }
 
 resource "google_compute_global_address" "lb" {
-  count = var.create_runtime ? 1 : 0
+  count = var.create_runtime && local.is_external ? 1 : 0
 
   name   = "${local.name}-lb-ip"
   labels = local.labels
@@ -57,13 +64,14 @@ resource "google_compute_region_network_endpoint_group" "ui" {
   }
 }
 
-# Backend services wrap each NEG.
+# Backend services wrap each NEG. The selected load_balancing_scheme controls
+# whether these serve EXTERNAL_MANAGED or INTERNAL_MANAGED.
 resource "google_compute_backend_service" "gateway" {
   count = var.create_runtime ? 1 : 0
 
   name                  = "${local.name}-gateway-bs"
   protocol              = "HTTP"
-  load_balancing_scheme = "EXTERNAL_MANAGED"
+  load_balancing_scheme = var.load_balancing_scheme
 
   backend {
     group = google_compute_region_network_endpoint_group.gateway[0].id
@@ -75,7 +83,7 @@ resource "google_compute_backend_service" "backend" {
 
   name                  = "${local.name}-backend-bs"
   protocol              = "HTTP"
-  load_balancing_scheme = "EXTERNAL_MANAGED"
+  load_balancing_scheme = var.load_balancing_scheme
 
   backend {
     group = google_compute_region_network_endpoint_group.backend[0].id
@@ -87,7 +95,7 @@ resource "google_compute_backend_service" "ui" {
 
   name                  = "${local.name}-ui-bs"
   protocol              = "HTTP"
-  load_balancing_scheme = "EXTERNAL_MANAGED"
+  load_balancing_scheme = var.load_balancing_scheme
 
   backend {
     group = google_compute_region_network_endpoint_group.ui[0].id
@@ -153,6 +161,14 @@ resource "google_compute_target_http_proxy" "this" {
   # Default-deny on the HTTP-only path: TLS is the supported posture.
   # Operators must either supply DNS names or explicitly opt in.
   lifecycle {
+    precondition {
+      condition = !local.is_internal || (
+        (length(var.lb_domains) == 0 && length(var.certificate_manager_certificates) == 0) ||
+        (length(var.lb_domains) > 0 && length(var.certificate_manager_certificates) > 0)
+      )
+      error_message = "INTERNAL_MANAGED TLS requires both `lb_domains` and `certificate_manager_certificates` to be set (or both empty for HTTP-only)."
+    }
+
     precondition {
       condition     = local.tls_enabled || var.allow_plaintext_lb
       error_message = "LB has no HTTPS forwarding rule. Either set `lb_domains` to a list of DNS names you want a Google-managed cert for, or set `allow_plaintext_lb = true` to opt into HTTP-only (trial / dev only)."
@@ -161,17 +177,31 @@ resource "google_compute_target_http_proxy" "this" {
 }
 
 resource "google_compute_global_forwarding_rule" "http" {
-  count = var.create_runtime ? 1 : 0
+  count = var.create_runtime && local.is_external ? 1 : 0
 
   name                  = "${local.name}-http"
   ip_protocol           = "TCP"
   port_range            = "80"
-  load_balancing_scheme = "EXTERNAL_MANAGED"
+  load_balancing_scheme = var.load_balancing_scheme
   ip_address            = google_compute_global_address.lb[0].address
   target                = google_compute_target_http_proxy.this[0].id
   labels                = local.labels
 }
 
+resource "google_compute_global_forwarding_rule" "http_internal" {
+  count                 = var.create_runtime && local.is_internal ? 1 : 0
+  name                  = "${local.name}-http"
+  network               = google_compute_network.this[0].id
+  subnetwork            = google_compute_subnetwork.this[0].id
+  ip_protocol           = "TCP"
+  port_range            = "80"
+  load_balancing_scheme = var.load_balancing_scheme
+  target                = google_compute_target_http_proxy.this[0].id
+  labels                = local.labels
+
+  depends_on = [google_compute_subnetwork.managed_proxy]
+}
+
 # ---------- HTTPS (gated on var.lb_domains) ----------
 #
 # Google-managed certs require each listed domain to resolve to lb_ip
@@ -181,7 +211,7 @@ resource "google_compute_global_forwarding_rule" "http" {
 # transitions to ACTIVE.
 
 resource "google_compute_managed_ssl_certificate" "this" {
-  count = var.create_runtime && local.tls_enabled ? 1 : 0
+  count = var.create_runtime && local.external_tls_enabled ? 1 : 0
 
   # A managed cert's `domains` is immutable, so changing var.lb_domains
   # forces replacement, and the cert is referenced by the HTTPS target
@@ -201,19 +231,29 @@ resource "google_compute_managed_ssl_certificate" "this" {
 }
 
 resource "google_compute_target_https_proxy" "this" {
-  count            = var.create_runtime && local.tls_enabled ? 1 : 0
-  name             = "${local.name}-https"
-  url_map          = google_compute_url_map.this[0].id
-  ssl_certificates = [google_compute_managed_ssl_certificate.this[0].id]
+  count = var.create_runtime && local.tls_enabled ? 1 : 0
+
+  name                             = "${local.name}-https"
+  url_map                          = google_compute_url_map.this[0].id
+  ssl_certificates                 = local.is_external ? [google_compute_managed_ssl_certificate.this[0].id] : null
+  certificate_manager_certificates = local.is_internal ? var.certificate_manager_certificates : null
 }
 
 resource "google_compute_global_forwarding_rule" "https" {
-  count                 = var.create_runtime && local.tls_enabled ? 1 : 0
+  count                 = var.create_runtime && local.external_tls_enabled ? 1 : 0
   name                  = "${local.name}-https"
   ip_protocol           = "TCP"
   port_range            = "443"
-  load_balancing_scheme = "EXTERNAL_MANAGED"
-  ip_address            = google_compute_global_address.lb[0].address
+  load_balancing_scheme = var.load_balancing_scheme
   target                = google_compute_target_https_proxy.this[0].id
   labels                = local.labels
+
+  # Configuration for Global External LB
+  ip_address = local.is_external ? google_compute_global_address.lb[0].address : null
+
+  # Configuration for Global Internal LB
+  network    = local.is_internal ? google_compute_network.this[0].id : null
+  subnetwork = local.is_internal ? google_compute_subnetwork.this[0].id : null
+
+  depends_on = [google_compute_subnetwork.managed_proxy]
 }

terraform/litellm/gcp/network.tf

@@ -16,6 +16,17 @@ resource "google_compute_subnetwork" "this" {
   private_ip_google_access = true
 }
 
+resource "google_compute_subnetwork" "managed_proxy" {
+  count = local.is_internal ? 1 : 0
+
+  name          = "${local.name}-${var.region}-managed-proxy"
+  region        = var.region
+  network       = google_compute_network.this[0].id
+  ip_cidr_range = var.lb_proxy_only_subnet_cidr
+  purpose       = "GLOBAL_MANAGED_PROXY"
+  role          = "ACTIVE"
+}
+
 # Private Services Access (PSA) range for Cloud SQL + Memorystore. Both
 # managed services peer with the VPC over the connection below using
 # addresses from this range.

terraform/litellm/gcp/outputs.tf

@@ -1,11 +1,21 @@
 output "lb_ip" {
-  description = "Global anycast IP of the external HTTPS load balancer. Null when create_runtime is false."
-  value       = var.create_runtime ? one(google_compute_global_address.lb[*].address) : null
+  description = "Load balancer IP. Global anycast for EXTERNAL_MANAGED, global private IP for INTERNAL_MANAGED. Null when create_runtime is false."
+  value = var.create_runtime ? (
+    var.load_balancing_scheme == "EXTERNAL_MANAGED" ? google_compute_global_address.lb[0].address : google_compute_global_forwarding_rule.http_internal[0].ip_address
+    ) : (
+    null
+  )
 }
 
 output "lb_url" {
-  description = "Proxy URL, or null when create_runtime is false. Switches scheme based on whether lb_domains is set."
-  value       = var.create_runtime ? (local.tls_enabled ? "https://${var.lb_domains[0]}" : "http://${one(google_compute_global_address.lb[*].address)}") : null
+  description = "Proxy URL, or null when create_runtime is false. Switches scheme based on whether TLS is enabled; when TLS is enabled the URL points at the first domain in lb_domains. The dashboard is served at /, the API at /v1/*."
+  value = var.create_runtime ? (local.tls_enabled ? (
+    "https://${var.lb_domains[0]}"
+    ) : (
+    var.load_balancing_scheme == "EXTERNAL_MANAGED" ? "https://${google_compute_global_address.lb[0].address}" : "https://${google_compute_global_forwarding_rule.http_internal[0].ip_address}"
+    )) : (
+    null
+  )
 }
 
 output "gateway_service_url" {

terraform/litellm/gcp/variables.tf

@@ -119,6 +119,12 @@ variable "vpc_connector_cidr" {
   default     = "10.41.0.0/28"
 }
 
+variable "lb_proxy_only_subnet_cidr" {
+  description = "CIDR for the regional proxy-only subnet used by INTERNAL_MANAGED load balancing."
+  type        = string
+  default     = "10.42.0.0/23"
+}
+
 # ---------- Component images ----------
 #
 # Cloud Run only pulls from Artifact Registry, [region.]gcr.io, or
@@ -367,19 +373,50 @@ variable "db_username" {
 
 variable "lb_domains" {
   description = <<-EOT
-    DNS names for a Google-managed SSL certificate fronting the LB. When
-    non-empty, the stack provisions a 443 forwarding rule + HTTPS target
-    proxy + managed cert covering these domains, and the existing 80
-    forwarding rule serves a permanent 301 redirect to HTTPS. Leave empty
-    ([]) to disable TLS (must combine with `allow_plaintext_lb = true` for
-    the plan to succeed — see README.md "TLS"). Each domain must already
-    resolve to the LB's anycast IP (`lb_ip` output) for managed-cert
-    provisioning to succeed.
+    DNS names fronting the LB when TLS is enabled.
+
+    EXTERNAL_MANAGED: when non-empty, the stack provisions a Google-managed
+    SSL certificate covering these domains, creates a 443 forwarding rule,
+    and rewrites port 80 to a permanent HTTPS redirect.
+
+    INTERNAL_MANAGED: when enabling TLS, `lb_domains` must be non-empty and
+    `certificate_manager_certificates` must also be provided.
+
+    Leave empty ([]) to disable TLS (must combine with
+    `allow_plaintext_lb = true` for the plan to succeed — see README.md
+    "TLS").
   EOT
   type        = list(string)
   default     = []
 }
 
+variable "certificate_manager_certificates" {
+  description = <<-EOT
+    Pre-existing Certificate Manager certificate resource references for
+    INTERNAL_MANAGED TLS. Provide full certificate self_links.
+
+    This input is ignored for EXTERNAL_MANAGED, where `lb_domains` continues
+    to drive Google-managed SSL certificate creation in this module.
+  EOT
+  type        = list(string)
+  default     = []
+}
+
+variable "load_balancing_scheme" {
+  description = <<-EOT
+    Load balancer scheme for the GCP HTTP(S) load balancer resources.
+    Allowed values match the provider/API values directly:
+    `EXTERNAL_MANAGED` (public) and `INTERNAL_MANAGED` (private).
+  EOT
+  type        = string
+  default     = "EXTERNAL_MANAGED"
+
+  validation {
+    condition     = contains(["EXTERNAL_MANAGED", "INTERNAL_MANAGED"], var.load_balancing_scheme)
+    error_message = "load_balancing_scheme must be EXTERNAL_MANAGED or INTERNAL_MANAGED."
+  }
+}
+
 variable "allow_plaintext_lb" {
   description = <<-EOT
     Opt into HTTP-only mode on the load balancer (port 80, no TLS).