Exposes input to toggle load-balancer deployment between EXTERNAL_MANAGED and INTERNAL_MANAGED

terraform/litellm/gcp/README.md

@@ -16,8 +16,10 @@ Deploys the componentized LiteLLM proxy on GCP:
 - **Cloud Run v2** services for `gateway` (port 4000), `backend` (port 4001),
   and `ui` (port 3000), all using a shared runtime service account
 - **Cloud Run Job** (`litellm-migrations`) that runs `prisma migrate deploy` from the dedicated `ghcr.io/berriai/litellm-migrations` image
-- **External global HTTP(S) load balancer** with serverless NEGs and a URL
-  map mirroring the helm-chart ingress path routing:
+- **HTTP(S) load balancer** with serverless NEGs and a URL map mirroring the
+  helm-chart ingress path routing. `load_balancing_scheme` controls mode:
+  - `EXTERNAL_MANAGED`: global external LB
+  - `INTERNAL_MANAGED`: global internal managed LB (private IP)
   - LLM data-plane prefixes → `gateway`
   - UI asset paths → `ui`
   - Everything else → `backend`
@@ -440,6 +442,9 @@ vpcaccess, compute, servicenetworking, storage, artifactregistry).
 
 ## TLS
 
+TLS and `lb_domains` apply only when `load_balancing_scheme = "EXTERNAL_MANAGED"`.
+For `INTERNAL_MANAGED`, set `lb_domains = []` (or leave it unset) and `allow_plaintext_lb = true`.
+
 `terraform plan` refuses to provision an HTTP-only LB by default — TLS
 is the supported posture. Two paths:
 

terraform/litellm/gcp/examples/default/main.tf

@@ -43,6 +43,7 @@ module "litellm" {
   image_registry = var.image_registry
   image_tag      = var.image_tag
 
+  load_balancing_scheme        = var.load_balancing_scheme
   lb_domains                   = var.lb_domains
   allow_plaintext_lb           = var.allow_plaintext_lb
   cloudsql_deletion_protection = var.cloudsql_deletion_protection

terraform/litellm/gcp/examples/default/terraform.tfvars.example

@@ -27,7 +27,9 @@ env    = "stage"
 
 # TLS: provide DNS names already pointing at the LB IP for a Google-managed
 # cert. Without one, plan fails unless allow_plaintext_lb = true is set
-# explicitly (trial/dev only).
+# explicitly (trial/dev only). INTERNAL_MANAGED is private-IP only and
+# should keep lb_domains empty.
+# load_balancing_scheme = "EXTERNAL_MANAGED" # or "INTERNAL_MANAGED"
 # lb_domains         = ["proxy.example.com"]
 # allow_plaintext_lb = true
 

terraform/litellm/gcp/examples/default/variables.tf

@@ -90,11 +90,22 @@ variable "image_tag" {
 
 # TLS — provide DNS names for a managed cert, or opt into HTTP-only for dev.
 variable "lb_domains" {
-  description = "DNS names (already pointing at lb_ip) for a Google-managed cert. Empty → no TLS."
+  description = "DNS names (already pointing at lb_ip) for a Google-managed cert in EXTERNAL_MANAGED mode. Empty → no TLS."
   type        = list(string)
   default     = []
 }
 
+variable "load_balancing_scheme" {
+  description = "Load balancer scheme. Allowed values: EXTERNAL_MANAGED or INTERNAL_MANAGED."
+  type        = string
+  default     = "EXTERNAL_MANAGED"
+
+  validation {
+    condition     = contains(["EXTERNAL_MANAGED", "INTERNAL_MANAGED"], var.load_balancing_scheme)
+    error_message = "load_balancing_scheme must be EXTERNAL_MANAGED or INTERNAL_MANAGED."
+  }
+}
+
 variable "allow_plaintext_lb" {
   description = "Opt into HTTP-only LB (trial/dev only)."
   type        = bool

terraform/litellm/gcp/load_balancer.tf

@@ -1,5 +1,7 @@
-# External global HTTP(S) load balancer fronting all three Cloud Run
-# services. URL map mirrors the helm-chart ingress path routing:
+# HTTP(S) load balancer fronting all three Cloud Run services.
+# EXTERNAL_MANAGED uses the global external path, INTERNAL_MANAGED uses
+# the global (cross-region) internal managed path. URL map mirrors the helm-chart
+# ingress path routing:
 #   - LLM data-plane paths → gateway
 #   - UI asset paths → ui
 #   - Everything else → backend (management API: /key/*, /user/*, …)
@@ -11,10 +13,12 @@
 
 locals {
   tls_enabled = length(var.lb_domains) > 0
+  is_external = var.load_balancing_scheme == "EXTERNAL_MANAGED"
+  is_internal = var.load_balancing_scheme == "INTERNAL_MANAGED"
 }
 
 resource "google_compute_global_address" "lb" {
-  count = var.create_runtime ? 1 : 0
+  count = var.create_runtime && local.is_external ? 1 : 0
 
   name   = "${local.name}-lb-ip"
   labels = local.labels
@@ -57,13 +61,14 @@ resource "google_compute_region_network_endpoint_group" "ui" {
   }
 }
 
-# Backend services wrap each NEG.
+# Backend services wrap each NEG. The selected load_balancing_scheme controls
+# whether these serve EXTERNAL_MANAGED or INTERNAL_MANAGED.
 resource "google_compute_backend_service" "gateway" {
   count = var.create_runtime ? 1 : 0
 
   name                  = "${local.name}-gateway-bs"
   protocol              = "HTTP"
-  load_balancing_scheme = "EXTERNAL_MANAGED"
+  load_balancing_scheme = var.load_balancing_scheme
 
   backend {
     group = google_compute_region_network_endpoint_group.gateway[0].id
@@ -75,7 +80,7 @@ resource "google_compute_backend_service" "backend" {
 
   name                  = "${local.name}-backend-bs"
   protocol              = "HTTP"
-  load_balancing_scheme = "EXTERNAL_MANAGED"
+  load_balancing_scheme = var.load_balancing_scheme
 
   backend {
     group = google_compute_region_network_endpoint_group.backend[0].id
@@ -87,7 +92,7 @@ resource "google_compute_backend_service" "ui" {
 
   name                  = "${local.name}-ui-bs"
   protocol              = "HTTP"
-  load_balancing_scheme = "EXTERNAL_MANAGED"
+  load_balancing_scheme = var.load_balancing_scheme
 
   backend {
     group = google_compute_region_network_endpoint_group.ui[0].id
@@ -134,7 +139,7 @@ resource "google_compute_url_map" "this" {
 # target proxy when TLS is enabled; otherwise the regular path-routing
 # URL map is attached to the HTTP proxy and everything stays plaintext.
 resource "google_compute_url_map" "https_redirect" {
-  count = var.create_runtime && local.tls_enabled ? 1 : 0
+  count = var.create_runtime && local.tls_enabled && local.is_external ? 1 : 0
   name  = "${local.name}-redirect"
 
   default_url_redirect {
@@ -153,6 +158,11 @@ resource "google_compute_target_http_proxy" "this" {
   # Default-deny on the HTTP-only path: TLS is the supported posture.
   # Operators must either supply DNS names or explicitly opt in.
   lifecycle {
+    precondition {
+      condition     = var.load_balancing_scheme != "INTERNAL_MANAGED" || length(var.lb_domains) == 0
+      error_message = "INTERNAL_MANAGED does not support lb_domains/TLS in this module. Set load_balancing_scheme = \"EXTERNAL_MANAGED\" to use lb_domains, or leave lb_domains empty for INTERNAL_MANAGED."
+    }
+
     precondition {
       condition     = local.tls_enabled || var.allow_plaintext_lb
       error_message = "LB has no HTTPS forwarding rule. Either set `lb_domains` to a list of DNS names you want a Google-managed cert for, or set `allow_plaintext_lb = true` to opt into HTTP-only (trial / dev only)."
@@ -161,17 +171,31 @@ resource "google_compute_target_http_proxy" "this" {
 }
 
 resource "google_compute_global_forwarding_rule" "http" {
-  count = var.create_runtime ? 1 : 0
+  count = var.create_runtime && local.is_external ? 1 : 0
 
   name                  = "${local.name}-http"
   ip_protocol           = "TCP"
   port_range            = "80"
-  load_balancing_scheme = "EXTERNAL_MANAGED"
+  load_balancing_scheme = var.load_balancing_scheme
   ip_address            = google_compute_global_address.lb[0].address
   target                = google_compute_target_http_proxy.this[0].id
   labels                = local.labels
 }
 
+resource "google_compute_global_forwarding_rule" "http_internal" {
+  count                 = var.create_runtime && local.is_internal ? 1 : 0
+  name                  = "${local.name}-http"
+  network               = google_compute_network.this[0].id
+  subnetwork            = google_compute_subnetwork.this[0].id
+  ip_protocol           = "TCP"
+  port_range            = "80"
+  load_balancing_scheme = var.load_balancing_scheme
+  target                = google_compute_target_http_proxy.this[0].id
+  labels                = local.labels
+
+  depends_on = [google_compute_subnetwork.managed_proxy]
+}
+
 # ---------- HTTPS (gated on var.lb_domains) ----------
 #
 # Google-managed certs require each listed domain to resolve to lb_ip
@@ -181,7 +205,7 @@ resource "google_compute_global_forwarding_rule" "http" {
 # transitions to ACTIVE.
 
 resource "google_compute_managed_ssl_certificate" "this" {
-  count = var.create_runtime && local.tls_enabled ? 1 : 0
+  count = var.create_runtime && local.tls_enabled && local.is_external ? 1 : 0
 
   # A managed cert's `domains` is immutable, so changing var.lb_domains
   # forces replacement, and the cert is referenced by the HTTPS target
@@ -201,18 +225,18 @@ resource "google_compute_managed_ssl_certificate" "this" {
 }
 
 resource "google_compute_target_https_proxy" "this" {
-  count            = var.create_runtime && local.tls_enabled ? 1 : 0
+  count            = var.create_runtime && local.tls_enabled && local.is_external ? 1 : 0
   name             = "${local.name}-https"
   url_map          = google_compute_url_map.this[0].id
   ssl_certificates = [google_compute_managed_ssl_certificate.this[0].id]
 }
 
 resource "google_compute_global_forwarding_rule" "https" {
-  count                 = var.create_runtime && local.tls_enabled ? 1 : 0
+  count                 = var.create_runtime && local.tls_enabled && local.is_external ? 1 : 0
   name                  = "${local.name}-https"
   ip_protocol           = "TCP"
   port_range            = "443"
-  load_balancing_scheme = "EXTERNAL_MANAGED"
+  load_balancing_scheme = var.load_balancing_scheme
   ip_address            = google_compute_global_address.lb[0].address
   target                = google_compute_target_https_proxy.this[0].id
   labels                = local.labels

terraform/litellm/gcp/network.tf

@@ -16,6 +16,17 @@ resource "google_compute_subnetwork" "this" {
   private_ip_google_access = true
 }
 
+resource "google_compute_subnetwork" "managed_proxy" {
+  count = local.is_internal ? 1 : 0
+
+  name          = "${local.name}-${var.region}-managed-proxy"
+  region        = var.region
+  network       = google_compute_network.this[0].id
+  ip_cidr_range = var.lb_proxy_only_subnet_cidr
+  purpose       = "GLOBAL_MANAGED_PROXY"
+  role          = "ACTIVE"
+}
+
 # Private Services Access (PSA) range for Cloud SQL + Memorystore. Both
 # managed services peer with the VPC over the connection below using
 # addresses from this range.

terraform/litellm/gcp/outputs.tf

@@ -1,11 +1,21 @@
 output "lb_ip" {
-  description = "Global anycast IP of the external HTTPS load balancer. Null when create_runtime is false."
-  value       = var.create_runtime ? one(google_compute_global_address.lb[*].address) : null
+  description = "Load balancer IP. Global anycast for EXTERNAL_MANAGED, global private IP for INTERNAL_MANAGED. Null when create_runtime is false."
+  value       = var.create_runtime ? (
+    var.load_balancing_scheme == "EXTERNAL_MANAGED" ? google_compute_global_address.lb[0].address : google_compute_global_forwarding_rule.http_internal[0].ip_address
+  ) : (
+    null
+  )
 }
 
 output "lb_url" {
-  description = "Proxy URL, or null when create_runtime is false. Switches scheme based on whether lb_domains is set."
-  value       = var.create_runtime ? (local.tls_enabled ? "https://${var.lb_domains[0]}" : "http://${one(google_compute_global_address.lb[*].address)}") : null
+  description = "Proxy URL, or null when create_runtime is false. Switches scheme based on whether lb_domains is set; when TLS is enabled the URL points at the first listed domain (since managed certs are tied to the hostname, not the anycast IP). The dashboard is served at /, the API at /v1/*."
+  value = var.create_runtime ? (local.tls_enabled ? (
+    "https://${var.lb_domains[0]}"
+  ) : (
+    format("http://%s", var.load_balancing_scheme == "EXTERNAL_MANAGED" ? google_compute_global_address.lb[0].address : google_compute_global_forwarding_rule.http_internal[0].ip_address)
+  )) : (
+    null
+  )
 }
 
 output "gateway_service_url" {

terraform/litellm/gcp/variables.tf

@@ -119,6 +119,12 @@ variable "vpc_connector_cidr" {
   default     = "10.41.0.0/28"
 }
 
+variable "lb_proxy_only_subnet_cidr" {
+  description = "CIDR for the regional proxy-only subnet used by INTERNAL_MANAGED load balancing."
+  type        = string
+  default     = "10.42.0.0/23"
+}
+
 # ---------- Component images ----------
 #
 # Cloud Run only pulls from Artifact Registry, [region.]gcr.io, or
@@ -380,6 +386,21 @@ variable "lb_domains" {
   default     = []
 }
 
+variable "load_balancing_scheme" {
+  description = <<-EOT
+    Load balancer scheme for the GCP HTTP(S) load balancer resources.
+    Allowed values match the provider/API values directly:
+    `EXTERNAL_MANAGED` (public) and `INTERNAL_MANAGED` (private).
+  EOT
+  type        = string
+  default     = "EXTERNAL_MANAGED"
+
+  validation {
+    condition     = contains(["EXTERNAL_MANAGED", "INTERNAL_MANAGED"], var.load_balancing_scheme)
+    error_message = "load_balancing_scheme must be EXTERNAL_MANAGED or INTERNAL_MANAGED."
+  }
+}
+
 variable "allow_plaintext_lb" {
   description = <<-EOT
     Opt into HTTP-only mode on the load balancer (port 80, no TLS).