diff --git a/terraform/litellm/gcp/README.md b/terraform/litellm/gcp/README.md index 00ec7e8afd0..66095ec0108 100644 --- a/terraform/litellm/gcp/README.md +++ b/terraform/litellm/gcp/README.md @@ -238,6 +238,37 @@ this with `litellm_license`. To tune the export cadence, set Behavior matches the AWS stack 1:1; the variable names are identical +### Prometheus metrics sidecar + +`gateway_metrics_port` adds a `metrics` sidecar +(`python -m litellm.proxy.prometheus_metrics_server`) to the gateway Cloud Run +service that aggregates the workers' samples over an in-memory volume shared +with the gateway container, so the collector's scrape never runs on an +inference worker. Cloud Run only routes traffic to the gateway container, so +the load balancer keeps hitting port 4000 (including the gateway's own +authenticated `/metrics`, which stays as it was) and the sidecar port is +reachable on localhost inside the instance only. To get the series out, the +stack also adds Google's +[Managed Service for Prometheus sidecar](https://cloud.google.com/stackdriver/docs/managed-prometheus/cloudrun-sidecar) +(`gateway_metrics_collector_image`) with a `RunMonitoring` config stored in +Secret Manager that scrapes `localhost:/metrics` every 30s and writes to +Cloud Monitoring as `prometheus.googleapis.com/...` metrics. Enabling it grants +the runtime service account `roles/monitoring.metricWriter` and +`roles/logging.logWriter` on the project. Needs `gateway_image` v1.101.0 or +newer. See [Prometheus metrics](https://docs.litellm.ai/docs/proxy/prometheus) +for the metrics themselves + +```hcl +gateway_metrics_port = 4001 +``` + +The collector scrapes from inside the instance, so scrapes on an instance with +no in-flight requests can fail when CPU is throttled between requests. Keep +`gateway_min_instances` at 1 or more and, if you see gaps, enable +instance-based billing on the gateway service. Unlike the AWS stack there is +no `gateway_metrics_scrape_cidrs`: nothing outside the instance can reach the +sidecar port, so there is no network rule to open + ### Autoscaling Cloud Run scales the gateway on request concurrency (plus its built-in CPU diff --git a/terraform/litellm/gcp/cloudrun.tf b/terraform/litellm/gcp/cloudrun.tf index 84ae8b9247f..405893b132b 100644 --- a/terraform/litellm/gcp/cloudrun.tf +++ b/terraform/litellm/gcp/cloudrun.tf @@ -150,6 +150,21 @@ locals { [local.gateway_launch_cmd], )) + metrics_enabled = var.create_runtime && var.gateway_metrics_port != null + metrics_multiproc_dir = "/tmp/litellm_prometheus_multiproc" + metrics_volume = "prometheus-multiproc" + metrics_env_kv = local.metrics_enabled ? [{ name = "PROMETHEUS_MULTIPROC_DIR", value = local.metrics_multiproc_dir }] : [] + metrics_config_volume = "gmp-config" + + metrics_run_monitoring_yaml = local.metrics_enabled ? yamlencode({ + apiVersion = "monitoring.googleapis.com/v1beta" + kind = "RunMonitoring" + metadata = { name = "${local.name}-gateway" } + spec = { + endpoints = [{ port = var.gateway_metrics_port, path = "/metrics", interval = "30s" }] + } + }) : "" + backend_args = join(" && ", concat( local.redis_ca_fragment, local.database_url_fragment, @@ -197,6 +212,7 @@ resource "google_cloud_run_v2_service" "gateway" { } containers { + name = "gateway" image = local.gateway_image command = ["sh", "-c"] args = [local.gateway_args] @@ -213,7 +229,7 @@ resource "google_cloud_run_v2_service" "gateway" { } dynamic "env" { - for_each = concat(local.shared_env_kv, local.gateway_otel_env_kv, local.billing_metrics_env_kv, local.gateway_extra_env_kv, local.proxy_config_env) + for_each = concat(local.shared_env_kv, local.gateway_otel_env_kv, local.billing_metrics_env_kv, local.gateway_extra_env_kv, local.proxy_config_env, local.metrics_env_kv) content { name = env.value.name value = env.value.value @@ -241,6 +257,14 @@ resource "google_cloud_run_v2_service" "gateway" { } } + dynamic "volume_mounts" { + for_each = local.metrics_enabled ? [1] : [] + content { + name = local.metrics_volume + mount_path = local.metrics_multiproc_dir + } + } + startup_probe { http_get { path = "/health/readiness" @@ -262,6 +286,71 @@ resource "google_cloud_run_v2_service" "gateway" { } } + dynamic "containers" { + for_each = local.metrics_enabled ? [1] : [] + content { + name = "metrics" + image = local.gateway_image + command = ["python", "-m", "litellm.proxy.prometheus_metrics_server"] + args = ["--port", tostring(var.gateway_metrics_port)] + + dynamic "env" { + for_each = local.metrics_env_kv + content { + name = env.value.name + value = env.value.value + } + } + + volume_mounts { + name = local.metrics_volume + mount_path = local.metrics_multiproc_dir + } + + startup_probe { + http_get { + path = "/health" + port = var.gateway_metrics_port + } + period_seconds = 5 + timeout_seconds = 3 + failure_threshold = 12 + } + + liveness_probe { + http_get { + path = "/health" + port = var.gateway_metrics_port + } + period_seconds = 30 + timeout_seconds = 5 + } + } + } + + dynamic "containers" { + for_each = local.metrics_enabled ? [1] : [] + content { + name = "collector" + image = var.gateway_metrics_collector_image + depends_on = ["metrics"] + + volume_mounts { + name = local.metrics_config_volume + mount_path = "/etc/rungmp" + } + + liveness_probe { + http_get { + path = "/liveness" + port = 13133 + } + period_seconds = 30 + timeout_seconds = 30 + } + } + } + dynamic "volumes" { for_each = local.proxy_config_enabled ? [1] : [] content { @@ -272,6 +361,31 @@ resource "google_cloud_run_v2_service" "gateway" { } } } + + dynamic "volumes" { + for_each = local.metrics_enabled ? [1] : [] + content { + name = local.metrics_volume + empty_dir { + medium = "MEMORY" + size_limit = "256Mi" + } + } + } + + dynamic "volumes" { + for_each = local.metrics_enabled ? [1] : [] + content { + name = local.metrics_config_volume + secret { + secret = google_secret_manager_secret.metrics_run_monitoring[0].secret_id + items { + version = "latest" + path = "config.yaml" + } + } + } + } } depends_on = [ @@ -283,6 +397,8 @@ resource "google_cloud_run_v2_service" "gateway" { google_secret_manager_secret_iam_member.billing_metrics_client_cert, google_secret_manager_secret_iam_member.billing_metrics_client_key, google_secret_manager_secret_iam_member.billing_metrics_ca_cert, + google_secret_manager_secret_iam_member.metrics_run_monitoring, + google_project_iam_member.runtime_metric_writer, google_storage_bucket_iam_member.proxy_config_runtime, google_sql_user.app, # Don't go live until the schema is migrated; otherwise the proxy boots, @@ -332,7 +448,7 @@ resource "google_cloud_run_v2_service" "backend" { } dynamic "env" { - for_each = concat(local.shared_env_kv, local.backend_default_env_kv, local.backend_otel_env_kv, local.billing_metrics_env_kv, local.backend_extra_env_kv, local.proxy_config_env) + for_each = concat(local.shared_env_kv, local.backend_default_env_kv, local.backend_otel_env_kv, local.billing_metrics_env_kv, local.backend_extra_env_kv, local.proxy_config_env, local.metrics_env_kv) content { name = env.value.name value = env.value.value diff --git a/terraform/litellm/gcp/examples/default/main.tf b/terraform/litellm/gcp/examples/default/main.tf index f44b2a9a001..782d4ee4b65 100644 --- a/terraform/litellm/gcp/examples/default/main.tf +++ b/terraform/litellm/gcp/examples/default/main.tf @@ -53,4 +53,6 @@ module "litellm" { backend_extra_env = var.backend_extra_env gateway_extra_secrets = var.gateway_extra_secrets backend_extra_secrets = var.backend_extra_secrets + + gateway_metrics_port = var.gateway_metrics_port } diff --git a/terraform/litellm/gcp/examples/default/terraform.tfvars.example b/terraform/litellm/gcp/examples/default/terraform.tfvars.example index c35206503bb..ec6206734e2 100644 --- a/terraform/litellm/gcp/examples/default/terraform.tfvars.example +++ b/terraform/litellm/gcp/examples/default/terraform.tfvars.example @@ -107,3 +107,9 @@ env = "stage" # main.tf (otel_endpoint, otel_exporter, otel_environment_name, # otel_capture_message_content, otel_headers_secret). Full docs in # ../../variables.tf. + +# ---------- Prometheus metrics sidecar ---------- +# Serve /metrics from a sidecar in the gateway service instead of the inference +# workers. Scraped inside the instance by the Managed Service for Prometheus +# sidecar and written to Cloud Monitoring; see ../../README.md. +# gateway_metrics_port = 4001 diff --git a/terraform/litellm/gcp/examples/default/variables.tf b/terraform/litellm/gcp/examples/default/variables.tf index 88b57ce27eb..08b78346df3 100644 --- a/terraform/litellm/gcp/examples/default/variables.tf +++ b/terraform/litellm/gcp/examples/default/variables.tf @@ -142,3 +142,9 @@ variable "backend_extra_secrets" { type = map(string) default = {} } + +variable "gateway_metrics_port" { + description = "Port for the Prometheus metrics sidecar in the gateway service. Null keeps /metrics on the gateway port only." + type = number + default = null +} diff --git a/terraform/litellm/gcp/iam.tf b/terraform/litellm/gcp/iam.tf index 509e6d48ffd..5c2bc38bc12 100644 --- a/terraform/litellm/gcp/iam.tf +++ b/terraform/litellm/gcp/iam.tf @@ -116,3 +116,27 @@ resource "google_secret_manager_secret_iam_member" "billing_metrics_ca_cert" { role = "roles/secretmanager.secretAccessor" member = "serviceAccount:${google_service_account.runtime.email}" } + +resource "google_secret_manager_secret_iam_member" "metrics_run_monitoring" { + count = local.metrics_enabled ? 1 : 0 + + secret_id = google_secret_manager_secret.metrics_run_monitoring[0].id + role = "roles/secretmanager.secretAccessor" + member = "serviceAccount:${google_service_account.runtime.email}" +} + +resource "google_project_iam_member" "runtime_metric_writer" { + count = local.metrics_enabled ? 1 : 0 + + project = var.project_id + role = "roles/monitoring.metricWriter" + member = "serviceAccount:${google_service_account.runtime.email}" +} + +resource "google_project_iam_member" "runtime_log_writer" { + count = local.metrics_enabled ? 1 : 0 + + project = var.project_id + role = "roles/logging.logWriter" + member = "serviceAccount:${google_service_account.runtime.email}" +} diff --git a/terraform/litellm/gcp/secrets.tf b/terraform/litellm/gcp/secrets.tf index 6ec77139996..b3b656721ab 100644 --- a/terraform/litellm/gcp/secrets.tf +++ b/terraform/litellm/gcp/secrets.tf @@ -118,3 +118,20 @@ resource "google_secret_manager_secret_version" "billing_metrics_ca_cert" { secret = google_secret_manager_secret.billing_metrics_ca_cert[0].id secret_data = var.billing_metrics_ca_cert_pem } + +resource "google_secret_manager_secret" "metrics_run_monitoring" { + count = local.metrics_enabled ? 1 : 0 + + secret_id = "${local.name}-gateway-run-monitoring" + labels = local.labels + replication { + auto {} + } +} + +resource "google_secret_manager_secret_version" "metrics_run_monitoring" { + count = local.metrics_enabled ? 1 : 0 + + secret = google_secret_manager_secret.metrics_run_monitoring[0].id + secret_data = local.metrics_run_monitoring_yaml +} diff --git a/terraform/litellm/gcp/tests/metrics_sidecar.tftest.hcl b/terraform/litellm/gcp/tests/metrics_sidecar.tftest.hcl new file mode 100644 index 00000000000..7de62065d23 --- /dev/null +++ b/terraform/litellm/gcp/tests/metrics_sidecar.tftest.hcl @@ -0,0 +1,200 @@ +mock_provider "google" { + mock_resource "google_redis_instance" { + defaults = { + host = "10.0.0.4" + port = 6379 + server_ca_certs = [{ + cert = "-----BEGIN CERTIFICATE-----\nmock\n-----END CERTIFICATE-----" + }] + } + } +} + +mock_provider "google-beta" {} +mock_provider "random" {} + +variables { + project_id = "test-project" + tenant = "tenant" + env = "test" + allow_plaintext_lb = true + image_registry = "us-central1-docker.pkg.dev/test-project/litellm" +} + +run "metrics_sidecar_off_by_default" { + command = plan + + assert { + condition = length(google_cloud_run_v2_service.gateway[0].template[0].containers) == 1 + error_message = "The gateway service must run only the gateway container when gateway_metrics_port is null." + } + + assert { + condition = length([for e in google_cloud_run_v2_service.gateway[0].template[0].containers[0].env : e if e.name == "PROMETHEUS_MULTIPROC_DIR"]) == 0 + error_message = "PROMETHEUS_MULTIPROC_DIR must not be set when the metrics sidecar is off." + } + + assert { + condition = length(google_cloud_run_v2_service.gateway[0].template[0].volumes) == 0 + error_message = "No shared multiproc or collector config volume must exist when the metrics sidecar is off." + } + + assert { + condition = alltrue([ + length(google_secret_manager_secret.metrics_run_monitoring) == 0, + length(google_secret_manager_secret_version.metrics_run_monitoring) == 0, + length(google_secret_manager_secret_iam_member.metrics_run_monitoring) == 0, + length(google_project_iam_member.runtime_metric_writer) == 0, + length(google_project_iam_member.runtime_log_writer) == 0, + ]) + error_message = "No RunMonitoring secret or monitoring IAM must be created when the metrics sidecar is off." + } +} + +run "metrics_sidecar_enabled" { + command = plan + + variables { + gateway_metrics_port = 4001 + } + + assert { + condition = join(",", [for c in google_cloud_run_v2_service.gateway[0].template[0].containers : c.name]) == "gateway,metrics,collector" + error_message = "gateway_metrics_port must add the metrics and collector sidecars after the gateway container." + } + + assert { + condition = alltrue([ + google_cloud_run_v2_service.gateway[0].template[0].containers[1].image == local.gateway_image, + join(" ", google_cloud_run_v2_service.gateway[0].template[0].containers[1].command) == "python -m litellm.proxy.prometheus_metrics_server", + join(" ", google_cloud_run_v2_service.gateway[0].template[0].containers[1].args) == "--port 4001", + ]) + error_message = "The metrics sidecar must run the gateway image's prometheus_metrics_server on the configured port." + } + + assert { + condition = alltrue([ + length([for e in google_cloud_run_v2_service.gateway[0].template[0].containers[0].env : e if e.name == "PROMETHEUS_MULTIPROC_DIR" && e.value == "/tmp/litellm_prometheus_multiproc"]) == 1, + length([for e in google_cloud_run_v2_service.gateway[0].template[0].containers[1].env : e if e.name == "PROMETHEUS_MULTIPROC_DIR" && e.value == "/tmp/litellm_prometheus_multiproc"]) == 1, + ]) + error_message = "Gateway and metrics containers must share PROMETHEUS_MULTIPROC_DIR." + } + + assert { + condition = alltrue([ + length([for m in google_cloud_run_v2_service.gateway[0].template[0].containers[0].volume_mounts : m if m.name == "prometheus-multiproc" && m.mount_path == "/tmp/litellm_prometheus_multiproc"]) == 1, + length([for m in google_cloud_run_v2_service.gateway[0].template[0].containers[1].volume_mounts : m if m.name == "prometheus-multiproc" && m.mount_path == "/tmp/litellm_prometheus_multiproc"]) == 1, + length([for v in google_cloud_run_v2_service.gateway[0].template[0].volumes : v if v.name == "prometheus-multiproc" && length(v.empty_dir) == 1 && v.empty_dir[0].medium == "MEMORY"]) == 1, + ]) + error_message = "Gateway and metrics containers must mount the same in-memory empty_dir at the multiproc dir." + } + + assert { + condition = alltrue([ + google_cloud_run_v2_service.gateway[0].template[0].containers[1].startup_probe[0].http_get[0].path == "/health", + google_cloud_run_v2_service.gateway[0].template[0].containers[1].startup_probe[0].http_get[0].port == 4001, + google_cloud_run_v2_service.gateway[0].template[0].containers[1].liveness_probe[0].http_get[0].path == "/health", + google_cloud_run_v2_service.gateway[0].template[0].containers[1].liveness_probe[0].http_get[0].port == 4001, + ]) + error_message = "The metrics sidecar must be probed on /health at the configured port." + } + + assert { + condition = length(google_cloud_run_v2_service.gateway[0].template[0].containers[1].ports) == 0 && length(google_cloud_run_v2_service.gateway[0].template[0].containers[2].ports) == 0 + error_message = "Only the gateway container may declare a port; Cloud Run routes ingress to exactly one container." + } + + assert { + condition = alltrue([ + google_cloud_run_v2_service.gateway[0].template[0].containers[0].ports[0].container_port == 4000, + google_cloud_run_v2_service.gateway[0].template[0].containers[0].startup_probe[0].http_get[0].port == 4000, + google_cloud_run_v2_service.gateway[0].template[0].containers[0].liveness_probe[0].http_get[0].port == 4000, + google_compute_region_network_endpoint_group.gateway[0].cloud_run[0].service == "${local.name}-gateway", + ]) + error_message = "The gateway must stay on port 4000 and remain the load balancer's Cloud Run target." + } + + assert { + condition = alltrue([ + google_cloud_run_v2_service.gateway[0].template[0].containers[2].image == var.gateway_metrics_collector_image, + join(",", google_cloud_run_v2_service.gateway[0].template[0].containers[2].depends_on) == "metrics", + length([for m in google_cloud_run_v2_service.gateway[0].template[0].containers[2].volume_mounts : m if m.name == "gmp-config" && m.mount_path == "/etc/rungmp"]) == 1, + google_cloud_run_v2_service.gateway[0].template[0].containers[2].liveness_probe[0].http_get[0].port == 13133, + ]) + error_message = "The collector sidecar must start after the metrics server and read its RunMonitoring config from /etc/rungmp." + } + + assert { + condition = alltrue([ + length([for v in google_cloud_run_v2_service.gateway[0].template[0].volumes : v if v.name == "gmp-config" && length(v.secret) == 1 && v.secret[0].items[0].path == "config.yaml"]) == 1, + google_secret_manager_secret.metrics_run_monitoring[0].secret_id == "${local.name}-gateway-run-monitoring", + google_secret_manager_secret_iam_member.metrics_run_monitoring[0].role == "roles/secretmanager.secretAccessor", + ]) + error_message = "The RunMonitoring config must be mounted from a Secret Manager secret readable by the runtime SA." + } + + assert { + condition = alltrue([ + yamldecode(google_secret_manager_secret_version.metrics_run_monitoring[0].secret_data).kind == "RunMonitoring", + yamldecode(google_secret_manager_secret_version.metrics_run_monitoring[0].secret_data).spec.endpoints[0].port == 4001, + yamldecode(google_secret_manager_secret_version.metrics_run_monitoring[0].secret_data).spec.endpoints[0].path == "/metrics", + ]) + error_message = "The RunMonitoring config must scrape /metrics on the configured metrics port." + } + + assert { + condition = alltrue([ + google_project_iam_member.runtime_metric_writer[0].role == "roles/monitoring.metricWriter", + google_project_iam_member.runtime_log_writer[0].role == "roles/logging.logWriter", + google_project_iam_member.runtime_metric_writer[0].project == "test-project", + ]) + error_message = "The runtime SA must be able to write metrics and logs for the collector sidecar." + } +} + +run "metrics_sidecar_ignored_in_deps_only" { + command = plan + + variables { + create_runtime = false + gateway_metrics_port = 4001 + } + + assert { + condition = alltrue([ + length(google_secret_manager_secret.metrics_run_monitoring) == 0, + length(google_project_iam_member.runtime_metric_writer) == 0, + ]) + error_message = "Dependencies-only mode must not create metrics sidecar resources." + } +} + +run "metrics_port_rejects_gateway_port" { + command = plan + + variables { + gateway_metrics_port = 4000 + } + + expect_failures = [var.gateway_metrics_port] +} + +run "metrics_port_rejects_collector_health_port" { + command = plan + + variables { + gateway_metrics_port = 13133 + } + + expect_failures = [var.gateway_metrics_port] +} + +run "metrics_port_rejects_fractional_port" { + command = plan + + variables { + gateway_metrics_port = 4000.5 + } + + expect_failures = [var.gateway_metrics_port] +} diff --git a/terraform/litellm/gcp/variables.tf b/terraform/litellm/gcp/variables.tf index 9c68ed3db76..a753b4584a6 100644 --- a/terraform/litellm/gcp/variables.tf +++ b/terraform/litellm/gcp/variables.tf @@ -517,6 +517,44 @@ variable "otel_capture_message_content" { } } +# ---------- Prometheus metrics sidecar ---------- + +variable "gateway_metrics_port" { + description = <<-EOT + Serve Prometheus /metrics from a `metrics` sidecar container in the + gateway Cloud Run service on this port (a whole number 1-65535, not 4000 + or 13133), so the collector's scrape never runs on an inference worker. + The sidecar runs the gateway image with + `python -m litellm.proxy.prometheus_metrics_server` and aggregates the + workers' PROMETHEUS_MULTIPROC_DIR samples over an in-memory volume shared + with the gateway container. Cloud Run only routes ingress to the gateway + container, so the sidecar port is reachable on localhost inside the + instance only; a Managed Service for Prometheus collector sidecar + (gateway_metrics_collector_image) scrapes it and writes the series to + Cloud Monitoring. The load balancer keeps serving the authenticated + /metrics on the gateway port as before. Null (the default) leaves /metrics + on the gateway port only. Needs gateway_image v1.101.0 or newer. + EOT + type = number + default = null + + validation { + condition = var.gateway_metrics_port == null || (var.gateway_metrics_port >= 1 && var.gateway_metrics_port <= 65535 && floor(var.gateway_metrics_port) == var.gateway_metrics_port && !contains([4000, 13133], var.gateway_metrics_port)) + error_message = "gateway_metrics_port must be a whole number between 1 and 65535 and must not be 4000 (the gateway port) or 13133 (the collector health port)." + } +} + +variable "gateway_metrics_collector_image" { + description = <<-EOT + Managed Service for Prometheus sidecar image that scrapes + localhost:/metrics and writes to Cloud Monitoring. + Override only to pin a different release or pull through your own + Artifact Registry. Ignored when gateway_metrics_port is null. + EOT + type = string + default = "us-docker.pkg.dev/cloud-ops-agents-artifacts/cloud-run-gmp-sidecar/cloud-run-gmp-sidecar:1.9.2" +} + # ---------- Enterprise billing metrics ---------- # # License-gated request metering. Opt-in and gated entirely on