diff --git a/.github/workflows/test-terraform-modules.yml b/.github/workflows/test-terraform-modules.yml index 0e3e5330453..52006d9b578 100644 --- a/.github/workflows/test-terraform-modules.yml +++ b/.github/workflows/test-terraform-modules.yml @@ -4,6 +4,7 @@ on: push: paths: - "terraform/litellm/aws/**" + - "terraform/litellm/gcp/**" - ".github/workflows/test-terraform-modules.yml" pull_request: branches: @@ -13,6 +14,7 @@ on: - "litellm_**" paths: - "terraform/litellm/aws/**" + - "terraform/litellm/gcp/**" - ".github/workflows/test-terraform-modules.yml" permissions: @@ -52,3 +54,32 @@ jobs: # Plan-only, mock_provider-backed: no AWS credentials, no API calls. - name: test run: terraform test + + gcp-module: + name: fmt, validate, test (gcp) + runs-on: ubuntu-latest + timeout-minutes: 15 + defaults: + run: + working-directory: terraform/litellm/gcp + steps: + - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + persist-credentials: false + + - uses: hashicorp/setup-terraform@b9cd54a3c349d3f38e8881555d616ced269862dd # v3.1.2 + with: + terraform_version: 1.13.3 + terraform_wrapper: false + + - name: fmt + run: terraform fmt -recursive -check -diff + + - name: init + run: terraform init -backend=false -input=false + + - name: validate + run: terraform validate + + - name: test + run: terraform test diff --git a/terraform/litellm/gcp/README.md b/terraform/litellm/gcp/README.md index 88e9979148f..c93e5f6b303 100644 --- a/terraform/litellm/gcp/README.md +++ b/terraform/litellm/gcp/README.md @@ -392,6 +392,63 @@ with its own provider config (one `examples/default`-style root per project), or fork the module to add `configuration_aliases` and pass per-instance `providers = { ... }`. +## Dependencies only (run LiteLLM on GKE) + +Set `create_runtime = false` to provision Cloud SQL, Memorystore, GCS, +Secret Manager, and the runtime service account without Cloud Run or the +load balancer. For a Shared VPC, set the full host-project network ID and +skip PSA creation after the host project has configured it: + +```hcl +create_runtime = false +network_id = "projects//global/networks/" +create_psa_connection = false +``` + +The host project must already have Private Services Access configured on +that network and the Service Networking API enabled; the module cannot set +PSA up from a service project. GKE nodes must sit on the same Shared VPC so +the Cloud SQL and Memorystore private IPs are routable from the pods. Run +the root with its provider pointed at the project that should own the +dependencies. `create_runtime = true` with `network_id` set is also allowed, +but the Serverless VPC Access connector has to live in the same project as +the network, so that combination only works when the VPC is in the +deployment project + +Map the outputs into the Helm values as follows: + +```yaml +database: + writer: + host: + dbname: + passwordSecret: + name: + reader: + host: + dbname: + passwordSecret: + name: +redis: + host: + port: +masterKey: + secretName: +``` + +Create the database Secret with keys `username` (the `db_username` output) +and `password` (read it with `gcloud secrets versions access latest +--secret=`), and the master key Secret from +`master_key_secret_id` the same way. Memorystore only accepts TLS by +default, so store the `redis_server_ca_pem` output in a third Secret, +mount it into the gateway and backend pods via `volumes` / `volumeMounts`, +and add `REDIS_SSL=true` and `REDIS_SSL_CA_CERTS=` to each +component's `extraEnv`. Setting `redis_transit_encryption = false` removes +the CA plumbing at the cost of plaintext Redis traffic inside the VPC + +The chart's pre-install/pre-upgrade migration hook runs the Prisma +migration, so nothing replaces the Cloud Run migrations Job in this mode + ## Storage and database retention Two opt-in tripwires guard against accidental data loss on @@ -409,14 +466,15 @@ Flip `cloudsql_deletion_protection` to `false` or `gcs_force_destroy` to ## Redis encryption -Memorystore runs with `transit_encryption_mode = "SERVER_AUTHENTICATION"`, -so the proxy connects via `rediss://`. The instance's self-signed CA cert -(`server_ca_certs[0].cert`) is shipped to gateway + backend as -`REDIS_CA_PEM_B64`; their entrypoint shell decodes it to `/tmp/redis-ca.pem` -before uvicorn starts and points `REDIS_SSL_CA_CERTS` at that path. No -extra config needed — but if you ever swap Memorystore for an external -Redis, override `REDIS_HOST`/`REDIS_PORT` and either drop these env vars -or point them at your own CA. +By default, Memorystore runs with +`transit_encryption_mode = "SERVER_AUTHENTICATION"`, so Cloud Run connects +via `rediss://`. The instance's self-signed CA cert +(`server_ca_certs[0].cert`) is shipped to gateway and backend as +`REDIS_CA_PEM_B64`; their entrypoint shell decodes it to +`/tmp/redis-ca.pem` before uvicorn starts and points `REDIS_SSL_CA_CERTS` at +that path. Set `redis_transit_encryption = false` to use plaintext Redis. +For GKE, use `redis_server_ca_pem` as described in the dependencies-only +section, or accept the security tradeoff of disabling transit encryption ## Files @@ -434,4 +492,5 @@ or point them at your own CA. | `iam.tf` | Runtime SA + Cloud SQL client + Secret Manager accessor | | `cloudrun.tf` | 3 Cloud Run services + Cloud Run Job for migrations | | `load_balancer.tf`| External HTTPS LB, serverless NEGs, URL map for path routing | -| `outputs.tf` | LB IP, service URLs, secret IDs, migration `execute` command | +| `outputs.tf` | LB IP, service URLs, dependency endpoints, secret IDs, migration command | +| `tests/` | Plan-only mock-provider coverage for deployment modes and Redis encryption | diff --git a/terraform/litellm/gcp/bootstrap.tf b/terraform/litellm/gcp/bootstrap.tf index b929c4d76f3..dead5c41f6b 100644 --- a/terraform/litellm/gcp/bootstrap.tf +++ b/terraform/litellm/gcp/bootstrap.tf @@ -15,15 +15,17 @@ # enough to invoke Cloud Run admin APIs (`gcloud auth login`). resource "terraform_data" "migration" { + count = var.create_runtime ? 1 : 0 + triggers_replace = { - job_id = google_cloud_run_v2_job.migrations.id + job_id = google_cloud_run_v2_job.migrations[0].id job_image = local.migrations_image } provisioner "local-exec" { interpreter = ["bash", "-c"] environment = { - JOB = google_cloud_run_v2_job.migrations.name + JOB = google_cloud_run_v2_job.migrations[0].name REGION = var.region PROJECT = var.project_id } diff --git a/terraform/litellm/gcp/cloudrun.tf b/terraform/litellm/gcp/cloudrun.tf index 5a5c361b832..84ae8b9247f 100644 --- a/terraform/litellm/gcp/cloudrun.tf +++ b/terraform/litellm/gcp/cloudrun.tf @@ -6,25 +6,28 @@ locals { # Memorystore exposes a self-signed CA cert per instance; we ship it as # a base64 env var and decode it to a file at container startup so the # rediss:// connection can validate. Public cert, not sensitive. - redis_ca_pem_b64 = base64encode(google_redis_instance.this.server_ca_certs[0].cert) + redis_ca_pem_b64 = var.redis_transit_encryption ? base64encode(google_redis_instance.this.server_ca_certs[0].cert) : "" - shared_env_kv = [ - { name = "DATABASE_HOST", value = google_sql_database_instance.writer.private_ip_address }, - { name = "DATABASE_PORT", value = "5432" }, - { name = "DATABASE_USER", value = var.db_username }, - { name = "DATABASE_NAME", value = var.db_name }, - { name = "DATABASE_HOST_READ_REPLICA", value = google_sql_database_instance.reader.private_ip_address }, - { name = "DATABASE_PORT_READ_REPLICA", value = "5432" }, - { name = "REDIS_HOST", value = google_redis_instance.this.host }, - { name = "REDIS_PORT", value = tostring(google_redis_instance.this.port) }, - # _redis.get_redis_url_from_environment honors REDIS_SSL to flip the - # scheme to rediss://; REDIS_SSL_CA_CERTS is mapped via - # _get_redis_env_kwarg_mapping → ssl_ca_certs on the redis-py client. - { name = "REDIS_SSL", value = "true" }, - { name = "REDIS_SSL_CA_CERTS", value = "/tmp/redis-ca.pem" }, - { name = "REDIS_CA_PEM_B64", value = local.redis_ca_pem_b64 }, - { name = "GCS_BUCKET_NAME", value = google_storage_bucket.this.name }, - ] + shared_env_kv = concat( + [ + { name = "DATABASE_HOST", value = google_sql_database_instance.writer.private_ip_address }, + { name = "DATABASE_PORT", value = "5432" }, + { name = "DATABASE_USER", value = var.db_username }, + { name = "DATABASE_NAME", value = var.db_name }, + { name = "DATABASE_HOST_READ_REPLICA", value = google_sql_database_instance.reader.private_ip_address }, + { name = "DATABASE_PORT_READ_REPLICA", value = "5432" }, + { name = "REDIS_HOST", value = google_redis_instance.this.host }, + { name = "REDIS_PORT", value = tostring(google_redis_instance.this.port) }, + ], + var.redis_transit_encryption ? [ + { name = "REDIS_SSL", value = "true" }, + { name = "REDIS_SSL_CA_CERTS", value = "/tmp/redis-ca.pem" }, + { name = "REDIS_CA_PEM_B64", value = local.redis_ca_pem_b64 }, + ] : [], + [ + { name = "GCS_BUCKET_NAME", value = google_storage_bucket.this.name }, + ], + ) # OTel v2 is opt-in and gated on otel_endpoint, matching the AWS stack — # nothing OTel-related is added to the container env until an endpoint is @@ -126,9 +129,9 @@ locals { # Decode the Memorystore CA cert (passed as REDIS_CA_PEM_B64) to the # path REDIS_SSL_CA_CERTS points at, so the redis-py client can validate # the rediss:// handshake. - redis_ca_fragment = [ + redis_ca_fragment = var.redis_transit_encryption ? [ "python -c \"import os, base64, pathlib; pathlib.Path(os.environ['REDIS_SSL_CA_CERTS']).write_bytes(base64.b64decode(os.environ['REDIS_CA_PEM_B64']))\"" - ] + ] : [] database_url_fragment = [ "export DATABASE_URL=\"postgresql://$${DATABASE_USER}:$${DATABASE_PASSWORD}@$${DATABASE_HOST}:$${DATABASE_PORT}/$${DATABASE_NAME}\"", @@ -171,29 +174,7 @@ locals { # ---------- Gateway ---------- resource "google_cloud_run_v2_service" "gateway" { - # Metering needs a client certificate AND its key. Each secret is created only - # when its own PEM is supplied, so an endpoint set with a missing key would - # otherwise apply cleanly and leave the proxy logging "missing config" and - # never exporting. ca_cert_pem stays optional: empty means fall back to the - # system trust store. - # - # The guard lives here, on an unconditional resource, rather than on the cert - # secret: that secret is count-gated on the cert itself, so it has zero - # instances in exactly the case this must catch. Adding count or for_each to - # this resource would silently stop the guard from evaluating. - # - # endpoint cert key -> result - # "" any any -> metering off, no secrets created - # set set set -> metering on - # set any-missing -> plan fails here - lifecycle { - precondition { - condition = var.billing_metrics_endpoint == "" || ( - var.billing_metrics_client_cert_pem != "" && var.billing_metrics_client_key_pem != "" - ) - error_message = "billing_metrics_client_cert_pem and billing_metrics_client_key_pem are both required when billing_metrics_endpoint is set." - } - } + count = var.create_runtime ? 1 : 0 name = "${local.name}-gateway" location = var.region @@ -206,7 +187,7 @@ resource "google_cloud_run_v2_service" "gateway" { max_instance_request_concurrency = var.gateway_max_instance_request_concurrency vpc_access { - connector = google_vpc_access_connector.this.id + connector = google_vpc_access_connector.this[0].id egress = "PRIVATE_RANGES_ONLY" } @@ -312,17 +293,7 @@ resource "google_cloud_run_v2_service" "gateway" { # ---------- Backend ---------- resource "google_cloud_run_v2_service" "backend" { - # Same guard as the gateway: the backend meters too (it serves the named-server - # MCP transport), and a targeted apply of just this resource must not slip a - # billing endpoint through without the credentials to use it. - lifecycle { - precondition { - condition = var.billing_metrics_endpoint == "" || ( - var.billing_metrics_client_cert_pem != "" && var.billing_metrics_client_key_pem != "" - ) - error_message = "billing_metrics_client_cert_pem and billing_metrics_client_key_pem are both required when billing_metrics_endpoint is set." - } - } + count = var.create_runtime ? 1 : 0 name = "${local.name}-backend" location = var.region @@ -335,7 +306,7 @@ resource "google_cloud_run_v2_service" "backend" { max_instance_request_concurrency = var.backend_max_instance_request_concurrency vpc_access { - connector = google_vpc_access_connector.this.id + connector = google_vpc_access_connector.this[0].id egress = "PRIVATE_RANGES_ONLY" } @@ -443,6 +414,8 @@ resource "google_cloud_run_v2_service" "backend" { # with zero IAM bindings, so a compromised UI container can't pivot to # Secret Manager / Cloud SQL via the metadata service. resource "google_cloud_run_v2_service" "ui" { + count = var.create_runtime ? 1 : 0 + name = "${local.name}-ui" location = var.region ingress = "INGRESS_TRAFFIC_INTERNAL_LOAD_BALANCER" @@ -450,7 +423,7 @@ resource "google_cloud_run_v2_service" "ui" { deletion_protection = false template { - service_account = google_service_account.ui_runtime.email + service_account = google_service_account.ui_runtime[0].email max_instance_request_concurrency = var.ui_max_instance_request_concurrency scaling { @@ -491,25 +464,31 @@ resource "google_cloud_run_v2_service" "ui" { # (LITELLM_MASTER_KEY); these IAM bindings just open up Cloud Run's invoker # gate so the LB request makes it to the container. resource "google_cloud_run_v2_service_iam_member" "gateway_allusers" { + count = var.create_runtime ? 1 : 0 + project = var.project_id - location = google_cloud_run_v2_service.gateway.location - name = google_cloud_run_v2_service.gateway.name + location = google_cloud_run_v2_service.gateway[0].location + name = google_cloud_run_v2_service.gateway[0].name role = "roles/run.invoker" member = "allUsers" } resource "google_cloud_run_v2_service_iam_member" "backend_allusers" { + count = var.create_runtime ? 1 : 0 + project = var.project_id - location = google_cloud_run_v2_service.backend.location - name = google_cloud_run_v2_service.backend.name + location = google_cloud_run_v2_service.backend[0].location + name = google_cloud_run_v2_service.backend[0].name role = "roles/run.invoker" member = "allUsers" } resource "google_cloud_run_v2_service_iam_member" "ui_allusers" { + count = var.create_runtime ? 1 : 0 + project = var.project_id - location = google_cloud_run_v2_service.ui.location - name = google_cloud_run_v2_service.ui.name + location = google_cloud_run_v2_service.ui[0].location + name = google_cloud_run_v2_service.ui[0].name role = "roles/run.invoker" member = "allUsers" } @@ -519,6 +498,8 @@ resource "google_cloud_run_v2_service_iam_member" "ui_allusers" { # assembles DATABASE_URL from the DATABASE_* env vars and runs `prisma # migrate deploy`. No proxy_config, no master key, no shell wrapper. resource "google_cloud_run_v2_job" "migrations" { + count = var.create_runtime ? 1 : 0 + name = "${local.name}-migrations" location = var.region labels = local.labels @@ -529,7 +510,7 @@ resource "google_cloud_run_v2_job" "migrations" { service_account = google_service_account.runtime.email vpc_access { - connector = google_vpc_access_connector.this.id + connector = google_vpc_access_connector.this[0].id egress = "PRIVATE_RANGES_ONLY" } diff --git a/terraform/litellm/gcp/cloudsql.tf b/terraform/litellm/gcp/cloudsql.tf index c9c2d03b2de..777434b4727 100644 --- a/terraform/litellm/gcp/cloudsql.tf +++ b/terraform/litellm/gcp/cloudsql.tf @@ -36,7 +36,7 @@ resource "google_sql_database_instance" "writer" { ip_configuration { ipv4_enabled = false - private_network = google_compute_network.this.id + private_network = local.network_id } insights_config { @@ -55,6 +55,11 @@ resource "google_sql_database_instance" "writer" { # (full data loss). Set the initial size only; let Cloud SQL own it # thereafter. ignore_changes = [settings[0].disk_size] + + precondition { + condition = var.create_psa_connection || var.network_id != "" + error_message = "create_psa_connection must be true unless network_id references an existing VPC with Private Services Access configured." + } } } @@ -76,7 +81,7 @@ resource "google_sql_database_instance" "reader" { ip_configuration { ipv4_enabled = false - private_network = google_compute_network.this.id + private_network = local.network_id } } diff --git a/terraform/litellm/gcp/examples/default/main.tf b/terraform/litellm/gcp/examples/default/main.tf index 8760d445f0c..f44b2a9a001 100644 --- a/terraform/litellm/gcp/examples/default/main.tf +++ b/terraform/litellm/gcp/examples/default/main.tf @@ -31,6 +31,11 @@ module "litellm" { tenant = var.tenant env = var.env + create_runtime = var.create_runtime + network_id = var.network_id + create_psa_connection = var.create_psa_connection + redis_transit_encryption = var.redis_transit_encryption + litellm_master_key = var.litellm_master_key litellm_license = var.litellm_license ui_password = var.ui_password diff --git a/terraform/litellm/gcp/examples/default/outputs.tf b/terraform/litellm/gcp/examples/default/outputs.tf index 3a9343c4850..48cdc1af66e 100644 --- a/terraform/litellm/gcp/examples/default/outputs.tf +++ b/terraform/litellm/gcp/examples/default/outputs.tf @@ -38,6 +38,31 @@ output "redis_endpoint" { value = module.litellm.redis_endpoint } +output "redis_host" { + description = "Memorystore Redis host." + value = module.litellm.redis_host +} + +output "redis_port" { + description = "Memorystore Redis port." + value = module.litellm.redis_port +} + +output "redis_server_ca_pem" { + description = "Memorystore server CA PEM." + value = module.litellm.redis_server_ca_pem +} + +output "db_username" { + description = "Cloud SQL application username." + value = module.litellm.db_username +} + +output "db_name" { + description = "Cloud SQL database name." + value = module.litellm.db_name +} + output "gcs_bucket" { description = "GCS bucket name." value = module.litellm.gcs_bucket @@ -53,6 +78,11 @@ output "db_password_secret_id" { value = module.litellm.db_password_secret_id } +output "runtime_service_account_email" { + description = "Runtime service account email." + value = module.litellm.runtime_service_account_email +} + output "migration_run_command" { description = "Break-glass command to re-run the one-off migration job." value = module.litellm.migration_run_command diff --git a/terraform/litellm/gcp/examples/default/terraform.tfvars.example b/terraform/litellm/gcp/examples/default/terraform.tfvars.example index 4416cf0ee5d..c35206503bb 100644 --- a/terraform/litellm/gcp/examples/default/terraform.tfvars.example +++ b/terraform/litellm/gcp/examples/default/terraform.tfvars.example @@ -8,6 +8,14 @@ region = "us-central1" tenant = "acme" env = "stage" +# Deployment mode. For dependencies only on a Shared VPC, set +# create_runtime = false, network_id to the full host-project network ID, and +# create_psa_connection = false after configuring PSA on that network. +# create_runtime = true +# network_id = "" +# create_psa_connection = true +# redis_transit_encryption = true + # Tenant-supplied secrets. Prefer TF_VAR_litellm_master_key / # TF_VAR_litellm_license / TF_VAR_ui_password env vars so the values don't # end up in a committed tfvars file. All three are optional — when diff --git a/terraform/litellm/gcp/examples/default/variables.tf b/terraform/litellm/gcp/examples/default/variables.tf index 56e5ec88ef8..88b57ce27eb 100644 --- a/terraform/litellm/gcp/examples/default/variables.tf +++ b/terraform/litellm/gcp/examples/default/variables.tf @@ -26,6 +26,30 @@ variable "env" { type = string } +variable "create_runtime" { + description = "Create Cloud Run and load balancer resources." + type = bool + default = true +} + +variable "network_id" { + description = "Existing VPC network resource ID. Empty creates a VPC." + type = string + default = "" +} + +variable "create_psa_connection" { + description = "Create Private Services Access resources." + type = bool + default = true +} + +variable "redis_transit_encryption" { + description = "Enable Memorystore transit encryption." + type = bool + default = true +} + # Sensitive — prefer TF_VAR_litellm_master_key / TF_VAR_litellm_license / # TF_VAR_ui_password so values stay out of any committed tfvars file. variable "litellm_master_key" { diff --git a/terraform/litellm/gcp/iam.tf b/terraform/litellm/gcp/iam.tf index 09df5e7dff0..509e6d48ffd 100644 --- a/terraform/litellm/gcp/iam.tf +++ b/terraform/litellm/gcp/iam.tf @@ -6,6 +6,15 @@ resource "google_service_account" "runtime" { account_id = "${local.name}-runtime" display_name = "LiteLLM Cloud Run runtime" + + lifecycle { + precondition { + condition = !var.create_runtime || var.billing_metrics_endpoint == "" || ( + var.billing_metrics_client_cert_pem != "" && var.billing_metrics_client_key_pem != "" + ) + error_message = "billing_metrics_client_cert_pem and billing_metrics_client_key_pem are both required when billing_metrics_endpoint is set and create_runtime is true." + } + } } # UI runtime SA — no role bindings. The UI is static nginx with no DB, @@ -14,6 +23,8 @@ resource "google_service_account" "runtime" { # project's serverless service agent (not this SA), so it doesn't need # artifactregistry.reader either. resource "google_service_account" "ui_runtime" { + count = var.create_runtime ? 1 : 0 + account_id = "${local.name}-ui-runtime" display_name = "LiteLLM Cloud Run UI runtime (no data-plane access)" } diff --git a/terraform/litellm/gcp/load_balancer.tf b/terraform/litellm/gcp/load_balancer.tf index 11f30d0f944..57e8af8210f 100644 --- a/terraform/litellm/gcp/load_balancer.tf +++ b/terraform/litellm/gcp/load_balancer.tf @@ -14,77 +14,93 @@ locals { } resource "google_compute_global_address" "lb" { + count = var.create_runtime ? 1 : 0 + name = "${local.name}-lb-ip" labels = local.labels } # Serverless NEGs — one per Cloud Run service. resource "google_compute_region_network_endpoint_group" "gateway" { + count = var.create_runtime ? 1 : 0 + name = "${local.name}-gateway-neg" region = var.region network_endpoint_type = "SERVERLESS" cloud_run { - service = google_cloud_run_v2_service.gateway.name + service = google_cloud_run_v2_service.gateway[0].name } } resource "google_compute_region_network_endpoint_group" "backend" { + count = var.create_runtime ? 1 : 0 + name = "${local.name}-backend-neg" region = var.region network_endpoint_type = "SERVERLESS" cloud_run { - service = google_cloud_run_v2_service.backend.name + service = google_cloud_run_v2_service.backend[0].name } } resource "google_compute_region_network_endpoint_group" "ui" { + count = var.create_runtime ? 1 : 0 + name = "${local.name}-ui-neg" region = var.region network_endpoint_type = "SERVERLESS" cloud_run { - service = google_cloud_run_v2_service.ui.name + service = google_cloud_run_v2_service.ui[0].name } } # Backend services wrap each NEG. resource "google_compute_backend_service" "gateway" { + count = var.create_runtime ? 1 : 0 + name = "${local.name}-gateway-bs" protocol = "HTTP" load_balancing_scheme = "EXTERNAL_MANAGED" backend { - group = google_compute_region_network_endpoint_group.gateway.id + group = google_compute_region_network_endpoint_group.gateway[0].id } } resource "google_compute_backend_service" "backend" { + count = var.create_runtime ? 1 : 0 + name = "${local.name}-backend-bs" protocol = "HTTP" load_balancing_scheme = "EXTERNAL_MANAGED" backend { - group = google_compute_region_network_endpoint_group.backend.id + group = google_compute_region_network_endpoint_group.backend[0].id } } resource "google_compute_backend_service" "ui" { + count = var.create_runtime ? 1 : 0 + name = "${local.name}-ui-bs" protocol = "HTTP" load_balancing_scheme = "EXTERNAL_MANAGED" backend { - group = google_compute_region_network_endpoint_group.ui.id + group = google_compute_region_network_endpoint_group.ui[0].id } } # URL map. Default → backend (management API). Path matchers route the # gateway and UI prefixes elsewhere. resource "google_compute_url_map" "this" { + count = var.create_runtime ? 1 : 0 + name = local.name - default_service = google_compute_backend_service.backend.id + default_service = google_compute_backend_service.backend[0].id host_rule { hosts = ["*"] @@ -93,13 +109,13 @@ resource "google_compute_url_map" "this" { path_matcher { name = "main" - default_service = google_compute_backend_service.backend.id + default_service = google_compute_backend_service.backend[0].id # UI paths (catch them before any /v1/* gateway rules so /favicon.ico # and / take precedence). path_rule { paths = local.ui_path_prefixes - service = google_compute_backend_service.ui.id + service = google_compute_backend_service.ui[0].id } # Gateway path prefixes. GCP URL maps cap a path_rule at 10 path globs, @@ -108,7 +124,7 @@ resource "google_compute_url_map" "this" { for_each = { for idx, chunk in chunklist(local.gateway_path_prefixes, 10) : idx => chunk } content { paths = path_rule.value - service = google_compute_backend_service.gateway.id + service = google_compute_backend_service.gateway[0].id } } } @@ -118,7 +134,7 @@ resource "google_compute_url_map" "this" { # target proxy when TLS is enabled; otherwise the regular path-routing # URL map is attached to the HTTP proxy and everything stays plaintext. resource "google_compute_url_map" "https_redirect" { - count = local.tls_enabled ? 1 : 0 + count = var.create_runtime && local.tls_enabled ? 1 : 0 name = "${local.name}-redirect" default_url_redirect { @@ -129,8 +145,10 @@ resource "google_compute_url_map" "https_redirect" { } resource "google_compute_target_http_proxy" "this" { + count = var.create_runtime ? 1 : 0 + name = "${local.name}-http" - url_map = local.tls_enabled ? google_compute_url_map.https_redirect[0].id : google_compute_url_map.this.id + url_map = local.tls_enabled ? google_compute_url_map.https_redirect[0].id : google_compute_url_map.this[0].id # Default-deny on the HTTP-only path: TLS is the supported posture. # Operators must either supply DNS names or explicitly opt in. @@ -143,12 +161,14 @@ resource "google_compute_target_http_proxy" "this" { } resource "google_compute_global_forwarding_rule" "http" { + count = var.create_runtime ? 1 : 0 + name = "${local.name}-http" ip_protocol = "TCP" port_range = "80" load_balancing_scheme = "EXTERNAL_MANAGED" - ip_address = google_compute_global_address.lb.address - target = google_compute_target_http_proxy.this.id + ip_address = google_compute_global_address.lb[0].address + target = google_compute_target_http_proxy.this[0].id labels = local.labels } @@ -161,7 +181,7 @@ resource "google_compute_global_forwarding_rule" "http" { # transitions to ACTIVE. resource "google_compute_managed_ssl_certificate" "this" { - count = local.tls_enabled ? 1 : 0 + count = var.create_runtime && local.tls_enabled ? 1 : 0 # A managed cert's `domains` is immutable, so changing var.lb_domains # forces replacement, and the cert is referenced by the HTTPS target @@ -181,19 +201,19 @@ resource "google_compute_managed_ssl_certificate" "this" { } resource "google_compute_target_https_proxy" "this" { - count = local.tls_enabled ? 1 : 0 + count = var.create_runtime && local.tls_enabled ? 1 : 0 name = "${local.name}-https" - url_map = google_compute_url_map.this.id + url_map = google_compute_url_map.this[0].id ssl_certificates = [google_compute_managed_ssl_certificate.this[0].id] } resource "google_compute_global_forwarding_rule" "https" { - count = local.tls_enabled ? 1 : 0 + count = var.create_runtime && local.tls_enabled ? 1 : 0 name = "${local.name}-https" ip_protocol = "TCP" port_range = "443" load_balancing_scheme = "EXTERNAL_MANAGED" - ip_address = google_compute_global_address.lb.address + ip_address = google_compute_global_address.lb[0].address target = google_compute_target_https_proxy.this[0].id labels = local.labels } diff --git a/terraform/litellm/gcp/locals.tf b/terraform/litellm/gcp/locals.tf index 9a817eba605..3861413d496 100644 --- a/terraform/litellm/gcp/locals.tf +++ b/terraform/litellm/gcp/locals.tf @@ -21,6 +21,9 @@ locals { var.labels, ) + create_network = var.network_id == "" + network_id = local.create_network ? google_compute_network.this[0].id : var.network_id + gateway_path_prefixes = [ "/v1/chat/*", "/chat/*", "/v1/completions*", "/completions*", @@ -74,7 +77,7 @@ locals { "/ui/*", ] - proxy_config_enabled = length(keys(var.proxy_config)) > 0 + proxy_config_enabled = var.create_runtime && length(keys(var.proxy_config)) > 0 proxy_config_yaml = local.proxy_config_enabled ? yamlencode(var.proxy_config) : "" proxy_config_mount_path = "/etc/litellm" diff --git a/terraform/litellm/gcp/network.tf b/terraform/litellm/gcp/network.tf index a1ccaed02f9..47c7bf94a2b 100644 --- a/terraform/litellm/gcp/network.tf +++ b/terraform/litellm/gcp/network.tf @@ -1,13 +1,17 @@ resource "google_compute_network" "this" { + count = local.create_network ? 1 : 0 + name = local.name auto_create_subnetworks = false routing_mode = "REGIONAL" } resource "google_compute_subnetwork" "this" { + count = local.create_network ? 1 : 0 + name = "${local.name}-${var.region}" region = var.region - network = google_compute_network.this.id + network = google_compute_network.this[0].id ip_cidr_range = var.subnet_cidr private_ip_google_access = true } @@ -16,17 +20,21 @@ resource "google_compute_subnetwork" "this" { # managed services peer with the VPC over the connection below using # addresses from this range. resource "google_compute_global_address" "psa" { + count = var.create_psa_connection ? 1 : 0 + name = "${local.name}-psa" purpose = "VPC_PEERING" address_type = "INTERNAL" prefix_length = 16 - network = google_compute_network.this.id + network = local.network_id } resource "google_service_networking_connection" "psa" { - network = google_compute_network.this.id + count = var.create_psa_connection ? 1 : 0 + + network = local.network_id service = "servicenetworking.googleapis.com" - reserved_peering_ranges = [google_compute_global_address.psa.name] + reserved_peering_ranges = [google_compute_global_address.psa[0].name] } # Serverless VPC Access connector — required so Cloud Run can reach @@ -37,9 +45,11 @@ resource "google_service_networking_connection" "psa" { # for low-to-moderate Cloud Run egress; bump max if your services push # heavy private-network traffic. resource "google_vpc_access_connector" "this" { + count = var.create_runtime ? 1 : 0 + name = "${local.name}-conn" region = var.region - network = google_compute_network.this.name + network = local.network_id ip_cidr_range = var.vpc_connector_cidr min_instances = 2 max_instances = 3 diff --git a/terraform/litellm/gcp/outputs.tf b/terraform/litellm/gcp/outputs.tf index 6f1f1d5ccf4..2a4742f42cf 100644 --- a/terraform/litellm/gcp/outputs.tf +++ b/terraform/litellm/gcp/outputs.tf @@ -1,26 +1,26 @@ output "lb_ip" { - description = "Global anycast IP of the external HTTPS load balancer." - value = google_compute_global_address.lb.address + description = "Global anycast IP of the external HTTPS load balancer. Null when create_runtime is false." + value = var.create_runtime ? one(google_compute_global_address.lb[*].address) : null } output "lb_url" { - description = "Proxy URL. Switches scheme based on whether lb_domains is set; when TLS is enabled the URL points at the first listed domain (since managed certs are tied to the hostname, not the anycast IP). The dashboard is served at /, the API at /v1/*." - value = local.tls_enabled ? "https://${var.lb_domains[0]}" : "http://${google_compute_global_address.lb.address}" + description = "Proxy URL, or null when create_runtime is false. Switches scheme based on whether lb_domains is set." + value = var.create_runtime ? (local.tls_enabled ? "https://${var.lb_domains[0]}" : "http://${one(google_compute_global_address.lb[*].address)}") : null } output "gateway_service_url" { - description = "Default Cloud Run URL for the gateway (bypasses the LB)." - value = google_cloud_run_v2_service.gateway.uri + description = "Default Cloud Run URL for the gateway, or null when create_runtime is false." + value = var.create_runtime ? one(google_cloud_run_v2_service.gateway[*].uri) : null } output "backend_service_url" { - description = "Default Cloud Run URL for the backend (bypasses the LB)." - value = google_cloud_run_v2_service.backend.uri + description = "Default Cloud Run URL for the backend, or null when create_runtime is false." + value = var.create_runtime ? one(google_cloud_run_v2_service.backend[*].uri) : null } output "ui_service_url" { - description = "Default Cloud Run URL for the UI (bypasses the LB)." - value = google_cloud_run_v2_service.ui.uri + description = "Default Cloud Run URL for the UI, or null when create_runtime is false." + value = var.create_runtime ? one(google_cloud_run_v2_service.ui[*].uri) : null } output "cloudsql_writer_ip" { @@ -38,6 +38,36 @@ output "redis_endpoint" { value = "${google_redis_instance.this.host}:${google_redis_instance.this.port}" } +output "runtime_service_account_email" { + description = "Runtime service account email for Cloud Run or GKE Workload Identity." + value = google_service_account.runtime.email +} + +output "redis_host" { + description = "Memorystore Redis host." + value = google_redis_instance.this.host +} + +output "redis_port" { + description = "Memorystore Redis port." + value = google_redis_instance.this.port +} + +output "redis_server_ca_pem" { + description = "Memorystore server CA PEM. Mount it in the pod and set REDIS_SSL=true and REDIS_SSL_CA_CERTS= via extraEnv when transit encryption is enabled." + value = var.redis_transit_encryption ? google_redis_instance.this.server_ca_certs[0].cert : null +} + +output "db_username" { + description = "Cloud SQL application username." + value = var.db_username +} + +output "db_name" { + description = "Cloud SQL database name." + value = var.db_name +} + output "gcs_bucket" { description = "GCS bucket name. Exposed to gateway + backend as GCS_BUCKET_NAME. Reference from proxy_config via `os.environ/GCS_BUCKET_NAME`." value = google_storage_bucket.this.name @@ -54,11 +84,11 @@ output "db_password_secret_id" { } output "migration_run_command" { - description = "Shell command that executes the one-off migration job against Cloud SQL. Run this once after the first apply." - value = format( + description = "Shell command that executes the one-off migration job against Cloud SQL, or null when create_runtime is false." + value = var.create_runtime ? format( "gcloud run jobs execute %s --region %s --project %s --wait", - google_cloud_run_v2_job.migrations.name, + one(google_cloud_run_v2_job.migrations[*].name), var.region, var.project_id, - ) + ) : null } diff --git a/terraform/litellm/gcp/redis.tf b/terraform/litellm/gcp/redis.tf index 0e07c416e85..0602758f090 100644 --- a/terraform/litellm/gcp/redis.tf +++ b/terraform/litellm/gcp/redis.tf @@ -4,7 +4,7 @@ resource "google_redis_instance" "this" { memory_size_gb = var.redis_memory_size_gb region = var.region - authorized_network = google_compute_network.this.id + authorized_network = local.network_id connect_mode = "PRIVATE_SERVICE_ACCESS" redis_version = "REDIS_7_0" @@ -16,7 +16,7 @@ resource "google_redis_instance" "this" { # and passed to the proxy as REDIS_CA_PEM_B64); the proxy decodes it to # /tmp/redis-ca.pem at startup and uses it to validate the rediss:// # handshake. Mirrors `transit_encryption_enabled = true` on AWS. - transit_encryption_mode = "SERVER_AUTHENTICATION" + transit_encryption_mode = var.redis_transit_encryption ? "SERVER_AUTHENTICATION" : "DISABLED" depends_on = [google_service_networking_connection.psa] } diff --git a/terraform/litellm/gcp/tests/deps_only.tftest.hcl b/terraform/litellm/gcp/tests/deps_only.tftest.hcl new file mode 100644 index 00000000000..610c49d5b52 --- /dev/null +++ b/terraform/litellm/gcp/tests/deps_only.tftest.hcl @@ -0,0 +1,175 @@ +mock_provider "google" { + mock_resource "google_redis_instance" { + defaults = { + host = "10.0.0.4" + port = 6379 + server_ca_certs = [{ + cert = "-----BEGIN CERTIFICATE-----\nmock\n-----END CERTIFICATE-----" + }] + } + } +} + +mock_provider "google-beta" {} +mock_provider "random" {} + +variables { + project_id = "test-project" + tenant = "tenant" + env = "test" + allow_plaintext_lb = true + image_registry = "us-central1-docker.pkg.dev/test-project/litellm" +} + +run "default_creates_everything" { + command = plan + + assert { + condition = alltrue([ + length(google_compute_network.this) == 1, + length(google_compute_subnetwork.this) == 1, + length(google_compute_global_address.psa) == 1, + length(google_service_networking_connection.psa) == 1, + length(google_vpc_access_connector.this) == 1, + length(google_cloud_run_v2_service.gateway) == 1, + length(google_cloud_run_v2_service.backend) == 1, + length(google_cloud_run_v2_service.ui) == 1, + length(google_cloud_run_v2_job.migrations) == 1, + length(google_compute_global_address.lb) == 1, + length(terraform_data.migration) == 1, + ]) + error_message = "The default mode must create networking, runtime services, the load balancer, and migrations." + } + + assert { + condition = google_redis_instance.this.transit_encryption_mode == "SERVER_AUTHENTICATION" + error_message = "Redis transit encryption must remain enabled by default." + } + + assert { + condition = length(local.shared_env_kv) == 12 + error_message = "The default runtime environment must include GCS and the three Redis TLS entries." + } +} + +run "deps_only_creates_no_runtime" { + command = plan + + variables { + create_runtime = false + proxy_config = { + model_list = [] + } + } + + assert { + condition = alltrue([ + length(google_cloud_run_v2_service.gateway) == 0, + length(google_cloud_run_v2_service.backend) == 0, + length(google_cloud_run_v2_service.ui) == 0, + length(google_cloud_run_v2_job.migrations) == 0, + length(google_cloud_run_v2_service_iam_member.gateway_allusers) == 0, + length(google_cloud_run_v2_service_iam_member.backend_allusers) == 0, + length(google_cloud_run_v2_service_iam_member.ui_allusers) == 0, + length(google_compute_global_address.lb) == 0, + length(google_compute_region_network_endpoint_group.gateway) == 0, + length(google_compute_region_network_endpoint_group.backend) == 0, + length(google_compute_region_network_endpoint_group.ui) == 0, + length(google_compute_backend_service.gateway) == 0, + length(google_compute_backend_service.backend) == 0, + length(google_compute_backend_service.ui) == 0, + length(google_compute_url_map.this) == 0, + length(google_compute_url_map.https_redirect) == 0, + length(google_compute_target_http_proxy.this) == 0, + length(google_compute_global_forwarding_rule.http) == 0, + length(google_compute_managed_ssl_certificate.this) == 0, + length(google_compute_target_https_proxy.this) == 0, + length(google_compute_global_forwarding_rule.https) == 0, + length(terraform_data.migration) == 0, + length(google_vpc_access_connector.this) == 0, + length(google_service_account.ui_runtime) == 0, + length(google_storage_bucket.proxy_config) == 0, + ]) + error_message = "Dependencies-only mode must omit all runtime, load balancer, connector, UI identity, and proxy config resources." + } + + assert { + condition = alltrue([ + google_sql_database_instance.writer.name == "tenant-litellm-test", + google_sql_database_instance.reader.name == "tenant-litellm-test-reader", + google_redis_instance.this.name == "tenant-litellm-test", + google_storage_bucket.this.force_destroy == false, + google_secret_manager_secret.master_key.secret_id == "tenant-litellm-test-master-key", + google_secret_manager_secret.db_password.secret_id == "tenant-litellm-test-db-password", + google_service_account.runtime.account_id == "tenant-litellm-test-runtime", + ]) + error_message = "Dependencies-only mode must retain data stores, secrets, and the runtime service account." + } + + assert { + condition = output.lb_url == null && output.migration_run_command == null + error_message = "Runtime outputs must be null while dependency outputs remain available." + } +} + +run "existing_network_attaches_data_stores" { + command = plan + + variables { + network_id = "projects/host-proj/global/networks/shared" + create_psa_connection = false + create_runtime = false + } + + assert { + condition = alltrue([ + length(google_compute_network.this) == 0, + length(google_compute_subnetwork.this) == 0, + length(google_compute_global_address.psa) == 0, + length(google_service_networking_connection.psa) == 0, + google_sql_database_instance.writer.settings[0].ip_configuration[0].private_network == var.network_id, + google_redis_instance.this.authorized_network == var.network_id, + ]) + error_message = "An existing VPC must receive the Cloud SQL and Memorystore private-network attachments." + } +} + +run "psa_required_without_existing_network" { + command = plan + + variables { + create_psa_connection = false + } + + expect_failures = [ + google_sql_database_instance.writer, + ] +} + +run "redis_plaintext_drops_tls_env" { + command = plan + + variables { + redis_transit_encryption = false + } + + assert { + condition = google_redis_instance.this.transit_encryption_mode == "DISABLED" + error_message = "Redis transit encryption must be disabled when requested." + } + + assert { + condition = length(local.shared_env_kv) == 9 + error_message = "Plaintext Redis mode must include GCS and omit the three Redis TLS entries." + } + + assert { + condition = length([for env in local.shared_env_kv : env if env.name == "REDIS_SSL"]) == 0 + error_message = "Plaintext Redis mode must not set REDIS_SSL." + } + + assert { + condition = length(local.redis_ca_fragment) == 0 + error_message = "Plaintext Redis mode must not decode a Redis CA at startup." + } +} diff --git a/terraform/litellm/gcp/variables.tf b/terraform/litellm/gcp/variables.tf index 1162e100bb2..9c68ed3db76 100644 --- a/terraform/litellm/gcp/variables.tf +++ b/terraform/litellm/gcp/variables.tf @@ -79,16 +79,42 @@ variable "ui_password" { sensitive = true } +# ---------- Deployment mode ---------- + +variable "create_runtime" { + description = "Create Cloud Run, load balancer, VPC connector, runtime support resources, and the migration job. Set false for GKE or another external runtime." + type = bool + default = true +} + +variable "network_id" { + description = "Existing VPC network resource ID (`projects//global/networks/`). When set, no VPC or subnet is created. A VPC connector requires this network to be in the deployment project when create_runtime is true." + type = string + default = "" +} + +variable "create_psa_connection" { + description = "Create the Private Services Access range and connection for Cloud SQL and Memorystore. Set false when the existing network already has PSA configured." + type = bool + default = true +} + +variable "redis_transit_encryption" { + description = "Enable Memorystore transit encryption and inject Redis TLS settings into Cloud Run. Set false to use plaintext Redis." + type = bool + default = true +} + # ---------- Networking ---------- variable "subnet_cidr" { - description = "Primary CIDR block for the LiteLLM subnet." + description = "Primary CIDR block for the LiteLLM subnet. Unused when network_id is set." type = string default = "10.40.0.0/16" } variable "vpc_connector_cidr" { - description = "CIDR for the Serverless VPC Access connector. /28 required." + description = "CIDR for the Serverless VPC Access connector. /28 required. Unused when create_runtime is false." type = string default = "10.41.0.0/28" } diff --git a/tests/e2e/access_control/test_access_control_e2e.py b/tests/e2e/access_control/test_access_control_e2e.py index af7e9a099fd..9d01f2915e7 100644 --- a/tests/e2e/access_control/test_access_control_e2e.py +++ b/tests/e2e/access_control/test_access_control_e2e.py @@ -24,7 +24,7 @@ from access_control_client import ( from e2e_config import unique_marker from e2e_http import Success, UnauthorizedError, UnknownApiError, unwrap from lifecycle import ResourceManager -from models import ChatBody, ChatMessage, ChatResponse, LiteLLMParamsBody +from models import ChatBody, ChatMessage, ChatResponse, EmbedBody, LiteLLMParamsBody from proxy_client import ProxyClient pytestmark = pytest.mark.e2e @@ -32,6 +32,7 @@ pytestmark = pytest.mark.e2e ALLOWED_MODEL = "gemini-2.5-flash" DISALLOWED_MODEL = "gpt-5.5" VIRTUAL_KEY_BACKEND = "anthropic/claude-haiku-4-5-20251001" +EMBEDDING_MODEL = "openai-text-embedding-3-small" class TestAccessControl: @@ -71,6 +72,31 @@ class TestAccessControl: f"403 body must be a model-access denial, got: {result.body[:300]}" ) + @pytest.mark.covers("other.auth.virtual_key.route_group_allowed") + def test_llm_api_routes_group_grants_every_llm_endpoint( + self, client: AccessControlClient, resources: ResourceManager + ) -> None: + key = client.llm_only_key() + resources.defer(lambda: client.delete_key(key)) + + chat = client.chat_status(key, ALLOWED_MODEL, f"capital of France? {unique_marker()}") + assert chat.status_code == 200, ( + f"llm_api_routes key must reach /chat/completions, got {chat.status_code}: {chat.body[:300]}" + ) + assert ChatResponse.model_validate_json(chat.body).choices, ( + f"200 must carry a real completion, not an error envelope: {chat.body[:300]}" + ) + + embedding = unwrap( + client.proxy.embed(key, EmbedBody(model=EMBEDDING_MODEL, input=f"route group {unique_marker()}")) + ) + assert embedding.model, f"llm_api_routes key reached /embeddings but got no model back: {embedding}" + + denied = client.create_model_status(key, f"e2e-route-group-{unique_marker()}") + assert denied.status_code == 403 and ROUTE_NOT_ALLOWED_MARKER in denied.body, ( + f"the same key must still be shut out of /model/new, got {denied.status_code}: {denied.body[:300]}" + ) + def test_llm_only_key_forbidden_from_management_route_403( self, client: AccessControlClient, resources: ResourceManager ) -> None: diff --git a/tests/e2e/llm_translation/test_cache_control.py b/tests/e2e/llm_translation/test_cache_control.py index a2c17b0fb66..0d224061381 100644 --- a/tests/e2e/llm_translation/test_cache_control.py +++ b/tests/e2e/llm_translation/test_cache_control.py @@ -10,6 +10,11 @@ Each case asserts the feature actually happened, not just a 200. Coverage matrix intentionally not covered here. - Vertex (gemini-2.5-flash): prompt caching via ``cache_control`` context caching; the second identical call must report cached prompt tokens > 0. +- Anthropic (claude-haiku-4-5, direct): the same ``cache_control`` prefix over + the OpenAI-compatible route; the second call must report cache-read tokens > 0. +- OpenAI (gpt-5.6): automatic prompt caching needs no request marker, so the + cacheable prefix goes out as a plain system string with a ``prompt_cache_key`` + and the second call must report ``prompt_tokens_details.cached_tokens`` > 0. service_tier lives in test_provider_features_e2e.py. @@ -21,6 +26,7 @@ built from the typed content blocks shared in ``endpoints_client.py``. from __future__ import annotations import time +from collections.abc import Callable import pytest from pydantic import BaseModel @@ -29,7 +35,7 @@ from e2e_config import unique_marker from e2e_http import Result, unwrap from endpoints_client import CacheControl, RichMessage, TextBlock from lifecycle import ResourceManager -from models import ChatResponse, LiteLLMParamsBody, Usage +from models import ChatBody, ChatMessage, ChatResponse, LiteLLMParamsBody, Usage from passthrough_client import PassthroughClient import os @@ -37,6 +43,8 @@ pytestmark = pytest.mark.e2e BEDROCK_MODEL = "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0" VERTEX_MODEL = "vertex_ai/gemini-2.5-flash" +ANTHROPIC_MODEL = "anthropic/claude-haiku-4-5-20251001" +OPENAI_MODEL = "openai/gpt-5.6" class CacheChatBody(BaseModel): @@ -89,17 +97,36 @@ def _cache_chat( ) +def _plain_cache_chat( + client: PassthroughClient, key: str, model: str, prefix: str, cache_key: str +) -> Result[ChatResponse]: + """The same cacheable prefix as a plain system string, for providers that cache + automatically and take no per-block marker (OpenAI).""" + return client.proxy.chat( + key, + ChatBody( + model=model, + messages=[ + ChatMessage(role="system", content=prefix), + ChatMessage(role="user", content="Reply with one word."), + ], + max_tokens=64, + prompt_cache_key=cache_key, + ), + ) + + def _assert_cache_read_on_second_call( - client: PassthroughClient, key: str, model: str + model: str, send: Callable[[str], Result[ChatResponse]] ) -> None: prefix = _cacheable_prefix() - first = unwrap(_cache_chat(client, key, model, prefix)) + first = unwrap(send(prefix)) assert first.choices, f"{model}: first cache-priming call returned no choices: {first}" deadline = time.monotonic() + 30.0 while True: - second = unwrap(_cache_chat(client, key, model, prefix)) + second = unwrap(send(prefix)) read_tokens = _cached_read_tokens(second.usage) if read_tokens > 0 or time.monotonic() >= deadline: break @@ -125,7 +152,8 @@ class TestCacheControl: LiteLLMParamsBody(model=BEDROCK_MODEL, aws_region_name="us-east-1"), ) resources.defer(lambda: client.proxy.delete_model(model_id)) - _assert_cache_read_on_second_call(client, resources.key(), model) + key = resources.key() + _assert_cache_read_on_second_call(model, lambda prefix: _cache_chat(client, key, model, prefix)) @pytest.mark.covers( "llm.chat_completions.vertex.prompt_cache_5m.nonstream.works", @@ -145,4 +173,40 @@ class TestCacheControl: ), ) resources.defer(lambda: client.proxy.delete_model(model_id)) - _assert_cache_read_on_second_call(client, resources.key(), model) + key = resources.key() + _assert_cache_read_on_second_call(model, lambda prefix: _cache_chat(client, key, model, prefix)) + + @pytest.mark.covers( + "llm.chat_completions.anthropic.prompt_cache_5m.nonstream.works", + exercised_on=[], + ) + def test_anthropic_prompt_caching_reads_cache( + self, client: PassthroughClient, resources: ResourceManager + ) -> None: + model = f"e2e-anthropic-cache-{unique_marker()}" + model_id = client.proxy.create_model( + model, + LiteLLMParamsBody(model=ANTHROPIC_MODEL, api_key="os.environ/ANTHROPIC_API_KEY"), + ) + resources.defer(lambda: client.proxy.delete_model(model_id)) + key = resources.key() + _assert_cache_read_on_second_call(model, lambda prefix: _cache_chat(client, key, model, prefix)) + + @pytest.mark.covers( + "llm.chat_completions.openai.prompt_cache_5m.nonstream.works", + exercised_on=[], + ) + def test_openai_prompt_caching_reads_cache( + self, client: PassthroughClient, resources: ResourceManager + ) -> None: + model = f"e2e-openai-cache-{unique_marker()}" + model_id = client.proxy.create_model( + model, + LiteLLMParamsBody(model=OPENAI_MODEL, api_key="os.environ/OPENAI_API_KEY"), + ) + resources.defer(lambda: client.proxy.delete_model(model_id)) + key = resources.key() + cache_key = f"e2e-openai-cache-{unique_marker()}" + _assert_cache_read_on_second_call( + model, lambda prefix: _plain_cache_chat(client, key, model, prefix, cache_key) + ) diff --git a/tests/e2e/llm_translation/test_embeddings_endpoint_e2e.py b/tests/e2e/llm_translation/test_embeddings_endpoint_e2e.py index f951eb328f5..5520ca0cee5 100644 --- a/tests/e2e/llm_translation/test_embeddings_endpoint_e2e.py +++ b/tests/e2e/llm_translation/test_embeddings_endpoint_e2e.py @@ -1,4 +1,4 @@ -"""Live e2e: POST /embeddings returns a real vector across OpenAI, Bedrock, Vertex. +"""Live e2e: POST /embeddings returns a real vector across OpenAI, Bedrock, Vertex, Cohere. Each test registers the deployment it needs at runtime (deleted on teardown) and asserts a non-empty, non-zero vector came back. The LIT-3167 guard in @@ -86,6 +86,26 @@ class TestEmbeddingsEndpoint: f"embedding vector is all zeros: {result.body[:300]}" ) + @pytest.mark.covers("llm.embeddings.cohere.basic.nonstream.works") + def test_cohere_embeddings_returns_vector( + self, endpoints_client: EndpointsClient, resources: ResourceManager + ) -> None: + model = f"e2e-embeddings-cohere-{unique_marker()}" + model_id = endpoints_client.create_model( + model, + LiteLLMParamsBody(model="cohere/embed-v4.0", api_key="os.environ/COHERE_API_KEY"), + ) + resources.defer(lambda: endpoints_client.delete_model(model_id)) + key = resources.key() + + result = endpoints_client.embeddings(key, model, "Say this is a test!") + require_successful_call(result) + parsed = EmbeddingsResult.model_validate_json(result.body) + assert parsed.first_vector, f"/embeddings returned no vector: {result.body[:300]}" + assert any(component != 0.0 for component in parsed.first_vector), ( + f"embedding vector is all zeros: {result.body[:300]}" + ) + @pytest.mark.covers("llm.embeddings.vertex.basic.nonstream.works") def test_vertex_embeddings_returns_vector( self, endpoints_client: EndpointsClient, resources: ResourceManager diff --git a/tests/e2e/llm_translation/test_passthrough_e2e.py b/tests/e2e/llm_translation/test_passthrough_e2e.py index 50ea8f4b4df..447fe7d30d9 100644 --- a/tests/e2e/llm_translation/test_passthrough_e2e.py +++ b/tests/e2e/llm_translation/test_passthrough_e2e.py @@ -17,7 +17,7 @@ import pytest from e2e_config import CHEAP_OPENAI_MODEL, unique_marker from e2e_http import require_successful_call, unwrap from lifecycle import ResourceManager -from models import KeyGenerateBody, SpendLogRow +from models import ChatResponse, KeyGenerateBody, SpendLogRow from passthrough_client import ( AnthropicTool, GeminiFunctionDeclaration, @@ -344,6 +344,44 @@ class TestOpenAIPassthroughSpend: ) +class TestOpenAIProviderPrefixChat: + """OpenAI-format chat through the raw `/openai/{endpoint}` passthrough (LIT-4752). + + The body goes to OpenAI untranslated with the proxy's own OPENAI_API_KEY swapped + in, so the customer gets OpenAI's real completion back, and the gateway must + still write a costed pass_through_endpoint row for it. + """ + + @pytest.mark.covers("llm.chat_completions.openai.passthrough.nonstream.cost_logged") + def test_openai_prefix_chat_returns_completion_and_logs_its_cost( + self, client: PassthroughClient, scoped_key: str + ) -> None: + result = client.openai_chat(scoped_key, CHEAP_OPENAI_MODEL, f"Say hi in one word. {unique_marker()}") + require_successful_call(result) + + completion = ChatResponse.model_validate_json(result.body) + assert completion.id, f"/openai/v1/chat/completions relayed no completion id: {result.body[:300]}" + content = ( + completion.choices[0].message.content + if completion.choices and completion.choices[0].message + else None + ) + assert content and content.strip(), ( + f"/openai/v1/chat/completions relayed an empty completion: {result.body[:300]}" + ) + assert completion.usage is not None, f"the completion carried no usage to price from: {completion}" + + row = _fetch_cost_breakdown(client, completion.id) + assert row.prompt_tokens == completion.usage.prompt_tokens, ( + f"logged {row.prompt_tokens} prompt tokens, the completion the customer read " + f"reported {completion.usage.prompt_tokens}" + ) + assert row.completion_tokens == completion.usage.completion_tokens, ( + f"logged {row.completion_tokens} completion tokens, the completion the customer read " + f"reported {completion.usage.completion_tokens}" + ) + + class TestOpenAIPassthroughWebsocket: """The OpenAI passthrough prefixes must answer a websocket upgrade, not only a POST. diff --git a/tests/e2e/management/management_client.py b/tests/e2e/management/management_client.py index 387280c8023..2b897f5f07f 100644 --- a/tests/e2e/management/management_client.py +++ b/tests/e2e/management/management_client.py @@ -40,6 +40,8 @@ from models import ( KeyListParams, KeyListResponse, KeyRegenerateBody, + KeyResetSpendBody, + KeyResetSpendResponse, KeyUpdateBody, ModelDeleteBody, OrgDeleteBody, @@ -191,16 +193,26 @@ class ManagementClient: response_type=NoBody, ) ) - def regenerate_key(self, key: str) -> str: + def regenerate_key(self, key: str, *, grace_period: str | None = None) -> str: return unwrap( self.proxy.transport.post( "/key/regenerate", headers=self.proxy.transport.master, - json=KeyRegenerateBody(key=key), + json=KeyRegenerateBody(key=key, grace_period=grace_period), response_type=KeyGenerateResponse, ) ).key + def reset_key_spend(self, key: str, reset_to: float) -> KeyResetSpendResponse: + return unwrap( + self.proxy.transport.post( + f"/key/{key}/reset_spend", + headers=self.proxy.transport.master, + json=KeyResetSpendBody(reset_to=reset_to), + response_type=KeyResetSpendResponse, + ) + ) + def key_list(self, key_alias: str, *, caller_key: str | None = None) -> Result[KeyListResponse]: """GET /key/list, the Virtual Keys page's own inventory call. `caller_key` is who is asking: the master key by default, or a virtual key.""" diff --git a/tests/e2e/management/test_key_management_e2e.py b/tests/e2e/management/test_key_management_e2e.py index 711175abb0d..8b7d5f0eb6f 100644 --- a/tests/e2e/management/test_key_management_e2e.py +++ b/tests/e2e/management/test_key_management_e2e.py @@ -18,7 +18,7 @@ from typing import Literal import pytest from e2e_config import unique_marker -from e2e_http import NoBody, unwrap +from e2e_http import NoBody, StreamingResponse, unwrap from lifecycle import ResourceManager from management_client import ManagementClient from models import KeyDeleteBody, KeyGenerateBody, KeyUpdateBody @@ -26,6 +26,9 @@ from pydantic import BaseModel pytestmark = pytest.mark.e2e +TINY_BUDGET = 3e-6 +SPEND_MODEL = "claude-haiku-4-5" + class KeyToggleBlockBody(BaseModel): key: str @@ -82,6 +85,30 @@ def _generate_key(client: ManagementClient, resources: ResourceManager, body: Ke return key +def _is_budget_block(outcome: StreamingResponse) -> bool: + return not outcome.ok and "budget_exceeded" in outcome.body + + +def _spend_until_budget_blocks(client: ManagementClient, key: str) -> None: + for _ in range(40): + outcome = client.chat_status(key, SPEND_MODEL, f"spend {unique_marker()}") + if _is_budget_block(outcome): + assert outcome.status_code == 429, ( + f"budget refusal must be 429, got {outcome.status_code}: {outcome.body[:200]}" + ) + return + assert outcome.ok, f"paid call failed before the budget tripped ({outcome.status_code}): {outcome.body[:300]}" + time.sleep(2) + pytest.fail(f"max_budget={TINY_BUDGET} never blocked a call on the key") + + +def _settled_spend(client: ManagementClient, key: str) -> float | None: + first = client.proxy.key_info(key).spend or 0.0 + time.sleep(client.proxy.poll_interval) + second = client.proxy.key_info(key).spend or 0.0 + return second if first > 0 and first == second else None + + def _block(client: ManagementClient, key: str) -> None: _ = unwrap( client.proxy.transport.post( @@ -197,6 +224,32 @@ class TestKeyManagementRoutes: "/key/info never reported max_budget 42.0 after /key/bulk_update before the deadline", ) + @pytest.mark.covers("other.key_mgmt.spend_reset.resets_to_value") + def test_reset_spend_zeroes_recorded_spend_and_lifts_the_budget_block( + self, client: ManagementClient, resources: ResourceManager + ) -> None: + key = _generate_key(client, resources, KeyGenerateBody(models=[SPEND_MODEL], max_budget=TINY_BUDGET)) + _spend_until_budget_blocks(client, key) + recorded = _poll( + client, lambda: _settled_spend(client, key), "key spend never landed in /key/info before the deadline" + ) + + reset = client.reset_key_spend(key, reset_to=0.0) + assert reset.previous_spend == recorded, ( + f"reset_spend reported previous_spend {reset.previous_spend}, /key/info had recorded {recorded}" + ) + assert reset.spend == 0.0, f"reset_spend to 0 reported spend {reset.spend}" + assert client.proxy.key_info(key).spend == 0.0, "/key/info still reports spend after the reset to 0" + + def call_allowed_again() -> bool | None: + outcome = client.chat_status(key, SPEND_MODEL, f"after reset {unique_marker()}") + if _is_budget_block(outcome): + return None + assert outcome.ok, f"post-reset call failed ({outcome.status_code}): {outcome.body[:300]}" + return True + + _ = _poll(client, call_allowed_again, "the key stayed budget-blocked after its spend was reset to 0") + @pytest.mark.covers("mgmt.key.generate.admin_only") def test_generate_forbidden_for_non_admin_key( self, client: ManagementClient, resources: ResourceManager diff --git a/tests/e2e/management/test_management_e2e.py b/tests/e2e/management/test_management_e2e.py index a56eb853823..476165b715d 100644 --- a/tests/e2e/management/test_management_e2e.py +++ b/tests/e2e/management/test_management_e2e.py @@ -12,6 +12,7 @@ from __future__ import annotations import math import time from collections.abc import Callable +from typing import Final import pytest @@ -42,6 +43,10 @@ from models import ( pytestmark = pytest.mark.e2e +REGENERATE_GRACE_PERIOD = "15s" +REGENERATE_GRACE_SECONDS = 15.0 + + def _poll[T](client: ManagementClient, attempt: Callable[[], T | None], failure: str) -> T: deadline = time.monotonic() + client.proxy.poll_timeout while time.monotonic() < deadline: @@ -365,6 +370,36 @@ class TestKeyRegeneration: client, old_rejected, "old key was still accepted after regeneration (never rejected 401) at the deadline" ) + @pytest.mark.covers("other.key_mgmt.regenerate.grace_period_honored") + def test_regenerate_with_grace_period_keeps_old_key_until_revoked( + self, client: ManagementClient, resources: ResourceManager + ) -> None: + old_key = _generate_key(client, resources, KeyGenerateBody(models=["gpt-5.5"])) + + new_key = client.regenerate_key(old_key, grace_period=REGENERATE_GRACE_PERIOD) + resources.defer(lambda: client.proxy.delete_key(new_key)) + revoke_at: Final = time.monotonic() + REGENERATE_GRACE_SECONDS + assert new_key != old_key, "regenerate returned the same key string, so no rotation happened" + + def old_accepted() -> bool | None: + outcome = client.chat_status(old_key, "gpt-5.5", f"say hi {unique_marker()}") + return True if outcome.ok else None + + _ = _poll(client, old_accepted, "old key was rejected 401 inside its grace period at the deadline") + assert time.monotonic() < revoke_at, ( + f"old key was only accepted after its {REGENERATE_GRACE_PERIOD} grace period had elapsed" + ) + + def old_rejected() -> bool | None: + outcome = client.chat_status(old_key, "gpt-5.5", f"say hi {unique_marker()}") + return True if outcome.status_code == 401 else None + + _ = _poll( + client, + old_rejected, + f"old key was still accepted past its {REGENERATE_GRACE_PERIOD} grace period (never 401) at the deadline", + ) + class TestTeamRoutes: @pytest.mark.covers("mgmt.team.new.persists") diff --git a/tests/e2e/models.py b/tests/e2e/models.py index b5229744d6f..5de49ead3ed 100644 --- a/tests/e2e/models.py +++ b/tests/e2e/models.py @@ -83,6 +83,16 @@ class KeyGenerateResponse(BaseModel): class KeyRegenerateBody(BaseModel): key: str + grace_period: str | None = None + + +class KeyResetSpendBody(BaseModel): + reset_to: float + + +class KeyResetSpendResponse(BaseModel): + spend: float + previous_spend: float class KeyDeleteBody(BaseModel): diff --git a/tests/e2e/ui/tests/tables/tableScrolling.spec.ts b/tests/e2e/ui/tests/tables/tableScrolling.spec.ts index 5c7438cfd11..0537ba193be 100644 --- a/tests/e2e/ui/tests/tables/tableScrolling.spec.ts +++ b/tests/e2e/ui/tests/tables/tableScrolling.spec.ts @@ -184,6 +184,7 @@ test.describe("Admin tables scroll inside the page", () => { ); try { await navigateToPage(page, Page.TagManagement); + await setRowsPerPage(page, "50"); await expectRowsAtLeast(page, SEED_ROWS); expect(await rowsPaintingPastAnAncestor(page)).toEqual([]); } finally { @@ -207,6 +208,7 @@ test.describe("Admin tables scroll inside the page", () => { ); try { await navigateToPage(page, Page.ModelHubTable); + await setRowsPerPage(page, "50"); await expectRowsAtLeast(page, SEED_ROWS); expect(await rowsPaintingPastAnAncestor(page)).toEqual([]); } finally { diff --git a/tests/router_unit_tests/test_router_index_management.py b/tests/router_unit_tests/test_router_index_management.py index 35d295d581a..291badd91b4 100644 --- a/tests/router_unit_tests/test_router_index_management.py +++ b/tests/router_unit_tests/test_router_index_management.py @@ -241,7 +241,7 @@ class TestRouterIndexManagement: "_get_deployment_by_litellm_model": "lookup by litellm_params.model, which is not indexed", "_finalize_adaptive_router_if_configured": 'init-time prefix scan for "auto_router/adaptive_router"; no index for prefix match', "config_deployments": "filters the whole list on model_info.db_model; admin path only (model add/upsert)", - "heuristic_v2_router_limit_violation": "counts heuristic_v2 routers across the whole list; admin path only (auto-router init/upsert)", + "auto_router_capability_violation": "counts gated auto-routers across the whole list; admin path only (auto-router init/upsert)", } # Get path to router.py diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailsMonitorView.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailsMonitorView.test.tsx index 9f27daab6b3..4e0590df72d 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailsMonitorView.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailsMonitorView.test.tsx @@ -1,36 +1,61 @@ -import { render, screen, waitFor } from "@testing-library/react"; -import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; -import { describe, expect, it, vi } from "vitest"; +import { type UrlUpdateEvent } from "nuqs/adapters/testing"; +import { beforeEach, describe, expect, it, vi } from "vitest"; +import userEvent from "@testing-library/user-event"; import GuardrailsMonitorView from "./GuardrailsMonitorView"; import * as networking from "@/components/networking"; +import { renderWithProviders, screen, testQueryClient, waitFor } from "@/../tests/test-utils"; vi.mock("@/components/networking", () => ({ getGuardrailsUsageOverview: vi.fn(), + getGuardrailsUsageDetail: vi.fn(), + getGuardrailsUsageLogs: vi.fn(), formatDate: vi.fn((d: Date) => d.toISOString().slice(0, 10)), })); -const mockGetGuardrailsUsageOverview = vi.mocked(networking.getGuardrailsUsageOverview); +vi.mock("@/components/GuardrailsMonitor/LogViewer", () => ({ + LogViewer: ({ guardrailName }: { guardrailName: string }) =>
{guardrailName}
, +})); -function wrapper({ children }: { children: React.ReactNode }) { - const queryClient = new QueryClient({ - defaultOptions: { - queries: { retry: false }, - }, - }); - return {children}; -} +const mockGetGuardrailsUsageOverview = vi.mocked(networking.getGuardrailsUsageOverview); +const mockGetGuardrailsUsageDetail = vi.mocked(networking.getGuardrailsUsageDetail); +const mockGetGuardrailsUsageLogs = vi.mocked(networking.getGuardrailsUsageLogs); + +const emptyOverview = { rows: [], chart: [], totalRequests: 0, totalBlocked: 0, passRate: 100 }; + +const piiRow = { + id: "gr-pii", + name: "PII Guard", + type: "pii", + provider: "LiteLLM", + requestsEvaluated: 10, + failRate: 10, + status: "healthy" as const, + trend: "stable" as const, +}; + +const piiDetail = { + guardrail_name: "PII Guard", + description: "", + status: "healthy", + provider: "LiteLLM", + type: "pii", + requestsEvaluated: 10, + failRate: 10, + avgScore: 0.5, + avgLatency: 20, +}; describe("GuardrailsMonitorView", () => { - it("should render overview and fetch guardrails usage when accessToken is provided", async () => { - mockGetGuardrailsUsageOverview.mockResolvedValue({ - rows: [], - chart: [], - totalRequests: 0, - totalBlocked: 0, - passRate: 100, - }); + beforeEach(() => { + testQueryClient.clear(); + vi.clearAllMocks(); + mockGetGuardrailsUsageOverview.mockResolvedValue(emptyOverview); + mockGetGuardrailsUsageDetail.mockResolvedValue(piiDetail); + mockGetGuardrailsUsageLogs.mockResolvedValue({ logs: [], total: 0 }); + }); - render(, { wrapper }); + it("should render overview and fetch guardrails usage when accessToken is provided", async () => { + renderWithProviders(); expect(await screen.findByRole("heading", { name: /Guardrails Monitor/i })).toBeInTheDocument(); await waitFor(() => { @@ -39,7 +64,54 @@ describe("GuardrailsMonitorView", () => { }); it("should render without crashing when accessToken is null", async () => { - render(, { wrapper }); + renderWithProviders(); expect(await screen.findByRole("heading", { name: /Guardrails Monitor/i })).toBeInTheDocument(); }); + + describe("guardrail detail deep link (?guardrail=)", () => { + it("should open the detail view directly from a ?guardrail= deep link", async () => { + renderWithProviders(, { searchParams: "?guardrail=gr-pii" }); + + expect(await screen.findByRole("heading", { name: "PII Guard" })).toBeInTheDocument(); + expect(mockGetGuardrailsUsageDetail).toHaveBeenCalledWith( + "test-token", + "gr-pii", + expect.any(String), + expect.any(String), + ); + expect(screen.queryByRole("heading", { name: /Guardrails Monitor/i })).not.toBeInTheDocument(); + }); + + it("should push ?guardrail= as a new history entry when a guardrail is selected", async () => { + const user = userEvent.setup(); + const onUrlUpdate = vi.fn<(event: UrlUpdateEvent) => void>(); + mockGetGuardrailsUsageOverview.mockResolvedValue({ ...emptyOverview, rows: [piiRow] }); + renderWithProviders(, { onUrlUpdate }); + + await user.click(await screen.findByRole("button", { name: "PII Guard" })); + + await waitFor(() => expect(onUrlUpdate).toHaveBeenCalled()); + const lastUpdate = onUrlUpdate.mock.calls.at(-1)![0]; + expect(lastUpdate.searchParams.get("guardrail")).toBe("gr-pii"); + expect(lastUpdate.options.history).toBe("push"); + expect(await screen.findByRole("heading", { name: "PII Guard" })).toBeInTheDocument(); + }); + + it("should clear ?guardrail= by replacing history when going back to the overview", async () => { + const user = userEvent.setup(); + const onUrlUpdate = vi.fn<(event: UrlUpdateEvent) => void>(); + renderWithProviders(, { + searchParams: "?guardrail=gr-pii", + onUrlUpdate, + }); + + await user.click(await screen.findByRole("button", { name: /back to overview/i })); + + await waitFor(() => expect(onUrlUpdate).toHaveBeenCalled()); + const lastUpdate = onUrlUpdate.mock.calls.at(-1)![0]; + expect(lastUpdate.searchParams.has("guardrail")).toBe(false); + expect(lastUpdate.options.history).toBe("replace"); + expect(await screen.findByRole("heading", { name: /Guardrails Monitor/i })).toBeInTheDocument(); + }); + }); }); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailsMonitorView.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailsMonitorView.tsx index a9acf3e6377..f90a46e19e4 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailsMonitorView.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailsMonitorView.tsx @@ -1,12 +1,11 @@ import type { DateRangePickerValue } from "@/components/shared/date_picker_types"; +import { parseAsString, useQueryState } from "nuqs"; import React, { useCallback, useMemo, useState } from "react"; import { formatDate } from "@/components/networking"; import AdvancedDatePicker from "@/components/shared/advanced_date_picker"; import { GuardrailDetail } from "./GuardrailDetail"; import { GuardrailsOverview } from "./GuardrailsOverview"; -type View = { type: "overview" } | { type: "detail"; guardrailId: string }; - interface GuardrailsMonitorViewProps { accessToken?: string | null; } @@ -16,7 +15,10 @@ const defaultStart = new Date(); defaultStart.setDate(defaultStart.getDate() - 7); export default function GuardrailsMonitorView({ accessToken = null }: GuardrailsMonitorViewProps) { - const [view, setView] = useState({ type: "overview" }); + const [selectedGuardrailId, setSelectedGuardrailId] = useQueryState( + "guardrail", + parseAsString.withOptions({ history: "push" }), + ); const initialFrom = useMemo(() => new Date(defaultStart), []); const initialTo = useMemo(() => new Date(defaultEnd), []); @@ -34,11 +36,11 @@ export default function GuardrailsMonitorView({ accessToken = null }: Guardrails }, []); const handleSelectGuardrail = (id: string) => { - setView({ type: "detail", guardrailId: id }); + void setSelectedGuardrailId(id); }; const handleBack = () => { - setView({ type: "overview" }); + void setSelectedGuardrailId(null, { history: "replace" }); }; const dateRangeControl = ( @@ -47,7 +49,7 @@ export default function GuardrailsMonitorView({ accessToken = null }: Guardrails return (
- {view.type === "overview" ? ( + {!selectedGuardrailId ? (
{dateRangeControl}
({ getGuardrailsList: vi.fn(), @@ -15,16 +16,21 @@ vi.mock("./add_guardrail_form", () => ({ vi.mock("./guardrail_table", () => ({ __esModule: true, - default: ({ guardrailsList, onDeleteClick }: any) => ( + default: ({ guardrailsList, onDeleteClick, onGuardrailClick }: any) => (
Mock Guardrail Table
{guardrailsList.length > 0 && ( - + <> + + + )}
), @@ -32,7 +38,12 @@ vi.mock("./guardrail_table", () => ({ vi.mock("./guardrail_info", () => ({ __esModule: true, - default: () =>
Mock Guardrail Info View
, + default: ({ guardrailId, onClose }: { guardrailId: string; onClose: () => void }) => ( +
+
Mock Guardrail Info View {guardrailId}
+ +
+ ), })); vi.mock("./GuardrailTestPlayground", async () => { @@ -112,7 +123,7 @@ describe("GuardrailsPanel", () => { }); it("should render the component", async () => { - render(); + renderWithProviders(); expect(screen.getByText("Guardrails")).toBeInTheDocument(); // Activate the Guardrails tab so its content (including the Add button) is rendered fireEvent.click(screen.getByText("Guardrails")); @@ -120,7 +131,7 @@ describe("GuardrailsPanel", () => { }); it("should delete the clicked guardrail after confirming in the modal", async () => { - render(); + renderWithProviders(); fireEvent.click(screen.getByText("Guardrails")); fireEvent.click(await screen.findByTestId("delete-button")); @@ -139,14 +150,14 @@ describe("GuardrailsPanel", () => { }); it("should mount every tab panel up front so panel state survives tab switches", async () => { - render(); + renderWithProviders(); expect(await screen.findByLabelText("playground draft")).toBeInTheDocument(); expect(screen.getByText("Mock Team Guardrails Tab")).toBeInTheDocument(); }); it("should keep test playground state when switching tabs away and back", async () => { - render(); + renderWithProviders(); fireEvent.click(screen.getByText("Test Playground")); @@ -161,7 +172,7 @@ describe("GuardrailsPanel", () => { }); it("should not delete anything when the modal is cancelled", async () => { - render(); + renderWithProviders(); fireEvent.click(screen.getByText("Guardrails")); fireEvent.click(await screen.findByTestId("delete-button")); @@ -171,4 +182,42 @@ describe("GuardrailsPanel", () => { expect(mockDeleteGuardrailCall).not.toHaveBeenCalled(); }); + + describe("guardrail detail deep link (?guardrail=)", () => { + it("should open the guardrail info view directly from a ?guardrail= deep link", async () => { + renderWithProviders(, { searchParams: "?guardrail=test-guardrail-1" }); + + expect(await screen.findByTestId("guardrail-info-view")).toHaveTextContent("test-guardrail-1"); + expect(screen.queryByText("Mock Guardrail Table")).not.toBeInTheDocument(); + }); + + it("should push ?guardrail= as a new history entry when a guardrail row is clicked", async () => { + const onUrlUpdate = vi.fn<(event: UrlUpdateEvent) => void>(); + renderWithProviders(, { onUrlUpdate }); + + fireEvent.click(await screen.findByTestId("open-button")); + + await waitFor(() => expect(onUrlUpdate).toHaveBeenCalled()); + const lastUpdate = onUrlUpdate.mock.calls.at(-1)![0]; + expect(lastUpdate.searchParams.get("guardrail")).toBe("test-guardrail-1"); + expect(lastUpdate.options.history).toBe("push"); + expect(await screen.findByTestId("guardrail-info-view")).toHaveTextContent("test-guardrail-1"); + }); + + it("should clear ?guardrail= by replacing history when the info view is closed", async () => { + const onUrlUpdate = vi.fn<(event: UrlUpdateEvent) => void>(); + renderWithProviders(, { + searchParams: "?guardrail=test-guardrail-1", + onUrlUpdate, + }); + + fireEvent.click(await screen.findByRole("button", { name: "Close Guardrail Info" })); + + await waitFor(() => expect(onUrlUpdate).toHaveBeenCalled()); + const lastUpdate = onUrlUpdate.mock.calls.at(-1)![0]; + expect(lastUpdate.searchParams.has("guardrail")).toBe(false); + expect(lastUpdate.options.history).toBe("replace"); + expect(await screen.findByText("Mock Guardrail Table")).toBeInTheDocument(); + }); + }); }); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/GuardrailsPanel.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/GuardrailsPanel.tsx index 7e59abf8e3d..901e39004f3 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/GuardrailsPanel.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/GuardrailsPanel.tsx @@ -1,3 +1,4 @@ +import { parseAsString, useQueryState } from "nuqs"; import React, { useState, useEffect } from "react"; import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs"; import { ChevronDown, Code, Plus } from "lucide-react"; @@ -40,7 +41,10 @@ const GuardrailsPanel: React.FC = ({ accessToken, userRole const [isDeleting, setIsDeleting] = useState(false); const [guardrailToDelete, setGuardrailToDelete] = useState(null); const [isDeleteModalOpen, setIsDeleteModalOpen] = useState(false); - const [selectedGuardrailId, setSelectedGuardrailId] = useState(null); + const [selectedGuardrailId, setSelectedGuardrailId] = useQueryState( + "guardrail", + parseAsString.withOptions({ history: "push" }), + ); const isAdmin = userRole ? isAdminRole(userRole) : false; const fetchGuardrails = async () => { @@ -63,16 +67,20 @@ const GuardrailsPanel: React.FC = ({ accessToken, userRole fetchGuardrails(); }, [accessToken]); + const closeGuardrailDetail = () => { + void setSelectedGuardrailId(null, { history: "replace" }); + }; + const handleAddGuardrail = () => { if (selectedGuardrailId) { - setSelectedGuardrailId(null); + closeGuardrailDetail(); } setIsAddModalVisible(true); }; const handleAddCustomCodeGuardrail = () => { if (selectedGuardrailId) { - setSelectedGuardrailId(null); + closeGuardrailDetail(); } setIsCustomCodeModalVisible(true); }; @@ -175,7 +183,7 @@ const GuardrailsPanel: React.FC = ({ accessToken, userRole {selectedGuardrailId ? ( setSelectedGuardrailId(null)} + onClose={closeGuardrailDetail} accessToken={accessToken} isAdmin={isAdmin} /> @@ -184,7 +192,7 @@ const GuardrailsPanel: React.FC = ({ accessToken, userRole guardrailsList={guardrailsList} isLoading={isLoading} onDeleteClick={handleDeleteClick} - onGuardrailClick={(id) => setSelectedGuardrailId(id)} + onGuardrailClick={(id) => void setSelectedGuardrailId(id)} /> )} diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info.test.tsx index fcffc2122e7..836921104ff 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info.test.tsx @@ -338,3 +338,27 @@ describe("Guardrail Info", () => { expect(screen.getByText("Guardrail Settings")).toBeInTheDocument(); }); }); + +describe("Guardrail Info when the guardrail cannot be loaded", () => { + afterEach(() => { + vi.clearAllMocks(); + }); + + it("should keep Back to Guardrails reachable so a stale ?guardrail= link is not a dead end", async () => { + vi.mocked(networking.getGuardrailInfo).mockRejectedValue(new Error("Guardrail stale-id not found")); + vi.mocked(networking.getGuardrailUISettings).mockResolvedValue({ + supported_entities: [], + supported_actions: [], + pii_entity_categories: [], + supported_modes: [], + }); + vi.mocked(networking.getGuardrailProviderSpecificParams).mockResolvedValue({}); + const onClose = vi.fn(); + + render(); + + expect(await screen.findByText("Guardrail not found")).toBeInTheDocument(); + fireEvent.click(screen.getByRole("button", { name: /back to guardrails/i })); + expect(onClose).toHaveBeenCalledTimes(1); + }); +}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info.tsx index aaa656d15bb..c9162d99934 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info.tsx @@ -481,16 +481,18 @@ const GuardrailInfoView: React.FC = ({ guardrailId, onClose, return
Loading...
; } + const backButton = ( + + ); + if (!guardrailData) { - return
Guardrail not found
; + return
{backButton}Guardrail not found
; } - // Format date helper function - const formatDate = (dateString?: string) => { - if (!dateString) return "-"; - const date = new Date(dateString); - return date.toLocaleString(); - }; + const formatDate = (dateString?: string) => (dateString ? new Date(dateString).toLocaleString() : "-"); // Format the provider display name and logo const { logo, displayName } = getGuardrailLogoAndName(guardrailData.litellm_params?.guardrail || ""); @@ -510,10 +512,7 @@ const GuardrailInfoView: React.FC = ({ guardrailId, onClose, return (
- + {backButton}

{guardrailData.guardrail_name || "Unnamed Guardrail"}

{guardrailData.guardrail_id}