mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-15 09:12:18 +00:00
Merge remote-tracking branch 'origin/litellm_internal_staging' into litellm_semantic_search_over_skills
This commit is contained in:
yassin
commit
9ae68a6d18
33 changed files with 1026 additions and 192 deletions
31
.github/workflows/test-terraform-modules.yml
vendored
31
.github/workflows/test-terraform-modules.yml
vendored
|
|
@ -4,6 +4,7 @@ on:
|
|||
push:
|
||||
paths:
|
||||
- "terraform/litellm/aws/**"
|
||||
- "terraform/litellm/gcp/**"
|
||||
- ".github/workflows/test-terraform-modules.yml"
|
||||
pull_request:
|
||||
branches:
|
||||
|
|
@ -13,6 +14,7 @@ on:
|
|||
- "litellm_**"
|
||||
paths:
|
||||
- "terraform/litellm/aws/**"
|
||||
- "terraform/litellm/gcp/**"
|
||||
- ".github/workflows/test-terraform-modules.yml"
|
||||
|
||||
permissions:
|
||||
|
|
@ -52,3 +54,32 @@ jobs:
|
|||
# Plan-only, mock_provider-backed: no AWS credentials, no API calls.
|
||||
- name: test
|
||||
run: terraform test
|
||||
|
||||
gcp-module:
|
||||
name: fmt, validate, test (gcp)
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 15
|
||||
defaults:
|
||||
run:
|
||||
working-directory: terraform/litellm/gcp
|
||||
steps:
|
||||
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- uses: hashicorp/setup-terraform@b9cd54a3c349d3f38e8881555d616ced269862dd # v3.1.2
|
||||
with:
|
||||
terraform_version: 1.13.3
|
||||
terraform_wrapper: false
|
||||
|
||||
- name: fmt
|
||||
run: terraform fmt -recursive -check -diff
|
||||
|
||||
- name: init
|
||||
run: terraform init -backend=false -input=false
|
||||
|
||||
- name: validate
|
||||
run: terraform validate
|
||||
|
||||
- name: test
|
||||
run: terraform test
|
||||
|
|
|
|||
|
|
@ -392,6 +392,63 @@ with its own provider config (one `examples/default`-style root per project),
|
|||
or fork the module to add `configuration_aliases` and pass per-instance
|
||||
`providers = { ... }`.
|
||||
|
||||
## Dependencies only (run LiteLLM on GKE)
|
||||
|
||||
Set `create_runtime = false` to provision Cloud SQL, Memorystore, GCS,
|
||||
Secret Manager, and the runtime service account without Cloud Run or the
|
||||
load balancer. For a Shared VPC, set the full host-project network ID and
|
||||
skip PSA creation after the host project has configured it:
|
||||
|
||||
```hcl
|
||||
create_runtime = false
|
||||
network_id = "projects/<host>/global/networks/<vpc>"
|
||||
create_psa_connection = false
|
||||
```
|
||||
|
||||
The host project must already have Private Services Access configured on
|
||||
that network and the Service Networking API enabled; the module cannot set
|
||||
PSA up from a service project. GKE nodes must sit on the same Shared VPC so
|
||||
the Cloud SQL and Memorystore private IPs are routable from the pods. Run
|
||||
the root with its provider pointed at the project that should own the
|
||||
dependencies. `create_runtime = true` with `network_id` set is also allowed,
|
||||
but the Serverless VPC Access connector has to live in the same project as
|
||||
the network, so that combination only works when the VPC is in the
|
||||
deployment project
|
||||
|
||||
Map the outputs into the Helm values as follows:
|
||||
|
||||
```yaml
|
||||
database:
|
||||
writer:
|
||||
host: <cloudsql_writer_ip>
|
||||
dbname: <db_name>
|
||||
passwordSecret:
|
||||
name: <kubernetes-secret-with-db-credentials>
|
||||
reader:
|
||||
host: <cloudsql_reader_ip>
|
||||
dbname: <db_name>
|
||||
passwordSecret:
|
||||
name: <kubernetes-secret-with-db-credentials>
|
||||
redis:
|
||||
host: <redis_host>
|
||||
port: <redis_port>
|
||||
masterKey:
|
||||
secretName: <kubernetes-secret-with-master-key>
|
||||
```
|
||||
|
||||
Create the database Secret with keys `username` (the `db_username` output)
|
||||
and `password` (read it with `gcloud secrets versions access latest
|
||||
--secret=<db_password_secret_id>`), and the master key Secret from
|
||||
`master_key_secret_id` the same way. Memorystore only accepts TLS by
|
||||
default, so store the `redis_server_ca_pem` output in a third Secret,
|
||||
mount it into the gateway and backend pods via `volumes` / `volumeMounts`,
|
||||
and add `REDIS_SSL=true` and `REDIS_SSL_CA_CERTS=<mount path>` to each
|
||||
component's `extraEnv`. Setting `redis_transit_encryption = false` removes
|
||||
the CA plumbing at the cost of plaintext Redis traffic inside the VPC
|
||||
|
||||
The chart's pre-install/pre-upgrade migration hook runs the Prisma
|
||||
migration, so nothing replaces the Cloud Run migrations Job in this mode
|
||||
|
||||
## Storage and database retention
|
||||
|
||||
Two opt-in tripwires guard against accidental data loss on
|
||||
|
|
@ -409,14 +466,15 @@ Flip `cloudsql_deletion_protection` to `false` or `gcs_force_destroy` to
|
|||
|
||||
## Redis encryption
|
||||
|
||||
Memorystore runs with `transit_encryption_mode = "SERVER_AUTHENTICATION"`,
|
||||
so the proxy connects via `rediss://`. The instance's self-signed CA cert
|
||||
(`server_ca_certs[0].cert`) is shipped to gateway + backend as
|
||||
`REDIS_CA_PEM_B64`; their entrypoint shell decodes it to `/tmp/redis-ca.pem`
|
||||
before uvicorn starts and points `REDIS_SSL_CA_CERTS` at that path. No
|
||||
extra config needed — but if you ever swap Memorystore for an external
|
||||
Redis, override `REDIS_HOST`/`REDIS_PORT` and either drop these env vars
|
||||
or point them at your own CA.
|
||||
By default, Memorystore runs with
|
||||
`transit_encryption_mode = "SERVER_AUTHENTICATION"`, so Cloud Run connects
|
||||
via `rediss://`. The instance's self-signed CA cert
|
||||
(`server_ca_certs[0].cert`) is shipped to gateway and backend as
|
||||
`REDIS_CA_PEM_B64`; their entrypoint shell decodes it to
|
||||
`/tmp/redis-ca.pem` before uvicorn starts and points `REDIS_SSL_CA_CERTS` at
|
||||
that path. Set `redis_transit_encryption = false` to use plaintext Redis.
|
||||
For GKE, use `redis_server_ca_pem` as described in the dependencies-only
|
||||
section, or accept the security tradeoff of disabling transit encryption
|
||||
|
||||
## Files
|
||||
|
||||
|
|
@ -434,4 +492,5 @@ or point them at your own CA.
|
|||
| `iam.tf` | Runtime SA + Cloud SQL client + Secret Manager accessor |
|
||||
| `cloudrun.tf` | 3 Cloud Run services + Cloud Run Job for migrations |
|
||||
| `load_balancer.tf`| External HTTPS LB, serverless NEGs, URL map for path routing |
|
||||
| `outputs.tf` | LB IP, service URLs, secret IDs, migration `execute` command |
|
||||
| `outputs.tf` | LB IP, service URLs, dependency endpoints, secret IDs, migration command |
|
||||
| `tests/` | Plan-only mock-provider coverage for deployment modes and Redis encryption |
|
||||
|
|
|
|||
|
|
@ -15,15 +15,17 @@
|
|||
# enough to invoke Cloud Run admin APIs (`gcloud auth login`).
|
||||
|
||||
resource "terraform_data" "migration" {
|
||||
count = var.create_runtime ? 1 : 0
|
||||
|
||||
triggers_replace = {
|
||||
job_id = google_cloud_run_v2_job.migrations.id
|
||||
job_id = google_cloud_run_v2_job.migrations[0].id
|
||||
job_image = local.migrations_image
|
||||
}
|
||||
|
||||
provisioner "local-exec" {
|
||||
interpreter = ["bash", "-c"]
|
||||
environment = {
|
||||
JOB = google_cloud_run_v2_job.migrations.name
|
||||
JOB = google_cloud_run_v2_job.migrations[0].name
|
||||
REGION = var.region
|
||||
PROJECT = var.project_id
|
||||
}
|
||||
|
|
|
|||
|
|
@ -6,25 +6,28 @@ locals {
|
|||
# Memorystore exposes a self-signed CA cert per instance; we ship it as
|
||||
# a base64 env var and decode it to a file at container startup so the
|
||||
# rediss:// connection can validate. Public cert, not sensitive.
|
||||
redis_ca_pem_b64 = base64encode(google_redis_instance.this.server_ca_certs[0].cert)
|
||||
redis_ca_pem_b64 = var.redis_transit_encryption ? base64encode(google_redis_instance.this.server_ca_certs[0].cert) : ""
|
||||
|
||||
shared_env_kv = [
|
||||
{ name = "DATABASE_HOST", value = google_sql_database_instance.writer.private_ip_address },
|
||||
{ name = "DATABASE_PORT", value = "5432" },
|
||||
{ name = "DATABASE_USER", value = var.db_username },
|
||||
{ name = "DATABASE_NAME", value = var.db_name },
|
||||
{ name = "DATABASE_HOST_READ_REPLICA", value = google_sql_database_instance.reader.private_ip_address },
|
||||
{ name = "DATABASE_PORT_READ_REPLICA", value = "5432" },
|
||||
{ name = "REDIS_HOST", value = google_redis_instance.this.host },
|
||||
{ name = "REDIS_PORT", value = tostring(google_redis_instance.this.port) },
|
||||
# _redis.get_redis_url_from_environment honors REDIS_SSL to flip the
|
||||
# scheme to rediss://; REDIS_SSL_CA_CERTS is mapped via
|
||||
# _get_redis_env_kwarg_mapping → ssl_ca_certs on the redis-py client.
|
||||
{ name = "REDIS_SSL", value = "true" },
|
||||
{ name = "REDIS_SSL_CA_CERTS", value = "/tmp/redis-ca.pem" },
|
||||
{ name = "REDIS_CA_PEM_B64", value = local.redis_ca_pem_b64 },
|
||||
{ name = "GCS_BUCKET_NAME", value = google_storage_bucket.this.name },
|
||||
]
|
||||
shared_env_kv = concat(
|
||||
[
|
||||
{ name = "DATABASE_HOST", value = google_sql_database_instance.writer.private_ip_address },
|
||||
{ name = "DATABASE_PORT", value = "5432" },
|
||||
{ name = "DATABASE_USER", value = var.db_username },
|
||||
{ name = "DATABASE_NAME", value = var.db_name },
|
||||
{ name = "DATABASE_HOST_READ_REPLICA", value = google_sql_database_instance.reader.private_ip_address },
|
||||
{ name = "DATABASE_PORT_READ_REPLICA", value = "5432" },
|
||||
{ name = "REDIS_HOST", value = google_redis_instance.this.host },
|
||||
{ name = "REDIS_PORT", value = tostring(google_redis_instance.this.port) },
|
||||
],
|
||||
var.redis_transit_encryption ? [
|
||||
{ name = "REDIS_SSL", value = "true" },
|
||||
{ name = "REDIS_SSL_CA_CERTS", value = "/tmp/redis-ca.pem" },
|
||||
{ name = "REDIS_CA_PEM_B64", value = local.redis_ca_pem_b64 },
|
||||
] : [],
|
||||
[
|
||||
{ name = "GCS_BUCKET_NAME", value = google_storage_bucket.this.name },
|
||||
],
|
||||
)
|
||||
|
||||
# OTel v2 is opt-in and gated on otel_endpoint, matching the AWS stack —
|
||||
# nothing OTel-related is added to the container env until an endpoint is
|
||||
|
|
@ -126,9 +129,9 @@ locals {
|
|||
# Decode the Memorystore CA cert (passed as REDIS_CA_PEM_B64) to the
|
||||
# path REDIS_SSL_CA_CERTS points at, so the redis-py client can validate
|
||||
# the rediss:// handshake.
|
||||
redis_ca_fragment = [
|
||||
redis_ca_fragment = var.redis_transit_encryption ? [
|
||||
"python -c \"import os, base64, pathlib; pathlib.Path(os.environ['REDIS_SSL_CA_CERTS']).write_bytes(base64.b64decode(os.environ['REDIS_CA_PEM_B64']))\""
|
||||
]
|
||||
] : []
|
||||
|
||||
database_url_fragment = [
|
||||
"export DATABASE_URL=\"postgresql://$${DATABASE_USER}:$${DATABASE_PASSWORD}@$${DATABASE_HOST}:$${DATABASE_PORT}/$${DATABASE_NAME}\"",
|
||||
|
|
@ -171,29 +174,7 @@ locals {
|
|||
|
||||
# ---------- Gateway ----------
|
||||
resource "google_cloud_run_v2_service" "gateway" {
|
||||
# Metering needs a client certificate AND its key. Each secret is created only
|
||||
# when its own PEM is supplied, so an endpoint set with a missing key would
|
||||
# otherwise apply cleanly and leave the proxy logging "missing config" and
|
||||
# never exporting. ca_cert_pem stays optional: empty means fall back to the
|
||||
# system trust store.
|
||||
#
|
||||
# The guard lives here, on an unconditional resource, rather than on the cert
|
||||
# secret: that secret is count-gated on the cert itself, so it has zero
|
||||
# instances in exactly the case this must catch. Adding count or for_each to
|
||||
# this resource would silently stop the guard from evaluating.
|
||||
#
|
||||
# endpoint cert key -> result
|
||||
# "" any any -> metering off, no secrets created
|
||||
# set set set -> metering on
|
||||
# set any-missing -> plan fails here
|
||||
lifecycle {
|
||||
precondition {
|
||||
condition = var.billing_metrics_endpoint == "" || (
|
||||
var.billing_metrics_client_cert_pem != "" && var.billing_metrics_client_key_pem != ""
|
||||
)
|
||||
error_message = "billing_metrics_client_cert_pem and billing_metrics_client_key_pem are both required when billing_metrics_endpoint is set."
|
||||
}
|
||||
}
|
||||
count = var.create_runtime ? 1 : 0
|
||||
|
||||
name = "${local.name}-gateway"
|
||||
location = var.region
|
||||
|
|
@ -206,7 +187,7 @@ resource "google_cloud_run_v2_service" "gateway" {
|
|||
max_instance_request_concurrency = var.gateway_max_instance_request_concurrency
|
||||
|
||||
vpc_access {
|
||||
connector = google_vpc_access_connector.this.id
|
||||
connector = google_vpc_access_connector.this[0].id
|
||||
egress = "PRIVATE_RANGES_ONLY"
|
||||
}
|
||||
|
||||
|
|
@ -312,17 +293,7 @@ resource "google_cloud_run_v2_service" "gateway" {
|
|||
|
||||
# ---------- Backend ----------
|
||||
resource "google_cloud_run_v2_service" "backend" {
|
||||
# Same guard as the gateway: the backend meters too (it serves the named-server
|
||||
# MCP transport), and a targeted apply of just this resource must not slip a
|
||||
# billing endpoint through without the credentials to use it.
|
||||
lifecycle {
|
||||
precondition {
|
||||
condition = var.billing_metrics_endpoint == "" || (
|
||||
var.billing_metrics_client_cert_pem != "" && var.billing_metrics_client_key_pem != ""
|
||||
)
|
||||
error_message = "billing_metrics_client_cert_pem and billing_metrics_client_key_pem are both required when billing_metrics_endpoint is set."
|
||||
}
|
||||
}
|
||||
count = var.create_runtime ? 1 : 0
|
||||
|
||||
name = "${local.name}-backend"
|
||||
location = var.region
|
||||
|
|
@ -335,7 +306,7 @@ resource "google_cloud_run_v2_service" "backend" {
|
|||
max_instance_request_concurrency = var.backend_max_instance_request_concurrency
|
||||
|
||||
vpc_access {
|
||||
connector = google_vpc_access_connector.this.id
|
||||
connector = google_vpc_access_connector.this[0].id
|
||||
egress = "PRIVATE_RANGES_ONLY"
|
||||
}
|
||||
|
||||
|
|
@ -443,6 +414,8 @@ resource "google_cloud_run_v2_service" "backend" {
|
|||
# with zero IAM bindings, so a compromised UI container can't pivot to
|
||||
# Secret Manager / Cloud SQL via the metadata service.
|
||||
resource "google_cloud_run_v2_service" "ui" {
|
||||
count = var.create_runtime ? 1 : 0
|
||||
|
||||
name = "${local.name}-ui"
|
||||
location = var.region
|
||||
ingress = "INGRESS_TRAFFIC_INTERNAL_LOAD_BALANCER"
|
||||
|
|
@ -450,7 +423,7 @@ resource "google_cloud_run_v2_service" "ui" {
|
|||
deletion_protection = false
|
||||
|
||||
template {
|
||||
service_account = google_service_account.ui_runtime.email
|
||||
service_account = google_service_account.ui_runtime[0].email
|
||||
max_instance_request_concurrency = var.ui_max_instance_request_concurrency
|
||||
|
||||
scaling {
|
||||
|
|
@ -491,25 +464,31 @@ resource "google_cloud_run_v2_service" "ui" {
|
|||
# (LITELLM_MASTER_KEY); these IAM bindings just open up Cloud Run's invoker
|
||||
# gate so the LB request makes it to the container.
|
||||
resource "google_cloud_run_v2_service_iam_member" "gateway_allusers" {
|
||||
count = var.create_runtime ? 1 : 0
|
||||
|
||||
project = var.project_id
|
||||
location = google_cloud_run_v2_service.gateway.location
|
||||
name = google_cloud_run_v2_service.gateway.name
|
||||
location = google_cloud_run_v2_service.gateway[0].location
|
||||
name = google_cloud_run_v2_service.gateway[0].name
|
||||
role = "roles/run.invoker"
|
||||
member = "allUsers"
|
||||
}
|
||||
|
||||
resource "google_cloud_run_v2_service_iam_member" "backend_allusers" {
|
||||
count = var.create_runtime ? 1 : 0
|
||||
|
||||
project = var.project_id
|
||||
location = google_cloud_run_v2_service.backend.location
|
||||
name = google_cloud_run_v2_service.backend.name
|
||||
location = google_cloud_run_v2_service.backend[0].location
|
||||
name = google_cloud_run_v2_service.backend[0].name
|
||||
role = "roles/run.invoker"
|
||||
member = "allUsers"
|
||||
}
|
||||
|
||||
resource "google_cloud_run_v2_service_iam_member" "ui_allusers" {
|
||||
count = var.create_runtime ? 1 : 0
|
||||
|
||||
project = var.project_id
|
||||
location = google_cloud_run_v2_service.ui.location
|
||||
name = google_cloud_run_v2_service.ui.name
|
||||
location = google_cloud_run_v2_service.ui[0].location
|
||||
name = google_cloud_run_v2_service.ui[0].name
|
||||
role = "roles/run.invoker"
|
||||
member = "allUsers"
|
||||
}
|
||||
|
|
@ -519,6 +498,8 @@ resource "google_cloud_run_v2_service_iam_member" "ui_allusers" {
|
|||
# assembles DATABASE_URL from the DATABASE_* env vars and runs `prisma
|
||||
# migrate deploy`. No proxy_config, no master key, no shell wrapper.
|
||||
resource "google_cloud_run_v2_job" "migrations" {
|
||||
count = var.create_runtime ? 1 : 0
|
||||
|
||||
name = "${local.name}-migrations"
|
||||
location = var.region
|
||||
labels = local.labels
|
||||
|
|
@ -529,7 +510,7 @@ resource "google_cloud_run_v2_job" "migrations" {
|
|||
service_account = google_service_account.runtime.email
|
||||
|
||||
vpc_access {
|
||||
connector = google_vpc_access_connector.this.id
|
||||
connector = google_vpc_access_connector.this[0].id
|
||||
egress = "PRIVATE_RANGES_ONLY"
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -36,7 +36,7 @@ resource "google_sql_database_instance" "writer" {
|
|||
|
||||
ip_configuration {
|
||||
ipv4_enabled = false
|
||||
private_network = google_compute_network.this.id
|
||||
private_network = local.network_id
|
||||
}
|
||||
|
||||
insights_config {
|
||||
|
|
@ -55,6 +55,11 @@ resource "google_sql_database_instance" "writer" {
|
|||
# (full data loss). Set the initial size only; let Cloud SQL own it
|
||||
# thereafter.
|
||||
ignore_changes = [settings[0].disk_size]
|
||||
|
||||
precondition {
|
||||
condition = var.create_psa_connection || var.network_id != ""
|
||||
error_message = "create_psa_connection must be true unless network_id references an existing VPC with Private Services Access configured."
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -76,7 +81,7 @@ resource "google_sql_database_instance" "reader" {
|
|||
|
||||
ip_configuration {
|
||||
ipv4_enabled = false
|
||||
private_network = google_compute_network.this.id
|
||||
private_network = local.network_id
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -31,6 +31,11 @@ module "litellm" {
|
|||
tenant = var.tenant
|
||||
env = var.env
|
||||
|
||||
create_runtime = var.create_runtime
|
||||
network_id = var.network_id
|
||||
create_psa_connection = var.create_psa_connection
|
||||
redis_transit_encryption = var.redis_transit_encryption
|
||||
|
||||
litellm_master_key = var.litellm_master_key
|
||||
litellm_license = var.litellm_license
|
||||
ui_password = var.ui_password
|
||||
|
|
|
|||
|
|
@ -38,6 +38,31 @@ output "redis_endpoint" {
|
|||
value = module.litellm.redis_endpoint
|
||||
}
|
||||
|
||||
output "redis_host" {
|
||||
description = "Memorystore Redis host."
|
||||
value = module.litellm.redis_host
|
||||
}
|
||||
|
||||
output "redis_port" {
|
||||
description = "Memorystore Redis port."
|
||||
value = module.litellm.redis_port
|
||||
}
|
||||
|
||||
output "redis_server_ca_pem" {
|
||||
description = "Memorystore server CA PEM."
|
||||
value = module.litellm.redis_server_ca_pem
|
||||
}
|
||||
|
||||
output "db_username" {
|
||||
description = "Cloud SQL application username."
|
||||
value = module.litellm.db_username
|
||||
}
|
||||
|
||||
output "db_name" {
|
||||
description = "Cloud SQL database name."
|
||||
value = module.litellm.db_name
|
||||
}
|
||||
|
||||
output "gcs_bucket" {
|
||||
description = "GCS bucket name."
|
||||
value = module.litellm.gcs_bucket
|
||||
|
|
@ -53,6 +78,11 @@ output "db_password_secret_id" {
|
|||
value = module.litellm.db_password_secret_id
|
||||
}
|
||||
|
||||
output "runtime_service_account_email" {
|
||||
description = "Runtime service account email."
|
||||
value = module.litellm.runtime_service_account_email
|
||||
}
|
||||
|
||||
output "migration_run_command" {
|
||||
description = "Break-glass command to re-run the one-off migration job."
|
||||
value = module.litellm.migration_run_command
|
||||
|
|
|
|||
|
|
@ -8,6 +8,14 @@ region = "us-central1"
|
|||
tenant = "acme"
|
||||
env = "stage"
|
||||
|
||||
# Deployment mode. For dependencies only on a Shared VPC, set
|
||||
# create_runtime = false, network_id to the full host-project network ID, and
|
||||
# create_psa_connection = false after configuring PSA on that network.
|
||||
# create_runtime = true
|
||||
# network_id = ""
|
||||
# create_psa_connection = true
|
||||
# redis_transit_encryption = true
|
||||
|
||||
# Tenant-supplied secrets. Prefer TF_VAR_litellm_master_key /
|
||||
# TF_VAR_litellm_license / TF_VAR_ui_password env vars so the values don't
|
||||
# end up in a committed tfvars file. All three are optional — when
|
||||
|
|
|
|||
|
|
@ -26,6 +26,30 @@ variable "env" {
|
|||
type = string
|
||||
}
|
||||
|
||||
variable "create_runtime" {
|
||||
description = "Create Cloud Run and load balancer resources."
|
||||
type = bool
|
||||
default = true
|
||||
}
|
||||
|
||||
variable "network_id" {
|
||||
description = "Existing VPC network resource ID. Empty creates a VPC."
|
||||
type = string
|
||||
default = ""
|
||||
}
|
||||
|
||||
variable "create_psa_connection" {
|
||||
description = "Create Private Services Access resources."
|
||||
type = bool
|
||||
default = true
|
||||
}
|
||||
|
||||
variable "redis_transit_encryption" {
|
||||
description = "Enable Memorystore transit encryption."
|
||||
type = bool
|
||||
default = true
|
||||
}
|
||||
|
||||
# Sensitive — prefer TF_VAR_litellm_master_key / TF_VAR_litellm_license /
|
||||
# TF_VAR_ui_password so values stay out of any committed tfvars file.
|
||||
variable "litellm_master_key" {
|
||||
|
|
|
|||
|
|
@ -6,6 +6,15 @@
|
|||
resource "google_service_account" "runtime" {
|
||||
account_id = "${local.name}-runtime"
|
||||
display_name = "LiteLLM Cloud Run runtime"
|
||||
|
||||
lifecycle {
|
||||
precondition {
|
||||
condition = !var.create_runtime || var.billing_metrics_endpoint == "" || (
|
||||
var.billing_metrics_client_cert_pem != "" && var.billing_metrics_client_key_pem != ""
|
||||
)
|
||||
error_message = "billing_metrics_client_cert_pem and billing_metrics_client_key_pem are both required when billing_metrics_endpoint is set and create_runtime is true."
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# UI runtime SA — no role bindings. The UI is static nginx with no DB,
|
||||
|
|
@ -14,6 +23,8 @@ resource "google_service_account" "runtime" {
|
|||
# project's serverless service agent (not this SA), so it doesn't need
|
||||
# artifactregistry.reader either.
|
||||
resource "google_service_account" "ui_runtime" {
|
||||
count = var.create_runtime ? 1 : 0
|
||||
|
||||
account_id = "${local.name}-ui-runtime"
|
||||
display_name = "LiteLLM Cloud Run UI runtime (no data-plane access)"
|
||||
}
|
||||
|
|
|
|||
|
|
@ -14,77 +14,93 @@ locals {
|
|||
}
|
||||
|
||||
resource "google_compute_global_address" "lb" {
|
||||
count = var.create_runtime ? 1 : 0
|
||||
|
||||
name = "${local.name}-lb-ip"
|
||||
labels = local.labels
|
||||
}
|
||||
|
||||
# Serverless NEGs — one per Cloud Run service.
|
||||
resource "google_compute_region_network_endpoint_group" "gateway" {
|
||||
count = var.create_runtime ? 1 : 0
|
||||
|
||||
name = "${local.name}-gateway-neg"
|
||||
region = var.region
|
||||
network_endpoint_type = "SERVERLESS"
|
||||
|
||||
cloud_run {
|
||||
service = google_cloud_run_v2_service.gateway.name
|
||||
service = google_cloud_run_v2_service.gateway[0].name
|
||||
}
|
||||
}
|
||||
|
||||
resource "google_compute_region_network_endpoint_group" "backend" {
|
||||
count = var.create_runtime ? 1 : 0
|
||||
|
||||
name = "${local.name}-backend-neg"
|
||||
region = var.region
|
||||
network_endpoint_type = "SERVERLESS"
|
||||
|
||||
cloud_run {
|
||||
service = google_cloud_run_v2_service.backend.name
|
||||
service = google_cloud_run_v2_service.backend[0].name
|
||||
}
|
||||
}
|
||||
|
||||
resource "google_compute_region_network_endpoint_group" "ui" {
|
||||
count = var.create_runtime ? 1 : 0
|
||||
|
||||
name = "${local.name}-ui-neg"
|
||||
region = var.region
|
||||
network_endpoint_type = "SERVERLESS"
|
||||
|
||||
cloud_run {
|
||||
service = google_cloud_run_v2_service.ui.name
|
||||
service = google_cloud_run_v2_service.ui[0].name
|
||||
}
|
||||
}
|
||||
|
||||
# Backend services wrap each NEG.
|
||||
resource "google_compute_backend_service" "gateway" {
|
||||
count = var.create_runtime ? 1 : 0
|
||||
|
||||
name = "${local.name}-gateway-bs"
|
||||
protocol = "HTTP"
|
||||
load_balancing_scheme = "EXTERNAL_MANAGED"
|
||||
|
||||
backend {
|
||||
group = google_compute_region_network_endpoint_group.gateway.id
|
||||
group = google_compute_region_network_endpoint_group.gateway[0].id
|
||||
}
|
||||
}
|
||||
|
||||
resource "google_compute_backend_service" "backend" {
|
||||
count = var.create_runtime ? 1 : 0
|
||||
|
||||
name = "${local.name}-backend-bs"
|
||||
protocol = "HTTP"
|
||||
load_balancing_scheme = "EXTERNAL_MANAGED"
|
||||
|
||||
backend {
|
||||
group = google_compute_region_network_endpoint_group.backend.id
|
||||
group = google_compute_region_network_endpoint_group.backend[0].id
|
||||
}
|
||||
}
|
||||
|
||||
resource "google_compute_backend_service" "ui" {
|
||||
count = var.create_runtime ? 1 : 0
|
||||
|
||||
name = "${local.name}-ui-bs"
|
||||
protocol = "HTTP"
|
||||
load_balancing_scheme = "EXTERNAL_MANAGED"
|
||||
|
||||
backend {
|
||||
group = google_compute_region_network_endpoint_group.ui.id
|
||||
group = google_compute_region_network_endpoint_group.ui[0].id
|
||||
}
|
||||
}
|
||||
|
||||
# URL map. Default → backend (management API). Path matchers route the
|
||||
# gateway and UI prefixes elsewhere.
|
||||
resource "google_compute_url_map" "this" {
|
||||
count = var.create_runtime ? 1 : 0
|
||||
|
||||
name = local.name
|
||||
default_service = google_compute_backend_service.backend.id
|
||||
default_service = google_compute_backend_service.backend[0].id
|
||||
|
||||
host_rule {
|
||||
hosts = ["*"]
|
||||
|
|
@ -93,13 +109,13 @@ resource "google_compute_url_map" "this" {
|
|||
|
||||
path_matcher {
|
||||
name = "main"
|
||||
default_service = google_compute_backend_service.backend.id
|
||||
default_service = google_compute_backend_service.backend[0].id
|
||||
|
||||
# UI paths (catch them before any /v1/* gateway rules so /favicon.ico
|
||||
# and / take precedence).
|
||||
path_rule {
|
||||
paths = local.ui_path_prefixes
|
||||
service = google_compute_backend_service.ui.id
|
||||
service = google_compute_backend_service.ui[0].id
|
||||
}
|
||||
|
||||
# Gateway path prefixes. GCP URL maps cap a path_rule at 10 path globs,
|
||||
|
|
@ -108,7 +124,7 @@ resource "google_compute_url_map" "this" {
|
|||
for_each = { for idx, chunk in chunklist(local.gateway_path_prefixes, 10) : idx => chunk }
|
||||
content {
|
||||
paths = path_rule.value
|
||||
service = google_compute_backend_service.gateway.id
|
||||
service = google_compute_backend_service.gateway[0].id
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -118,7 +134,7 @@ resource "google_compute_url_map" "this" {
|
|||
# target proxy when TLS is enabled; otherwise the regular path-routing
|
||||
# URL map is attached to the HTTP proxy and everything stays plaintext.
|
||||
resource "google_compute_url_map" "https_redirect" {
|
||||
count = local.tls_enabled ? 1 : 0
|
||||
count = var.create_runtime && local.tls_enabled ? 1 : 0
|
||||
name = "${local.name}-redirect"
|
||||
|
||||
default_url_redirect {
|
||||
|
|
@ -129,8 +145,10 @@ resource "google_compute_url_map" "https_redirect" {
|
|||
}
|
||||
|
||||
resource "google_compute_target_http_proxy" "this" {
|
||||
count = var.create_runtime ? 1 : 0
|
||||
|
||||
name = "${local.name}-http"
|
||||
url_map = local.tls_enabled ? google_compute_url_map.https_redirect[0].id : google_compute_url_map.this.id
|
||||
url_map = local.tls_enabled ? google_compute_url_map.https_redirect[0].id : google_compute_url_map.this[0].id
|
||||
|
||||
# Default-deny on the HTTP-only path: TLS is the supported posture.
|
||||
# Operators must either supply DNS names or explicitly opt in.
|
||||
|
|
@ -143,12 +161,14 @@ resource "google_compute_target_http_proxy" "this" {
|
|||
}
|
||||
|
||||
resource "google_compute_global_forwarding_rule" "http" {
|
||||
count = var.create_runtime ? 1 : 0
|
||||
|
||||
name = "${local.name}-http"
|
||||
ip_protocol = "TCP"
|
||||
port_range = "80"
|
||||
load_balancing_scheme = "EXTERNAL_MANAGED"
|
||||
ip_address = google_compute_global_address.lb.address
|
||||
target = google_compute_target_http_proxy.this.id
|
||||
ip_address = google_compute_global_address.lb[0].address
|
||||
target = google_compute_target_http_proxy.this[0].id
|
||||
labels = local.labels
|
||||
}
|
||||
|
||||
|
|
@ -161,7 +181,7 @@ resource "google_compute_global_forwarding_rule" "http" {
|
|||
# transitions to ACTIVE.
|
||||
|
||||
resource "google_compute_managed_ssl_certificate" "this" {
|
||||
count = local.tls_enabled ? 1 : 0
|
||||
count = var.create_runtime && local.tls_enabled ? 1 : 0
|
||||
|
||||
# A managed cert's `domains` is immutable, so changing var.lb_domains
|
||||
# forces replacement, and the cert is referenced by the HTTPS target
|
||||
|
|
@ -181,19 +201,19 @@ resource "google_compute_managed_ssl_certificate" "this" {
|
|||
}
|
||||
|
||||
resource "google_compute_target_https_proxy" "this" {
|
||||
count = local.tls_enabled ? 1 : 0
|
||||
count = var.create_runtime && local.tls_enabled ? 1 : 0
|
||||
name = "${local.name}-https"
|
||||
url_map = google_compute_url_map.this.id
|
||||
url_map = google_compute_url_map.this[0].id
|
||||
ssl_certificates = [google_compute_managed_ssl_certificate.this[0].id]
|
||||
}
|
||||
|
||||
resource "google_compute_global_forwarding_rule" "https" {
|
||||
count = local.tls_enabled ? 1 : 0
|
||||
count = var.create_runtime && local.tls_enabled ? 1 : 0
|
||||
name = "${local.name}-https"
|
||||
ip_protocol = "TCP"
|
||||
port_range = "443"
|
||||
load_balancing_scheme = "EXTERNAL_MANAGED"
|
||||
ip_address = google_compute_global_address.lb.address
|
||||
ip_address = google_compute_global_address.lb[0].address
|
||||
target = google_compute_target_https_proxy.this[0].id
|
||||
labels = local.labels
|
||||
}
|
||||
|
|
|
|||
|
|
@ -21,6 +21,9 @@ locals {
|
|||
var.labels,
|
||||
)
|
||||
|
||||
create_network = var.network_id == ""
|
||||
network_id = local.create_network ? google_compute_network.this[0].id : var.network_id
|
||||
|
||||
gateway_path_prefixes = [
|
||||
"/v1/chat/*", "/chat/*",
|
||||
"/v1/completions*", "/completions*",
|
||||
|
|
@ -74,7 +77,7 @@ locals {
|
|||
"/ui/*",
|
||||
]
|
||||
|
||||
proxy_config_enabled = length(keys(var.proxy_config)) > 0
|
||||
proxy_config_enabled = var.create_runtime && length(keys(var.proxy_config)) > 0
|
||||
proxy_config_yaml = local.proxy_config_enabled ? yamlencode(var.proxy_config) : ""
|
||||
|
||||
proxy_config_mount_path = "/etc/litellm"
|
||||
|
|
|
|||
|
|
@ -1,13 +1,17 @@
|
|||
resource "google_compute_network" "this" {
|
||||
count = local.create_network ? 1 : 0
|
||||
|
||||
name = local.name
|
||||
auto_create_subnetworks = false
|
||||
routing_mode = "REGIONAL"
|
||||
}
|
||||
|
||||
resource "google_compute_subnetwork" "this" {
|
||||
count = local.create_network ? 1 : 0
|
||||
|
||||
name = "${local.name}-${var.region}"
|
||||
region = var.region
|
||||
network = google_compute_network.this.id
|
||||
network = google_compute_network.this[0].id
|
||||
ip_cidr_range = var.subnet_cidr
|
||||
private_ip_google_access = true
|
||||
}
|
||||
|
|
@ -16,17 +20,21 @@ resource "google_compute_subnetwork" "this" {
|
|||
# managed services peer with the VPC over the connection below using
|
||||
# addresses from this range.
|
||||
resource "google_compute_global_address" "psa" {
|
||||
count = var.create_psa_connection ? 1 : 0
|
||||
|
||||
name = "${local.name}-psa"
|
||||
purpose = "VPC_PEERING"
|
||||
address_type = "INTERNAL"
|
||||
prefix_length = 16
|
||||
network = google_compute_network.this.id
|
||||
network = local.network_id
|
||||
}
|
||||
|
||||
resource "google_service_networking_connection" "psa" {
|
||||
network = google_compute_network.this.id
|
||||
count = var.create_psa_connection ? 1 : 0
|
||||
|
||||
network = local.network_id
|
||||
service = "servicenetworking.googleapis.com"
|
||||
reserved_peering_ranges = [google_compute_global_address.psa.name]
|
||||
reserved_peering_ranges = [google_compute_global_address.psa[0].name]
|
||||
}
|
||||
|
||||
# Serverless VPC Access connector — required so Cloud Run can reach
|
||||
|
|
@ -37,9 +45,11 @@ resource "google_service_networking_connection" "psa" {
|
|||
# for low-to-moderate Cloud Run egress; bump max if your services push
|
||||
# heavy private-network traffic.
|
||||
resource "google_vpc_access_connector" "this" {
|
||||
count = var.create_runtime ? 1 : 0
|
||||
|
||||
name = "${local.name}-conn"
|
||||
region = var.region
|
||||
network = google_compute_network.this.name
|
||||
network = local.network_id
|
||||
ip_cidr_range = var.vpc_connector_cidr
|
||||
min_instances = 2
|
||||
max_instances = 3
|
||||
|
|
|
|||
|
|
@ -1,26 +1,26 @@
|
|||
output "lb_ip" {
|
||||
description = "Global anycast IP of the external HTTPS load balancer."
|
||||
value = google_compute_global_address.lb.address
|
||||
description = "Global anycast IP of the external HTTPS load balancer. Null when create_runtime is false."
|
||||
value = var.create_runtime ? one(google_compute_global_address.lb[*].address) : null
|
||||
}
|
||||
|
||||
output "lb_url" {
|
||||
description = "Proxy URL. Switches scheme based on whether lb_domains is set; when TLS is enabled the URL points at the first listed domain (since managed certs are tied to the hostname, not the anycast IP). The dashboard is served at /, the API at /v1/*."
|
||||
value = local.tls_enabled ? "https://${var.lb_domains[0]}" : "http://${google_compute_global_address.lb.address}"
|
||||
description = "Proxy URL, or null when create_runtime is false. Switches scheme based on whether lb_domains is set."
|
||||
value = var.create_runtime ? (local.tls_enabled ? "https://${var.lb_domains[0]}" : "http://${one(google_compute_global_address.lb[*].address)}") : null
|
||||
}
|
||||
|
||||
output "gateway_service_url" {
|
||||
description = "Default Cloud Run URL for the gateway (bypasses the LB)."
|
||||
value = google_cloud_run_v2_service.gateway.uri
|
||||
description = "Default Cloud Run URL for the gateway, or null when create_runtime is false."
|
||||
value = var.create_runtime ? one(google_cloud_run_v2_service.gateway[*].uri) : null
|
||||
}
|
||||
|
||||
output "backend_service_url" {
|
||||
description = "Default Cloud Run URL for the backend (bypasses the LB)."
|
||||
value = google_cloud_run_v2_service.backend.uri
|
||||
description = "Default Cloud Run URL for the backend, or null when create_runtime is false."
|
||||
value = var.create_runtime ? one(google_cloud_run_v2_service.backend[*].uri) : null
|
||||
}
|
||||
|
||||
output "ui_service_url" {
|
||||
description = "Default Cloud Run URL for the UI (bypasses the LB)."
|
||||
value = google_cloud_run_v2_service.ui.uri
|
||||
description = "Default Cloud Run URL for the UI, or null when create_runtime is false."
|
||||
value = var.create_runtime ? one(google_cloud_run_v2_service.ui[*].uri) : null
|
||||
}
|
||||
|
||||
output "cloudsql_writer_ip" {
|
||||
|
|
@ -38,6 +38,36 @@ output "redis_endpoint" {
|
|||
value = "${google_redis_instance.this.host}:${google_redis_instance.this.port}"
|
||||
}
|
||||
|
||||
output "runtime_service_account_email" {
|
||||
description = "Runtime service account email for Cloud Run or GKE Workload Identity."
|
||||
value = google_service_account.runtime.email
|
||||
}
|
||||
|
||||
output "redis_host" {
|
||||
description = "Memorystore Redis host."
|
||||
value = google_redis_instance.this.host
|
||||
}
|
||||
|
||||
output "redis_port" {
|
||||
description = "Memorystore Redis port."
|
||||
value = google_redis_instance.this.port
|
||||
}
|
||||
|
||||
output "redis_server_ca_pem" {
|
||||
description = "Memorystore server CA PEM. Mount it in the pod and set REDIS_SSL=true and REDIS_SSL_CA_CERTS=<path> via extraEnv when transit encryption is enabled."
|
||||
value = var.redis_transit_encryption ? google_redis_instance.this.server_ca_certs[0].cert : null
|
||||
}
|
||||
|
||||
output "db_username" {
|
||||
description = "Cloud SQL application username."
|
||||
value = var.db_username
|
||||
}
|
||||
|
||||
output "db_name" {
|
||||
description = "Cloud SQL database name."
|
||||
value = var.db_name
|
||||
}
|
||||
|
||||
output "gcs_bucket" {
|
||||
description = "GCS bucket name. Exposed to gateway + backend as GCS_BUCKET_NAME. Reference from proxy_config via `os.environ/GCS_BUCKET_NAME`."
|
||||
value = google_storage_bucket.this.name
|
||||
|
|
@ -54,11 +84,11 @@ output "db_password_secret_id" {
|
|||
}
|
||||
|
||||
output "migration_run_command" {
|
||||
description = "Shell command that executes the one-off migration job against Cloud SQL. Run this once after the first apply."
|
||||
value = format(
|
||||
description = "Shell command that executes the one-off migration job against Cloud SQL, or null when create_runtime is false."
|
||||
value = var.create_runtime ? format(
|
||||
"gcloud run jobs execute %s --region %s --project %s --wait",
|
||||
google_cloud_run_v2_job.migrations.name,
|
||||
one(google_cloud_run_v2_job.migrations[*].name),
|
||||
var.region,
|
||||
var.project_id,
|
||||
)
|
||||
) : null
|
||||
}
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ resource "google_redis_instance" "this" {
|
|||
memory_size_gb = var.redis_memory_size_gb
|
||||
region = var.region
|
||||
|
||||
authorized_network = google_compute_network.this.id
|
||||
authorized_network = local.network_id
|
||||
connect_mode = "PRIVATE_SERVICE_ACCESS"
|
||||
|
||||
redis_version = "REDIS_7_0"
|
||||
|
|
@ -16,7 +16,7 @@ resource "google_redis_instance" "this" {
|
|||
# and passed to the proxy as REDIS_CA_PEM_B64); the proxy decodes it to
|
||||
# /tmp/redis-ca.pem at startup and uses it to validate the rediss://
|
||||
# handshake. Mirrors `transit_encryption_enabled = true` on AWS.
|
||||
transit_encryption_mode = "SERVER_AUTHENTICATION"
|
||||
transit_encryption_mode = var.redis_transit_encryption ? "SERVER_AUTHENTICATION" : "DISABLED"
|
||||
|
||||
depends_on = [google_service_networking_connection.psa]
|
||||
}
|
||||
|
|
|
|||
175
terraform/litellm/gcp/tests/deps_only.tftest.hcl
Normal file
175
terraform/litellm/gcp/tests/deps_only.tftest.hcl
Normal file
|
|
@ -0,0 +1,175 @@
|
|||
mock_provider "google" {
|
||||
mock_resource "google_redis_instance" {
|
||||
defaults = {
|
||||
host = "10.0.0.4"
|
||||
port = 6379
|
||||
server_ca_certs = [{
|
||||
cert = "-----BEGIN CERTIFICATE-----\nmock\n-----END CERTIFICATE-----"
|
||||
}]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
mock_provider "google-beta" {}
|
||||
mock_provider "random" {}
|
||||
|
||||
variables {
|
||||
project_id = "test-project"
|
||||
tenant = "tenant"
|
||||
env = "test"
|
||||
allow_plaintext_lb = true
|
||||
image_registry = "us-central1-docker.pkg.dev/test-project/litellm"
|
||||
}
|
||||
|
||||
run "default_creates_everything" {
|
||||
command = plan
|
||||
|
||||
assert {
|
||||
condition = alltrue([
|
||||
length(google_compute_network.this) == 1,
|
||||
length(google_compute_subnetwork.this) == 1,
|
||||
length(google_compute_global_address.psa) == 1,
|
||||
length(google_service_networking_connection.psa) == 1,
|
||||
length(google_vpc_access_connector.this) == 1,
|
||||
length(google_cloud_run_v2_service.gateway) == 1,
|
||||
length(google_cloud_run_v2_service.backend) == 1,
|
||||
length(google_cloud_run_v2_service.ui) == 1,
|
||||
length(google_cloud_run_v2_job.migrations) == 1,
|
||||
length(google_compute_global_address.lb) == 1,
|
||||
length(terraform_data.migration) == 1,
|
||||
])
|
||||
error_message = "The default mode must create networking, runtime services, the load balancer, and migrations."
|
||||
}
|
||||
|
||||
assert {
|
||||
condition = google_redis_instance.this.transit_encryption_mode == "SERVER_AUTHENTICATION"
|
||||
error_message = "Redis transit encryption must remain enabled by default."
|
||||
}
|
||||
|
||||
assert {
|
||||
condition = length(local.shared_env_kv) == 12
|
||||
error_message = "The default runtime environment must include GCS and the three Redis TLS entries."
|
||||
}
|
||||
}
|
||||
|
||||
run "deps_only_creates_no_runtime" {
|
||||
command = plan
|
||||
|
||||
variables {
|
||||
create_runtime = false
|
||||
proxy_config = {
|
||||
model_list = []
|
||||
}
|
||||
}
|
||||
|
||||
assert {
|
||||
condition = alltrue([
|
||||
length(google_cloud_run_v2_service.gateway) == 0,
|
||||
length(google_cloud_run_v2_service.backend) == 0,
|
||||
length(google_cloud_run_v2_service.ui) == 0,
|
||||
length(google_cloud_run_v2_job.migrations) == 0,
|
||||
length(google_cloud_run_v2_service_iam_member.gateway_allusers) == 0,
|
||||
length(google_cloud_run_v2_service_iam_member.backend_allusers) == 0,
|
||||
length(google_cloud_run_v2_service_iam_member.ui_allusers) == 0,
|
||||
length(google_compute_global_address.lb) == 0,
|
||||
length(google_compute_region_network_endpoint_group.gateway) == 0,
|
||||
length(google_compute_region_network_endpoint_group.backend) == 0,
|
||||
length(google_compute_region_network_endpoint_group.ui) == 0,
|
||||
length(google_compute_backend_service.gateway) == 0,
|
||||
length(google_compute_backend_service.backend) == 0,
|
||||
length(google_compute_backend_service.ui) == 0,
|
||||
length(google_compute_url_map.this) == 0,
|
||||
length(google_compute_url_map.https_redirect) == 0,
|
||||
length(google_compute_target_http_proxy.this) == 0,
|
||||
length(google_compute_global_forwarding_rule.http) == 0,
|
||||
length(google_compute_managed_ssl_certificate.this) == 0,
|
||||
length(google_compute_target_https_proxy.this) == 0,
|
||||
length(google_compute_global_forwarding_rule.https) == 0,
|
||||
length(terraform_data.migration) == 0,
|
||||
length(google_vpc_access_connector.this) == 0,
|
||||
length(google_service_account.ui_runtime) == 0,
|
||||
length(google_storage_bucket.proxy_config) == 0,
|
||||
])
|
||||
error_message = "Dependencies-only mode must omit all runtime, load balancer, connector, UI identity, and proxy config resources."
|
||||
}
|
||||
|
||||
assert {
|
||||
condition = alltrue([
|
||||
google_sql_database_instance.writer.name == "tenant-litellm-test",
|
||||
google_sql_database_instance.reader.name == "tenant-litellm-test-reader",
|
||||
google_redis_instance.this.name == "tenant-litellm-test",
|
||||
google_storage_bucket.this.force_destroy == false,
|
||||
google_secret_manager_secret.master_key.secret_id == "tenant-litellm-test-master-key",
|
||||
google_secret_manager_secret.db_password.secret_id == "tenant-litellm-test-db-password",
|
||||
google_service_account.runtime.account_id == "tenant-litellm-test-runtime",
|
||||
])
|
||||
error_message = "Dependencies-only mode must retain data stores, secrets, and the runtime service account."
|
||||
}
|
||||
|
||||
assert {
|
||||
condition = output.lb_url == null && output.migration_run_command == null
|
||||
error_message = "Runtime outputs must be null while dependency outputs remain available."
|
||||
}
|
||||
}
|
||||
|
||||
run "existing_network_attaches_data_stores" {
|
||||
command = plan
|
||||
|
||||
variables {
|
||||
network_id = "projects/host-proj/global/networks/shared"
|
||||
create_psa_connection = false
|
||||
create_runtime = false
|
||||
}
|
||||
|
||||
assert {
|
||||
condition = alltrue([
|
||||
length(google_compute_network.this) == 0,
|
||||
length(google_compute_subnetwork.this) == 0,
|
||||
length(google_compute_global_address.psa) == 0,
|
||||
length(google_service_networking_connection.psa) == 0,
|
||||
google_sql_database_instance.writer.settings[0].ip_configuration[0].private_network == var.network_id,
|
||||
google_redis_instance.this.authorized_network == var.network_id,
|
||||
])
|
||||
error_message = "An existing VPC must receive the Cloud SQL and Memorystore private-network attachments."
|
||||
}
|
||||
}
|
||||
|
||||
run "psa_required_without_existing_network" {
|
||||
command = plan
|
||||
|
||||
variables {
|
||||
create_psa_connection = false
|
||||
}
|
||||
|
||||
expect_failures = [
|
||||
google_sql_database_instance.writer,
|
||||
]
|
||||
}
|
||||
|
||||
run "redis_plaintext_drops_tls_env" {
|
||||
command = plan
|
||||
|
||||
variables {
|
||||
redis_transit_encryption = false
|
||||
}
|
||||
|
||||
assert {
|
||||
condition = google_redis_instance.this.transit_encryption_mode == "DISABLED"
|
||||
error_message = "Redis transit encryption must be disabled when requested."
|
||||
}
|
||||
|
||||
assert {
|
||||
condition = length(local.shared_env_kv) == 9
|
||||
error_message = "Plaintext Redis mode must include GCS and omit the three Redis TLS entries."
|
||||
}
|
||||
|
||||
assert {
|
||||
condition = length([for env in local.shared_env_kv : env if env.name == "REDIS_SSL"]) == 0
|
||||
error_message = "Plaintext Redis mode must not set REDIS_SSL."
|
||||
}
|
||||
|
||||
assert {
|
||||
condition = length(local.redis_ca_fragment) == 0
|
||||
error_message = "Plaintext Redis mode must not decode a Redis CA at startup."
|
||||
}
|
||||
}
|
||||
|
|
@ -79,16 +79,42 @@ variable "ui_password" {
|
|||
sensitive = true
|
||||
}
|
||||
|
||||
# ---------- Deployment mode ----------
|
||||
|
||||
variable "create_runtime" {
|
||||
description = "Create Cloud Run, load balancer, VPC connector, runtime support resources, and the migration job. Set false for GKE or another external runtime."
|
||||
type = bool
|
||||
default = true
|
||||
}
|
||||
|
||||
variable "network_id" {
|
||||
description = "Existing VPC network resource ID (`projects/<host-project>/global/networks/<name>`). When set, no VPC or subnet is created. A VPC connector requires this network to be in the deployment project when create_runtime is true."
|
||||
type = string
|
||||
default = ""
|
||||
}
|
||||
|
||||
variable "create_psa_connection" {
|
||||
description = "Create the Private Services Access range and connection for Cloud SQL and Memorystore. Set false when the existing network already has PSA configured."
|
||||
type = bool
|
||||
default = true
|
||||
}
|
||||
|
||||
variable "redis_transit_encryption" {
|
||||
description = "Enable Memorystore transit encryption and inject Redis TLS settings into Cloud Run. Set false to use plaintext Redis."
|
||||
type = bool
|
||||
default = true
|
||||
}
|
||||
|
||||
# ---------- Networking ----------
|
||||
|
||||
variable "subnet_cidr" {
|
||||
description = "Primary CIDR block for the LiteLLM subnet."
|
||||
description = "Primary CIDR block for the LiteLLM subnet. Unused when network_id is set."
|
||||
type = string
|
||||
default = "10.40.0.0/16"
|
||||
}
|
||||
|
||||
variable "vpc_connector_cidr" {
|
||||
description = "CIDR for the Serverless VPC Access connector. /28 required."
|
||||
description = "CIDR for the Serverless VPC Access connector. /28 required. Unused when create_runtime is false."
|
||||
type = string
|
||||
default = "10.41.0.0/28"
|
||||
}
|
||||
|
|
|
|||
|
|
@ -24,7 +24,7 @@ from access_control_client import (
|
|||
from e2e_config import unique_marker
|
||||
from e2e_http import Success, UnauthorizedError, UnknownApiError, unwrap
|
||||
from lifecycle import ResourceManager
|
||||
from models import ChatBody, ChatMessage, ChatResponse, LiteLLMParamsBody
|
||||
from models import ChatBody, ChatMessage, ChatResponse, EmbedBody, LiteLLMParamsBody
|
||||
from proxy_client import ProxyClient
|
||||
|
||||
pytestmark = pytest.mark.e2e
|
||||
|
|
@ -32,6 +32,7 @@ pytestmark = pytest.mark.e2e
|
|||
ALLOWED_MODEL = "gemini-2.5-flash"
|
||||
DISALLOWED_MODEL = "gpt-5.5"
|
||||
VIRTUAL_KEY_BACKEND = "anthropic/claude-haiku-4-5-20251001"
|
||||
EMBEDDING_MODEL = "openai-text-embedding-3-small"
|
||||
|
||||
|
||||
class TestAccessControl:
|
||||
|
|
@ -71,6 +72,31 @@ class TestAccessControl:
|
|||
f"403 body must be a model-access denial, got: {result.body[:300]}"
|
||||
)
|
||||
|
||||
@pytest.mark.covers("other.auth.virtual_key.route_group_allowed")
|
||||
def test_llm_api_routes_group_grants_every_llm_endpoint(
|
||||
self, client: AccessControlClient, resources: ResourceManager
|
||||
) -> None:
|
||||
key = client.llm_only_key()
|
||||
resources.defer(lambda: client.delete_key(key))
|
||||
|
||||
chat = client.chat_status(key, ALLOWED_MODEL, f"capital of France? {unique_marker()}")
|
||||
assert chat.status_code == 200, (
|
||||
f"llm_api_routes key must reach /chat/completions, got {chat.status_code}: {chat.body[:300]}"
|
||||
)
|
||||
assert ChatResponse.model_validate_json(chat.body).choices, (
|
||||
f"200 must carry a real completion, not an error envelope: {chat.body[:300]}"
|
||||
)
|
||||
|
||||
embedding = unwrap(
|
||||
client.proxy.embed(key, EmbedBody(model=EMBEDDING_MODEL, input=f"route group {unique_marker()}"))
|
||||
)
|
||||
assert embedding.model, f"llm_api_routes key reached /embeddings but got no model back: {embedding}"
|
||||
|
||||
denied = client.create_model_status(key, f"e2e-route-group-{unique_marker()}")
|
||||
assert denied.status_code == 403 and ROUTE_NOT_ALLOWED_MARKER in denied.body, (
|
||||
f"the same key must still be shut out of /model/new, got {denied.status_code}: {denied.body[:300]}"
|
||||
)
|
||||
|
||||
def test_llm_only_key_forbidden_from_management_route_403(
|
||||
self, client: AccessControlClient, resources: ResourceManager
|
||||
) -> None:
|
||||
|
|
|
|||
|
|
@ -10,6 +10,11 @@ Each case asserts the feature actually happened, not just a 200. Coverage matrix
|
|||
intentionally not covered here.
|
||||
- Vertex (gemini-2.5-flash): prompt caching via ``cache_control`` context
|
||||
caching; the second identical call must report cached prompt tokens > 0.
|
||||
- Anthropic (claude-haiku-4-5, direct): the same ``cache_control`` prefix over
|
||||
the OpenAI-compatible route; the second call must report cache-read tokens > 0.
|
||||
- OpenAI (gpt-5.6): automatic prompt caching needs no request marker, so the
|
||||
cacheable prefix goes out as a plain system string with a ``prompt_cache_key``
|
||||
and the second call must report ``prompt_tokens_details.cached_tokens`` > 0.
|
||||
|
||||
service_tier lives in test_provider_features_e2e.py.
|
||||
|
||||
|
|
@ -21,6 +26,7 @@ built from the typed content blocks shared in ``endpoints_client.py``.
|
|||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
from collections.abc import Callable
|
||||
|
||||
import pytest
|
||||
from pydantic import BaseModel
|
||||
|
|
@ -29,7 +35,7 @@ from e2e_config import unique_marker
|
|||
from e2e_http import Result, unwrap
|
||||
from endpoints_client import CacheControl, RichMessage, TextBlock
|
||||
from lifecycle import ResourceManager
|
||||
from models import ChatResponse, LiteLLMParamsBody, Usage
|
||||
from models import ChatBody, ChatMessage, ChatResponse, LiteLLMParamsBody, Usage
|
||||
from passthrough_client import PassthroughClient
|
||||
import os
|
||||
|
||||
|
|
@ -37,6 +43,8 @@ pytestmark = pytest.mark.e2e
|
|||
|
||||
BEDROCK_MODEL = "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0"
|
||||
VERTEX_MODEL = "vertex_ai/gemini-2.5-flash"
|
||||
ANTHROPIC_MODEL = "anthropic/claude-haiku-4-5-20251001"
|
||||
OPENAI_MODEL = "openai/gpt-5.6"
|
||||
|
||||
|
||||
class CacheChatBody(BaseModel):
|
||||
|
|
@ -89,17 +97,36 @@ def _cache_chat(
|
|||
)
|
||||
|
||||
|
||||
def _plain_cache_chat(
|
||||
client: PassthroughClient, key: str, model: str, prefix: str, cache_key: str
|
||||
) -> Result[ChatResponse]:
|
||||
"""The same cacheable prefix as a plain system string, for providers that cache
|
||||
automatically and take no per-block marker (OpenAI)."""
|
||||
return client.proxy.chat(
|
||||
key,
|
||||
ChatBody(
|
||||
model=model,
|
||||
messages=[
|
||||
ChatMessage(role="system", content=prefix),
|
||||
ChatMessage(role="user", content="Reply with one word."),
|
||||
],
|
||||
max_tokens=64,
|
||||
prompt_cache_key=cache_key,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _assert_cache_read_on_second_call(
|
||||
client: PassthroughClient, key: str, model: str
|
||||
model: str, send: Callable[[str], Result[ChatResponse]]
|
||||
) -> None:
|
||||
prefix = _cacheable_prefix()
|
||||
|
||||
first = unwrap(_cache_chat(client, key, model, prefix))
|
||||
first = unwrap(send(prefix))
|
||||
assert first.choices, f"{model}: first cache-priming call returned no choices: {first}"
|
||||
|
||||
deadline = time.monotonic() + 30.0
|
||||
while True:
|
||||
second = unwrap(_cache_chat(client, key, model, prefix))
|
||||
second = unwrap(send(prefix))
|
||||
read_tokens = _cached_read_tokens(second.usage)
|
||||
if read_tokens > 0 or time.monotonic() >= deadline:
|
||||
break
|
||||
|
|
@ -125,7 +152,8 @@ class TestCacheControl:
|
|||
LiteLLMParamsBody(model=BEDROCK_MODEL, aws_region_name="us-east-1"),
|
||||
)
|
||||
resources.defer(lambda: client.proxy.delete_model(model_id))
|
||||
_assert_cache_read_on_second_call(client, resources.key(), model)
|
||||
key = resources.key()
|
||||
_assert_cache_read_on_second_call(model, lambda prefix: _cache_chat(client, key, model, prefix))
|
||||
|
||||
@pytest.mark.covers(
|
||||
"llm.chat_completions.vertex.prompt_cache_5m.nonstream.works",
|
||||
|
|
@ -145,4 +173,40 @@ class TestCacheControl:
|
|||
),
|
||||
)
|
||||
resources.defer(lambda: client.proxy.delete_model(model_id))
|
||||
_assert_cache_read_on_second_call(client, resources.key(), model)
|
||||
key = resources.key()
|
||||
_assert_cache_read_on_second_call(model, lambda prefix: _cache_chat(client, key, model, prefix))
|
||||
|
||||
@pytest.mark.covers(
|
||||
"llm.chat_completions.anthropic.prompt_cache_5m.nonstream.works",
|
||||
exercised_on=[],
|
||||
)
|
||||
def test_anthropic_prompt_caching_reads_cache(
|
||||
self, client: PassthroughClient, resources: ResourceManager
|
||||
) -> None:
|
||||
model = f"e2e-anthropic-cache-{unique_marker()}"
|
||||
model_id = client.proxy.create_model(
|
||||
model,
|
||||
LiteLLMParamsBody(model=ANTHROPIC_MODEL, api_key="os.environ/ANTHROPIC_API_KEY"),
|
||||
)
|
||||
resources.defer(lambda: client.proxy.delete_model(model_id))
|
||||
key = resources.key()
|
||||
_assert_cache_read_on_second_call(model, lambda prefix: _cache_chat(client, key, model, prefix))
|
||||
|
||||
@pytest.mark.covers(
|
||||
"llm.chat_completions.openai.prompt_cache_5m.nonstream.works",
|
||||
exercised_on=[],
|
||||
)
|
||||
def test_openai_prompt_caching_reads_cache(
|
||||
self, client: PassthroughClient, resources: ResourceManager
|
||||
) -> None:
|
||||
model = f"e2e-openai-cache-{unique_marker()}"
|
||||
model_id = client.proxy.create_model(
|
||||
model,
|
||||
LiteLLMParamsBody(model=OPENAI_MODEL, api_key="os.environ/OPENAI_API_KEY"),
|
||||
)
|
||||
resources.defer(lambda: client.proxy.delete_model(model_id))
|
||||
key = resources.key()
|
||||
cache_key = f"e2e-openai-cache-{unique_marker()}"
|
||||
_assert_cache_read_on_second_call(
|
||||
model, lambda prefix: _plain_cache_chat(client, key, model, prefix, cache_key)
|
||||
)
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
"""Live e2e: POST /embeddings returns a real vector across OpenAI, Bedrock, Vertex.
|
||||
"""Live e2e: POST /embeddings returns a real vector across OpenAI, Bedrock, Vertex, Cohere.
|
||||
|
||||
Each test registers the deployment it needs at runtime (deleted on teardown) and
|
||||
asserts a non-empty, non-zero vector came back. The LIT-3167 guard in
|
||||
|
|
@ -86,6 +86,26 @@ class TestEmbeddingsEndpoint:
|
|||
f"embedding vector is all zeros: {result.body[:300]}"
|
||||
)
|
||||
|
||||
@pytest.mark.covers("llm.embeddings.cohere.basic.nonstream.works")
|
||||
def test_cohere_embeddings_returns_vector(
|
||||
self, endpoints_client: EndpointsClient, resources: ResourceManager
|
||||
) -> None:
|
||||
model = f"e2e-embeddings-cohere-{unique_marker()}"
|
||||
model_id = endpoints_client.create_model(
|
||||
model,
|
||||
LiteLLMParamsBody(model="cohere/embed-v4.0", api_key="os.environ/COHERE_API_KEY"),
|
||||
)
|
||||
resources.defer(lambda: endpoints_client.delete_model(model_id))
|
||||
key = resources.key()
|
||||
|
||||
result = endpoints_client.embeddings(key, model, "Say this is a test!")
|
||||
require_successful_call(result)
|
||||
parsed = EmbeddingsResult.model_validate_json(result.body)
|
||||
assert parsed.first_vector, f"/embeddings returned no vector: {result.body[:300]}"
|
||||
assert any(component != 0.0 for component in parsed.first_vector), (
|
||||
f"embedding vector is all zeros: {result.body[:300]}"
|
||||
)
|
||||
|
||||
@pytest.mark.covers("llm.embeddings.vertex.basic.nonstream.works")
|
||||
def test_vertex_embeddings_returns_vector(
|
||||
self, endpoints_client: EndpointsClient, resources: ResourceManager
|
||||
|
|
|
|||
|
|
@ -17,7 +17,7 @@ import pytest
|
|||
from e2e_config import CHEAP_OPENAI_MODEL, unique_marker
|
||||
from e2e_http import require_successful_call, unwrap
|
||||
from lifecycle import ResourceManager
|
||||
from models import KeyGenerateBody, SpendLogRow
|
||||
from models import ChatResponse, KeyGenerateBody, SpendLogRow
|
||||
from passthrough_client import (
|
||||
AnthropicTool,
|
||||
GeminiFunctionDeclaration,
|
||||
|
|
@ -344,6 +344,44 @@ class TestOpenAIPassthroughSpend:
|
|||
)
|
||||
|
||||
|
||||
class TestOpenAIProviderPrefixChat:
|
||||
"""OpenAI-format chat through the raw `/openai/{endpoint}` passthrough (LIT-4752).
|
||||
|
||||
The body goes to OpenAI untranslated with the proxy's own OPENAI_API_KEY swapped
|
||||
in, so the customer gets OpenAI's real completion back, and the gateway must
|
||||
still write a costed pass_through_endpoint row for it.
|
||||
"""
|
||||
|
||||
@pytest.mark.covers("llm.chat_completions.openai.passthrough.nonstream.cost_logged")
|
||||
def test_openai_prefix_chat_returns_completion_and_logs_its_cost(
|
||||
self, client: PassthroughClient, scoped_key: str
|
||||
) -> None:
|
||||
result = client.openai_chat(scoped_key, CHEAP_OPENAI_MODEL, f"Say hi in one word. {unique_marker()}")
|
||||
require_successful_call(result)
|
||||
|
||||
completion = ChatResponse.model_validate_json(result.body)
|
||||
assert completion.id, f"/openai/v1/chat/completions relayed no completion id: {result.body[:300]}"
|
||||
content = (
|
||||
completion.choices[0].message.content
|
||||
if completion.choices and completion.choices[0].message
|
||||
else None
|
||||
)
|
||||
assert content and content.strip(), (
|
||||
f"/openai/v1/chat/completions relayed an empty completion: {result.body[:300]}"
|
||||
)
|
||||
assert completion.usage is not None, f"the completion carried no usage to price from: {completion}"
|
||||
|
||||
row = _fetch_cost_breakdown(client, completion.id)
|
||||
assert row.prompt_tokens == completion.usage.prompt_tokens, (
|
||||
f"logged {row.prompt_tokens} prompt tokens, the completion the customer read "
|
||||
f"reported {completion.usage.prompt_tokens}"
|
||||
)
|
||||
assert row.completion_tokens == completion.usage.completion_tokens, (
|
||||
f"logged {row.completion_tokens} completion tokens, the completion the customer read "
|
||||
f"reported {completion.usage.completion_tokens}"
|
||||
)
|
||||
|
||||
|
||||
class TestOpenAIPassthroughWebsocket:
|
||||
"""The OpenAI passthrough prefixes must answer a websocket upgrade, not only a POST.
|
||||
|
||||
|
|
|
|||
|
|
@ -40,6 +40,8 @@ from models import (
|
|||
KeyListParams,
|
||||
KeyListResponse,
|
||||
KeyRegenerateBody,
|
||||
KeyResetSpendBody,
|
||||
KeyResetSpendResponse,
|
||||
KeyUpdateBody,
|
||||
ModelDeleteBody,
|
||||
OrgDeleteBody,
|
||||
|
|
@ -191,16 +193,26 @@ class ManagementClient:
|
|||
response_type=NoBody,
|
||||
)
|
||||
)
|
||||
def regenerate_key(self, key: str) -> str:
|
||||
def regenerate_key(self, key: str, *, grace_period: str | None = None) -> str:
|
||||
return unwrap(
|
||||
self.proxy.transport.post(
|
||||
"/key/regenerate",
|
||||
headers=self.proxy.transport.master,
|
||||
json=KeyRegenerateBody(key=key),
|
||||
json=KeyRegenerateBody(key=key, grace_period=grace_period),
|
||||
response_type=KeyGenerateResponse,
|
||||
)
|
||||
).key
|
||||
|
||||
def reset_key_spend(self, key: str, reset_to: float) -> KeyResetSpendResponse:
|
||||
return unwrap(
|
||||
self.proxy.transport.post(
|
||||
f"/key/{key}/reset_spend",
|
||||
headers=self.proxy.transport.master,
|
||||
json=KeyResetSpendBody(reset_to=reset_to),
|
||||
response_type=KeyResetSpendResponse,
|
||||
)
|
||||
)
|
||||
|
||||
def key_list(self, key_alias: str, *, caller_key: str | None = None) -> Result[KeyListResponse]:
|
||||
"""GET /key/list, the Virtual Keys page's own inventory call. `caller_key` is
|
||||
who is asking: the master key by default, or a virtual key."""
|
||||
|
|
|
|||
|
|
@ -18,7 +18,7 @@ from typing import Literal
|
|||
import pytest
|
||||
|
||||
from e2e_config import unique_marker
|
||||
from e2e_http import NoBody, unwrap
|
||||
from e2e_http import NoBody, StreamingResponse, unwrap
|
||||
from lifecycle import ResourceManager
|
||||
from management_client import ManagementClient
|
||||
from models import KeyDeleteBody, KeyGenerateBody, KeyUpdateBody
|
||||
|
|
@ -26,6 +26,9 @@ from pydantic import BaseModel
|
|||
|
||||
pytestmark = pytest.mark.e2e
|
||||
|
||||
TINY_BUDGET = 3e-6
|
||||
SPEND_MODEL = "claude-haiku-4-5"
|
||||
|
||||
|
||||
class KeyToggleBlockBody(BaseModel):
|
||||
key: str
|
||||
|
|
@ -82,6 +85,30 @@ def _generate_key(client: ManagementClient, resources: ResourceManager, body: Ke
|
|||
return key
|
||||
|
||||
|
||||
def _is_budget_block(outcome: StreamingResponse) -> bool:
|
||||
return not outcome.ok and "budget_exceeded" in outcome.body
|
||||
|
||||
|
||||
def _spend_until_budget_blocks(client: ManagementClient, key: str) -> None:
|
||||
for _ in range(40):
|
||||
outcome = client.chat_status(key, SPEND_MODEL, f"spend {unique_marker()}")
|
||||
if _is_budget_block(outcome):
|
||||
assert outcome.status_code == 429, (
|
||||
f"budget refusal must be 429, got {outcome.status_code}: {outcome.body[:200]}"
|
||||
)
|
||||
return
|
||||
assert outcome.ok, f"paid call failed before the budget tripped ({outcome.status_code}): {outcome.body[:300]}"
|
||||
time.sleep(2)
|
||||
pytest.fail(f"max_budget={TINY_BUDGET} never blocked a call on the key")
|
||||
|
||||
|
||||
def _settled_spend(client: ManagementClient, key: str) -> float | None:
|
||||
first = client.proxy.key_info(key).spend or 0.0
|
||||
time.sleep(client.proxy.poll_interval)
|
||||
second = client.proxy.key_info(key).spend or 0.0
|
||||
return second if first > 0 and first == second else None
|
||||
|
||||
|
||||
def _block(client: ManagementClient, key: str) -> None:
|
||||
_ = unwrap(
|
||||
client.proxy.transport.post(
|
||||
|
|
@ -197,6 +224,32 @@ class TestKeyManagementRoutes:
|
|||
"/key/info never reported max_budget 42.0 after /key/bulk_update before the deadline",
|
||||
)
|
||||
|
||||
@pytest.mark.covers("other.key_mgmt.spend_reset.resets_to_value")
|
||||
def test_reset_spend_zeroes_recorded_spend_and_lifts_the_budget_block(
|
||||
self, client: ManagementClient, resources: ResourceManager
|
||||
) -> None:
|
||||
key = _generate_key(client, resources, KeyGenerateBody(models=[SPEND_MODEL], max_budget=TINY_BUDGET))
|
||||
_spend_until_budget_blocks(client, key)
|
||||
recorded = _poll(
|
||||
client, lambda: _settled_spend(client, key), "key spend never landed in /key/info before the deadline"
|
||||
)
|
||||
|
||||
reset = client.reset_key_spend(key, reset_to=0.0)
|
||||
assert reset.previous_spend == recorded, (
|
||||
f"reset_spend reported previous_spend {reset.previous_spend}, /key/info had recorded {recorded}"
|
||||
)
|
||||
assert reset.spend == 0.0, f"reset_spend to 0 reported spend {reset.spend}"
|
||||
assert client.proxy.key_info(key).spend == 0.0, "/key/info still reports spend after the reset to 0"
|
||||
|
||||
def call_allowed_again() -> bool | None:
|
||||
outcome = client.chat_status(key, SPEND_MODEL, f"after reset {unique_marker()}")
|
||||
if _is_budget_block(outcome):
|
||||
return None
|
||||
assert outcome.ok, f"post-reset call failed ({outcome.status_code}): {outcome.body[:300]}"
|
||||
return True
|
||||
|
||||
_ = _poll(client, call_allowed_again, "the key stayed budget-blocked after its spend was reset to 0")
|
||||
|
||||
@pytest.mark.covers("mgmt.key.generate.admin_only")
|
||||
def test_generate_forbidden_for_non_admin_key(
|
||||
self, client: ManagementClient, resources: ResourceManager
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@ from __future__ import annotations
|
|||
import math
|
||||
import time
|
||||
from collections.abc import Callable
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
|
||||
|
|
@ -42,6 +43,10 @@ from models import (
|
|||
|
||||
pytestmark = pytest.mark.e2e
|
||||
|
||||
REGENERATE_GRACE_PERIOD = "15s"
|
||||
REGENERATE_GRACE_SECONDS = 15.0
|
||||
|
||||
|
||||
def _poll[T](client: ManagementClient, attempt: Callable[[], T | None], failure: str) -> T:
|
||||
deadline = time.monotonic() + client.proxy.poll_timeout
|
||||
while time.monotonic() < deadline:
|
||||
|
|
@ -365,6 +370,36 @@ class TestKeyRegeneration:
|
|||
client, old_rejected, "old key was still accepted after regeneration (never rejected 401) at the deadline"
|
||||
)
|
||||
|
||||
@pytest.mark.covers("other.key_mgmt.regenerate.grace_period_honored")
|
||||
def test_regenerate_with_grace_period_keeps_old_key_until_revoked(
|
||||
self, client: ManagementClient, resources: ResourceManager
|
||||
) -> None:
|
||||
old_key = _generate_key(client, resources, KeyGenerateBody(models=["gpt-5.5"]))
|
||||
|
||||
new_key = client.regenerate_key(old_key, grace_period=REGENERATE_GRACE_PERIOD)
|
||||
resources.defer(lambda: client.proxy.delete_key(new_key))
|
||||
revoke_at: Final = time.monotonic() + REGENERATE_GRACE_SECONDS
|
||||
assert new_key != old_key, "regenerate returned the same key string, so no rotation happened"
|
||||
|
||||
def old_accepted() -> bool | None:
|
||||
outcome = client.chat_status(old_key, "gpt-5.5", f"say hi {unique_marker()}")
|
||||
return True if outcome.ok else None
|
||||
|
||||
_ = _poll(client, old_accepted, "old key was rejected 401 inside its grace period at the deadline")
|
||||
assert time.monotonic() < revoke_at, (
|
||||
f"old key was only accepted after its {REGENERATE_GRACE_PERIOD} grace period had elapsed"
|
||||
)
|
||||
|
||||
def old_rejected() -> bool | None:
|
||||
outcome = client.chat_status(old_key, "gpt-5.5", f"say hi {unique_marker()}")
|
||||
return True if outcome.status_code == 401 else None
|
||||
|
||||
_ = _poll(
|
||||
client,
|
||||
old_rejected,
|
||||
f"old key was still accepted past its {REGENERATE_GRACE_PERIOD} grace period (never 401) at the deadline",
|
||||
)
|
||||
|
||||
|
||||
class TestTeamRoutes:
|
||||
@pytest.mark.covers("mgmt.team.new.persists")
|
||||
|
|
|
|||
|
|
@ -83,6 +83,16 @@ class KeyGenerateResponse(BaseModel):
|
|||
|
||||
class KeyRegenerateBody(BaseModel):
|
||||
key: str
|
||||
grace_period: str | None = None
|
||||
|
||||
|
||||
class KeyResetSpendBody(BaseModel):
|
||||
reset_to: float
|
||||
|
||||
|
||||
class KeyResetSpendResponse(BaseModel):
|
||||
spend: float
|
||||
previous_spend: float
|
||||
|
||||
|
||||
class KeyDeleteBody(BaseModel):
|
||||
|
|
|
|||
|
|
@ -184,6 +184,7 @@ test.describe("Admin tables scroll inside the page", () => {
|
|||
);
|
||||
try {
|
||||
await navigateToPage(page, Page.TagManagement);
|
||||
await setRowsPerPage(page, "50");
|
||||
await expectRowsAtLeast(page, SEED_ROWS);
|
||||
expect(await rowsPaintingPastAnAncestor(page)).toEqual([]);
|
||||
} finally {
|
||||
|
|
@ -207,6 +208,7 @@ test.describe("Admin tables scroll inside the page", () => {
|
|||
);
|
||||
try {
|
||||
await navigateToPage(page, Page.ModelHubTable);
|
||||
await setRowsPerPage(page, "50");
|
||||
await expectRowsAtLeast(page, SEED_ROWS);
|
||||
expect(await rowsPaintingPastAnAncestor(page)).toEqual([]);
|
||||
} finally {
|
||||
|
|
|
|||
|
|
@ -241,7 +241,7 @@ class TestRouterIndexManagement:
|
|||
"_get_deployment_by_litellm_model": "lookup by litellm_params.model, which is not indexed",
|
||||
"_finalize_adaptive_router_if_configured": 'init-time prefix scan for "auto_router/adaptive_router"; no index for prefix match',
|
||||
"config_deployments": "filters the whole list on model_info.db_model; admin path only (model add/upsert)",
|
||||
"heuristic_v2_router_limit_violation": "counts heuristic_v2 routers across the whole list; admin path only (auto-router init/upsert)",
|
||||
"auto_router_capability_violation": "counts gated auto-routers across the whole list; admin path only (auto-router init/upsert)",
|
||||
}
|
||||
|
||||
# Get path to router.py
|
||||
|
|
|
|||
|
|
@ -1,36 +1,61 @@
|
|||
import { render, screen, waitFor } from "@testing-library/react";
|
||||
import { QueryClient, QueryClientProvider } from "@tanstack/react-query";
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { type UrlUpdateEvent } from "nuqs/adapters/testing";
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import userEvent from "@testing-library/user-event";
|
||||
import GuardrailsMonitorView from "./GuardrailsMonitorView";
|
||||
import * as networking from "@/components/networking";
|
||||
import { renderWithProviders, screen, testQueryClient, waitFor } from "@/../tests/test-utils";
|
||||
|
||||
vi.mock("@/components/networking", () => ({
|
||||
getGuardrailsUsageOverview: vi.fn(),
|
||||
getGuardrailsUsageDetail: vi.fn(),
|
||||
getGuardrailsUsageLogs: vi.fn(),
|
||||
formatDate: vi.fn((d: Date) => d.toISOString().slice(0, 10)),
|
||||
}));
|
||||
|
||||
const mockGetGuardrailsUsageOverview = vi.mocked(networking.getGuardrailsUsageOverview);
|
||||
vi.mock("@/components/GuardrailsMonitor/LogViewer", () => ({
|
||||
LogViewer: ({ guardrailName }: { guardrailName: string }) => <div data-testid="log-viewer">{guardrailName}</div>,
|
||||
}));
|
||||
|
||||
function wrapper({ children }: { children: React.ReactNode }) {
|
||||
const queryClient = new QueryClient({
|
||||
defaultOptions: {
|
||||
queries: { retry: false },
|
||||
},
|
||||
});
|
||||
return <QueryClientProvider client={queryClient}>{children}</QueryClientProvider>;
|
||||
}
|
||||
const mockGetGuardrailsUsageOverview = vi.mocked(networking.getGuardrailsUsageOverview);
|
||||
const mockGetGuardrailsUsageDetail = vi.mocked(networking.getGuardrailsUsageDetail);
|
||||
const mockGetGuardrailsUsageLogs = vi.mocked(networking.getGuardrailsUsageLogs);
|
||||
|
||||
const emptyOverview = { rows: [], chart: [], totalRequests: 0, totalBlocked: 0, passRate: 100 };
|
||||
|
||||
const piiRow = {
|
||||
id: "gr-pii",
|
||||
name: "PII Guard",
|
||||
type: "pii",
|
||||
provider: "LiteLLM",
|
||||
requestsEvaluated: 10,
|
||||
failRate: 10,
|
||||
status: "healthy" as const,
|
||||
trend: "stable" as const,
|
||||
};
|
||||
|
||||
const piiDetail = {
|
||||
guardrail_name: "PII Guard",
|
||||
description: "",
|
||||
status: "healthy",
|
||||
provider: "LiteLLM",
|
||||
type: "pii",
|
||||
requestsEvaluated: 10,
|
||||
failRate: 10,
|
||||
avgScore: 0.5,
|
||||
avgLatency: 20,
|
||||
};
|
||||
|
||||
describe("GuardrailsMonitorView", () => {
|
||||
it("should render overview and fetch guardrails usage when accessToken is provided", async () => {
|
||||
mockGetGuardrailsUsageOverview.mockResolvedValue({
|
||||
rows: [],
|
||||
chart: [],
|
||||
totalRequests: 0,
|
||||
totalBlocked: 0,
|
||||
passRate: 100,
|
||||
});
|
||||
beforeEach(() => {
|
||||
testQueryClient.clear();
|
||||
vi.clearAllMocks();
|
||||
mockGetGuardrailsUsageOverview.mockResolvedValue(emptyOverview);
|
||||
mockGetGuardrailsUsageDetail.mockResolvedValue(piiDetail);
|
||||
mockGetGuardrailsUsageLogs.mockResolvedValue({ logs: [], total: 0 });
|
||||
});
|
||||
|
||||
render(<GuardrailsMonitorView accessToken="test-token" />, { wrapper });
|
||||
it("should render overview and fetch guardrails usage when accessToken is provided", async () => {
|
||||
renderWithProviders(<GuardrailsMonitorView accessToken="test-token" />);
|
||||
|
||||
expect(await screen.findByRole("heading", { name: /Guardrails Monitor/i })).toBeInTheDocument();
|
||||
await waitFor(() => {
|
||||
|
|
@ -39,7 +64,54 @@ describe("GuardrailsMonitorView", () => {
|
|||
});
|
||||
|
||||
it("should render without crashing when accessToken is null", async () => {
|
||||
render(<GuardrailsMonitorView accessToken={null} />, { wrapper });
|
||||
renderWithProviders(<GuardrailsMonitorView accessToken={null} />);
|
||||
expect(await screen.findByRole("heading", { name: /Guardrails Monitor/i })).toBeInTheDocument();
|
||||
});
|
||||
|
||||
describe("guardrail detail deep link (?guardrail=)", () => {
|
||||
it("should open the detail view directly from a ?guardrail= deep link", async () => {
|
||||
renderWithProviders(<GuardrailsMonitorView accessToken="test-token" />, { searchParams: "?guardrail=gr-pii" });
|
||||
|
||||
expect(await screen.findByRole("heading", { name: "PII Guard" })).toBeInTheDocument();
|
||||
expect(mockGetGuardrailsUsageDetail).toHaveBeenCalledWith(
|
||||
"test-token",
|
||||
"gr-pii",
|
||||
expect.any(String),
|
||||
expect.any(String),
|
||||
);
|
||||
expect(screen.queryByRole("heading", { name: /Guardrails Monitor/i })).not.toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("should push ?guardrail= as a new history entry when a guardrail is selected", async () => {
|
||||
const user = userEvent.setup();
|
||||
const onUrlUpdate = vi.fn<(event: UrlUpdateEvent) => void>();
|
||||
mockGetGuardrailsUsageOverview.mockResolvedValue({ ...emptyOverview, rows: [piiRow] });
|
||||
renderWithProviders(<GuardrailsMonitorView accessToken="test-token" />, { onUrlUpdate });
|
||||
|
||||
await user.click(await screen.findByRole("button", { name: "PII Guard" }));
|
||||
|
||||
await waitFor(() => expect(onUrlUpdate).toHaveBeenCalled());
|
||||
const lastUpdate = onUrlUpdate.mock.calls.at(-1)![0];
|
||||
expect(lastUpdate.searchParams.get("guardrail")).toBe("gr-pii");
|
||||
expect(lastUpdate.options.history).toBe("push");
|
||||
expect(await screen.findByRole("heading", { name: "PII Guard" })).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("should clear ?guardrail= by replacing history when going back to the overview", async () => {
|
||||
const user = userEvent.setup();
|
||||
const onUrlUpdate = vi.fn<(event: UrlUpdateEvent) => void>();
|
||||
renderWithProviders(<GuardrailsMonitorView accessToken="test-token" />, {
|
||||
searchParams: "?guardrail=gr-pii",
|
||||
onUrlUpdate,
|
||||
});
|
||||
|
||||
await user.click(await screen.findByRole("button", { name: /back to overview/i }));
|
||||
|
||||
await waitFor(() => expect(onUrlUpdate).toHaveBeenCalled());
|
||||
const lastUpdate = onUrlUpdate.mock.calls.at(-1)![0];
|
||||
expect(lastUpdate.searchParams.has("guardrail")).toBe(false);
|
||||
expect(lastUpdate.options.history).toBe("replace");
|
||||
expect(await screen.findByRole("heading", { name: /Guardrails Monitor/i })).toBeInTheDocument();
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -1,12 +1,11 @@
|
|||
import type { DateRangePickerValue } from "@/components/shared/date_picker_types";
|
||||
import { parseAsString, useQueryState } from "nuqs";
|
||||
import React, { useCallback, useMemo, useState } from "react";
|
||||
import { formatDate } from "@/components/networking";
|
||||
import AdvancedDatePicker from "@/components/shared/advanced_date_picker";
|
||||
import { GuardrailDetail } from "./GuardrailDetail";
|
||||
import { GuardrailsOverview } from "./GuardrailsOverview";
|
||||
|
||||
type View = { type: "overview" } | { type: "detail"; guardrailId: string };
|
||||
|
||||
interface GuardrailsMonitorViewProps {
|
||||
accessToken?: string | null;
|
||||
}
|
||||
|
|
@ -16,7 +15,10 @@ const defaultStart = new Date();
|
|||
defaultStart.setDate(defaultStart.getDate() - 7);
|
||||
|
||||
export default function GuardrailsMonitorView({ accessToken = null }: GuardrailsMonitorViewProps) {
|
||||
const [view, setView] = useState<View>({ type: "overview" });
|
||||
const [selectedGuardrailId, setSelectedGuardrailId] = useQueryState(
|
||||
"guardrail",
|
||||
parseAsString.withOptions({ history: "push" }),
|
||||
);
|
||||
|
||||
const initialFrom = useMemo(() => new Date(defaultStart), []);
|
||||
const initialTo = useMemo(() => new Date(defaultEnd), []);
|
||||
|
|
@ -34,11 +36,11 @@ export default function GuardrailsMonitorView({ accessToken = null }: Guardrails
|
|||
}, []);
|
||||
|
||||
const handleSelectGuardrail = (id: string) => {
|
||||
setView({ type: "detail", guardrailId: id });
|
||||
void setSelectedGuardrailId(id);
|
||||
};
|
||||
|
||||
const handleBack = () => {
|
||||
setView({ type: "overview" });
|
||||
void setSelectedGuardrailId(null, { history: "replace" });
|
||||
};
|
||||
|
||||
const dateRangeControl = (
|
||||
|
|
@ -47,7 +49,7 @@ export default function GuardrailsMonitorView({ accessToken = null }: Guardrails
|
|||
|
||||
return (
|
||||
<main className="w-full min-w-0 flex-1 p-8">
|
||||
{view.type === "overview" ? (
|
||||
{!selectedGuardrailId ? (
|
||||
<GuardrailsOverview
|
||||
accessToken={accessToken}
|
||||
startDate={startDate}
|
||||
|
|
@ -59,7 +61,7 @@ export default function GuardrailsMonitorView({ accessToken = null }: Guardrails
|
|||
<>
|
||||
<div className="mb-4 flex items-center justify-end">{dateRangeControl}</div>
|
||||
<GuardrailDetail
|
||||
guardrailId={view.guardrailId}
|
||||
guardrailId={selectedGuardrailId}
|
||||
onBack={handleBack}
|
||||
accessToken={accessToken}
|
||||
startDate={startDate}
|
||||
|
|
|
|||
|
|
@ -1,7 +1,8 @@
|
|||
import { render, screen, fireEvent, waitFor, within } from "@testing-library/react";
|
||||
import { type UrlUpdateEvent } from "nuqs/adapters/testing";
|
||||
import { beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import GuardrailsPanel from "./GuardrailsPanel";
|
||||
import { getGuardrailsList, deleteGuardrailCall } from "@/components/networking";
|
||||
import { fireEvent, renderWithProviders, screen, waitFor, within } from "@/../tests/test-utils";
|
||||
|
||||
vi.mock("@/components/networking", () => ({
|
||||
getGuardrailsList: vi.fn(),
|
||||
|
|
@ -15,16 +16,21 @@ vi.mock("./add_guardrail_form", () => ({
|
|||
|
||||
vi.mock("./guardrail_table", () => ({
|
||||
__esModule: true,
|
||||
default: ({ guardrailsList, onDeleteClick }: any) => (
|
||||
default: ({ guardrailsList, onDeleteClick, onGuardrailClick }: any) => (
|
||||
<div>
|
||||
<div>Mock Guardrail Table</div>
|
||||
{guardrailsList.length > 0 && (
|
||||
<button
|
||||
data-testid="delete-button"
|
||||
onClick={() => onDeleteClick(guardrailsList[0].guardrail_id, guardrailsList[0].guardrail_name)}
|
||||
>
|
||||
Delete
|
||||
</button>
|
||||
<>
|
||||
<button
|
||||
data-testid="delete-button"
|
||||
onClick={() => onDeleteClick(guardrailsList[0].guardrail_id, guardrailsList[0].guardrail_name)}
|
||||
>
|
||||
Delete
|
||||
</button>
|
||||
<button data-testid="open-button" onClick={() => onGuardrailClick(guardrailsList[0].guardrail_id)}>
|
||||
Open
|
||||
</button>
|
||||
</>
|
||||
)}
|
||||
</div>
|
||||
),
|
||||
|
|
@ -32,7 +38,12 @@ vi.mock("./guardrail_table", () => ({
|
|||
|
||||
vi.mock("./guardrail_info", () => ({
|
||||
__esModule: true,
|
||||
default: () => <div>Mock Guardrail Info View</div>,
|
||||
default: ({ guardrailId, onClose }: { guardrailId: string; onClose: () => void }) => (
|
||||
<div>
|
||||
<div data-testid="guardrail-info-view">Mock Guardrail Info View {guardrailId}</div>
|
||||
<button onClick={onClose}>Close Guardrail Info</button>
|
||||
</div>
|
||||
),
|
||||
}));
|
||||
|
||||
vi.mock("./GuardrailTestPlayground", async () => {
|
||||
|
|
@ -112,7 +123,7 @@ describe("GuardrailsPanel", () => {
|
|||
});
|
||||
|
||||
it("should render the component", async () => {
|
||||
render(<GuardrailsPanel {...defaultProps} />);
|
||||
renderWithProviders(<GuardrailsPanel {...defaultProps} />);
|
||||
expect(screen.getByText("Guardrails")).toBeInTheDocument();
|
||||
// Activate the Guardrails tab so its content (including the Add button) is rendered
|
||||
fireEvent.click(screen.getByText("Guardrails"));
|
||||
|
|
@ -120,7 +131,7 @@ describe("GuardrailsPanel", () => {
|
|||
});
|
||||
|
||||
it("should delete the clicked guardrail after confirming in the modal", async () => {
|
||||
render(<GuardrailsPanel {...defaultProps} />);
|
||||
renderWithProviders(<GuardrailsPanel {...defaultProps} />);
|
||||
fireEvent.click(screen.getByText("Guardrails"));
|
||||
|
||||
fireEvent.click(await screen.findByTestId("delete-button"));
|
||||
|
|
@ -139,14 +150,14 @@ describe("GuardrailsPanel", () => {
|
|||
});
|
||||
|
||||
it("should mount every tab panel up front so panel state survives tab switches", async () => {
|
||||
render(<GuardrailsPanel {...defaultProps} />);
|
||||
renderWithProviders(<GuardrailsPanel {...defaultProps} />);
|
||||
|
||||
expect(await screen.findByLabelText("playground draft")).toBeInTheDocument();
|
||||
expect(screen.getByText("Mock Team Guardrails Tab")).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("should keep test playground state when switching tabs away and back", async () => {
|
||||
render(<GuardrailsPanel {...defaultProps} />);
|
||||
renderWithProviders(<GuardrailsPanel {...defaultProps} />);
|
||||
|
||||
fireEvent.click(screen.getByText("Test Playground"));
|
||||
|
||||
|
|
@ -161,7 +172,7 @@ describe("GuardrailsPanel", () => {
|
|||
});
|
||||
|
||||
it("should not delete anything when the modal is cancelled", async () => {
|
||||
render(<GuardrailsPanel {...defaultProps} />);
|
||||
renderWithProviders(<GuardrailsPanel {...defaultProps} />);
|
||||
fireEvent.click(screen.getByText("Guardrails"));
|
||||
|
||||
fireEvent.click(await screen.findByTestId("delete-button"));
|
||||
|
|
@ -171,4 +182,42 @@ describe("GuardrailsPanel", () => {
|
|||
|
||||
expect(mockDeleteGuardrailCall).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
describe("guardrail detail deep link (?guardrail=)", () => {
|
||||
it("should open the guardrail info view directly from a ?guardrail= deep link", async () => {
|
||||
renderWithProviders(<GuardrailsPanel {...defaultProps} />, { searchParams: "?guardrail=test-guardrail-1" });
|
||||
|
||||
expect(await screen.findByTestId("guardrail-info-view")).toHaveTextContent("test-guardrail-1");
|
||||
expect(screen.queryByText("Mock Guardrail Table")).not.toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("should push ?guardrail= as a new history entry when a guardrail row is clicked", async () => {
|
||||
const onUrlUpdate = vi.fn<(event: UrlUpdateEvent) => void>();
|
||||
renderWithProviders(<GuardrailsPanel {...defaultProps} />, { onUrlUpdate });
|
||||
|
||||
fireEvent.click(await screen.findByTestId("open-button"));
|
||||
|
||||
await waitFor(() => expect(onUrlUpdate).toHaveBeenCalled());
|
||||
const lastUpdate = onUrlUpdate.mock.calls.at(-1)![0];
|
||||
expect(lastUpdate.searchParams.get("guardrail")).toBe("test-guardrail-1");
|
||||
expect(lastUpdate.options.history).toBe("push");
|
||||
expect(await screen.findByTestId("guardrail-info-view")).toHaveTextContent("test-guardrail-1");
|
||||
});
|
||||
|
||||
it("should clear ?guardrail= by replacing history when the info view is closed", async () => {
|
||||
const onUrlUpdate = vi.fn<(event: UrlUpdateEvent) => void>();
|
||||
renderWithProviders(<GuardrailsPanel {...defaultProps} />, {
|
||||
searchParams: "?guardrail=test-guardrail-1",
|
||||
onUrlUpdate,
|
||||
});
|
||||
|
||||
fireEvent.click(await screen.findByRole("button", { name: "Close Guardrail Info" }));
|
||||
|
||||
await waitFor(() => expect(onUrlUpdate).toHaveBeenCalled());
|
||||
const lastUpdate = onUrlUpdate.mock.calls.at(-1)![0];
|
||||
expect(lastUpdate.searchParams.has("guardrail")).toBe(false);
|
||||
expect(lastUpdate.options.history).toBe("replace");
|
||||
expect(await screen.findByText("Mock Guardrail Table")).toBeInTheDocument();
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
import { parseAsString, useQueryState } from "nuqs";
|
||||
import React, { useState, useEffect } from "react";
|
||||
import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs";
|
||||
import { ChevronDown, Code, Plus } from "lucide-react";
|
||||
|
|
@ -40,7 +41,10 @@ const GuardrailsPanel: React.FC<GuardrailsPanelProps> = ({ accessToken, userRole
|
|||
const [isDeleting, setIsDeleting] = useState(false);
|
||||
const [guardrailToDelete, setGuardrailToDelete] = useState<Guardrail | null>(null);
|
||||
const [isDeleteModalOpen, setIsDeleteModalOpen] = useState(false);
|
||||
const [selectedGuardrailId, setSelectedGuardrailId] = useState<string | null>(null);
|
||||
const [selectedGuardrailId, setSelectedGuardrailId] = useQueryState(
|
||||
"guardrail",
|
||||
parseAsString.withOptions({ history: "push" }),
|
||||
);
|
||||
const isAdmin = userRole ? isAdminRole(userRole) : false;
|
||||
|
||||
const fetchGuardrails = async () => {
|
||||
|
|
@ -63,16 +67,20 @@ const GuardrailsPanel: React.FC<GuardrailsPanelProps> = ({ accessToken, userRole
|
|||
fetchGuardrails();
|
||||
}, [accessToken]);
|
||||
|
||||
const closeGuardrailDetail = () => {
|
||||
void setSelectedGuardrailId(null, { history: "replace" });
|
||||
};
|
||||
|
||||
const handleAddGuardrail = () => {
|
||||
if (selectedGuardrailId) {
|
||||
setSelectedGuardrailId(null);
|
||||
closeGuardrailDetail();
|
||||
}
|
||||
setIsAddModalVisible(true);
|
||||
};
|
||||
|
||||
const handleAddCustomCodeGuardrail = () => {
|
||||
if (selectedGuardrailId) {
|
||||
setSelectedGuardrailId(null);
|
||||
closeGuardrailDetail();
|
||||
}
|
||||
setIsCustomCodeModalVisible(true);
|
||||
};
|
||||
|
|
@ -175,7 +183,7 @@ const GuardrailsPanel: React.FC<GuardrailsPanelProps> = ({ accessToken, userRole
|
|||
{selectedGuardrailId ? (
|
||||
<GuardrailInfoView
|
||||
guardrailId={selectedGuardrailId}
|
||||
onClose={() => setSelectedGuardrailId(null)}
|
||||
onClose={closeGuardrailDetail}
|
||||
accessToken={accessToken}
|
||||
isAdmin={isAdmin}
|
||||
/>
|
||||
|
|
@ -184,7 +192,7 @@ const GuardrailsPanel: React.FC<GuardrailsPanelProps> = ({ accessToken, userRole
|
|||
guardrailsList={guardrailsList}
|
||||
isLoading={isLoading}
|
||||
onDeleteClick={handleDeleteClick}
|
||||
onGuardrailClick={(id) => setSelectedGuardrailId(id)}
|
||||
onGuardrailClick={(id) => void setSelectedGuardrailId(id)}
|
||||
/>
|
||||
)}
|
||||
|
||||
|
|
|
|||
|
|
@ -338,3 +338,27 @@ describe("Guardrail Info", () => {
|
|||
expect(screen.getByText("Guardrail Settings")).toBeInTheDocument();
|
||||
});
|
||||
});
|
||||
|
||||
describe("Guardrail Info when the guardrail cannot be loaded", () => {
|
||||
afterEach(() => {
|
||||
vi.clearAllMocks();
|
||||
});
|
||||
|
||||
it("should keep Back to Guardrails reachable so a stale ?guardrail= link is not a dead end", async () => {
|
||||
vi.mocked(networking.getGuardrailInfo).mockRejectedValue(new Error("Guardrail stale-id not found"));
|
||||
vi.mocked(networking.getGuardrailUISettings).mockResolvedValue({
|
||||
supported_entities: [],
|
||||
supported_actions: [],
|
||||
pii_entity_categories: [],
|
||||
supported_modes: [],
|
||||
});
|
||||
vi.mocked(networking.getGuardrailProviderSpecificParams).mockResolvedValue({});
|
||||
const onClose = vi.fn();
|
||||
|
||||
render(<GuardrailInfoView guardrailId="stale-id" onClose={onClose} accessToken="123" isAdmin={true} />);
|
||||
|
||||
expect(await screen.findByText("Guardrail not found")).toBeInTheDocument();
|
||||
fireEvent.click(screen.getByRole("button", { name: /back to guardrails/i }));
|
||||
expect(onClose).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -481,16 +481,18 @@ const GuardrailInfoView: React.FC<GuardrailInfoProps> = ({ guardrailId, onClose,
|
|||
return <div className="p-4">Loading...</div>;
|
||||
}
|
||||
|
||||
const backButton = (
|
||||
<Button variant="ghost" onClick={onClose} className="mb-4">
|
||||
<ArrowLeft className="w-4 h-4" />
|
||||
Back to Guardrails
|
||||
</Button>
|
||||
);
|
||||
|
||||
if (!guardrailData) {
|
||||
return <div className="p-4">Guardrail not found</div>;
|
||||
return <div className="p-4">{backButton}Guardrail not found</div>;
|
||||
}
|
||||
|
||||
// Format date helper function
|
||||
const formatDate = (dateString?: string) => {
|
||||
if (!dateString) return "-";
|
||||
const date = new Date(dateString);
|
||||
return date.toLocaleString();
|
||||
};
|
||||
const formatDate = (dateString?: string) => (dateString ? new Date(dateString).toLocaleString() : "-");
|
||||
|
||||
// Format the provider display name and logo
|
||||
const { logo, displayName } = getGuardrailLogoAndName(guardrailData.litellm_params?.guardrail || "");
|
||||
|
|
@ -510,10 +512,7 @@ const GuardrailInfoView: React.FC<GuardrailInfoProps> = ({ guardrailId, onClose,
|
|||
return (
|
||||
<div className="p-4">
|
||||
<div>
|
||||
<Button variant="ghost" onClick={onClose} className="mb-4">
|
||||
<ArrowLeft className="w-4 h-4" />
|
||||
Back to Guardrails
|
||||
</Button>
|
||||
{backButton}
|
||||
<h1 className="text-2xl font-semibold">{guardrailData.guardrail_name || "Unnamed Guardrail"}</h1>
|
||||
<div className="flex items-center cursor-pointer">
|
||||
<p className="text-muted-foreground font-mono">{guardrailData.guardrail_id}</p>
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue