diff --git a/.github/workflows/test-rust.yml b/.github/workflows/test-rust.yml index 6f8599daf75..4bf41dc249c 100644 --- a/.github/workflows/test-rust.yml +++ b/.github/workflows/test-rust.yml @@ -130,7 +130,7 @@ jobs: - name: Test secret manager feature combinations run: | cargo test -p litellm-auth-gcp --locked --no-default-features - for features in '' aws google azure cyberark aws,google aws,azure google,azure aws,google,azure aws,google,cyberark aws,google,azure,cyberark; do + for features in '' aws google hashicorp azure cyberark aws,google aws,azure google,azure aws,google,azure aws,google,cyberark aws,google,azure,cyberark aws,google,hashicorp,azure,cyberark; do cargo test -p litellm-secrets --locked --no-default-features --features "$features" done diff --git a/cookbook/litellm_proxy_server/grafana_dashboard/dashboard_all_metrics/grafana_dashboard.json b/cookbook/litellm_proxy_server/grafana_dashboard/dashboard_all_metrics/grafana_dashboard.json index d8cb122417a..af88708166f 100644 --- a/cookbook/litellm_proxy_server/grafana_dashboard/dashboard_all_metrics/grafana_dashboard.json +++ b/cookbook/litellm_proxy_server/grafana_dashboard/dashboard_all_metrics/grafana_dashboard.json @@ -6267,6 +6267,63 @@ ], "title": "Spend update queue sizes (litellm__size)", "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Requests that carried usage but were logged at $0 on a model whose pricing entry has a non-zero rate, by requested model and reason", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "showPoints": "never", + "spanNulls": false + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 430 + }, + "id": 110, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(rate(litellm_zero_cost_requests_total[$__rate_interval])) by (requested_model, reason)", + "legendFormat": "{{requested_model}} / {{reason}}", + "range": true, + "refId": "A" + } + ], + "title": "litellm_zero_cost_requests rate", + "type": "timeseries" } ], "preload": false, diff --git a/docker/README.md b/docker/README.md index 26d8c9a37b0..376dc7b2d97 100644 --- a/docker/README.md +++ b/docker/README.md @@ -2,6 +2,17 @@ This guide provides instructions for building and running the LiteLLM application using Docker and Docker Compose. +> **Just want to run LiteLLM?** This guide builds from source. To run the published +> image instead, use `docker-compose.quickstart.yml` in this directory — the +> two-service stack (gateway + Postgres) that the +> [Docker quickstart](https://docs.litellm.ai/docs/proxy/docker_quick_start) documents: +> +> ```bash +> curl -sSLO https://github.com/BerriAI/litellm/raw/main/docker/docker-compose.quickstart.yml +> printf 'LITELLM_MASTER_KEY=sk-%s\nLITELLM_SALT_KEY=sk-%s\n' "$(openssl rand -hex 32)" "$(openssl rand -hex 32)" > .env +> docker compose -f docker-compose.quickstart.yml up -d +> ``` + ## Prerequisites - Docker diff --git a/docker/docker-compose.quickstart.yml b/docker/docker-compose.quickstart.yml new file mode 100644 index 00000000000..11631603a72 --- /dev/null +++ b/docker/docker-compose.quickstart.yml @@ -0,0 +1,41 @@ +# LiteLLM quickstart stack: the gateway plus a Postgres database that stores +# models, virtual keys, and spend logs. Used by +# https://docs.litellm.ai/docs/proxy/docker_quick_start +# +# curl -sSLO https://github.com/BerriAI/litellm/raw/main/docker/docker-compose.quickstart.yml +# printf 'LITELLM_MASTER_KEY=sk-%s\nLITELLM_SALT_KEY=sk-%s\n' "$(openssl rand -hex 32)" "$(openssl rand -hex 32)" > .env +# docker compose -f docker-compose.quickstart.yml up -d +# +# Compose reads .env from this directory. Keep it: regenerating LITELLM_SALT_KEY +# makes credentials already stored in the database unreadable. For anything +# beyond local evaluation, pin the image to a specific release tag. +services: + litellm: + image: docker.litellm.ai/berriai/litellm:main-stable + ports: + - "4000:4000" + environment: + LITELLM_MASTER_KEY: ${LITELLM_MASTER_KEY:?set it in .env - see the header of this file} + LITELLM_SALT_KEY: ${LITELLM_SALT_KEY:?set it in .env - see the header of this file} + DATABASE_URL: postgresql://litellm:litellm@db:5432/litellm + STORE_MODEL_IN_DB: "True" + depends_on: + db: + condition: service_healthy + + db: + image: postgres:16 + environment: + POSTGRES_USER: litellm + POSTGRES_PASSWORD: litellm + POSTGRES_DB: litellm + healthcheck: + test: ["CMD-SHELL", "pg_isready -U litellm"] + interval: 5s + timeout: 5s + retries: 10 + volumes: + - postgres_data:/var/lib/postgresql/data + +volumes: + postgres_data: diff --git a/gateway/routes/allowlist.py b/gateway/routes/allowlist.py index f7557983b91..01ba9da3364 100644 --- a/gateway/routes/allowlist.py +++ b/gateway/routes/allowlist.py @@ -96,10 +96,12 @@ GATEWAY_PATH_PREFIXES: tuple[str, ...] = ( "/assemblyai/", "/eu.assemblyai/", "/deepgram/", + "/fal_ai/", "/langfuse/", "/vllm/", "/mistral/", "/typesafe/", + "/openrouter/", "/nvidia_nim/", "/groq/", "/voyage/", diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20260917000000_add_agent_access_group_ids/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260917000000_add_agent_access_group_ids/migration.sql new file mode 100644 index 00000000000..d594b0056df --- /dev/null +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260917000000_add_agent_access_group_ids/migration.sql @@ -0,0 +1,2 @@ +-- AlterTable +ALTER TABLE "LiteLLM_AgentsTable" ADD COLUMN IF NOT EXISTS "access_group_ids" TEXT[] DEFAULT ARRAY[]::TEXT[]; diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20260919000000_add_autorouter_user_session_rollup/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260919000000_add_autorouter_user_session_rollup/migration.sql new file mode 100644 index 00000000000..2b864131ab2 --- /dev/null +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260919000000_add_autorouter_user_session_rollup/migration.sql @@ -0,0 +1,42 @@ +CREATE TABLE IF NOT EXISTS "LiteLLM_AutoRouterUserSession" ( + "user_id" TEXT NOT NULL, + "api_key" TEXT NOT NULL, + "session_id" TEXT NOT NULL, + "router_name" TEXT NOT NULL, + "router_type" TEXT NOT NULL, + "first_turn_at" TIMESTAMP(3) NOT NULL, + "last_turn_at" TIMESTAMP(3) NOT NULL, + "last_model" TEXT NOT NULL, + "models" JSONB NOT NULL DEFAULT '{}', + "turns" INTEGER NOT NULL DEFAULT 0, + "unordered_turns" INTEGER NOT NULL DEFAULT 0, + "covered_turns" INTEGER NOT NULL DEFAULT 0, + "cache_hits" INTEGER NOT NULL DEFAULT 0, + "same_model_turns" INTEGER NOT NULL DEFAULT 0, + "same_model_hits" INTEGER NOT NULL DEFAULT 0, + "first_visit_turns" INTEGER NOT NULL DEFAULT 0, + "first_visit_hits" INTEGER NOT NULL DEFAULT 0, + "return_turns" INTEGER NOT NULL DEFAULT 0, + "return_hits" INTEGER NOT NULL DEFAULT 0, + "return_expired_misses" INTEGER NOT NULL DEFAULT 0, + "return_within_ttl_misses" INTEGER NOT NULL DEFAULT 0, + "ttl_5m_turns" INTEGER NOT NULL DEFAULT 0, + "ttl_1h_turns" INTEGER NOT NULL DEFAULT 0, + "total_tokens" BIGINT NOT NULL DEFAULT 0, + "spend" DOUBLE PRECISION NOT NULL DEFAULT 0, + "saved_spend" DOUBLE PRECISION NOT NULL DEFAULT 0, + "savings_estimated_turns" INTEGER NOT NULL DEFAULT 0, + "savings_estimated_actual_spend" DOUBLE PRECISION NOT NULL DEFAULT 0, + "savings_estimated_saved_spend" DOUBLE PRECISION NOT NULL DEFAULT 0, + "savings_estimated_baseline_models" JSONB NOT NULL DEFAULT '{}', + "classifier_cost" DOUBLE PRECISION NOT NULL DEFAULT 0, + "classifier_cost_recorded_turns" INTEGER NOT NULL DEFAULT 0, + "tier_turns" JSONB NOT NULL DEFAULT '{}', + "baseline_models" JSONB NOT NULL DEFAULT '{}', + + CONSTRAINT "LiteLLM_AutoRouterUserSession_pkey" PRIMARY KEY ("user_id", "api_key", "session_id", "router_name") +); + +CREATE INDEX IF NOT EXISTS "idx_autorouter_user_session_last_turn" ON "LiteLLM_AutoRouterUserSession"("last_turn_at"); + +CREATE INDEX IF NOT EXISTS "idx_autorouter_user_session_user_last_turn" ON "LiteLLM_AutoRouterUserSession"("user_id", "last_turn_at"); diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20260921000000_add_password_reset_columns/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260921000000_add_password_reset_columns/migration.sql new file mode 100644 index 00000000000..960b0d4d7eb --- /dev/null +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260921000000_add_password_reset_columns/migration.sql @@ -0,0 +1,3 @@ +ALTER TABLE "LiteLLM_UserTable" ADD COLUMN IF NOT EXISTS "password_reset_required" BOOLEAN; + +ALTER TABLE "LiteLLM_UserTable" ADD COLUMN IF NOT EXISTS "last_breach_check_at" TIMESTAMP(3); diff --git a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma index 2d7e557a9d1..85996430bc5 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma +++ b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma @@ -73,6 +73,7 @@ model LiteLLM_AgentsTable { static_headers Json? @default("{}") extra_headers String[] @default([]) agent_access_groups String[] @default([]) + access_group_ids String[] @default([]) object_permission_id String? object_permission LiteLLM_ObjectPermissionTable? @relation(fields: [object_permission_id], references: [object_permission_id]) spend Float @default(0.0) @@ -246,6 +247,8 @@ model LiteLLM_UserTable { organization_id String? object_permission_id String? password String? + password_reset_required Boolean? + last_breach_check_at DateTime? teams String[] @default([]) user_role String? max_budget Float? @@ -1621,6 +1624,47 @@ model LiteLLM_AutoRouterSession { @@index([last_turn_at], map: "idx_autorouter_session_last_turn") } +model LiteLLM_AutoRouterUserSession { + user_id String + api_key String + session_id String + router_name String + router_type String + first_turn_at DateTime + last_turn_at DateTime + last_model String + models Json @default("{}") + turns Int @default(0) + unordered_turns Int @default(0) + covered_turns Int @default(0) + cache_hits Int @default(0) + same_model_turns Int @default(0) + same_model_hits Int @default(0) + first_visit_turns Int @default(0) + first_visit_hits Int @default(0) + return_turns Int @default(0) + return_hits Int @default(0) + return_expired_misses Int @default(0) + return_within_ttl_misses Int @default(0) + ttl_5m_turns Int @default(0) + ttl_1h_turns Int @default(0) + total_tokens BigInt @default(0) + spend Float @default(0) + saved_spend Float @default(0) + savings_estimated_turns Int @default(0) + savings_estimated_actual_spend Float @default(0) + savings_estimated_saved_spend Float @default(0) + savings_estimated_baseline_models Json @default("{}") + classifier_cost Float @default(0) + classifier_cost_recorded_turns Int @default(0) + tier_turns Json @default("{}") + baseline_models Json @default("{}") + + @@id([user_id, api_key, session_id, router_name]) + @@index([last_turn_at], map: "idx_autorouter_user_session_last_turn") + @@index([user_id, last_turn_at], map: "idx_autorouter_user_session_user_last_turn") +} + // Shadow eval: evaluation of an auto-router against one or more keys' live traffic, in // either direction. forward duplicates the requests the keys did not route through the // router through it, answering whether they should adopt it; reverse duplicates the diff --git a/litellm-rust/Cargo.lock b/litellm-rust/Cargo.lock index b37202dce1b..464edb3b104 100644 --- a/litellm-rust/Cargo.lock +++ b/litellm-rust/Cargo.lock @@ -40,6 +40,12 @@ dependencies = [ "cc", ] +[[package]] +name = "allocator-api2" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" + [[package]] name = "android_system_properties" version = "0.1.6" @@ -230,6 +236,7 @@ dependencies = [ "aws-credential-types", "aws-sigv4", "aws-smithy-async", + "aws-smithy-eventstream", "aws-smithy-http", "aws-smithy-runtime", "aws-smithy-runtime-api", @@ -238,7 +245,9 @@ dependencies = [ "bytes", "bytes-utils", "fastrand", + "http 0.2.12", "http 1.4.2", + "http-body 0.4.6", "http-body 1.1.0", "percent-encoding", "pin-project-lite", @@ -272,6 +281,43 @@ dependencies = [ "tracing", ] +[[package]] +name = "aws-sdk-s3" +version = "1.146.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2cd651b4400d4011b8927b83a9552bf90ff11e6e5da0b9f0a7583247aceec971" +dependencies = [ + "arc-swap", + "aws-credential-types", + "aws-runtime", + "aws-sigv4", + "aws-smithy-async", + "aws-smithy-checksums", + "aws-smithy-eventstream", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-observability", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-schema", + "aws-smithy-types", + "aws-smithy-xml 0.62.1", + "aws-types", + "bytes", + "fastrand", + "hex", + "hmac", + "http 0.2.12", + "http 1.4.2", + "http-body 1.1.0", + "lru", + "percent-encoding", + "regex-lite", + "sha2 0.11.0", + "tracing", + "url", +] + [[package]] name = "aws-sdk-secretsmanager" version = "1.117.0" @@ -316,7 +362,7 @@ dependencies = [ "aws-smithy-runtime-api", "aws-smithy-schema", "aws-smithy-types", - "aws-smithy-xml", + "aws-smithy-xml 0.61.1", "aws-types", "fastrand", "http 0.2.12", @@ -332,6 +378,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "31d955e76ff96acd555bf06fa0fa6d5bf9335fa84ae7c64481b20ae61d231f70" dependencies = [ "aws-credential-types", + "aws-smithy-eventstream", "aws-smithy-http", "aws-smithy-runtime-api", "aws-smithy-types", @@ -359,10 +406,31 @@ dependencies = [ ] [[package]] -name = "aws-smithy-eventstream" -version = "0.61.1" +name = "aws-smithy-checksums" +version = "0.65.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a9381123ab62d20c13082b151f30f962a3b112b727345394536dfa39a482944" +checksum = "b67ecd999972b58e67cab052f5129906c08c25883bd0788ceefc55ef97d61307" +dependencies = [ + "aws-smithy-http", + "aws-smithy-types", + "bytes", + "crc-fast", + "hex", + "http 1.4.2", + "http-body 1.1.0", + "http-body-util", + "md-5", + "pin-project-lite", + "sha1 0.11.0", + "sha2 0.11.0", + "tracing", +] + +[[package]] +name = "aws-smithy-eventstream" +version = "0.61.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "80c2051c2f1016fb8e6548dd07b8bc2ac9c3fe583721444b92f515e856d31609" dependencies = [ "aws-smithy-types", "bytes", @@ -375,6 +443,7 @@ version = "0.64.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37843d9add67c3aff5856f409c6dc315d3cdff60f9c0cb5b670dab1e9920306d" dependencies = [ + "aws-smithy-eventstream", "aws-smithy-runtime-api", "aws-smithy-types", "bytes", @@ -554,6 +623,18 @@ dependencies = [ "xmlparser", ] +[[package]] +name = "aws-smithy-xml" +version = "0.62.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b932c8d6dc127fc980eecd78f8694ae9b9551b69a93a7def2a199c1c0033daf" +dependencies = [ + "aws-smithy-runtime-api", + "aws-smithy-schema", + "aws-smithy-types", + "xmlparser", +] + [[package]] name = "aws-types" version = "1.6.0" @@ -569,6 +650,49 @@ dependencies = [ "tracing", ] +[[package]] +name = "axum" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31b698c5f9a010f6573133b09e0de5408834d0c82f8d7475a89fc1867a71cd90" +dependencies = [ + "axum-core", + "bytes", + "futures-util", + "http 1.4.2", + "http-body 1.1.0", + "http-body-util", + "itoa", + "matchit", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "serde_core", + "sync_wrapper", + "tower", + "tower-layer", + "tower-service", +] + +[[package]] +name = "axum-core" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08c78f31d7b1291f7ee735c1c6780ccde7785daae9a9206026862dab7d8792d1" +dependencies = [ + "bytes", + "futures-core", + "http 1.4.2", + "http-body 1.1.0", + "http-body-util", + "mime", + "pin-project-lite", + "sync_wrapper", + "tower-layer", + "tower-service", +] + [[package]] name = "azure_core" version = "1.1.0" @@ -980,6 +1104,16 @@ dependencies = [ "libc", ] +[[package]] +name = "crc-fast" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e75b2483e97a5a7da73ac68a05b629f9c53cff58d8ed1c77866079e18b00dba5" +dependencies = [ + "digest 0.10.7", + "spin", +] + [[package]] name = "crc16" version = "0.4.0" @@ -1377,6 +1511,18 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "fallible-iterator" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649" + +[[package]] +name = "fallible-streaming-iterator" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" + [[package]] name = "fancy-regex" version = "0.17.0" @@ -1428,6 +1574,12 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "foldhash" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" + [[package]] name = "form_urlencoded" version = "1.2.2" @@ -1892,11 +2044,34 @@ version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" +[[package]] +name = "hashbrown" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +dependencies = [ + "foldhash", +] + [[package]] name = "hashbrown" version = "0.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash", +] + +[[package]] +name = "hashlink" +version = "0.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a596f1b20ed2cc5ecac41a164aaebc7258057060f06c0cf7a2ba3991ee7990fb" +dependencies = [ + "hashbrown 0.17.1", +] [[package]] name = "heck" @@ -2277,6 +2452,12 @@ version = "2.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2" +[[package]] +name = "iter-read" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071ed4cc1afd86650602c7b11aa2e1ce30762a1c27193201cb5cee9c6ebb1294" + [[package]] name = "itertools" version = "0.13.0" @@ -2435,6 +2616,17 @@ version = "0.2.186" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" +[[package]] +name = "libsqlite3-sys" +version = "0.38.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1d20bef17f513b9b3004532233187769cd072d790971f4e4da0e346eb6401e8" +dependencies = [ + "cc", + "pkg-config", + "vcpkg", +] + [[package]] name = "linux-raw-sys" version = "0.12.1" @@ -2540,6 +2732,36 @@ dependencies = [ "url", ] +[[package]] +name = "litellm-cache-disk" +version = "0.1.0" +dependencies = [ + "litellm-cache", + "py_literal", + "rand 0.8.7", + "rstest", + "rusqlite", + "serde-pickle", + "serde_json", + "tempfile", + "tokio", +] + +[[package]] +name = "litellm-cache-gcs" +version = "0.1.0" +dependencies = [ + "futures-util", + "litellm-auth-gcp", + "litellm-auth-types", + "litellm-cache", + "percent-encoding", + "reqwest 0.12.28", + "serde_json", + "tokio", + "wiremock", +] + [[package]] name = "litellm-cache-memory" version = "0.1.0" @@ -2550,6 +2772,26 @@ dependencies = [ "tokio", ] +[[package]] +name = "litellm-cache-qdrant-semantic" +version = "0.1.0" +dependencies = [ + "futures-util", + "litellm-cache", + "litellm-cache-response", + "qdrant-client", + "reqwest 0.12.28", + "rstest", + "serde", + "serde_json", + "thiserror 2.0.19", + "tokio", + "tokio-stream", + "tonic", + "tonic-prost", + "uuid", +] + [[package]] name = "litellm-cache-redis" version = "0.1.0" @@ -2562,6 +2804,21 @@ dependencies = [ "tokio", ] +[[package]] +name = "litellm-cache-redis-semantic" +version = "0.1.0" +dependencies = [ + "litellm-cache", + "litellm-cache-redis", + "litellm-cache-response", + "r2d2", + "redis", + "redis-test", + "serde_json", + "sha2 0.10.9", + "tokio", +] + [[package]] name = "litellm-cache-response" version = "0.1.0" @@ -2578,6 +2835,37 @@ dependencies = [ "tokio", ] +[[package]] +name = "litellm-cache-s3" +version = "0.1.0" +dependencies = [ + "aws-credential-types", + "aws-sdk-s3", + "aws-smithy-types", + "aws-types", + "litellm-auth-aws", + "litellm-cache", + "serde_json", + "tokio", + "wiremock", +] + +[[package]] +name = "litellm-cache-valkey-semantic" +version = "0.1.0" +dependencies = [ + "litellm-cache", + "litellm-cache-redis", + "litellm-cache-response", + "redis", + "redis-test", + "rstest", + "serde_json", + "sha2 0.10.9", + "tokio", + "uuid", +] + [[package]] name = "litellm-callbacks-legacy-python" version = "0.1.0" @@ -2740,12 +3028,19 @@ dependencies = [ "criterion", "futures-util", "litellm-auth", + "litellm-auth-aws", "litellm-auth-gcp", "litellm-cache", "litellm-cache-azure-blob", + "litellm-cache-disk", + "litellm-cache-gcs", "litellm-cache-memory", + "litellm-cache-qdrant-semantic", "litellm-cache-redis", + "litellm-cache-redis-semantic", "litellm-cache-response", + "litellm-cache-s3", + "litellm-cache-valkey-semantic", "litellm-callbacks-legacy-python", "litellm-core", "litellm-core-utils", @@ -2756,12 +3051,17 @@ dependencies = [ "litellm-types", "pyo3", "pyo3-async-runtimes", + "qdrant-client", + "redis", + "reqwest 0.12.28", "rstest", "serde", "serde_json", "serde_with", + "sha2 0.10.9", "tokio", "tokio-tungstenite", + "url", ] [[package]] @@ -2778,6 +3078,7 @@ dependencies = [ "litellm-secrets-azure", "litellm-secrets-cyberark", "litellm-secrets-google", + "litellm-secrets-hashicorp", "litellm-secrets-types", "moka", "reqwest 0.12.28", @@ -2875,6 +3176,26 @@ dependencies = [ "wiremock", ] +[[package]] +name = "litellm-secrets-hashicorp" +version = "0.1.0" +dependencies = [ + "litellm-core-utils", + "litellm-secrets-types", + "moka", + "rstest", + "rustify", + "rustify_derive", + "serde", + "serde_json", + "tempfile", + "thiserror 2.0.19", + "tokio", + "vaultrs", + "veil", + "wiremock", +] + [[package]] name = "litellm-secrets-types" version = "0.1.0" @@ -2966,6 +3287,15 @@ version = "0.4.33" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" +[[package]] +name = "lru" +version = "0.18.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff9840bcc50b71349309900da0ce7279aa336ae71d73250b07998932c7d97c25" +dependencies = [ + "hashbrown 0.17.1", +] + [[package]] name = "lru-slab" version = "0.1.2" @@ -2988,6 +3318,22 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc04a4c58212d57930a24bf47d3fa87485264a3a054e9c10e042eb373573ad3c" +[[package]] +name = "matchit" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" + +[[package]] +name = "md-5" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69b6441f590336821bb897fb28fc622898ccceb1d6cea3fde5ea86b090c4de98" +dependencies = [ + "cfg-if", + "digest 0.11.3", +] + [[package]] name = "memchr" version = "2.8.3" @@ -3601,6 +3947,27 @@ dependencies = [ "serde", ] +[[package]] +name = "qdrant-client" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dddc19df129bad7346ebd027288621ab1ac7e52678371f906b9a8622d7aaf87e" +dependencies = [ + "anyhow", + "derive_builder", + "futures", + "parking_lot", + "prost", + "prost-types", + "semver", + "serde", + "serde_json", + "thiserror 2.0.19", + "tokio", + "tonic", + "tonic-prost", +] + [[package]] name = "quick-error" version = "1.2.3" @@ -4033,6 +4400,16 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "rsqlite-vfs" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c51c9ae4df8a7fba42103df5c621fa3c37eccf3a3c650879e90fc48b11cc192c" +dependencies = [ + "hashbrown 0.16.1", + "thiserror 2.0.19", +] + [[package]] name = "rstest" version = "0.26.1" @@ -4073,6 +4450,21 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "rusqlite" +version = "0.40.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23f2a97da3e3873c73cb2a2e71b35c40ff95e0b1eefa8d72d8499a6928c3b5b3" +dependencies = [ + "bitflags 2.13.1", + "fallible-iterator", + "fallible-streaming-iterator", + "hashlink", + "libsqlite3-sys", + "smallvec", + "sqlite-wasm-rs", +] + [[package]] name = "rustc-hash" version = "2.1.3" @@ -4088,6 +4480,40 @@ dependencies = [ "semver", ] +[[package]] +name = "rustify" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4800ce4c1cc2fec12c559dae2ddbf0e17fcee7569b796e6d75898efef443368b" +dependencies = [ + "anyhow", + "async-trait", + "bytes", + "http 1.4.2", + "reqwest 0.13.5", + "rustify_derive", + "serde", + "serde_json", + "serde_urlencoded", + "thiserror 1.0.69", + "tracing", + "url", +] + +[[package]] +name = "rustify_derive" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78ea7fda74240f7410d0198b603a8a2f662acc7d76b6667a49f9b162cd8d9b4f" +dependencies = [ + "proc-macro2", + "quote", + "regex", + "serde_urlencoded", + "syn 1.0.109", + "synstructure 0.12.6", +] + [[package]] name = "rustix" version = "1.1.5" @@ -4330,6 +4756,19 @@ dependencies = [ "serde_derive", ] +[[package]] +name = "serde-pickle" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b641fdc8bcf2781ee78b30c599700d64ad4f412976143e4c5d0b9df906bb4843" +dependencies = [ + "byteorder", + "iter-read", + "num-bigint 0.4.8", + "num-traits", + "serde", +] + [[package]] name = "serde_core" version = "1.0.229" @@ -4429,6 +4868,17 @@ dependencies = [ "digest 0.10.7", ] +[[package]] +name = "sha1" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aacc4cc499359472b4abe1bf11d0b12e688af9a805fa5e3016f9a386dc2d0214" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", + "digest 0.11.3", +] + [[package]] name = "sha1_smol" version = "1.0.1" @@ -4545,6 +4995,12 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "spin" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "023a211cb3138dbc438680b32560ad89f699977624c9f8dbb95a47d5b4c07dd3" + [[package]] name = "spm_precompiled" version = "0.1.4" @@ -4557,6 +5013,18 @@ dependencies = [ "unicode-segmentation", ] +[[package]] +name = "sqlite-wasm-rs" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc3efc0da82635d7e1ced0053bbbfa8c7ab9645d0bf36ceb4f7127bb85315d75" +dependencies = [ + "cc", + "js-sys", + "rsqlite-vfs", + "wasm-bindgen", +] + [[package]] name = "sse-stream" version = "0.2.6" @@ -4615,6 +5083,17 @@ version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" +[[package]] +name = "syn" +version = "1.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + [[package]] name = "syn" version = "2.0.119" @@ -4646,6 +5125,18 @@ dependencies = [ "futures-core", ] +[[package]] +name = "synstructure" +version = "0.12.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f36bdaa60a83aca3921b5259d5400cbf5e90fc51931376a9bd4a0eb79aa7210f" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", + "unicode-xid", +] + [[package]] name = "synstructure" version = "0.13.2" @@ -4968,8 +5459,12 @@ version = "0.14.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac2a5518c70fa84342385732db33fb3f44bc4cc748936eb5833d2df34d6445ef" dependencies = [ + "async-trait", + "axum", "base64 0.22.1", "bytes", + "flate2", + "h2 0.4.15", "http 1.4.2", "http-body 1.1.0", "http-body-util", @@ -4979,6 +5474,7 @@ dependencies = [ "percent-encoding", "pin-project", "rustls-native-certs", + "socket2 0.6.5", "sync_wrapper", "tokio", "tokio-rustls 0.26.4", @@ -5060,6 +5556,7 @@ version = "0.1.44" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" dependencies = [ + "log", "pin-project-lite", "tracing-attributes", "tracing-core", @@ -5142,7 +5639,7 @@ dependencies = [ "rand 0.8.7", "rustls 0.23.42", "rustls-pki-types", - "sha1", + "sha1 0.10.7", "thiserror 1.0.69", "utf-8", ] @@ -5244,6 +5741,12 @@ version = "1.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8" +[[package]] +name = "unicode-xid" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" + [[package]] name = "unicode_categories" version = "0.1.1" @@ -5303,6 +5806,31 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" +[[package]] +name = "vaultrs" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30ffcc0e81025065dda612ec1e26a3d81bb16ef3062354873d17a35965d68522" +dependencies = [ + "async-trait", + "derive_builder", + "http 1.4.2", + "reqwest 0.13.5", + "rustify", + "rustify_derive", + "serde", + "serde_json", + "thiserror 2.0.19", + "tracing", + "url", +] + +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + [[package]] name = "veil" version = "0.3.0" @@ -5746,7 +6274,7 @@ dependencies = [ "proc-macro2", "quote", "syn 2.0.119", - "synstructure", + "synstructure 0.13.2", ] [[package]] @@ -5787,7 +6315,7 @@ dependencies = [ "proc-macro2", "quote", "syn 2.0.119", - "synstructure", + "synstructure 0.13.2", ] [[package]] diff --git a/litellm-rust/Cargo.toml b/litellm-rust/Cargo.toml index 7d94d31db3e..0e8941cb8e6 100644 --- a/litellm-rust/Cargo.toml +++ b/litellm-rust/Cargo.toml @@ -22,6 +22,7 @@ litellm-secrets = { path = "crates/secrets" } litellm-secrets-types = { path = "crates/secrets-types" } litellm-secrets-aws = { path = "crates/secrets-aws" } litellm-secrets-google = { path = "crates/secrets-google" } +litellm-secrets-hashicorp = { path = "crates/secrets-hashicorp" } litellm-secrets-azure = { path = "crates/secrets-azure" } litellm-secrets-cyberark = { path = "crates/secrets-cyberark" } litellm-http = { path = "crates/http" } @@ -32,7 +33,12 @@ litellm-cache = { path = "crates/cache" } litellm-cache-azure-blob = { path = "crates/cache-azure-blob" } litellm-cache-memory = { path = "crates/cache-memory" } litellm-cache-redis = { path = "crates/cache-redis" } +litellm-cache-s3 = { path = "crates/cache-s3" } +litellm-cache-gcs = { path = "crates/cache-gcs" } +litellm-cache-disk = { path = "crates/cache-disk" } +litellm-cache-redis-semantic = { path = "crates/cache-redis-semantic" } litellm-cache-response = { path = "crates/cache-response" } +litellm-cache-qdrant-semantic = { path = "crates/cache-qdrant-semantic" } litellm-token-counter = { path = "crates/token-counter" } litellm-token-counter-fast = { path = "crates/token-counter-fast" } litellm-token-counter-huggingface = { path = "crates/token-counter-huggingface" } @@ -50,9 +56,14 @@ pyo3-async-runtimes = { version = "0.29.0", features = ["tokio-runtime"] } pythonize = "0.29.0" rand = "0.8" reqwest = { version = "0.12", default-features = false, features = ["json", "multipart", "rustls-tls", "http2", "stream"] } +qdrant-client = { version = "1.19.0", default-features = false } +uuid = { version = "1", features = ["v4"] } rstest = "0.26.1" rstest_reuse = "0.7.0" rustls = { version = "0.23", default-features = false, features = ["ring", "std", "tls12"] } +rustify = "=0.7.0" +rustify_derive = "=0.5.5" +vaultrs = { version = "=0.8.0", default-features = false, features = ["rustls"] } rustls-native-certs = "0.8" serde = { version = "1.0", features = ["derive"] } serde_json = { version = "1.0", features = ["float_roundtrip"] } @@ -69,6 +80,7 @@ base64 = "0.22" moka = { version = "0.12.16", features = ["future"] } strum = { version = "0.28.0", features = ["derive"] } url = "2.5.8" +percent-encoding = "2.3" webpki-roots = "1" time = { version = "0.3.53", features = ["parsing"] } criterion = "0.8.2" diff --git a/litellm-rust/crates/auth-gcp/src/lib.rs b/litellm-rust/crates/auth-gcp/src/lib.rs index 8aeddae9efc..534d85acdb0 100644 --- a/litellm-rust/crates/auth-gcp/src/lib.rs +++ b/litellm-rust/crates/auth-gcp/src/lib.rs @@ -128,6 +128,14 @@ impl VertexAuth { } } + pub async fn access_token( + &self, + config: &VertexConfig, + env_lookup: &(dyn Fn(&str) -> Option + Sync), + ) -> Result { + self.load_provider(config, env_lookup).await?.token().await + } + pub async fn validate_environment( &self, headers: Vec<(String, String)>, diff --git a/litellm-rust/crates/cache-disk/Cargo.toml b/litellm-rust/crates/cache-disk/Cargo.toml new file mode 100644 index 00000000000..b96994b3b55 --- /dev/null +++ b/litellm-rust/crates/cache-disk/Cargo.toml @@ -0,0 +1,19 @@ +[package] +name = "litellm-cache-disk" +version = "0.1.0" +edition.workspace = true +license.workspace = true +repository.workspace = true + +[dependencies] +litellm-cache.workspace = true +py_literal = "0.4.0" +rand.workspace = true +rusqlite = { version = "0.40", features = ["bundled"] } +serde-pickle = "1.2" +serde_json.workspace = true +tokio.workspace = true + +[dev-dependencies] +rstest.workspace = true +tempfile = "3.27.0" diff --git a/litellm-rust/crates/cache-disk/src/adapter.rs b/litellm-rust/crates/cache-disk/src/adapter.rs new file mode 100644 index 00000000000..b6d318d5509 --- /dev/null +++ b/litellm-rust/crates/cache-disk/src/adapter.rs @@ -0,0 +1,10 @@ +use litellm_cache::Error; + +use crate::StoredValue; + +pub trait ValueAdapter: Send + Sync + 'static { + fn read(&self, value: StoredValue) -> Result>, Error>; + fn write(&self, payload: Vec) -> StoredValue; + fn counter_seed(&self, value: Option) -> Result; + fn counter_value(&self, value: f64) -> StoredValue; +} diff --git a/litellm-rust/crates/cache-disk/src/cache.rs b/litellm-rust/crates/cache-disk/src/cache.rs new file mode 100644 index 00000000000..8e1223309b4 --- /dev/null +++ b/litellm-rust/crates/cache-disk/src/cache.rs @@ -0,0 +1,301 @@ +use std::{ + path::Path, + sync::Arc, + time::{Duration, SystemTime, UNIX_EPOCH}, +}; + +use litellm_cache::{ + BaseCache, BatchCache, BatchEntry, CacheCodec, CacheConnectionResult, CacheConnectionStatus, + CounterCache, DeleteCache, Error, ExactCacheContext, FlushCache, +}; + +use crate::{DiskStore, DiskcacheSqliteStore, PythonDiskCacheAdapter, StoredValue, ValueAdapter}; + +pub struct DiskCache { + store: Arc, + adapter: Arc, + codec: S, +} + +impl DiskCache { + pub fn open(directory: impl AsRef, codec: S) -> Result { + Ok(Self { + store: Arc::new(DiskcacheSqliteStore::open(directory)?), + adapter: Arc::new(PythonDiskCacheAdapter), + codec, + }) + } +} + +impl DiskCache { + pub fn with_store(store: D, codec: S) -> Self { + Self { + store: Arc::new(store), + adapter: Arc::new(PythonDiskCacheAdapter), + codec, + } + } +} + +impl DiskCache { + pub fn with_adapter(store: D, adapter: A, codec: S) -> Self { + Self { + store: Arc::new(store), + adapter: Arc::new(adapter), + codec, + } + } + + pub fn directory(&self) -> &Path { + self.store.directory() + } + + fn decode_stored(&self, value: StoredValue) -> Result, Error> { + let Some(bytes) = self.adapter.read(value)? else { + return Ok(None); + }; + self.codec.decode(&bytes).map(Some) + } + + async fn run_blocking(store: Arc, operation: F) -> Result + where + T: Send + 'static, + F: FnOnce(&D) -> Result + Send + 'static, + { + tokio::task::spawn_blocking(move || operation(&store)) + .await + .map_err(|_| Error::Unavailable)? + } +} + +impl BaseCache for DiskCache { + type Value = S::Value; + type Context = ExactCacheContext; + + fn get_ttl(&self, context: &Self::Context) -> Option { + context.ttl + } + + fn set_cache( + &self, + key: &str, + value: Self::Value, + context: &Self::Context, + ) -> Result<(), Error> { + let value = self.adapter.write(self.codec.encode(&value)?); + let expire_time = context.ttl.map(|ttl| unix_now() + ttl.as_secs_f64()); + self.store.set(key, value, expire_time, unix_now()) + } + + fn get_cache(&self, key: &str, _: &Self::Context) -> Result, Error> { + self.store + .get(key, unix_now())? + .map(|value| self.decode_stored(value)) + .transpose() + .map(|value| value.flatten()) + } + + async fn async_set_cache( + &self, + key: &str, + value: Self::Value, + context: ExactCacheContext, + ) -> Result<(), Error> { + let value = self.adapter.write(self.codec.encode(&value)?); + let ttl = context.ttl; + let key = key.to_string(); + Self::run_blocking(Arc::clone(&self.store), move |store| { + let expire_time = ttl.map(|ttl| unix_now() + ttl.as_secs_f64()); + store.set(&key, value, expire_time, unix_now()) + }) + .await + } + + async fn async_get_cache( + &self, + key: &str, + _: &ExactCacheContext, + ) -> Result, Error> { + let key = key.to_string(); + let value = Self::run_blocking(Arc::clone(&self.store), move |store| { + store.get(&key, unix_now()) + }) + .await?; + value + .map(|value| self.decode_stored(value)) + .transpose() + .map(|value| value.flatten()) + } + + async fn async_set_cache_pipeline( + &self, + entries: Vec<(String, Self::Value)>, + context: ExactCacheContext, + ) -> Result<(), Error> { + let entries = entries + .into_iter() + .map(|(key, value)| { + self.codec + .encode(&value) + .map(|value| (key, self.adapter.write(value))) + }) + .collect::, _>>()?; + let expire_after = context.ttl; + Self::run_blocking(Arc::clone(&self.store), move |store| { + for (key, value) in entries { + let expire_time = expire_after.map(|ttl| unix_now() + ttl.as_secs_f64()); + store.set(&key, value, expire_time, unix_now())?; + } + Ok(()) + }) + .await + } + + async fn disconnect(&self) -> Result<(), Error> { + Ok(()) + } + + async fn test_connection(&self) -> Result { + let result = Self::run_blocking(Arc::clone(&self.store), |store| { + store.probe().map(|_| CacheConnectionResult { + status: CacheConnectionStatus::Success, + message: "Disk cache connection test successful".into(), + error: None, + }) + }) + .await; + Ok(match result { + Ok(result) => result, + Err(error) => CacheConnectionResult { + status: CacheConnectionStatus::Failed, + message: format!("Disk cache connection failed: {error}"), + error: Some(error.to_string()), + }, + }) + } +} + +impl BatchCache for DiskCache { + fn batch_get_cache( + &self, + keys: &[String], + context: &ExactCacheContext, + ) -> Result>, Error> { + keys.iter() + .map(|key| match self.get_cache(key, context) { + Ok(Some(value)) => Ok(BatchEntry::Hit(value)), + Ok(None) => Ok(BatchEntry::Miss), + Err(Error::InvalidEntry) => Ok(BatchEntry::Invalid), + Err(error) => Err(error), + }) + .collect() + } + + async fn async_batch_get_cache( + &self, + keys: Vec, + _: ExactCacheContext, + ) -> Result>, Error> { + let values = Self::run_blocking(Arc::clone(&self.store), move |store| { + keys.into_iter() + .map(|key| store.get(&key, unix_now()).map(|value| (key, value))) + .collect::, _>>() + }) + .await?; + values + .into_iter() + .map(|(_, value)| match value { + None => Ok(BatchEntry::Miss), + Some(value) => match self.decode_stored(value) { + Ok(Some(value)) => Ok(BatchEntry::Hit(value)), + Ok(None) => Ok(BatchEntry::Miss), + Err(Error::InvalidEntry) => Ok(BatchEntry::Invalid), + Err(error) => Err(error), + }, + }) + .collect() + } +} + +impl DeleteCache for DiskCache { + fn delete_cache(&self, key: &str) -> Result<(), Error> { + self.store.pop(key, unix_now()).map(|_| ()) + } + + async fn async_delete_cache(&self, key: &str) -> Result<(), Error> { + let key = key.to_string(); + Self::run_blocking(Arc::clone(&self.store), move |store| { + store.pop(&key, unix_now()).map(|_| ()) + }) + .await + } +} + +impl FlushCache for DiskCache { + fn flush_cache(&self) -> Result<(), Error> { + self.store.clear() + } + + async fn async_flush_cache(&self) -> Result<(), Error> { + Self::run_blocking(Arc::clone(&self.store), |store| store.clear()).await + } +} + +impl, D: DiskStore, A: ValueAdapter> CounterCache + for DiskCache +{ + fn increment_cache( + &self, + key: &str, + amount: f64, + context: ExactCacheContext, + ) -> Result { + increment( + self.adapter.as_ref(), + self.store.as_ref(), + key, + amount, + context.ttl, + ) + } + + async fn async_increment( + &self, + key: &str, + amount: f64, + context: ExactCacheContext, + ) -> Result { + let key = key.to_string(); + let adapter = Arc::clone(&self.adapter); + Self::run_blocking(Arc::clone(&self.store), move |store| { + increment(adapter.as_ref(), store, &key, amount, context.ttl) + }) + .await + } +} + +fn increment( + adapter: &A, + store: &D, + key: &str, + amount: f64, + ttl: Option, +) -> Result { + let mut result = None; + let mut apply = |current: Option| { + let initial = adapter.counter_seed(current)?; + let value = initial + amount; + let stored = adapter.counter_value(value); + result = Some(value); + Ok((stored, ttl.map(|ttl| unix_now() + ttl.as_secs_f64()))) + }; + store.update(key, unix_now(), &mut apply)?; + result.ok_or(Error::InvalidEntry) +} + +fn unix_now() -> f64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_secs_f64() +} diff --git a/litellm-rust/crates/cache-disk/src/lib.rs b/litellm-rust/crates/cache-disk/src/lib.rs new file mode 100644 index 00000000000..9b2ffc24915 --- /dev/null +++ b/litellm-rust/crates/cache-disk/src/lib.rs @@ -0,0 +1,11 @@ +mod adapter; +mod cache; +mod python; +mod sqlite; +mod store; + +pub use adapter::ValueAdapter; +pub use cache::DiskCache; +pub use python::PythonDiskCacheAdapter; +pub use sqlite::DiskcacheSqliteStore; +pub use store::{DiskStore, StoredValue}; diff --git a/litellm-rust/crates/cache-disk/src/python/mod.rs b/litellm-rust/crates/cache-disk/src/python/mod.rs new file mode 100644 index 00000000000..7a370db357c --- /dev/null +++ b/litellm-rust/crates/cache-disk/src/python/mod.rs @@ -0,0 +1,77 @@ +mod value; + +use litellm_cache::Error; +use py_literal::Value; + +use crate::{StoredValue, ValueAdapter}; + +#[derive(Clone, Copy, Debug, Default)] +pub struct PythonDiskCacheAdapter; + +impl PythonDiskCacheAdapter { + fn python_get_cache(value: StoredValue) -> Result, Error> { + let value = match value { + StoredValue::Bytes(value) => Value::Bytes(value), + StoredValue::Text(value) => Value::String(value), + StoredValue::Integer(value) => Value::Integer(value.into()), + StoredValue::Float(value) => Value::Float(value), + StoredValue::Pickle(value) => value::from_pickle(&value)?, + }; + if !value::is_truthy(&value) { + return Ok(None); + } + match value { + Value::String(text) => Ok(Some( + value::from_json_text(&text).unwrap_or(Value::String(text)), + )), + Value::Bytes(bytes) => match std::str::from_utf8(&bytes) { + Ok(text) => Ok(Some( + value::from_json_text(text).unwrap_or(Value::Bytes(bytes)), + )), + Err(_) => Ok(Some(Value::Bytes(bytes))), + }, + value => Ok(Some(value)), + } + } +} + +impl ValueAdapter for PythonDiskCacheAdapter { + fn read(&self, value: StoredValue) -> Result>, Error> { + match value { + StoredValue::Text(value) => Ok((!value.is_empty()).then(|| value.into_bytes())), + StoredValue::Bytes(value) => Ok((!value.is_empty()).then_some(value)), + value => { + let Some(value) = Self::python_get_cache(value)? else { + return Ok(None); + }; + value::to_json(&value).map(Some) + } + } + } + + fn write(&self, payload: Vec) -> StoredValue { + StoredValue::Bytes(payload) + } + + fn counter_seed(&self, value: Option) -> Result { + let Some(value) = value else { + return Ok(0.0); + }; + let Some(value) = Self::python_get_cache(value)? else { + return Ok(0.0); + }; + Ok(if value::is_int(&value) { + value::to_f64(&value).unwrap_or(0.0) + } else { + 0.0 + }) + } + + fn counter_value(&self, value: f64) -> StoredValue { + if value.fract() == 0.0 && value >= i64::MIN as f64 && value <= i64::MAX as f64 { + StoredValue::Integer(value as i64) + } else { + StoredValue::Float(value) + } + } +} diff --git a/litellm-rust/crates/cache-disk/src/python/value.rs b/litellm-rust/crates/cache-disk/src/python/value.rs new file mode 100644 index 00000000000..645eba7b757 --- /dev/null +++ b/litellm-rust/crates/cache-disk/src/python/value.rs @@ -0,0 +1,173 @@ +use litellm_cache::Error; +use py_literal::Value; +use serde_json::{Map, Number}; + +pub(crate) fn from_pickle(bytes: &[u8]) -> Result { + let value = serde_pickle::value_from_slice(bytes, Default::default()) + .map_err(|_| Error::InvalidEntry)?; + from_pickle_value(value) +} + +fn from_pickle_value(value: serde_pickle::Value) -> Result { + match value { + serde_pickle::Value::None => Ok(Value::None), + serde_pickle::Value::Bool(value) => Ok(Value::Boolean(value)), + serde_pickle::Value::I64(value) => integer(value.to_string()), + serde_pickle::Value::Int(value) => integer(value.to_string()), + serde_pickle::Value::F64(value) => Ok(Value::Float(value)), + serde_pickle::Value::String(value) => Ok(Value::String(value)), + serde_pickle::Value::Bytes(value) => Ok(Value::Bytes(value)), + serde_pickle::Value::List(values) => values + .into_iter() + .map(from_pickle_value) + .collect::, _>>() + .map(Value::List), + serde_pickle::Value::Tuple(values) => values + .into_iter() + .map(from_pickle_value) + .collect::, _>>() + .map(Value::Tuple), + serde_pickle::Value::Set(values) => values + .into_iter() + .map(from_pickle_hashable) + .collect::, _>>() + .map(Value::Set), + serde_pickle::Value::FrozenSet(values) => values + .into_iter() + .map(from_pickle_hashable) + .collect::, _>>() + .map(Value::Set), + serde_pickle::Value::Dict(values) => values + .into_iter() + .map(|(key, value)| Ok((from_pickle_hashable(key)?, from_pickle_value(value)?))) + .collect::, Error>>() + .map(Value::Dict), + } +} + +fn from_pickle_hashable(value: serde_pickle::HashableValue) -> Result { + Ok(match value { + serde_pickle::HashableValue::None => Value::None, + serde_pickle::HashableValue::Bool(value) => Value::Boolean(value), + serde_pickle::HashableValue::I64(value) => integer(value.to_string())?, + serde_pickle::HashableValue::Int(value) => integer(value.to_string())?, + serde_pickle::HashableValue::F64(value) => Value::Float(value), + serde_pickle::HashableValue::Bytes(value) => Value::Bytes(value), + serde_pickle::HashableValue::String(value) => Value::String(value), + serde_pickle::HashableValue::Tuple(values) => Value::Tuple( + values + .into_iter() + .map(from_pickle_hashable) + .collect::, _>>()?, + ), + serde_pickle::HashableValue::FrozenSet(values) => Value::Set( + values + .into_iter() + .map(from_pickle_hashable) + .collect::, _>>()?, + ), + }) +} + +fn integer(value: String) -> Result { + value.parse().map_err(|_| Error::InvalidEntry) +} + +pub(crate) fn from_json(value: serde_json::Value) -> Value { + match value { + serde_json::Value::Null => Value::None, + serde_json::Value::Bool(value) => Value::Boolean(value), + serde_json::Value::Number(value) => { + if value.is_i64() || value.is_u64() { + integer(value.to_string()) + .unwrap_or(Value::Float(value.as_f64().unwrap_or(f64::NAN))) + } else { + Value::Float(value.as_f64().unwrap_or(f64::NAN)) + } + } + serde_json::Value::String(value) => Value::String(value), + serde_json::Value::Array(values) => { + Value::List(values.into_iter().map(from_json).collect()) + } + serde_json::Value::Object(values) => Value::Dict( + values + .into_iter() + .map(|(key, value)| (Value::String(key), from_json(value))) + .collect(), + ), + } +} + +pub(crate) fn from_json_text(value: &str) -> Result { + serde_json::from_str(value) + .map(from_json) + .map_err(|_| Error::InvalidEntry) +} + +pub(crate) fn is_truthy(value: &Value) -> bool { + match value { + Value::None => false, + Value::Boolean(value) => *value, + Value::Integer(value) => value.to_string() != "0", + Value::Float(value) => *value != 0.0, + Value::Complex(value) => value.re != 0.0 || value.im != 0.0, + Value::String(value) => !value.is_empty(), + Value::Bytes(value) => !value.is_empty(), + Value::Tuple(value) | Value::List(value) | Value::Set(value) => !value.is_empty(), + Value::Dict(value) => !value.is_empty(), + } +} + +pub(crate) fn is_int(value: &Value) -> bool { + matches!(value, Value::Integer(_) | Value::Boolean(_)) +} + +pub(crate) fn to_f64(value: &Value) -> Option { + match value { + Value::Integer(value) => value.to_string().parse().ok(), + Value::Boolean(value) => Some(if *value { 1.0 } else { 0.0 }), + _ => None, + } +} + +pub(crate) fn to_json(value: &Value) -> Result, Error> { + serde_json::to_vec(&to_json_value(value)?).map_err(|_| Error::InvalidEntry) +} + +fn to_json_value(value: &Value) -> Result { + Ok(match value { + Value::None => serde_json::Value::Null, + Value::Boolean(value) => serde_json::Value::Bool(*value), + Value::Integer(value) => serde_json::Value::Number( + value + .to_string() + .parse::() + .map_err(|_| Error::InvalidEntry)?, + ), + Value::Float(value) => { + serde_json::Value::Number(Number::from_f64(*value).ok_or(Error::InvalidEntry)?) + } + Value::Complex(_) | Value::Bytes(_) => return Err(Error::InvalidEntry), + Value::String(value) => serde_json::Value::String(value.clone()), + Value::Tuple(values) | Value::List(values) | Value::Set(values) => { + serde_json::Value::Array( + values + .iter() + .map(to_json_value) + .collect::, _>>()?, + ) + } + Value::Dict(values) => { + let values = values + .iter() + .map(|(key, value)| { + let Value::String(key) = key else { + return Err(Error::InvalidEntry); + }; + Ok((key.clone(), to_json_value(value)?)) + }) + .collect::, _>>()?; + serde_json::Value::Object(values) + } + }) +} diff --git a/litellm-rust/crates/cache-disk/src/sqlite.rs b/litellm-rust/crates/cache-disk/src/sqlite.rs new file mode 100644 index 00000000000..9a36f8af6ad --- /dev/null +++ b/litellm-rust/crates/cache-disk/src/sqlite.rs @@ -0,0 +1,817 @@ +use std::{ + collections::HashMap, + fs::{self, OpenOptions}, + io::Write, + path::{Path, PathBuf}, + sync::Mutex, +}; + +use litellm_cache::Error; +use rand::RngCore; +use rusqlite::{Connection, OptionalExtension, params, types::Value}; + +use crate::{DiskStore, StoredValue}; + +const MODE_RAW: i64 = 1; +const MODE_BINARY: i64 = 2; +const MODE_TEXT: i64 = 3; +const MODE_PICKLE: i64 = 4; + +const DEFAULT_DISK_MIN_FILE_SIZE: i64 = 2_i64.pow(15); +const DEFAULT_SIZE_LIMIT: i64 = 2_i64.pow(30); +const DEFAULT_CULL_LIMIT: i64 = 10; + +pub struct DiskcacheSqliteStore { + directory: PathBuf, + connection: Mutex, + min_file_size: usize, + eviction_policy: String, + size_limit: i64, + cull_limit: i64, + statistics: bool, +} + +struct StoredColumns { + size: i64, + mode: i64, + filename: Option, + value: Option, +} + +struct Row { + rowid: i64, + mode: i64, + filename: Option, + value: Value, +} + +impl DiskcacheSqliteStore { + pub fn open(directory: impl AsRef) -> Result { + let directory = directory.as_ref().to_path_buf(); + fs::create_dir_all(&directory).map_err(|_| Error::Unavailable)?; + let directory = std::path::absolute(&directory).map_err(|_| Error::Unavailable)?; + let database = directory.join("cache.db"); + let connection = Connection::open(database).map_err(|_| Error::Unavailable)?; + connection + .busy_timeout(std::time::Duration::from_secs(60)) + .map_err(|_| Error::Unavailable)?; + + let mut settings = read_settings(&connection)?; + for (key, value) in default_settings() { + settings.entry(key).or_insert(value); + } + for (key, value) in settings + .iter() + .filter(|(key, _)| key.starts_with("sqlite_")) + { + apply_pragma(&connection, key, value)?; + } + + connection + .execute_batch( + "CREATE TABLE IF NOT EXISTS Settings ( + key TEXT NOT NULL UNIQUE, + value + )", + ) + .map_err(|_| Error::Unavailable)?; + for (key, value) in &settings { + if !matches!(key.as_str(), "count" | "size" | "hits" | "misses") { + connection + .execute( + "INSERT OR REPLACE INTO Settings VALUES (?, ?)", + params![key, value], + ) + .map_err(|_| Error::Unavailable)?; + } + } + for (key, value) in [ + ("count", Value::Integer(0)), + ("size", Value::Integer(0)), + ("hits", Value::Integer(0)), + ("misses", Value::Integer(0)), + ] { + connection + .execute( + "INSERT OR IGNORE INTO Settings VALUES (?, ?)", + params![key, value], + ) + .map_err(|_| Error::Unavailable)?; + } + connection + .execute_batch( + "CREATE TABLE IF NOT EXISTS Cache ( + rowid INTEGER PRIMARY KEY, + key BLOB, + raw INTEGER, + store_time REAL, + expire_time REAL, + access_time REAL, + access_count INTEGER DEFAULT 0, + tag BLOB, + size INTEGER DEFAULT 0, + mode INTEGER DEFAULT 0, + filename TEXT, + value BLOB + ); + CREATE UNIQUE INDEX IF NOT EXISTS Cache_key_raw ON Cache(key, raw); + CREATE INDEX IF NOT EXISTS Cache_expire_time ON Cache(expire_time);", + ) + .map_err(|_| Error::Unavailable)?; + + let eviction_policy = setting_string(&settings, "eviction_policy") + .unwrap_or_else(|| "least-recently-stored".to_string()); + match eviction_policy.as_str() { + "none" => {} + "least-recently-stored" => { + connection + .execute_batch( + "CREATE INDEX IF NOT EXISTS Cache_store_time ON Cache(store_time)", + ) + .map_err(|_| Error::Unavailable)?; + } + "least-recently-used" => { + connection + .execute_batch( + "CREATE INDEX IF NOT EXISTS Cache_access_time ON Cache(access_time)", + ) + .map_err(|_| Error::Unavailable)?; + } + "least-frequently-used" => { + connection + .execute_batch( + "CREATE INDEX IF NOT EXISTS Cache_access_count ON Cache(access_count)", + ) + .map_err(|_| Error::Unavailable)?; + } + _ => return Err(Error::Unavailable), + } + connection + .execute_batch( + "CREATE TRIGGER IF NOT EXISTS Settings_count_insert + AFTER INSERT ON Cache FOR EACH ROW BEGIN + UPDATE Settings SET value = value + 1 + WHERE key = \"count\"; END; + CREATE TRIGGER IF NOT EXISTS Settings_count_delete + AFTER DELETE ON Cache FOR EACH ROW BEGIN + UPDATE Settings SET value = value - 1 + WHERE key = \"count\"; END; + CREATE TRIGGER IF NOT EXISTS Settings_size_insert + AFTER INSERT ON Cache FOR EACH ROW BEGIN + UPDATE Settings SET value = value + NEW.size + WHERE key = \"size\"; END; + CREATE TRIGGER IF NOT EXISTS Settings_size_update + AFTER UPDATE ON Cache FOR EACH ROW BEGIN + UPDATE Settings + SET value = value + NEW.size - OLD.size + WHERE key = \"size\"; END; + CREATE TRIGGER IF NOT EXISTS Settings_size_delete + AFTER DELETE ON Cache FOR EACH ROW BEGIN + UPDATE Settings SET value = value - OLD.size + WHERE key = \"size\"; END;", + ) + .map_err(|_| Error::Unavailable)?; + + let min_file_size = setting_i64(&settings, "disk_min_file_size") + .unwrap_or(DEFAULT_DISK_MIN_FILE_SIZE) + .try_into() + .map_err(|_| Error::Unavailable)?; + let size_limit = setting_i64(&settings, "size_limit").unwrap_or(DEFAULT_SIZE_LIMIT); + let cull_limit = setting_i64(&settings, "cull_limit").unwrap_or(DEFAULT_CULL_LIMIT); + let statistics = setting_i64(&settings, "statistics").unwrap_or_default() != 0; + + Ok(Self { + directory, + connection: Mutex::new(connection), + min_file_size, + eviction_policy, + size_limit, + cull_limit, + statistics, + }) + } + + fn set_locked( + &self, + connection: &Connection, + key: &str, + columns: StoredColumns, + expire_time: Option, + now: f64, + ) -> Result, Error> { + let mut cleanup = Vec::new(); + if let Some(old_filename) = connection + .query_row( + "SELECT filename FROM Cache WHERE key = ? AND raw = 1", + params![key], + |row| row.get::<_, Option>(0), + ) + .optional() + .map_err(|_| Error::Unavailable)? + .flatten() + { + cleanup.push(old_filename); + } + let (size, mode, filename, value) = + (columns.size, columns.mode, columns.filename, columns.value); + let rowid = connection + .query_row( + "SELECT rowid FROM Cache WHERE key = ? AND raw = 1", + params![key], + |row| row.get::<_, i64>(0), + ) + .optional() + .map_err(|_| Error::Unavailable)?; + if let Some(rowid) = rowid { + connection + .execute( + "UPDATE Cache SET store_time = ?, expire_time = ?, access_time = ?, + access_count = 0, tag = NULL, size = ?, mode = ?, filename = ?, value = ? + WHERE rowid = ?", + params![now, expire_time, now, size, mode, filename, value, rowid], + ) + .map_err(|_| Error::Unavailable)?; + } else { + connection + .execute( + "INSERT INTO Cache( + key, raw, store_time, expire_time, access_time, access_count, + tag, size, mode, filename, value + ) VALUES (?, 1, ?, ?, ?, 0, NULL, ?, ?, ?, ?)", + params![key, now, expire_time, now, size, mode, filename, value], + ) + .map_err(|_| Error::Unavailable)?; + } + cleanup.extend(self.cull(connection, now)?); + Ok(cleanup) + } + + fn cull(&self, connection: &Connection, now: f64) -> Result, Error> { + if self.cull_limit <= 0 { + return Ok(Vec::new()); + } + let mut cleanup = Vec::new(); + let expired = connection + .prepare( + "SELECT rowid, filename FROM Cache + WHERE expire_time IS NOT NULL AND expire_time < ? + ORDER BY expire_time LIMIT ?", + ) + .map_err(|_| Error::Unavailable)? + .query_map(params![now, self.cull_limit], |row| { + Ok((row.get::<_, i64>(0)?, row.get::<_, Option>(1)?)) + }) + .map_err(|_| Error::Unavailable)? + .collect::, _>>() + .map_err(|_| Error::Unavailable)?; + for (_, filename) in &expired { + if let Some(filename) = filename { + cleanup.push(filename.clone()); + } + } + for (rowid, _) in &expired { + connection + .execute("DELETE FROM Cache WHERE rowid = ?", params![rowid]) + .map_err(|_| Error::Unavailable)?; + } + let remaining = self.cull_limit - i64::try_from(expired.len()).unwrap_or(self.cull_limit); + if remaining <= 0 || self.volume(connection)? < self.size_limit { + return Ok(cleanup); + } + let order = match self.eviction_policy.as_str() { + "none" => return Ok(cleanup), + "least-recently-stored" => "store_time", + "least-recently-used" => "access_time", + "least-frequently-used" => "access_count", + _ => return Err(Error::Unavailable), + }; + let rows = connection + .prepare(&format!( + "SELECT rowid, filename FROM Cache ORDER BY {order} LIMIT ?" + )) + .map_err(|_| Error::Unavailable)? + .query_map(params![remaining], |row| { + Ok((row.get::<_, i64>(0)?, row.get::<_, Option>(1)?)) + }) + .map_err(|_| Error::Unavailable)? + .collect::, _>>() + .map_err(|_| Error::Unavailable)?; + for (_, filename) in &rows { + if let Some(filename) = filename { + cleanup.push(filename.clone()); + } + } + for (rowid, _) in rows { + connection + .execute("DELETE FROM Cache WHERE rowid = ?", params![rowid]) + .map_err(|_| Error::Unavailable)?; + } + Ok(cleanup) + } + + fn volume(&self, connection: &Connection) -> Result { + let page_count: i64 = connection + .query_row("PRAGMA page_count", [], |row| row.get(0)) + .map_err(|_| Error::Unavailable)?; + let page_size: i64 = connection + .query_row("PRAGMA page_size", [], |row| row.get(0)) + .map_err(|_| Error::Unavailable)?; + let size: i64 = connection + .query_row("SELECT value FROM Settings WHERE key = 'size'", [], |row| { + row.get(0) + }) + .map_err(|_| Error::Unavailable)?; + Ok(page_count.saturating_mul(page_size).saturating_add(size)) + } +} + +impl DiskStore for DiskcacheSqliteStore { + fn directory(&self) -> &Path { + &self.directory + } + + fn get(&self, key: &str, now: f64) -> Result, Error> { + let connection = self.connection.lock().map_err(|_| Error::Unavailable)?; + let select = "SELECT rowid, expire_time, mode, filename, value FROM Cache + WHERE key = ? AND raw = 1 AND (expire_time IS NULL OR expire_time > ?)"; + let row = connection + .query_row(select, params![key, now], row_from_query) + .optional() + .map_err(|_| Error::Unavailable)?; + if !self.statistics && !has_get_update(&self.eviction_policy) { + return row + .map(|row| fetch_row(&self.directory, row)) + .transpose() + .map(|value| value.flatten()); + } + transactional(&connection, |connection| { + let row = connection + .query_row(select, params![key, now], row_from_query) + .optional() + .map_err(|_| Error::Unavailable)?; + let Some(row) = row else { + if self.statistics { + connection + .execute( + "UPDATE Settings SET value = value + 1 WHERE key = 'misses'", + [], + ) + .map_err(|_| Error::Unavailable)?; + } + return Ok(None); + }; + let rowid = row.rowid; + let value = fetch_row(&self.directory, row); + let hit = value.as_ref().is_ok_and(Option::is_some); + if hit && self.statistics { + connection + .execute( + "UPDATE Settings SET value = value + 1 WHERE key = 'hits'", + [], + ) + .map_err(|_| Error::Unavailable)?; + } else if !hit && self.statistics { + connection + .execute( + "UPDATE Settings SET value = value + 1 WHERE key = 'misses'", + [], + ) + .map_err(|_| Error::Unavailable)?; + } + if has_get_update(&self.eviction_policy) && hit { + let update = match self.eviction_policy.as_str() { + "least-recently-used" => "UPDATE Cache SET access_time = ? WHERE rowid = ?", + "least-frequently-used" => { + "UPDATE Cache SET access_count = access_count + 1 WHERE rowid = ?" + } + _ => return Err(Error::Unavailable), + }; + if self.eviction_policy == "least-recently-used" { + connection + .execute(update, params![now, rowid]) + .map_err(|_| Error::Unavailable)?; + } else { + connection + .execute(update, params![rowid]) + .map_err(|_| Error::Unavailable)?; + } + } + value + }) + } + + fn set( + &self, + key: &str, + value: StoredValue, + expire_time: Option, + now: f64, + ) -> Result<(), Error> { + let columns = store_value(&self.directory, self.min_file_size, value)?; + let new_filename = columns.filename.clone(); + let connection = self.connection.lock().map_err(|_| Error::Unavailable)?; + let result = transactional(&connection, |connection| { + self.set_locked(connection, key, columns, expire_time, now) + }); + match result { + Ok(cleanup) => { + cleanup_files(&self.directory, cleanup); + Ok(()) + } + Err(error) => { + if let Some(filename) = new_filename { + remove_file(&self.directory, &filename); + } + Err(error) + } + } + } + + fn pop(&self, key: &str, now: f64) -> Result, Error> { + let connection = self.connection.lock().map_err(|_| Error::Unavailable)?; + let selected = transactional(&connection, |connection| { + let row = connection + .query_row( + "SELECT rowid, expire_time, mode, filename, value FROM Cache + WHERE key = ? AND raw = 1 + AND (expire_time IS NULL OR expire_time > ?)", + params![key, now], + row_from_query, + ) + .optional() + .map_err(|_| Error::Unavailable)?; + let Some(row) = row else { + return Ok(None); + }; + connection + .execute("DELETE FROM Cache WHERE rowid = ?", params![row.rowid]) + .map_err(|_| Error::Unavailable)?; + Ok(Some(row)) + })?; + let Some(row) = selected else { + return Ok(None); + }; + let filename = row.filename.clone(); + let result = fetch_row(&self.directory, row)?; + if let Some(filename) = filename { + remove_file(&self.directory, &filename); + } + Ok(result) + } + + fn clear(&self) -> Result<(), Error> { + let connection = self.connection.lock().map_err(|_| Error::Unavailable)?; + let mut last_rowid = 0_i64; + loop { + let batch = transactional(&connection, |connection| { + let rows = connection + .prepare( + "SELECT rowid, filename FROM Cache + WHERE rowid > ? ORDER BY rowid LIMIT 100", + ) + .map_err(|_| Error::Unavailable)? + .query_map(params![last_rowid], |row| { + Ok((row.get::<_, i64>(0)?, row.get::<_, Option>(1)?)) + }) + .map_err(|_| Error::Unavailable)? + .collect::, _>>() + .map_err(|_| Error::Unavailable)?; + if rows.is_empty() { + return Ok(rows); + } + let ids = rows + .iter() + .map(|(rowid, _)| rowid.to_string()) + .collect::>() + .join(","); + connection + .execute(&format!("DELETE FROM Cache WHERE rowid IN ({ids})"), []) + .map_err(|_| Error::Unavailable)?; + Ok(rows) + })?; + if batch.is_empty() { + return Ok(()); + } + last_rowid = batch.last().map(|(rowid, _)| *rowid).unwrap_or(last_rowid); + cleanup_files( + &self.directory, + batch + .into_iter() + .filter_map(|(_, filename)| filename) + .collect(), + ); + } + } + + fn update( + &self, + key: &str, + now: f64, + apply: &mut dyn FnMut(Option) -> Result<(StoredValue, Option), Error>, + ) -> Result<(), Error> { + let connection = self.connection.lock().map_err(|_| Error::Unavailable)?; + let mut created_filename = None; + let result = transactional(&connection, |connection| { + let current = connection + .query_row( + "SELECT rowid, expire_time, mode, filename, value FROM Cache + WHERE key = ? AND raw = 1 + AND (expire_time IS NULL OR expire_time > ?)", + params![key, now], + row_from_query, + ) + .optional() + .map_err(|_| Error::Unavailable)? + .map(|row| fetch_row(&self.directory, row)) + .transpose()? + .flatten(); + let (value, expire_time) = apply(current)?; + let columns = store_value(&self.directory, self.min_file_size, value)?; + created_filename = columns.filename.clone(); + let cleanup = self.set_locked(connection, key, columns, expire_time, now)?; + Ok(cleanup) + }); + match result { + Ok(cleanup) => { + cleanup_files(&self.directory, cleanup); + Ok(()) + } + Err(error) => { + if let Some(filename) = created_filename { + remove_file(&self.directory, &filename); + } + Err(error) + } + } + } + + fn probe(&self) -> Result<(), Error> { + let connection = self.connection.lock().map_err(|_| Error::Unavailable)?; + connection + .query_row( + "SELECT value FROM Settings WHERE key = 'count'", + [], + |row| row.get::<_, i64>(0), + ) + .map(|_| ()) + .map_err(|_| Error::Unavailable) + } +} + +fn default_settings() -> HashMap { + HashMap::from([ + ("statistics".to_string(), Value::Integer(0)), + ("tag_index".to_string(), Value::Integer(0)), + ( + "eviction_policy".to_string(), + Value::Text("least-recently-stored".to_string()), + ), + ("size_limit".to_string(), Value::Integer(DEFAULT_SIZE_LIMIT)), + ("cull_limit".to_string(), Value::Integer(DEFAULT_CULL_LIMIT)), + ("sqlite_auto_vacuum".to_string(), Value::Integer(1)), + ("sqlite_cache_size".to_string(), Value::Integer(8192)), + ( + "sqlite_journal_mode".to_string(), + Value::Text("wal".to_string()), + ), + ( + "sqlite_mmap_size".to_string(), + Value::Integer(2_i64.pow(26)), + ), + ("sqlite_synchronous".to_string(), Value::Integer(1)), + ( + "disk_min_file_size".to_string(), + Value::Integer(DEFAULT_DISK_MIN_FILE_SIZE), + ), + ("disk_pickle_protocol".to_string(), Value::Integer(5)), + ]) +} + +fn read_settings(connection: &Connection) -> Result, Error> { + let mut statement = match connection.prepare("SELECT key, value FROM Settings") { + Ok(statement) => statement, + Err(_) => return Ok(HashMap::new()), + }; + statement + .query_map([], |row| Ok((row.get(0)?, row.get(1)?))) + .map_err(|_| Error::Unavailable)? + .collect::, _>>() + .map_err(|_| Error::Unavailable) +} + +fn apply_pragma(connection: &Connection, key: &str, value: &Value) -> Result<(), Error> { + let pragma = key.strip_prefix("sqlite_").ok_or(Error::Unavailable)?; + match value { + Value::Integer(value) => connection + .pragma_update(None, pragma, value) + .map_err(|_| Error::Unavailable), + Value::Text(value) => connection + .pragma_update(None, pragma, value) + .map_err(|_| Error::Unavailable), + _ => Err(Error::Unavailable), + } +} + +fn setting_i64(settings: &HashMap, key: &str) -> Option { + match settings.get(key) { + Some(Value::Integer(value)) => Some(*value), + _ => None, + } +} + +fn setting_string(settings: &HashMap, key: &str) -> Option { + match settings.get(key) { + Some(Value::Text(value)) => Some(value.clone()), + _ => None, + } +} + +fn has_get_update(policy: &str) -> bool { + matches!(policy, "least-recently-used" | "least-frequently-used") +} + +fn row_from_query(row: &rusqlite::Row<'_>) -> rusqlite::Result { + Ok(Row { + rowid: row.get(0)?, + mode: row.get(2)?, + filename: row.get(3)?, + value: row.get(4)?, + }) +} + +fn fetch_row(directory: &Path, row: Row) -> Result, Error> { + match row.mode { + MODE_RAW => match row.value { + Value::Blob(value) => Ok(Some(StoredValue::Bytes(value))), + Value::Text(value) => Ok(Some(StoredValue::Text(value))), + Value::Integer(value) => Ok(Some(StoredValue::Integer(value))), + Value::Real(value) => Ok(Some(StoredValue::Float(value))), + Value::Null => Err(Error::InvalidEntry), + }, + MODE_BINARY | MODE_PICKLE => { + let bytes = match row.value { + Value::Blob(value) => value, + Value::Null => { + let Some(value) = read_file(directory, row.filename.as_deref())? else { + return Ok(None); + }; + value + } + _ => return Err(Error::InvalidEntry), + }; + Ok(Some(if row.mode == MODE_BINARY { + StoredValue::Bytes(bytes) + } else { + StoredValue::Pickle(bytes) + })) + } + MODE_TEXT => { + let bytes = match row.value { + Value::Null => { + let Some(value) = read_file(directory, row.filename.as_deref())? else { + return Ok(None); + }; + value + } + Value::Blob(value) => value, + Value::Text(value) => value.into_bytes(), + _ => return Err(Error::InvalidEntry), + }; + Ok(Some(StoredValue::Text( + String::from_utf8(bytes).map_err(|_| Error::InvalidEntry)?, + ))) + } + _ => Err(Error::InvalidEntry), + } +} + +fn read_file(directory: &Path, filename: Option<&str>) -> Result>, Error> { + let Some(filename) = filename else { + return Err(Error::InvalidEntry); + }; + match fs::read(directory.join(filename)) { + Ok(value) => Ok(Some(value)), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(None), + Err(_) => Err(Error::Unavailable), + } +} + +fn store_value( + directory: &Path, + min_file_size: usize, + value: StoredValue, +) -> Result { + match value { + StoredValue::Integer(value) => Ok(StoredColumns { + size: 0, + mode: MODE_RAW, + filename: None, + value: Some(Value::Integer(value)), + }), + StoredValue::Float(value) => Ok(StoredColumns { + size: 0, + mode: MODE_RAW, + filename: None, + value: Some(Value::Real(value)), + }), + StoredValue::Text(value) if value.chars().count() < min_file_size => Ok(StoredColumns { + size: 0, + mode: MODE_RAW, + filename: None, + value: Some(Value::Text(value)), + }), + StoredValue::Text(value) => { + let bytes = value.into_bytes(); + let filename = write_file(directory, &bytes)?; + Ok(StoredColumns { + size: i64::try_from(bytes.len()).map_err(|_| Error::Unavailable)?, + mode: MODE_TEXT, + filename: Some(filename), + value: None, + }) + } + StoredValue::Bytes(value) if value.len() < min_file_size => Ok(StoredColumns { + size: 0, + mode: MODE_RAW, + filename: None, + value: Some(Value::Blob(value)), + }), + StoredValue::Bytes(value) => { + let filename = write_file(directory, &value)?; + Ok(StoredColumns { + size: i64::try_from(value.len()).map_err(|_| Error::Unavailable)?, + mode: MODE_BINARY, + filename: Some(filename), + value: None, + }) + } + StoredValue::Pickle(value) if value.len() < min_file_size => Ok(StoredColumns { + size: 0, + mode: MODE_PICKLE, + filename: None, + value: Some(Value::Blob(value)), + }), + StoredValue::Pickle(value) => { + let filename = write_file(directory, &value)?; + Ok(StoredColumns { + size: i64::try_from(value.len()).map_err(|_| Error::Unavailable)?, + mode: MODE_PICKLE, + filename: Some(filename), + value: None, + }) + } + } +} + +fn write_file(directory: &Path, bytes: &[u8]) -> Result { + let mut random = [0_u8; 16]; + rand::rngs::OsRng.fill_bytes(&mut random); + let hex = random + .iter() + .map(|byte| format!("{byte:02x}")) + .collect::(); + let filename = format!("{}/{}/{}.val", &hex[..2], &hex[2..4], &hex[4..]); + let path = directory.join(&filename); + if let Some(parent) = path.parent() { + fs::create_dir_all(parent).map_err(|_| Error::Unavailable)?; + } + let mut file = OpenOptions::new() + .write(true) + .create_new(true) + .open(path) + .map_err(|_| Error::Unavailable)?; + file.write_all(bytes).map_err(|_| Error::Unavailable)?; + Ok(filename) +} + +fn cleanup_files(directory: &Path, filenames: Vec) { + for filename in filenames { + remove_file(directory, &filename); + } +} + +fn remove_file(directory: &Path, filename: &str) { + let path = directory.join(filename); + let _ = fs::remove_file(&path); +} + +fn transactional( + connection: &Connection, + operation: impl FnOnce(&Connection) -> Result, +) -> Result { + connection + .execute_batch("BEGIN IMMEDIATE") + .map_err(|_| Error::Unavailable)?; + match operation(connection) { + Ok(value) => { + connection + .execute_batch("COMMIT") + .map_err(|_| Error::Unavailable)?; + Ok(value) + } + Err(error) => { + let _ = connection.execute_batch("ROLLBACK"); + Err(error) + } + } +} diff --git a/litellm-rust/crates/cache-disk/src/store.rs b/litellm-rust/crates/cache-disk/src/store.rs new file mode 100644 index 00000000000..ed167317cf0 --- /dev/null +++ b/litellm-rust/crates/cache-disk/src/store.rs @@ -0,0 +1,33 @@ +use std::path::Path; + +use litellm_cache::Error; + +#[derive(Clone, Debug, PartialEq)] +pub enum StoredValue { + Bytes(Vec), + Text(String), + Integer(i64), + Float(f64), + Pickle(Vec), +} + +pub trait DiskStore: Send + Sync + 'static { + fn directory(&self) -> &Path; + fn get(&self, key: &str, now: f64) -> Result, Error>; + fn set( + &self, + key: &str, + value: StoredValue, + expire_time: Option, + now: f64, + ) -> Result<(), Error>; + fn pop(&self, key: &str, now: f64) -> Result, Error>; + fn clear(&self) -> Result<(), Error>; + fn update( + &self, + key: &str, + now: f64, + apply: &mut dyn FnMut(Option) -> Result<(StoredValue, Option), Error>, + ) -> Result<(), Error>; + fn probe(&self) -> Result<(), Error>; +} diff --git a/litellm-rust/crates/cache-disk/tests/cache.rs b/litellm-rust/crates/cache-disk/tests/cache.rs new file mode 100644 index 00000000000..dd1f2b1f04e --- /dev/null +++ b/litellm-rust/crates/cache-disk/tests/cache.rs @@ -0,0 +1,431 @@ +use std::{ + fs, + path::{Path, PathBuf}, + sync::Arc, + thread, + time::Duration, +}; + +use litellm_cache::{ + BaseCache, BatchCache, BatchEntry, CacheCodec, CounterCache, DeleteCache, ExactCacheContext, + FlushCache, JsonCodec, +}; +use litellm_cache_disk::{DiskCache, DiskStore, DiskcacheSqliteStore, StoredValue, ValueAdapter}; +use rstest::{fixture, rstest}; +use rusqlite::Connection; +use serde_json::{Value, json}; +use tempfile::TempDir; + +struct Sandbox { + directory: TempDir, +} + +#[fixture] +fn sandbox() -> Sandbox { + Sandbox { + directory: tempfile::tempdir().unwrap(), + } +} + +impl Sandbox { + fn store(&self) -> DiskcacheSqliteStore { + DiskcacheSqliteStore::open(self.directory.path()).unwrap() + } + + fn cache(&self) -> DiskCache> + where + JsonCodec: CacheCodec, + { + DiskCache::open(self.directory.path(), JsonCodec::new()).unwrap() + } + + fn db(&self) -> Connection { + Connection::open(self.directory.path().join("cache.db")).unwrap() + } + + fn value_files(&self) -> Vec { + fn visit(directory: &Path, files: &mut Vec) { + for entry in fs::read_dir(directory).unwrap() { + let path = entry.unwrap().path(); + if path.is_dir() { + visit(&path, files); + } else if path.extension().is_some_and(|extension| extension == "val") { + files.push(path); + } + } + } + + let mut files = Vec::new(); + visit(self.directory.path(), &mut files); + files + } +} + +#[rstest] +fn relative_store_directory_is_absolutized(sandbox: Sandbox) { + let relative = PathBuf::from(format!( + ".litellm-cache-disk-{}", + sandbox + .directory + .path() + .file_name() + .unwrap() + .to_string_lossy() + )); + let store = DiskcacheSqliteStore::open(&relative).unwrap(); + assert!(store.directory().is_absolute()); + assert!(store.directory().ends_with(&relative)); + let directory = store.directory().to_path_buf(); + drop(store); + fs::remove_dir_all(directory).unwrap(); +} + +#[derive(Clone, Copy, Debug, Default)] +struct TextAdapter; + +impl ValueAdapter for TextAdapter { + fn read(&self, value: StoredValue) -> Result>, litellm_cache::Error> { + match value { + StoredValue::Text(value) => Ok(Some(value.into_bytes())), + _ => Ok(None), + } + } + + fn write(&self, payload: Vec) -> StoredValue { + StoredValue::Text(String::from_utf8(payload).unwrap()) + } + + fn counter_seed(&self, _: Option) -> Result { + Ok(0.0) + } + + fn counter_value(&self, value: f64) -> StoredValue { + if value.fract() == 0.0 { + StoredValue::Integer(value as i64) + } else { + StoredValue::Float(value) + } + } +} + +#[rstest] +fn roundtrip_persists_and_reopens(sandbox: Sandbox) { + let context = ExactCacheContext::default(); + let opened = sandbox.cache::(); + opened + .set_cache("key", json!({"answer": 42}), &context) + .unwrap(); + assert_eq!( + opened.get_cache("key", &context).unwrap(), + Some(json!({"answer": 42})) + ); + drop(opened); + let reopened = sandbox.cache::(); + assert_eq!( + reopened.get_cache("key", &context).unwrap(), + Some(json!({"answer": 42})) + ); +} + +#[rstest] +fn ttl_and_expired_culling_match_cache_contract(sandbox: Sandbox) { + let store = sandbox.store(); + store + .set( + "expired", + StoredValue::Bytes(b"old".to_vec()), + Some(10.0), + 0.0, + ) + .unwrap(); + assert_eq!(store.get("expired", 10.0).unwrap(), None); + store + .set("new", StoredValue::Bytes(b"new".to_vec()), None, 11.0) + .unwrap(); + assert_eq!( + sandbox + .db() + .query_row("SELECT COUNT(*) FROM Cache", [], |row| row.get::<_, i64>(0)) + .unwrap(), + 1 + ); + assert_eq!( + sandbox + .db() + .query_row( + "SELECT value FROM Settings WHERE key = 'count'", + [], + |row| row.get::<_, i64>(0) + ) + .unwrap(), + 1 + ); +} + +#[rstest] +fn batch_preserves_order_and_classifies_misses_and_invalid_values(sandbox: Sandbox) { + let store = sandbox.store(); + store + .set( + "hit", + StoredValue::Bytes(br#"{"ok":true}"#.to_vec()), + None, + 0.0, + ) + .unwrap(); + store + .set( + "invalid", + StoredValue::Pickle(vec![0x80, 0x05, 0x2e]), + None, + 0.0, + ) + .unwrap(); + let entries = sandbox + .cache::() + .batch_get_cache( + &["hit".into(), "missing".into(), "invalid".into()], + &ExactCacheContext::default(), + ) + .unwrap(); + assert_eq!( + entries, + vec![ + BatchEntry::Hit(json!({"ok": true})), + BatchEntry::Miss, + BatchEntry::Invalid + ] + ); +} + +#[rstest] +#[case(StoredValue::Bytes(Vec::new()))] +#[case(StoredValue::Text(String::new()))] +#[case(StoredValue::Integer(0))] +#[case(StoredValue::Float(0.0))] +#[case(StoredValue::Pickle(vec![0x80, 0x05, 0x4e, 0x2e]))] +#[case(StoredValue::Pickle(vec![0x80, 0x05, 0x89, 0x2e]))] +#[case(StoredValue::Pickle(vec![0x80, 0x05, 0x4b, 0x00, 0x2e]))] +#[case(StoredValue::Pickle(vec![0x80, 0x05, 0x95, 0x0a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x47, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x2e]))] +#[case(StoredValue::Pickle(vec![0x80, 0x05, 0x95, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8c, 0x00, 0x94, 0x2e]))] +#[case(StoredValue::Pickle(vec![0x80, 0x05, 0x5d, 0x94, 0x2e]))] +#[case(StoredValue::Pickle(vec![0x80, 0x05, 0x7d, 0x94, 0x2e]))] +#[case(StoredValue::Pickle(vec![0x80, 0x05, 0x29, 0x2e]))] +fn falsy_values_are_misses(sandbox: Sandbox, #[case] value: StoredValue) { + sandbox.store().set("key", value, None, 0.0).unwrap(); + assert_eq!( + sandbox + .cache::() + .get_cache("key", &ExactCacheContext::default()) + .unwrap(), + None + ); +} + +#[rstest] +#[case(Some(StoredValue::Integer(2)), 1.5, 3.5, "real")] +#[case(Some(StoredValue::Integer(2)), 1.0, 3.0, "integer")] +#[case(Some(StoredValue::Float(3.5)), 1.0, 1.0, "integer")] +#[case(Some(StoredValue::Text("not a number".into())), 2.0, 2.0, "integer")] +#[case(Some(StoredValue::Text("5".into())), 2.0, 7.0, "integer")] +#[case(Some(StoredValue::Text("3.5".into())), 2.0, 2.0, "integer")] +#[case(Some(StoredValue::Pickle(vec![0x80, 0x05, 0x88, 0x2e])), 1.0, 2.0, "integer")] +#[case(Some(StoredValue::Pickle(vec![0x80, 0x05, 0x4b, 0x02, 0x2e])), 1.0, 3.0, "integer")] +#[case(Some(StoredValue::Pickle(vec![0x80, 0x05, 0x95, 0x0a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7d, 0x94, 0x8c, 0x01, 0x61, 0x94, 0x4b, 0x01, 0x73, 0x2e])), 1.0, 1.0, "integer")] +#[case(Some(StoredValue::Pickle(vec![0x80, 0x05, 0x4e, 0x2e])), 4.0, 4.0, "integer")] +fn counters_follow_python_initialization( + sandbox: Sandbox, + #[case] initial: Option, + #[case] amount: f64, + #[case] expected: f64, + #[case] sqlite_type: &str, +) { + if let Some(initial) = initial { + sandbox.store().set("counter", initial, None, 0.0).unwrap(); + } + let cache = sandbox.cache::(); + assert_eq!( + cache + .increment_cache("counter", amount, ExactCacheContext::default()) + .unwrap(), + expected + ); + assert_eq!( + sandbox + .db() + .query_row( + "SELECT typeof(value) FROM Cache WHERE key = 'counter'", + [], + |row| row.get::<_, String>(0) + ) + .unwrap(), + sqlite_type + ); +} + +#[rstest] +fn counters_are_atomic_across_concurrent_callers(sandbox: Sandbox) { + let cache = Arc::new(sandbox.cache::()); + let workers = (0..8) + .map(|_| { + let cache = Arc::clone(&cache); + thread::spawn(move || { + for _ in 0..25 { + cache + .increment_cache("counter", 1.0, ExactCacheContext::default()) + .unwrap(); + } + }) + }) + .collect::>(); + for worker in workers { + worker.join().unwrap(); + } + assert_eq!( + cache + .increment_cache("counter", 0.0, ExactCacheContext::default()) + .unwrap(), + 200.0 + ); +} + +#[rstest] +fn fractional_then_integer_increment_follows_python_behavior(sandbox: Sandbox) { + let cache = sandbox.cache::(); + assert_eq!( + cache + .increment_cache("counter", 3.5, ExactCacheContext::default()) + .unwrap(), + 3.5 + ); + assert_eq!( + cache + .increment_cache("counter", 1.0, ExactCacheContext::default()) + .unwrap(), + 1.0 + ); +} + +#[rstest] +fn increment_ttl_replacement_clears_expiry_without_ttl(sandbox: Sandbox) { + let cache = sandbox.cache::(); + cache + .increment_cache( + "counter", + 1.0, + ExactCacheContext { + ttl: Some(Duration::from_secs(60)), + }, + ) + .unwrap(); + assert!( + sandbox + .db() + .query_row( + "SELECT expire_time IS NOT NULL FROM Cache WHERE key = 'counter'", + [], + |row| row.get::<_, bool>(0) + ) + .unwrap() + ); + cache + .increment_cache("counter", 1.0, ExactCacheContext::default()) + .unwrap(); + assert!( + !sandbox + .db() + .query_row( + "SELECT expire_time IS NOT NULL FROM Cache WHERE key = 'counter'", + [], + |row| row.get::<_, bool>(0) + ) + .unwrap() + ); +} + +#[rstest] +fn custom_adapter_controls_storage_and_reads(sandbox: Sandbox) { + let cache = DiskCache::with_adapter(sandbox.store(), TextAdapter, JsonCodec::::new()); + cache + .set_cache("key", json!({"answer": 42}), &ExactCacheContext::default()) + .unwrap(); + assert!(matches!( + sandbox.store().get("key", 0.0).unwrap(), + Some(StoredValue::Text(_)) + )); + assert_eq!( + cache + .get_cache("key", &ExactCacheContext::default()) + .unwrap(), + Some(json!({"answer": 42})) + ); +} + +#[rstest] +fn delete_flush_and_spilled_file_replacement_clean_up_storage(sandbox: Sandbox) { + let large = vec![b'x'; 32 * 1024]; + sandbox + .store() + .set("large", StoredValue::Bytes(large.clone()), None, 0.0) + .unwrap(); + assert_eq!(sandbox.value_files().len(), 1); + sandbox + .store() + .set( + "large", + StoredValue::Bytes(vec![b'y'; 32 * 1024]), + None, + 0.0, + ) + .unwrap(); + assert_eq!(sandbox.value_files().len(), 1); + sandbox.store().pop("large", 0.0).unwrap(); + assert!(sandbox.value_files().is_empty()); + sandbox + .store() + .set("a", StoredValue::Bytes(large.clone()), None, 0.0) + .unwrap(); + sandbox + .store() + .set("b", StoredValue::Bytes(large), None, 0.0) + .unwrap(); + sandbox.store().clear().unwrap(); + assert!(sandbox.value_files().is_empty()); +} + +#[rstest] +#[tokio::test] +async fn async_operations_connection_and_delete_match_sync_operations(sandbox: Sandbox) { + let cache = sandbox.cache::(); + let context = ExactCacheContext { + ttl: Some(Duration::from_secs(60)), + }; + cache + .async_set_cache("a", json!(1), context.clone()) + .await + .unwrap(); + cache + .async_set_cache_pipeline( + vec![("b".into(), json!(2)), ("c".into(), json!(3))], + context.clone(), + ) + .await + .unwrap(); + assert_eq!( + cache.async_get_cache("a", &context).await.unwrap(), + Some(json!(1)) + ); + assert_eq!( + cache + .async_batch_get_cache(vec!["c".into(), "missing".into()], context.clone()) + .await + .unwrap(), + vec![BatchEntry::Hit(json!(3)), BatchEntry::Miss] + ); + cache.async_delete_cache("a").await.unwrap(); + cache.async_flush_cache().await.unwrap(); + assert_eq!( + cache.test_connection().await.unwrap().status, + litellm_cache::CacheConnectionStatus::Success + ); +} diff --git a/litellm-rust/crates/cache-disk/tests/python_compat.rs b/litellm-rust/crates/cache-disk/tests/python_compat.rs new file mode 100644 index 00000000000..9cbef8573bd --- /dev/null +++ b/litellm-rust/crates/cache-disk/tests/python_compat.rs @@ -0,0 +1,113 @@ +use litellm_cache::Error; +use litellm_cache_disk::{PythonDiskCacheAdapter, StoredValue, ValueAdapter}; +use rstest::rstest; + +enum ReadExpectation { + Bytes(&'static [u8]), + Miss, + Invalid, +} + +#[rstest] +#[case::pickled_dictionary_with_string_keys( + StoredValue::Pickle(vec![0x80, 0x05, 0x95, 0x0a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7d, 0x94, 0x8c, 0x01, 0x61, 0x94, 0x4b, 0x01, 0x73, 0x2e]), + ReadExpectation::Bytes(br#"{"a":1}"#) +)] +#[case::pickled_list_of_integers( + StoredValue::Pickle(vec![0x80, 0x05, 0x95, 0x09, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x5d, 0x94, 0x28, 0x4b, 0x01, 0x4b, 0x02, 0x65, 0x2e]), + ReadExpectation::Bytes(br#"[1,2]"#) +)] +#[case::pickled_tuple_of_integers( + StoredValue::Pickle(vec![0x80, 0x05, 0x95, 0x07, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x4b, 0x01, 0x4b, 0x02, 0x86, 0x94, 0x2e]), + ReadExpectation::Bytes(br#"[1,2]"#) +)] +#[case::pickled_set_of_integers( + StoredValue::Pickle(vec![0x80, 0x05, 0x95, 0x09, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8f, 0x94, 0x28, 0x4b, 0x01, 0x4b, 0x02, 0x90, 0x2e]), + ReadExpectation::Bytes(br#"[1,2]"#) +)] +#[case::pickled_response_envelope( + StoredValue::Pickle(vec![0x80, 0x05, 0x95, 0x30, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7d, 0x94, 0x28, 0x8c, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x94, 0x47, 0x3f, 0xf8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8c, 0x08, 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x94, 0x8c, 0x08, 0x7b, 0x22, 0x61, 0x22, 0x3a, 0x20, 0x31, 0x7d, 0x94, 0x75, 0x2e]), + ReadExpectation::Bytes(br#"{"response":"{\"a\": 1}","timestamp":1.5}"#) +)] +#[case::non_json_text( + StoredValue::Text("not json".into()), + ReadExpectation::Bytes(b"not json") +)] +#[case::json_text( + StoredValue::Text("{\"a\": 1}".into()), + ReadExpectation::Bytes(br#"{"a": 1}"#) +)] +#[case::non_utf8_bytes( + StoredValue::Bytes(vec![0xff, 0xfe]), + ReadExpectation::Bytes(&[0xff, 0xfe]) +)] +#[case::integer_seven(StoredValue::Integer(7), ReadExpectation::Bytes(b"7"))] +#[case::float_one_point_five(StoredValue::Float(1.5), ReadExpectation::Bytes(b"1.5"))] +#[case::pickled_true( + StoredValue::Pickle(vec![0x80, 0x05, 0x88, 0x2e]), + ReadExpectation::Bytes(b"true") +)] +#[case::pickled_negative_integer( + StoredValue::Pickle(vec![0x80, 0x05, 0x95, 0x06, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x4a, 0xfd, 0xff, 0xff, 0xff, 0x2e]), + ReadExpectation::Bytes(b"-3") +)] +#[case::pickled_bytes( + StoredValue::Pickle(vec![0x80, 0x05, 0x95, 0x09, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x43, 0x05, 0x62, 0x79, 0x74, 0x65, 0x73, 0x94, 0x2e]), + ReadExpectation::Invalid +)] +#[case::pickled_dictionary_with_integer_key( + StoredValue::Pickle(vec![0x80, 0x05, 0x95, 0x0a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7d, 0x94, 0x4b, 0x01, 0x8c, 0x01, 0x61, 0x94, 0x73, 0x2e]), + ReadExpectation::Invalid +)] +#[case::pickled_complex( + StoredValue::Pickle(vec![0x80, 0x05, 0x95, 0x2e, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8c, 0x08, 0x62, 0x75, 0x69, 0x6c, 0x74, 0x69, 0x6e, 0x73, 0x94, 0x8c, 0x07, 0x63, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x78, 0x94, 0x93, 0x94, 0x47, 0x3f, 0xf0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x47, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x86, 0x94, 0x52, 0x94, 0x2e]), + ReadExpectation::Invalid +)] +#[case::truncated_pickle( + StoredValue::Pickle(vec![0x80, 0x05, 0x2e]), + ReadExpectation::Invalid +)] +#[case::empty_bytes(StoredValue::Bytes(Vec::new()), ReadExpectation::Miss)] +#[case::empty_text(StoredValue::Text(String::new()), ReadExpectation::Miss)] +#[case::zero_integer(StoredValue::Integer(0), ReadExpectation::Miss)] +#[case::zero_float(StoredValue::Float(0.0), ReadExpectation::Miss)] +#[case::pickled_none(StoredValue::Pickle(vec![0x80, 0x05, 0x4e, 0x2e]), ReadExpectation::Miss)] +#[case::pickled_false(StoredValue::Pickle(vec![0x80, 0x05, 0x89, 0x2e]), ReadExpectation::Miss)] +#[case::pickled_zero(StoredValue::Pickle(vec![0x80, 0x05, 0x4b, 0x00, 0x2e]), ReadExpectation::Miss)] +#[case::pickled_zero_float(StoredValue::Pickle(vec![0x80, 0x05, 0x95, 0x0a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x47, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x2e]), ReadExpectation::Miss)] +#[case::pickled_empty_string(StoredValue::Pickle(vec![0x80, 0x05, 0x95, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8c, 0x00, 0x94, 0x2e]), ReadExpectation::Miss)] +#[case::pickled_empty_list(StoredValue::Pickle(vec![0x80, 0x05, 0x5d, 0x94, 0x2e]), ReadExpectation::Miss)] +#[case::pickled_empty_dictionary(StoredValue::Pickle(vec![0x80, 0x05, 0x7d, 0x94, 0x2e]), ReadExpectation::Miss)] +#[case::pickled_empty_tuple(StoredValue::Pickle(vec![0x80, 0x05, 0x29, 0x2e]), ReadExpectation::Miss)] +fn python_read_cases(#[case] row: StoredValue, #[case] expected: ReadExpectation) { + let result = PythonDiskCacheAdapter.read(row); + match expected { + ReadExpectation::Bytes(expected) => assert_eq!(result.unwrap().unwrap(), expected), + ReadExpectation::Miss => assert_eq!(result.unwrap(), None), + ReadExpectation::Invalid => assert!(matches!(result, Err(Error::InvalidEntry))), + } +} + +#[rstest] +#[case::integer_two(Some(StoredValue::Integer(2)), 2.0)] +#[case::float_three_point_five(Some(StoredValue::Float(3.5)), 0.0)] +#[case::text_not_a_number(Some(StoredValue::Text("not a number".into())), 0.0)] +#[case::text_five(Some(StoredValue::Text("5".into())), 5.0)] +#[case::text_three_point_five(Some(StoredValue::Text("3.5".into())), 0.0)] +#[case::pickled_true(Some(StoredValue::Pickle(vec![0x80, 0x05, 0x88, 0x2e])), 1.0)] +#[case::pickled_two(Some(StoredValue::Pickle(vec![0x80, 0x05, 0x4b, 0x02, 0x2e])), 2.0)] +#[case::pickled_dictionary(Some(StoredValue::Pickle(vec![0x80, 0x05, 0x95, 0x0a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7d, 0x94, 0x8c, 0x01, 0x61, 0x94, 0x4b, 0x01, 0x73, 0x2e])), 0.0)] +#[case::missing(None, 0.0)] +#[case::pickled_none(Some(StoredValue::Pickle(vec![0x80, 0x05, 0x4e, 0x2e])), 0.0)] +fn python_counter_seed_cases(#[case] row: Option, #[case] expected: f64) { + assert_eq!(PythonDiskCacheAdapter.counter_seed(row).unwrap(), expected); +} + +#[rstest] +#[case::integer_three(3.0, StoredValue::Integer(3))] +#[case::fractional_three_point_five(3.5, StoredValue::Float(3.5))] +#[case::negative_zero(-0.0, StoredValue::Integer(0))] +#[case::large_float(1e300, StoredValue::Float(1e300))] +fn python_counter_value_cases(#[case] value: f64, #[case] expected: StoredValue) { + assert_eq!(PythonDiskCacheAdapter.counter_value(value), expected); +} diff --git a/litellm-rust/crates/cache-gcs/Cargo.toml b/litellm-rust/crates/cache-gcs/Cargo.toml new file mode 100644 index 00000000000..4ec60bcfa3b --- /dev/null +++ b/litellm-rust/crates/cache-gcs/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "litellm-cache-gcs" +version = "0.1.0" +edition.workspace = true +license.workspace = true +repository.workspace = true + +[dependencies] +futures-util.workspace = true +litellm-auth-gcp.workspace = true +litellm-auth-types.workspace = true +litellm-cache.workspace = true +percent-encoding.workspace = true +reqwest.workspace = true +tokio.workspace = true + +[dev-dependencies] +serde_json.workspace = true +tokio.workspace = true +wiremock = "0.6.5" diff --git a/litellm-rust/crates/cache-gcs/src/cache.rs b/litellm-rust/crates/cache-gcs/src/cache.rs new file mode 100644 index 00000000000..65282ac99d5 --- /dev/null +++ b/litellm-rust/crates/cache-gcs/src/cache.rs @@ -0,0 +1,260 @@ +use std::{future::Future, sync::Arc, time::Duration}; + +use futures_util::future::try_join_all; +use litellm_cache::{ + BaseCache, BatchCache, BatchEntry, CacheCodec, CacheConnectionResult, Error, ExactCacheContext, + FlushCache, +}; +use percent_encoding::{AsciiSet, NON_ALPHANUMERIC, percent_encode}; +use reqwest::Client; + +use crate::{GcpTokenSource, TokenSource}; + +pub const DEFAULT_ENDPOINT: &str = "https://storage.googleapis.com"; + +const OBJECT_NAME_ENCODE_SET: &AsciiSet = &NON_ALPHANUMERIC + .remove(b'-') + .remove(b'_') + .remove(b'.') + .remove(b'~'); + +pub fn key_prefix(gcs_path: Option<&str>) -> String { + match gcs_path { + Some(path) if !path.is_empty() => format!("{}/", path.trim_end_matches('/')), + _ => String::new(), + } +} + +#[derive(Clone, Debug)] +pub struct GcsConfig { + pub bucket_name: String, + pub gcs_path: Option, + pub path_service_account: Option, + pub endpoint: String, +} + +impl GcsConfig { + pub fn new(bucket_name: impl Into) -> Self { + Self { + bucket_name: bucket_name.into(), + gcs_path: None, + path_service_account: None, + endpoint: DEFAULT_ENDPOINT.to_string(), + } + } +} + +pub struct GcsCache { + config: GcsConfig, + key_prefix: String, + client: Client, + token: Arc, + codec: S, +} + +impl GcsCache { + pub fn new(config: GcsConfig, codec: S) -> Result { + let token = Arc::new(GcpTokenSource::new(config.path_service_account.clone())); + Self::with_token_source(config, codec, token) + } + + pub fn with_token_source( + config: GcsConfig, + codec: S, + token: Arc, + ) -> Result { + let client = Client::builder().build().map_err(|_| Error::Unavailable)?; + let key_prefix = key_prefix(config.gcs_path.as_deref()); + Ok(Self { + config, + key_prefix, + client, + token, + codec, + }) + } + + pub fn bucket_name(&self) -> &str { + &self.config.bucket_name + } + + pub fn key_prefix(&self) -> &str { + &self.key_prefix + } + + pub fn path_service_account(&self) -> Option<&str> { + self.config.path_service_account.as_deref() + } + + pub fn object_name(&self, key: &str) -> String { + format!("{}{}", self.key_prefix, key) + } + + fn encoded_object_name(&self, key: &str) -> String { + percent_encode(self.object_name(key).as_bytes(), OBJECT_NAME_ENCODE_SET).to_string() + } + + fn endpoint(&self, path: &str) -> String { + format!("{}{}", self.config.endpoint.trim_end_matches('/'), path) + } + + async fn async_set(&self, key: &str, value: S::Value) -> Result<(), Error> { + let token = self.token.bearer_token().await?; + let payload = self.codec.encode(&value)?; + let url = self.endpoint(&format!( + "/upload/storage/v1/b/{}/o?uploadType=media&name={}", + self.config.bucket_name, + self.encoded_object_name(key) + )); + let response = self + .client + .post(url) + .bearer_auth(token) + .header(reqwest::header::CONTENT_TYPE, "application/json") + .body(payload) + .send() + .await + .map_err(|_| Error::Unavailable)?; + if !response.status().is_success() { + return Err(Error::Unavailable); + } + Ok(()) + } + + async fn async_get(&self, key: &str) -> Result, Error> { + let token = self.token.bearer_token().await?; + let url = self.endpoint(&format!( + "/storage/v1/b/{}/o/{}?alt=media", + self.config.bucket_name, + self.encoded_object_name(key) + )); + let response = self + .client + .get(url) + .bearer_auth(token) + .header(reqwest::header::CONTENT_TYPE, "application/json") + .send() + .await + .map_err(|_| Error::Unavailable)?; + if response.status() == reqwest::StatusCode::NOT_FOUND { + return Ok(None); + } + if !response.status().is_success() { + return Err(Error::Unavailable); + } + let body = response.bytes().await.map_err(|_| Error::Unavailable)?; + self.codec + .decode(&body) + .map(Some) + .map_err(|_| Error::InvalidEntry) + } + + fn run_sync(future: F) -> Result + where + F: Future> + Send, + T: Send, + { + let run = || { + tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .map_err(|_| Error::Unavailable) + .and_then(|runtime| runtime.block_on(future)) + }; + if let Ok(handle) = tokio::runtime::Handle::try_current() { + if handle.runtime_flavor() == tokio::runtime::RuntimeFlavor::MultiThread { + return tokio::task::block_in_place(run); + } + return std::thread::scope(|scope| { + scope + .spawn(run) + .join() + .map_err(|_| Error::Unavailable) + .and_then(|result| result) + }); + } + run() + } +} + +impl BaseCache for GcsCache { + type Value = S::Value; + type Context = ExactCacheContext; + + fn get_ttl(&self, _: &Self::Context) -> Option { + None + } + + fn set_cache(&self, key: &str, value: Self::Value, _: &Self::Context) -> Result<(), Error> { + Self::run_sync(self.async_set(key, value)) + } + + fn get_cache(&self, key: &str, _: &Self::Context) -> Result, Error> { + Self::run_sync(self.async_get(key)) + } + + async fn async_set_cache( + &self, + key: &str, + value: Self::Value, + _: Self::Context, + ) -> Result<(), Error> { + self.async_set(key, value).await + } + + async fn async_get_cache( + &self, + key: &str, + _: &Self::Context, + ) -> Result, Error> { + self.async_get(key).await + } + + async fn async_set_cache_pipeline( + &self, + entries: Vec<(String, Self::Value)>, + context: Self::Context, + ) -> Result<(), Error> { + try_join_all(entries.into_iter().map(|(key, value)| { + let context = context.clone(); + async move { self.async_set_cache(&key, value, context).await } + })) + .await + .map(|_| ()) + } + + async fn disconnect(&self) -> Result<(), Error> { + Ok(()) + } + + async fn test_connection(&self) -> Result { + Err(Error::UnsupportedOperation) + } +} + +impl BatchCache for GcsCache { + async fn async_batch_get_cache( + &self, + keys: Vec, + context: Self::Context, + ) -> Result>, Error> { + try_join_all(keys.into_iter().map(|key| { + let context = context.clone(); + async move { + match self.async_get_cache(&key, &context).await { + Ok(Some(value)) => Ok(BatchEntry::Hit(value)), + Ok(None) => Ok(BatchEntry::Miss), + Err(Error::InvalidEntry) => Ok(BatchEntry::Invalid), + Err(error) => Err(error), + } + } + })) + .await + } +} + +impl FlushCache for GcsCache { + fn flush_cache(&self) -> Result<(), Error> { + Ok(()) + } +} diff --git a/litellm-rust/crates/cache-gcs/src/lib.rs b/litellm-rust/crates/cache-gcs/src/lib.rs new file mode 100644 index 00000000000..cbb61cf0685 --- /dev/null +++ b/litellm-rust/crates/cache-gcs/src/lib.rs @@ -0,0 +1,5 @@ +mod cache; +mod token; + +pub use cache::{DEFAULT_ENDPOINT, GcsCache, GcsConfig, key_prefix}; +pub use token::{GcpTokenSource, StaticTokenSource, TokenSource}; diff --git a/litellm-rust/crates/cache-gcs/src/token.rs b/litellm-rust/crates/cache-gcs/src/token.rs new file mode 100644 index 00000000000..adb601c276c --- /dev/null +++ b/litellm-rust/crates/cache-gcs/src/token.rs @@ -0,0 +1,44 @@ +use std::{future::Future, pin::Pin}; + +use litellm_auth_gcp::{VertexAuth, VertexConfig}; +use litellm_auth_types::{InputSource, SecretValue, Sourced}; +use litellm_cache::Error; + +pub trait TokenSource: Send + Sync + 'static { + fn bearer_token(&self) -> Pin> + Send + '_>>; +} + +pub struct GcpTokenSource { + auth: VertexAuth, + config: VertexConfig, +} + +impl GcpTokenSource { + pub fn new(path_service_account: Option) -> Self { + let credentials = path_service_account + .map(|path| Sourced::new(SecretValue::new(path), InputSource::Deployment)); + Self { + auth: VertexAuth::default(), + config: VertexConfig::new(credentials, None, None), + } + } +} + +impl TokenSource for GcpTokenSource { + fn bearer_token(&self) -> Pin> + Send + '_>> { + Box::pin(async move { + self.auth + .access_token(&self.config, &|name| std::env::var(name).ok()) + .await + .map_err(|_| Error::Unavailable) + }) + } +} + +pub struct StaticTokenSource(pub String); + +impl TokenSource for StaticTokenSource { + fn bearer_token(&self) -> Pin> + Send + '_>> { + Box::pin(async move { Ok(self.0.clone()) }) + } +} diff --git a/litellm-rust/crates/cache-gcs/tests/cache.rs b/litellm-rust/crates/cache-gcs/tests/cache.rs new file mode 100644 index 00000000000..45eecf01cec --- /dev/null +++ b/litellm-rust/crates/cache-gcs/tests/cache.rs @@ -0,0 +1,324 @@ +use std::{sync::Arc, time::Duration}; + +use litellm_cache::{ + BaseCache, BatchCache, BatchEntry, CacheContext, Error, ExactCacheContext, FlushCache, + JsonCodec, +}; +use litellm_cache_gcs::{GcsCache, GcsConfig, StaticTokenSource, TokenSource, key_prefix}; +use serde_json::json; +use wiremock::{ + Mock, MockServer, ResponseTemplate, + matchers::{body_bytes, header, method, path, query_param}, +}; + +fn config(server: &MockServer, gcs_path: Option<&str>) -> GcsConfig { + GcsConfig { + bucket_name: "bucket".into(), + gcs_path: gcs_path.map(str::to_string), + path_service_account: None, + endpoint: server.uri(), + } +} + +fn cache(server: &MockServer, gcs_path: Option<&str>) -> GcsCache> { + GcsCache::with_token_source( + config(server, gcs_path), + JsonCodec::new(), + Arc::new(StaticTokenSource("tok".into())), + ) + .unwrap() +} + +#[tokio::test] +async fn set_writes_encoded_object_and_headers() { + let server = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/upload/storage/v1/b/bucket/o")) + .and(query_param("uploadType", "media")) + .and(header("authorization", "Bearer tok")) + .and(header("content-type", "application/json")) + .and(body_bytes(br#"{"value":"entry"}"#)) + .respond_with(ResponseTemplate::new(200)) + .expect(1) + .mount(&server) + .await; + cache(&server, Some("cache/")) + .set_cache( + "team:a b/c", + json!({"value": "entry"}), + &ExactCacheContext::default(), + ) + .unwrap(); + let requests = server.received_requests().await.unwrap(); + assert_eq!(requests.len(), 1); + assert_eq!( + requests[0].url.query(), + Some("uploadType=media&name=cache%2Fteam%3Aa%20b%2Fc") + ); +} + +#[tokio::test] +async fn get_maps_statuses_and_decode_failures() { + let server = MockServer::start().await; + Mock::given(method("GET")) + .and(path("/storage/v1/b/bucket/o/hit")) + .and(query_param("alt", "media")) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({"value": "entry"}))) + .mount(&server) + .await; + Mock::given(method("GET")) + .and(path("/storage/v1/b/bucket/o/missing")) + .respond_with(ResponseTemplate::new(404)) + .mount(&server) + .await; + Mock::given(method("GET")) + .and(path("/storage/v1/b/bucket/o/server-error")) + .respond_with(ResponseTemplate::new(500)) + .mount(&server) + .await; + Mock::given(method("GET")) + .and(path("/storage/v1/b/bucket/o/invalid")) + .respond_with(ResponseTemplate::new(200).set_body_string("not json")) + .mount(&server) + .await; + + let cache = cache(&server, None); + assert_eq!( + cache + .get_cache("hit", &ExactCacheContext::default()) + .unwrap(), + Some(json!({"value": "entry"})) + ); + assert_eq!( + cache + .get_cache("missing", &ExactCacheContext::default()) + .unwrap(), + None + ); + assert_eq!( + cache + .get_cache("server-error", &ExactCacheContext::default()) + .unwrap_err(), + Error::Unavailable + ); + assert_eq!( + cache + .get_cache("invalid", &ExactCacheContext::default()) + .unwrap_err(), + Error::InvalidEntry + ); +} + +#[test] +fn key_prefix_normalizes_paths() { + assert_eq!(key_prefix(None), ""); + assert_eq!(key_prefix(Some("a/b/")), "a/b/"); + assert_eq!(key_prefix(Some("a/b")), "a/b/"); + assert_eq!(key_prefix(Some("")), ""); +} + +#[tokio::test] +async fn object_names_use_python_quote_encoding() { + let server = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/upload/storage/v1/b/bucket/o")) + .and(query_param("uploadType", "media")) + .respond_with(ResponseTemplate::new(200)) + .expect(2) + .mount(&server) + .await; + let cache = cache(&server, Some("p/")); + cache + .set_cache( + "a~b-c_d.e/f g%h", + json!({"value": "punctuation"}), + &ExactCacheContext::default(), + ) + .unwrap(); + cache + .set_cache( + "ключ", + json!({"value": "utf8"}), + &ExactCacheContext::default(), + ) + .unwrap(); + let requests = server.received_requests().await.unwrap(); + let queries: Vec<_> = requests + .iter() + .filter_map(|request| request.url.query()) + .collect(); + assert!(queries.contains(&"uploadType=media&name=p%2Fa~b-c_d.e%2Ff%20g%25h")); + assert!(queries.contains(&"uploadType=media&name=p%2F%D0%BA%D0%BB%D1%8E%D1%87")); +} + +#[tokio::test] +async fn ignores_ttl_and_writes_pipeline_concurrently() { + let server = MockServer::start().await; + for key in ["one", "two", "three"] { + Mock::given(method("POST")) + .and(path("/upload/storage/v1/b/bucket/o")) + .and(query_param("uploadType", "media")) + .and(query_param("name", key)) + .respond_with(ResponseTemplate::new(200)) + .expect(1) + .mount(&server) + .await; + } + let cache = cache(&server, None); + assert_eq!(cache.get_ttl(&ExactCacheContext::default()), None); + assert_eq!( + cache.get_ttl(&ExactCacheContext::default().with_ttl(Some(Duration::from_secs(5)))), + None + ); + cache + .async_set_cache_pipeline( + vec![ + ("one".into(), json!({"key": "one"})), + ("two".into(), json!({"key": "two"})), + ("three".into(), json!({"key": "three"})), + ], + ExactCacheContext::default().with_ttl(Some(Duration::from_secs(5))), + ) + .await + .unwrap(); +} + +#[tokio::test] +async fn async_batch_get_preserves_hits_misses_and_invalid_entries() { + let server = MockServer::start().await; + Mock::given(method("GET")) + .and(path("/storage/v1/b/bucket/o/hit")) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({"value": "entry"}))) + .mount(&server) + .await; + Mock::given(method("GET")) + .and(path("/storage/v1/b/bucket/o/missing")) + .respond_with(ResponseTemplate::new(404)) + .mount(&server) + .await; + Mock::given(method("GET")) + .and(path("/storage/v1/b/bucket/o/invalid")) + .respond_with(ResponseTemplate::new(200).set_body_string("not json")) + .mount(&server) + .await; + + assert_eq!( + cache(&server, None) + .async_batch_get_cache( + vec!["hit".into(), "missing".into(), "invalid".into()], + ExactCacheContext::default(), + ) + .await + .unwrap(), + vec![ + BatchEntry::Hit(json!({"value": "entry"})), + BatchEntry::Miss, + BatchEntry::Invalid, + ] + ); +} + +#[tokio::test] +async fn lifecycle_operations_are_noops_and_connection_test_is_unsupported() { + let server = MockServer::start().await; + let cache = cache(&server, None); + assert_eq!(cache.flush_cache(), Ok(())); + assert_eq!(cache.disconnect().await, Ok(())); + assert_eq!( + cache.test_connection().await, + Err(Error::UnsupportedOperation) + ); +} + +#[test] +fn sync_operations_work_without_an_active_runtime() { + let runtime = tokio::runtime::Builder::new_multi_thread() + .enable_all() + .build() + .unwrap(); + let server = runtime.block_on(MockServer::start()); + runtime.block_on( + Mock::given(method("POST")) + .and(path("/upload/storage/v1/b/bucket/o")) + .respond_with(ResponseTemplate::new(200)) + .mount(&server), + ); + runtime.block_on( + Mock::given(method("GET")) + .and(path("/storage/v1/b/bucket/o/key")) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({"value": "entry"}))) + .mount(&server), + ); + let cache = cache(&server, None); + cache + .set_cache( + "key", + json!({"value": "entry"}), + &ExactCacheContext::default(), + ) + .unwrap(); + assert_eq!( + cache + .get_cache("key", &ExactCacheContext::default()) + .unwrap(), + Some(json!({"value": "entry"})) + ); +} + +#[tokio::test(flavor = "multi_thread")] +async fn sync_operations_work_inside_a_multi_thread_runtime() { + let server = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/upload/storage/v1/b/bucket/o")) + .respond_with(ResponseTemplate::new(200)) + .mount(&server) + .await; + Mock::given(method("GET")) + .and(path("/storage/v1/b/bucket/o/key")) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({"value": "entry"}))) + .mount(&server) + .await; + let cache = cache(&server, None); + cache + .set_cache( + "key", + json!({"value": "entry"}), + &ExactCacheContext::default(), + ) + .unwrap(); + assert_eq!( + cache + .get_cache("key", &ExactCacheContext::default()) + .unwrap(), + Some(json!({"value": "entry"})) + ); +} + +struct FailingTokenSource; + +impl TokenSource for FailingTokenSource { + fn bearer_token( + &self, + ) -> std::pin::Pin> + Send + '_>> + { + Box::pin(async { Err(Error::Unavailable) }) + } +} + +#[tokio::test] +async fn token_source_failure_skips_http() { + let server = MockServer::start().await; + let cache = GcsCache::with_token_source( + config(&server, None), + JsonCodec::::new(), + Arc::new(FailingTokenSource), + ) + .unwrap(); + assert_eq!( + cache + .get_cache("key", &ExactCacheContext::default()) + .unwrap_err(), + Error::Unavailable + ); + assert_eq!(server.received_requests().await.unwrap().len(), 0); +} diff --git a/litellm-rust/crates/cache-qdrant-semantic/Cargo.toml b/litellm-rust/crates/cache-qdrant-semantic/Cargo.toml new file mode 100644 index 00000000000..09d6a9637f3 --- /dev/null +++ b/litellm-rust/crates/cache-qdrant-semantic/Cargo.toml @@ -0,0 +1,24 @@ +[package] +name = "litellm-cache-qdrant-semantic" +version = "0.1.0" +edition.workspace = true +license.workspace = true +repository.workspace = true + +[dependencies] +futures-util.workspace = true +litellm-cache.workspace = true +qdrant-client = { workspace = true, features = ["serde"] } +reqwest.workspace = true +serde.workspace = true +serde_json.workspace = true +thiserror.workspace = true +tokio.workspace = true +uuid.workspace = true + +[dev-dependencies] +litellm-cache-response.workspace = true +rstest.workspace = true +tonic = "0.14" +tonic-prost = "0.14" +tokio-stream = "0.1" diff --git a/litellm-rust/crates/cache-qdrant-semantic/src/embedder.rs b/litellm-rust/crates/cache-qdrant-semantic/src/embedder.rs new file mode 100644 index 00000000000..47b898d6f4e --- /dev/null +++ b/litellm-rust/crates/cache-qdrant-semantic/src/embedder.rs @@ -0,0 +1,75 @@ +use std::time::Duration; + +use litellm_cache::Error; +use reqwest::Client; +use serde_json::Value; + +use crate::Embedder; + +pub struct OpenAiEmbedder { + client: Client, + api_base: String, + api_key: String, + model: String, + timeout: Option, +} + +pub struct OpenAiEmbedderConfig { + pub api_base: String, + pub api_key: String, + pub model: String, + pub timeout: Option, +} + +impl OpenAiEmbedder { + pub fn new(client: Client, config: OpenAiEmbedderConfig) -> Self { + Self { + client, + api_base: config.api_base.trim_end_matches('/').to_owned(), + api_key: config.api_key, + model: config.model, + timeout: config.timeout, + } + } +} + +impl Embedder for OpenAiEmbedder { + fn model(&self) -> &str { + &self.model + } + + async fn embed(&self, input: &str) -> Result, Error> { + let request = self + .client + .post(format!("{}/embeddings", self.api_base)) + .bearer_auth(&self.api_key) + .json(&serde_json::json!({ + "model": self.model, + "input": input, + "encoding_format": "float", + })); + let response = if let Some(timeout) = self.timeout { + request.timeout(timeout) + } else { + request + } + .send() + .await + .map_err(|_| Error::Unavailable)? + .error_for_status() + .map_err(|_| Error::Unavailable)?; + let body: Value = response.json().await.map_err(|_| Error::Unavailable)?; + body.get("data") + .and_then(Value::as_array) + .and_then(|data| data.first()) + .and_then(|item| item.get("embedding")) + .and_then(Value::as_array) + .and_then(|embedding| { + embedding + .iter() + .map(|value| value.as_f64().map(|value| value as f32)) + .collect::>>() + }) + .ok_or(Error::Unavailable) + } +} diff --git a/litellm-rust/crates/cache-qdrant-semantic/src/lib.rs b/litellm-rust/crates/cache-qdrant-semantic/src/lib.rs new file mode 100644 index 00000000000..0f346a9155b --- /dev/null +++ b/litellm-rust/crates/cache-qdrant-semantic/src/lib.rs @@ -0,0 +1,7 @@ +mod embedder; +mod prompt; +mod semantic; + +pub use embedder::{OpenAiEmbedder, OpenAiEmbedderConfig}; +pub use prompt::prompt_from_messages; +pub use semantic::{Embedder, QdrantSemanticCache, QdrantSemanticConfig, Quantization}; diff --git a/litellm-rust/crates/cache-qdrant-semantic/src/prompt.rs b/litellm-rust/crates/cache-qdrant-semantic/src/prompt.rs new file mode 100644 index 00000000000..ef1a2306658 --- /dev/null +++ b/litellm-rust/crates/cache-qdrant-semantic/src/prompt.rs @@ -0,0 +1,59 @@ +use serde_json::Value; + +fn search_results_text(search_results: Option<&Value>) -> String { + let Some(Value::Array(results)) = search_results else { + return String::new(); + }; + results + .iter() + .filter_map(Value::as_object) + .flat_map(|result| { + let source = result + .get("source") + .and_then(Value::as_str) + .map(str::to_owned); + let title = result + .get("title") + .and_then(Value::as_str) + .map(str::to_owned); + let content = result + .get("content") + .and_then(Value::as_array) + .into_iter() + .flatten() + .filter_map(Value::as_object) + .filter_map(|block| block.get("text").and_then(Value::as_str).map(str::to_owned)); + let citations = result + .get("citations") + .filter(|value| !value.is_null()) + .map(|value| serde_json::to_string(value).unwrap_or_default()); + source + .into_iter() + .chain(title) + .chain(content) + .chain(citations) + }) + .collect() +} + +pub fn prompt_from_messages(messages: &[Value]) -> String { + messages + .iter() + .filter_map(Value::as_object) + .map(|message| { + let content = match message.get("content") { + Some(Value::String(content)) => content.clone(), + Some(Value::Array(parts)) => parts + .iter() + .filter_map(Value::as_object) + .filter_map(|part| part.get("text").and_then(Value::as_str)) + .collect(), + _ => String::new(), + }; + format!( + "{content}{}", + search_results_text(message.get("search_results")) + ) + }) + .collect() +} diff --git a/litellm-rust/crates/cache-qdrant-semantic/src/semantic.rs b/litellm-rust/crates/cache-qdrant-semantic/src/semantic.rs new file mode 100644 index 00000000000..fb165ed5a8e --- /dev/null +++ b/litellm-rust/crates/cache-qdrant-semantic/src/semantic.rs @@ -0,0 +1,262 @@ +use std::future::Future; + +use futures_util::future::try_join_all; +use litellm_cache::{BaseCache, CacheCodec, CacheConnectionResult, Error, SemanticCacheContext}; +use qdrant_client::{ + Payload, Qdrant, + qdrant::{ + BinaryQuantizationBuilder, CompressionRatio, Condition, CreateCollectionBuilder, + CreateFieldIndexCollectionBuilder, Distance, FieldType, Filter, PointStruct, + ProductQuantizationBuilder, QuantizationSearchParamsBuilder, ScalarQuantizationBuilder, + SearchParamsBuilder, SearchPointsBuilder, UpsertPointsBuilder, VectorParamsBuilder, + }, +}; +use serde_json::{Map, Value, json}; +use uuid::Uuid; + +use crate::prompt_from_messages; + +pub trait Embedder: Send + Sync + 'static { + fn model(&self) -> &str; + fn embed(&self, input: &str) -> impl Future, Error>> + Send; +} + +#[derive(Clone, Debug, PartialEq)] +pub enum Quantization { + Binary, + Scalar, + Product, +} + +pub struct QdrantSemanticConfig { + pub collection_name: String, + pub similarity_threshold: f64, + pub vector_size: u64, + pub quantization: Quantization, +} + +pub struct QdrantSemanticCache { + client: Qdrant, + embedder: E, + codec: C, + config: QdrantSemanticConfig, + runtime: tokio::runtime::Handle, +} + +impl QdrantSemanticCache { + pub async fn connect( + client: Qdrant, + embedder: E, + codec: C, + config: QdrantSemanticConfig, + runtime: tokio::runtime::Handle, + ) -> Result { + let exists = client + .collection_exists(config.collection_name.clone()) + .await + .map_err(|_| Error::Unavailable)?; + if !exists { + client + .create_collection( + CreateCollectionBuilder::new(config.collection_name.clone()) + .vectors_config(VectorParamsBuilder::new( + config.vector_size, + Distance::Cosine, + )) + .quantization_config(quantization(&config.quantization)), + ) + .await + .map_err(|_| Error::Unavailable)?; + } + let _ = client + .create_field_index(CreateFieldIndexCollectionBuilder::new( + config.collection_name.clone(), + "litellm_cache_key".to_owned(), + FieldType::Keyword, + )) + .await; + Ok(Self { + client, + embedder, + codec, + config, + runtime, + }) + } + + pub fn collection_name(&self) -> &str { + &self.config.collection_name + } + + pub fn similarity_threshold(&self) -> f64 { + self.config.similarity_threshold + } + + pub fn vector_size(&self) -> u64 { + self.config.vector_size + } + + pub fn embedder(&self) -> &E { + &self.embedder + } + + fn prompt(context: &SemanticCacheContext) -> Result { + let Some(messages) = context.messages.as_ref().and_then(Value::as_array) else { + return Err(Error::MissingPrompt); + }; + if messages.is_empty() { + return Err(Error::MissingPrompt); + } + Ok(prompt_from_messages(messages)) + } + + async fn set( + &self, + key: &str, + value: C::Value, + context: &SemanticCacheContext, + ) -> Result<(), Error> { + let prompt = Self::prompt(context)?; + let vector = self.embedder.embed(&prompt).await?; + let response = + String::from_utf8(self.codec.encode(&value)?).map_err(|_| Error::InvalidEntry)?; + let payload = Payload::try_from(json!({ + "litellm_cache_key": key, + "text": prompt, + "response": response, + })) + .map_err(|_| Error::InvalidEntry)?; + self.client + .upsert_points( + UpsertPointsBuilder::new( + self.collection_name(), + vec![PointStruct::new( + Uuid::new_v4().to_string(), + vector, + payload, + )], + ) + .wait(true), + ) + .await + .map_err(|_| Error::Unavailable)?; + Ok(()) + } + + async fn get( + &self, + key: &str, + context: &SemanticCacheContext, + ) -> Result, Error> { + let prompt = Self::prompt(context)?; + let vector = self.embedder.embed(&prompt).await?; + let result = self + .client + .search_points( + SearchPointsBuilder::new(self.collection_name(), vector, 1) + .with_payload(true) + .filter(Filter::must([Condition::matches( + "litellm_cache_key", + key.to_owned(), + )])) + .params( + SearchParamsBuilder::default().quantization( + QuantizationSearchParamsBuilder::default() + .ignore(false) + .rescore(true) + .oversampling(3.0), + ), + ), + ) + .await + .map_err(|_| Error::Unavailable)?; + let Some(point) = result.result.into_iter().next() else { + return Ok(None); + }; + let payload: Map = Payload::from(point.payload).into(); + if payload.get("litellm_cache_key").and_then(Value::as_str) != Some(key) { + return Ok(None); + } + if f64::from(point.score) < self.config.similarity_threshold { + return Ok(None); + } + let response = payload + .get("response") + .and_then(Value::as_str) + .ok_or(Error::InvalidEntry)?; + self.codec.decode(response.as_bytes()).map(Some) + } +} + +fn quantization(value: &Quantization) -> qdrant_client::qdrant::quantization_config::Quantization { + match value { + Quantization::Binary => BinaryQuantizationBuilder::new(false).into(), + Quantization::Scalar => ScalarQuantizationBuilder::default() + .quantile(0.99) + .always_ram(false) + .into(), + Quantization::Product => ProductQuantizationBuilder::new(CompressionRatio::X16.into()) + .always_ram(false) + .into(), + } +} + +impl BaseCache for QdrantSemanticCache { + type Value = C::Value; + type Context = SemanticCacheContext; + + fn get_ttl(&self, _: &Self::Context) -> Option { + None + } + + fn set_cache( + &self, + key: &str, + value: Self::Value, + context: &Self::Context, + ) -> Result<(), Error> { + self.runtime.block_on(self.set(key, value, context)) + } + + fn get_cache(&self, key: &str, context: &Self::Context) -> Result, Error> { + self.runtime.block_on(self.get(key, context)) + } + + async fn async_set_cache( + &self, + key: &str, + value: Self::Value, + context: Self::Context, + ) -> Result<(), Error> { + self.set(key, value, &context).await + } + + async fn async_get_cache( + &self, + key: &str, + context: &Self::Context, + ) -> Result, Error> { + self.get(key, context).await + } + + async fn async_set_cache_pipeline( + &self, + entries: Vec<(String, Self::Value)>, + context: Self::Context, + ) -> Result<(), Error> { + try_join_all(entries.into_iter().map(|(key, value)| { + let context = context.clone(); + async move { self.async_set_cache(&key, value, context).await } + })) + .await + .map(|_| ()) + } + + async fn disconnect(&self) -> Result<(), Error> { + Ok(()) + } + + async fn test_connection(&self) -> Result { + Err(Error::UnsupportedOperation) + } +} diff --git a/litellm-rust/crates/cache-qdrant-semantic/tests/embedder.rs b/litellm-rust/crates/cache-qdrant-semantic/tests/embedder.rs new file mode 100644 index 00000000000..6b09448fde8 --- /dev/null +++ b/litellm-rust/crates/cache-qdrant-semantic/tests/embedder.rs @@ -0,0 +1,166 @@ +use std::{ + sync::{Arc, Mutex}, + time::Duration, +}; + +use litellm_cache::Error; +use litellm_cache_qdrant_semantic::{Embedder, OpenAiEmbedder, OpenAiEmbedderConfig}; +use serde_json::Value; +use tokio::{ + io::{AsyncReadExt, AsyncWriteExt}, + net::TcpListener, +}; + +struct TestHttpServer { + address: std::net::SocketAddr, + request: Arc>>>, + task: tokio::task::JoinHandle<()>, +} + +impl TestHttpServer { + async fn response(status: &str, body: &str) -> Self { + Self::response_after(status, body, Duration::ZERO).await + } + + async fn response_after(status: &str, body: &str, delay: Duration) -> Self { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let request = Arc::new(Mutex::new(None)); + let captured = request.clone(); + let status = status.to_owned(); + let body = body.to_owned(); + let task = tokio::spawn(async move { + let (mut stream, _) = listener.accept().await.unwrap(); + let request_bytes = read_request(&mut stream).await; + *captured.lock().unwrap() = Some(request_bytes); + tokio::time::sleep(delay).await; + let response = format!( + "HTTP/1.1 {status}\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", + body.len() + ); + stream.write_all(response.as_bytes()).await.unwrap(); + }); + Self { + address, + request, + task, + } + } + + fn base_url(&self) -> String { + format!("http://{}", self.address) + } +} + +impl Drop for TestHttpServer { + fn drop(&mut self) { + self.task.abort(); + } +} + +async fn read_request(stream: &mut tokio::net::TcpStream) -> Vec { + let mut bytes = Vec::new(); + let header_end = loop { + let mut chunk = [0_u8; 1024]; + let count = stream.read(&mut chunk).await.unwrap(); + assert_ne!(count, 0); + bytes.extend_from_slice(&chunk[..count]); + if let Some(end) = bytes.windows(4).position(|window| window == b"\r\n\r\n") { + break end + 4; + } + }; + let headers = String::from_utf8_lossy(&bytes[..header_end]); + let content_length = headers + .lines() + .find_map(|line| { + line.split_once(':') + .filter(|(name, _)| name.eq_ignore_ascii_case("content-length")) + .map(|(_, value)| value.trim()) + }) + .unwrap() + .parse::() + .unwrap(); + while bytes.len() < header_end + content_length { + let mut chunk = [0_u8; 1024]; + let count = stream.read(&mut chunk).await.unwrap(); + assert_ne!(count, 0); + bytes.extend_from_slice(&chunk[..count]); + } + bytes +} + +fn config(base: String, timeout: Option) -> OpenAiEmbedderConfig { + OpenAiEmbedderConfig { + api_base: base, + api_key: "test-key".to_owned(), + model: "test-model".to_owned(), + timeout, + } +} + +#[tokio::test] +async fn posts_embeddings_request_and_parses_vector() { + let server = TestHttpServer::response("200 OK", r#"{"data":[{"embedding":[0.1,0.2]}]}"#).await; + let embedder = OpenAiEmbedder::new( + reqwest::Client::new(), + config( + format!("{}/", server.base_url()), + Some(Duration::from_secs(1)), + ), + ); + assert_eq!(embedder.embed("hello").await.unwrap(), vec![0.1, 0.2]); + let request = server.request.lock().unwrap().clone().unwrap(); + let request_text = String::from_utf8(request).unwrap(); + assert!(request_text.starts_with("POST /embeddings HTTP/1.1\r\n")); + assert!(request_text.contains("\r\nauthorization: Bearer test-key\r\n")); + let body = request_text.split("\r\n\r\n").nth(1).unwrap(); + let body: Value = serde_json::from_str(body).unwrap(); + assert_eq!(body["model"], "test-model"); + assert_eq!(body["input"], "hello"); + assert_eq!(body["encoding_format"], "float"); +} + +#[tokio::test] +async fn status_and_timeout_errors_are_unavailable() { + let server = TestHttpServer::response("500 Internal Server Error", "{}").await; + let embedder = OpenAiEmbedder::new(reqwest::Client::new(), config(server.base_url(), None)); + assert_eq!(embedder.embed("hello").await, Err(Error::Unavailable)); + + let server = TestHttpServer::response_after( + "200 OK", + r#"{"data":[{"embedding":[0.1,0.2]}]}"#, + Duration::from_millis(500), + ) + .await; + let embedder = OpenAiEmbedder::new( + reqwest::Client::new(), + config(server.base_url(), Some(Duration::from_millis(200))), + ); + assert_eq!(embedder.embed("hello").await, Err(Error::Unavailable)); + + let server = TestHttpServer::response_after( + "200 OK", + r#"{"data":[{"embedding":[0.1,0.2]}]}"#, + Duration::from_millis(100), + ) + .await; + let embedder = OpenAiEmbedder::new( + reqwest::Client::new(), + config(server.base_url(), Some(Duration::from_secs(1))), + ); + assert_eq!(embedder.embed("hello").await.unwrap(), vec![0.1, 0.2]); +} + +#[tokio::test] +async fn uses_the_injected_client() { + let server = TestHttpServer::response("200 OK", r#"{"data":[{"embedding":[0.1,0.2]}]}"#).await; + let client = reqwest::Client::builder() + .user_agent("litellm-embedder-test") + .build() + .unwrap(); + let embedder = OpenAiEmbedder::new(client, config(server.base_url(), None)); + assert_eq!(embedder.embed("hello").await.unwrap(), vec![0.1, 0.2]); + let request = server.request.lock().unwrap().clone().unwrap(); + let request_text = String::from_utf8(request).unwrap(); + assert!(request_text.contains("\r\nuser-agent: litellm-embedder-test\r\n")); +} diff --git a/litellm-rust/crates/cache-qdrant-semantic/tests/prompt.rs b/litellm-rust/crates/cache-qdrant-semantic/tests/prompt.rs new file mode 100644 index 00000000000..38cd9e2f908 --- /dev/null +++ b/litellm-rust/crates/cache-qdrant-semantic/tests/prompt.rs @@ -0,0 +1,38 @@ +use litellm_cache_qdrant_semantic::prompt_from_messages; +use serde_json::json; + +#[test] +fn prompt_matches_python_message_content_rules() { + let messages = vec![ + json!({"role": "user", "content": "hello"}), + json!({ + "role": "user", + "content": [ + {"type": "text", "text": "world"}, + {"type": "image_url", "image_url": {"url": "ignored"}}, + {"type": "text", "text": "!"}, + ], + }), + ]; + + assert_eq!(prompt_from_messages(&messages), "helloworld!"); +} + +#[test] +fn prompt_includes_search_result_text_and_compact_citations() { + let messages = vec![json!({ + "role": "tool", + "content": null, + "search_results": [{ + "source": "source", + "title": "title", + "content": [{"text": "body"}], + "citations": {"page": 1, "section": "intro"}, + }], + })]; + + assert_eq!( + prompt_from_messages(&messages), + r#"sourcetitlebody{"page":1,"section":"intro"}"# + ); +} diff --git a/litellm-rust/crates/cache-qdrant-semantic/tests/qdrant.rs b/litellm-rust/crates/cache-qdrant-semantic/tests/qdrant.rs new file mode 100644 index 00000000000..c7522c0b313 --- /dev/null +++ b/litellm-rust/crates/cache-qdrant-semantic/tests/qdrant.rs @@ -0,0 +1,422 @@ +#[path = "support/mod.rs"] +mod support; + +use std::{collections::HashMap, sync::Arc, time::Duration}; + +use litellm_cache::{BaseCache, CacheCodec, CacheContext, Error, SemanticCacheContext}; +use litellm_cache_qdrant_semantic::{ + Embedder, QdrantSemanticCache, QdrantSemanticConfig, Quantization, +}; +use litellm_cache_response::{ + CacheEntry, CacheKeyInput, ResponseCache, ResponseCacheCodec, ResponseCacheRequest, +}; +use qdrant_client::Payload; +use qdrant_client::{ + Qdrant, + qdrant::{self, CompressionRatio, Distance, PointId, QuantizationType, Value, VectorParams}, +}; +use serde_json::{Value as JsonValue, json}; + +use support::{FakeQdrant, FakeState, StoredPoint}; + +#[derive(Clone)] +struct FixedEmbedder { + vectors: Arc>>, +} + +impl FixedEmbedder { + fn new(vectors: impl IntoIterator)>) -> Self { + Self { + vectors: Arc::new( + vectors + .into_iter() + .map(|(prompt, vector)| (prompt.to_owned(), vector)) + .collect(), + ), + } + } +} + +impl Embedder for FixedEmbedder { + fn model(&self) -> &str { + "fixed" + } + + async fn embed(&self, input: &str) -> Result, Error> { + self.vectors.get(input).cloned().ok_or(Error::Unavailable) + } +} + +fn config(quantization: Quantization) -> QdrantSemanticConfig { + QdrantSemanticConfig { + collection_name: "semantic".to_owned(), + similarity_threshold: 0.9, + vector_size: 2, + quantization, + } +} + +fn context(prompt: &str) -> SemanticCacheContext { + SemanticCacheContext { + messages: Some(json!([{"role": "user", "content": prompt}])), + ..Default::default() + } +} + +fn value(response: JsonValue) -> CacheEntry { + CacheEntry { + timestamp: Some(1.0), + response, + } +} + +async fn connect( + server: &FakeQdrant, + vectors: impl IntoIterator)>, +) -> QdrantSemanticCache { + let client = Qdrant::from_url(&server.url()).build().unwrap(); + QdrantSemanticCache::connect( + client, + FixedEmbedder::new(vectors), + ResponseCacheCodec, + config(Quantization::Binary), + tokio::runtime::Handle::current(), + ) + .await + .unwrap() +} + +#[tokio::test(flavor = "multi_thread")] +#[expect( + deprecated, + reason = "the test verifies Qdrant's legacy always_ram quantization contract" +)] +async fn connect_sets_collection_quantization_and_index() { + for (quantization, expected) in [ + (Quantization::Binary, 0), + (Quantization::Scalar, 1), + (Quantization::Product, 2), + ] { + let server = FakeQdrant::start(FakeState::default()).await; + let client = Qdrant::from_url(&server.url()).build().unwrap(); + QdrantSemanticCache::connect( + client, + FixedEmbedder::new([]), + ResponseCacheCodec, + config(quantization), + tokio::runtime::Handle::current(), + ) + .await + .unwrap(); + let state = server.state.lock().unwrap(); + let request = &state.created_collections[0]; + let Some(qdrant::vectors_config::Config::Params(VectorParams { size, distance, .. })) = + request + .vectors_config + .as_ref() + .and_then(|config| config.config.clone()) + else { + panic!("missing vector params"); + }; + assert_eq!(size, 2); + assert_eq!(distance, Distance::Cosine as i32); + let quantization_config = request + .quantization_config + .as_ref() + .unwrap() + .quantization + .unwrap(); + match (expected, quantization_config) { + (0, qdrant::quantization_config::Quantization::Binary(binary)) => { + assert_eq!(binary.always_ram, Some(false)); + } + (1, qdrant::quantization_config::Quantization::Scalar(scalar)) => { + assert_eq!(scalar.r#type, QuantizationType::Int8 as i32); + assert_eq!(scalar.quantile, Some(0.99)); + assert_eq!(scalar.always_ram, Some(false)); + } + (2, qdrant::quantization_config::Quantization::Product(product)) => { + assert_eq!(product.compression, CompressionRatio::X16 as i32); + assert_eq!(product.always_ram, Some(false)); + } + _ => panic!("unexpected quantization"), + } + assert!(state.index_creations >= 1); + assert_eq!(state.field_indexes[0].collection_name, "semantic"); + assert_eq!(state.field_indexes[0].field_name, "litellm_cache_key"); + assert_eq!( + state.field_indexes[0].field_type, + Some(qdrant::FieldType::Keyword as i32) + ); + server.stop(); + } +} + +#[tokio::test(flavor = "multi_thread")] +async fn existing_collection_skips_create_and_index_failure_is_non_fatal() { + let server = FakeQdrant::start(FakeState { + collections: ["semantic".to_owned()].into_iter().collect(), + fail_field_index: true, + ..Default::default() + }) + .await; + let _cache = connect(&server, [("hello", vec![1.0, 0.0])]).await; + let state = server.state.lock().unwrap(); + assert!(state.created_collections.is_empty()); + assert!(state.index_creations >= 1); + server.stop(); +} + +#[tokio::test(flavor = "multi_thread")] +async fn async_and_sync_set_get_store_exact_payload() { + let server = FakeQdrant::start(FakeState::default()).await; + let cache = Arc::new(connect(&server, [("hello", vec![1.0, 0.0])]).await); + let ctx = context("hello"); + let entry = value(json!({"answer": 42})); + cache + .async_set_cache("key", entry.clone(), ctx.clone()) + .await + .unwrap(); + assert_eq!( + cache.async_get_cache("key", &ctx).await.unwrap().as_ref(), + Some(&entry) + ); + { + let state = server.state.lock().unwrap(); + let payload = &state.points[0].payload; + let mut payload_keys = payload.keys().cloned().collect::>(); + payload_keys.sort(); + assert_eq!(payload_keys, ["litellm_cache_key", "response", "text"]); + assert_eq!(payload["litellm_cache_key"], Value::from("key")); + assert_eq!( + payload["response"], + Value::from(String::from_utf8(ResponseCacheCodec.encode(&entry).unwrap()).unwrap()) + ); + } + let sync_entry = entry.clone(); + let sync_cache = cache.clone(); + let sync_ctx = ctx.clone(); + tokio::task::spawn_blocking(move || { + sync_cache + .set_cache("sync", sync_entry.clone(), &sync_ctx) + .unwrap(); + assert_eq!( + sync_cache.get_cache("sync", &sync_ctx).unwrap(), + Some(sync_entry) + ); + }) + .await + .unwrap(); + server.stop(); +} + +#[tokio::test(flavor = "multi_thread")] +async fn misses_and_payload_validation_are_safe() { + let server = FakeQdrant::start(FakeState::default()).await; + let cache = connect( + &server, + [("hello", vec![1.0, 0.0]), ("near", vec![0.7, 0.71414286])], + ) + .await; + let entry = value(json!({"answer": 1})); + cache + .async_set_cache("key", entry, context("hello")) + .await + .unwrap(); + assert_eq!( + cache + .async_get_cache("other", &context("hello")) + .await + .unwrap(), + None + ); + assert_eq!( + cache + .async_get_cache("key", &context("near")) + .await + .unwrap(), + None + ); + server.insert_point(StoredPoint { + id: Some(PointId::from(99_u64)), + vector: vec![1.0, 0.0], + payload: Payload::try_from(json!({ + "litellm_cache_key": 99, + "response": "{}", + })) + .unwrap() + .into(), + }); + assert_eq!( + cache + .async_get_cache("99", &context("hello")) + .await + .unwrap(), + None + ); + server.stop(); +} + +#[tokio::test(flavor = "multi_thread")] +async fn decoding_errors_missing_prompt_pipeline_and_ttl_behave_as_required() { + let server = FakeQdrant::start(FakeState::default()).await; + let cache = connect(&server, [("one", vec![1.0, 0.0]), ("two", vec![0.0, 1.0])]).await; + let empty = SemanticCacheContext::default(); + assert_eq!( + cache + .async_set_cache("key", value(json!({})), empty.clone()) + .await, + Err(Error::MissingPrompt) + ); + assert_eq!( + cache.async_get_cache("key", &empty).await, + Err(Error::MissingPrompt) + ); + assert_eq!( + cache.async_get_cache("key", &context("unknown")).await, + Err(Error::Unavailable) + ); + cache + .async_set_cache( + "ttl", + value(json!({"ttl": true})), + context("one").with_ttl(Some(Duration::from_secs(1))), + ) + .await + .unwrap(); + tokio::time::sleep(Duration::from_millis(1_100)).await; + assert!( + cache + .async_get_cache( + "ttl", + &context("one").with_ttl(Some(Duration::from_secs(1))), + ) + .await + .unwrap() + .is_some() + ); + cache + .async_set_cache_pipeline( + vec![ + ("one".to_owned(), value(json!({"n": 1}))), + ("two".to_owned(), value(json!({"n": 2}))), + ], + context("one"), + ) + .await + .unwrap(); + assert!( + cache + .async_get_cache("one", &context("one")) + .await + .unwrap() + .is_some() + ); + assert!( + cache + .async_get_cache("two", &context("one")) + .await + .unwrap() + .is_some() + ); + assert_eq!( + server.state.lock().unwrap().upsert_waits, + vec![Some(true), Some(true), Some(true)] + ); + assert_eq!(cache.get_ttl(&context("one")), None); + assert_eq!( + cache.test_connection().await, + Err(Error::UnsupportedOperation) + ); + server.stop(); +} + +#[tokio::test(flavor = "multi_thread")] +async fn response_payloads_decode_and_invalid_entries_fail() { + let server = FakeQdrant::start(FakeState::default()).await; + let cache = connect(&server, [("hello", vec![1.0, 0.0])]).await; + for (key, response) in [ + ("python", json!("{'timestamp': 1.0, 'response': {'a': 1}}")), + ("garbage", json!("not json")), + ("missing", json!("unused")), + ] { + let mut payload = serde_json::Map::new(); + payload.insert("litellm_cache_key".to_owned(), json!(key)); + if key != "missing" { + payload.insert("response".to_owned(), response); + } + server.insert_point(StoredPoint { + id: Some(PointId::from(key.len() as u64)), + vector: vec![1.0, 0.0], + payload: Payload::try_from(JsonValue::Object(payload)) + .unwrap() + .into(), + }); + } + assert_eq!( + cache + .async_get_cache("python", &context("hello")) + .await + .unwrap(), + Some(value(json!({"a": 1}))) + ); + assert_eq!( + cache.async_get_cache("garbage", &context("hello")).await, + Err(Error::InvalidEntry) + ); + assert_eq!( + cache.async_get_cache("missing", &context("hello")).await, + Err(Error::InvalidEntry) + ); + server.stop(); +} + +#[tokio::test(flavor = "multi_thread")] +async fn response_cache_facade_turns_invalid_entry_into_miss() { + let server = FakeQdrant::start(FakeState::default()).await; + let cache = Arc::new(connect(&server, [("hello", vec![1.0, 0.0])]).await); + let request = ResponseCacheRequest::::new(CacheKeyInput { + preset: Some("key".to_owned()), + ..Default::default() + }) + .with_context(context("hello")); + let response = json!({"answer": 42}); + let facade = ResponseCache::new(cache.clone()); + facade + .async_store(&request, response.clone(), Duration::from_secs(1)) + .await + .unwrap(); + assert_eq!( + facade + .async_lookup(&request, Duration::from_secs(1)) + .await + .unwrap(), + Some(response) + ); + { + let mut state = server.state.lock().unwrap(); + state.points[0] + .payload + .insert("response".to_owned(), Value::from("not json")); + } + assert_eq!( + facade + .async_lookup(&request, Duration::from_secs(1)) + .await + .unwrap(), + None + ); + server.stop(); +} + +#[tokio::test(flavor = "multi_thread")] +async fn stopped_qdrant_server_maps_to_unavailable() { + let server = FakeQdrant::start(FakeState::default()).await; + let cache = connect(&server, [("hello", vec![1.0, 0.0])]).await; + server.stop(); + tokio::time::sleep(Duration::from_millis(50)).await; + assert_eq!( + cache.async_get_cache("key", &context("hello")).await, + Err(Error::Unavailable) + ); +} diff --git a/litellm-rust/crates/cache-qdrant-semantic/tests/support/mod.rs b/litellm-rust/crates/cache-qdrant-semantic/tests/support/mod.rs new file mode 100644 index 00000000000..9a556ae7df5 --- /dev/null +++ b/litellm-rust/crates/cache-qdrant-semantic/tests/support/mod.rs @@ -0,0 +1,342 @@ +use std::{ + collections::{HashMap, HashSet}, + net::SocketAddr, + sync::{Arc, Mutex}, +}; + +use qdrant_client::qdrant::collections_server::CollectionsServer; +use qdrant_client::qdrant::{ + self, CollectionExists, CollectionExistsRequest, CollectionExistsResponse, + CollectionOperationResponse, CreateCollection, CreateFieldIndexCollection, Filter, PointId, + PointsOperationResponse, ScoredPoint, SearchPoints, SearchResponse, Value, Vector, Vectors, + collections_server::Collections, + points_server::{Points, PointsServer}, +}; +use tokio::sync::oneshot; +use tokio_stream::wrappers::TcpListenerStream; +use tonic::{Request, Response, Status, transport::Server}; + +#[derive(Clone, Debug)] +pub struct StoredPoint { + pub id: Option, + pub vector: Vec, + pub payload: HashMap, +} + +#[derive(Default)] +pub struct FakeState { + pub collections: HashSet, + pub created_collections: Vec, + pub field_indexes: Vec, + pub points: Vec, + pub upsert_waits: Vec>, + pub index_creations: usize, + pub fail_field_index: bool, +} + +#[derive(Clone)] +pub struct FakeQdrant { + pub state: Arc>, + pub address: SocketAddr, + shutdown: Arc>>>, +} + +impl FakeQdrant { + pub async fn start(state: FakeState) -> Self { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let state = Arc::new(Mutex::new(state)); + let service = FakeService { + state: state.clone(), + }; + let (shutdown_tx, shutdown_rx) = oneshot::channel(); + tokio::spawn(async move { + Server::builder() + .add_service(CollectionsServer::new(service.clone())) + .add_service(PointsServer::new(service)) + .serve_with_incoming_shutdown(TcpListenerStream::new(listener), async { + let _ = shutdown_rx.await; + }) + .await + .unwrap(); + }); + Self { + state, + address, + shutdown: Arc::new(Mutex::new(Some(shutdown_tx))), + } + } + + pub fn url(&self) -> String { + format!("http://{}", self.address) + } + + pub fn stop(&self) { + self.shutdown + .lock() + .unwrap() + .take() + .unwrap() + .send(()) + .unwrap(); + } + + pub fn insert_point(&self, point: StoredPoint) { + self.state.lock().unwrap().points.push(point); + } +} + +#[derive(Clone)] +struct FakeService { + state: Arc>, +} + +macro_rules! unimplemented_collections { + ($($name:ident, $request:ty, $response:ty);* $(;)?) => { + $( + fn $name<'life0, 'async_trait>( + &'life0 self, + _: Request<$request>, + ) -> std::pin::Pin< + Box< + dyn std::future::Future< + Output = Result, Status>, + > + Send + + 'async_trait, + >, + > + where + 'life0: 'async_trait, + Self: 'async_trait, + { + Box::pin(async { Err(Status::unimplemented(stringify!($name))) }) + } + )* + }; +} + +macro_rules! unimplemented_points { + ($($name:ident, $request:ty, $response:ty);* $(;)?) => { + $( + fn $name<'life0, 'async_trait>( + &'life0 self, + _: Request<$request>, + ) -> std::pin::Pin< + Box< + dyn std::future::Future< + Output = Result, Status>, + > + Send + + 'async_trait, + >, + > + where + 'life0: 'async_trait, + Self: 'async_trait, + { + Box::pin(async { Err(Status::unimplemented(stringify!($name))) }) + } + )* + }; +} + +#[tonic::async_trait] +impl Collections for FakeService { + async fn create( + &self, + request: Request, + ) -> Result, Status> { + let request = request.into_inner(); + let mut state = self.state.lock().unwrap(); + state.collections.insert(request.collection_name.clone()); + state.created_collections.push(request); + Ok(Response::new(CollectionOperationResponse { + result: true, + ..Default::default() + })) + } + + async fn collection_exists( + &self, + request: Request, + ) -> Result, Status> { + let exists = self + .state + .lock() + .unwrap() + .collections + .contains(&request.into_inner().collection_name); + Ok(Response::new(CollectionExistsResponse { + result: Some(CollectionExists { exists }), + ..Default::default() + })) + } + + unimplemented_collections!( + get, qdrant::GetCollectionInfoRequest, qdrant::GetCollectionInfoResponse; + list, qdrant::ListCollectionsRequest, qdrant::ListCollectionsResponse; + update, qdrant::UpdateCollection, qdrant::CollectionOperationResponse; + delete, qdrant::DeleteCollection, qdrant::CollectionOperationResponse; + update_aliases, qdrant::ChangeAliases, qdrant::CollectionOperationResponse; + list_collection_aliases, qdrant::ListCollectionAliasesRequest, qdrant::ListAliasesResponse; + list_aliases, qdrant::ListAliasesRequest, qdrant::ListAliasesResponse; + collection_cluster_info, qdrant::CollectionClusterInfoRequest, qdrant::CollectionClusterInfoResponse; + update_collection_cluster_setup, qdrant::UpdateCollectionClusterSetupRequest, qdrant::UpdateCollectionClusterSetupResponse; + create_shard_key, qdrant::CreateShardKeyRequest, qdrant::CreateShardKeyResponse; + delete_shard_key, qdrant::DeleteShardKeyRequest, qdrant::DeleteShardKeyResponse; + list_shard_keys, qdrant::ListShardKeysRequest, qdrant::ListShardKeysResponse; + ); +} + +#[tonic::async_trait] +impl Points for FakeService { + async fn create_field_index( + &self, + request: Request, + ) -> Result, Status> { + let mut state = self.state.lock().unwrap(); + state.index_creations += 1; + state.field_indexes.push(request.into_inner()); + if state.fail_field_index { + return Err(Status::internal("field index failure")); + } + Ok(Response::new(PointsOperationResponse::default())) + } + + async fn upsert( + &self, + request: Request, + ) -> Result, Status> { + let request = request.into_inner(); + let mut state = self.state.lock().unwrap(); + state.upsert_waits.push(request.wait); + for point in request.points { + let stored = StoredPoint { + id: point.id.clone(), + vector: dense_vector(point.vectors)?, + payload: point.payload, + }; + if let Some(existing) = state + .points + .iter_mut() + .find(|existing| existing.id == stored.id) + { + *existing = stored; + } else { + state.points.push(stored); + } + } + Ok(Response::new(PointsOperationResponse::default())) + } + + async fn search( + &self, + request: Request, + ) -> Result, Status> { + let request = request.into_inner(); + let key_filter = keyword_filter(request.filter.as_ref()); + let state = self.state.lock().unwrap(); + let mut results = state + .points + .iter() + .filter(|point| { + key_filter.as_ref().is_none_or(|(field, expected)| { + point + .payload + .get(field) + .and_then(|value| { + let value: serde_json::Value = value.clone().into(); + value + .as_str() + .map(str::to_owned) + .or_else(|| value.as_i64().map(|value| value.to_string())) + }) + .is_some_and(|value| value == *expected) + }) + }) + .map(|point| ScoredPoint { + id: point.id.clone(), + payload: point.payload.clone(), + score: cosine(&request.vector, &point.vector), + ..Default::default() + }) + .collect::>(); + results.sort_by(|left, right| right.score.total_cmp(&left.score)); + results.truncate(request.limit as usize); + Ok(Response::new(SearchResponse { + result: results, + ..Default::default() + })) + } + + unimplemented_points!( + delete, qdrant::DeletePoints, qdrant::PointsOperationResponse; + get, qdrant::GetPoints, qdrant::GetResponse; + update_vectors, qdrant::UpdatePointVectors, qdrant::PointsOperationResponse; + delete_vectors, qdrant::DeletePointVectors, qdrant::PointsOperationResponse; + set_payload, qdrant::SetPayloadPoints, qdrant::PointsOperationResponse; + overwrite_payload, qdrant::SetPayloadPoints, qdrant::PointsOperationResponse; + delete_payload, qdrant::DeletePayloadPoints, qdrant::PointsOperationResponse; + clear_payload, qdrant::ClearPayloadPoints, qdrant::PointsOperationResponse; + delete_field_index, qdrant::DeleteFieldIndexCollection, qdrant::PointsOperationResponse; + create_vector_name, qdrant::CreateVectorNameRequest, qdrant::PointsOperationResponse; + delete_vector_name, qdrant::DeleteVectorNameRequest, qdrant::PointsOperationResponse; + search_batch, qdrant::SearchBatchPoints, qdrant::SearchBatchResponse; + search_groups, qdrant::SearchPointGroups, qdrant::SearchGroupsResponse; + scroll, qdrant::ScrollPoints, qdrant::ScrollResponse; + recommend, qdrant::RecommendPoints, qdrant::RecommendResponse; + recommend_batch, qdrant::RecommendBatchPoints, qdrant::RecommendBatchResponse; + recommend_groups, qdrant::RecommendPointGroups, qdrant::RecommendGroupsResponse; + discover, qdrant::DiscoverPoints, qdrant::DiscoverResponse; + discover_batch, qdrant::DiscoverBatchPoints, qdrant::DiscoverBatchResponse; + count, qdrant::CountPoints, qdrant::CountResponse; + update_batch, qdrant::UpdateBatchPoints, qdrant::UpdateBatchResponse; + query, qdrant::QueryPoints, qdrant::QueryResponse; + query_batch, qdrant::QueryBatchPoints, qdrant::QueryBatchResponse; + query_groups, qdrant::QueryPointGroups, qdrant::QueryGroupsResponse; + facet, qdrant::FacetCounts, qdrant::FacetResponse; + search_matrix_pairs, qdrant::SearchMatrixPoints, qdrant::SearchMatrixPairsResponse; + search_matrix_offsets, qdrant::SearchMatrixPoints, qdrant::SearchMatrixOffsetsResponse; + ); +} + +fn dense_vector(vectors: Option) -> Result, Status> { + let Some(Vectors { + vectors_options: + Some(qdrant::vectors::VectorsOptions::Vector(Vector { + vector: Some(qdrant::vector::Vector::Dense(qdrant::DenseVector { data })), + .. + })), + }) = vectors + else { + return Err(Status::invalid_argument("expected dense vector")); + }; + Ok(data) +} + +fn keyword_filter(filter: Option<&Filter>) -> Option<(String, String)> { + filter? + .must + .iter() + .find_map(|condition| match condition.condition_one_of.as_ref()? { + qdrant::condition::ConditionOneOf::Field(field) => { + let qdrant::r#match::MatchValue::Keyword(value) = + field.r#match.as_ref()?.match_value.as_ref()? + else { + return None; + }; + Some((field.key.clone(), value.clone())) + } + _ => None, + }) +} + +fn cosine(left: &[f32], right: &[f32]) -> f32 { + let dot = left + .iter() + .zip(right) + .map(|(left, right)| left * right) + .sum::(); + let left_norm = left.iter().map(|value| value * value).sum::().sqrt(); + let right_norm = right.iter().map(|value| value * value).sum::().sqrt(); + dot / (left_norm * right_norm) +} diff --git a/litellm-rust/crates/cache-redis-semantic/Cargo.toml b/litellm-rust/crates/cache-redis-semantic/Cargo.toml new file mode 100644 index 00000000000..9a8755a189e --- /dev/null +++ b/litellm-rust/crates/cache-redis-semantic/Cargo.toml @@ -0,0 +1,21 @@ +[package] +name = "litellm-cache-redis-semantic" +version = "0.1.0" +edition.workspace = true +license.workspace = true +repository.workspace = true + +[dependencies] +litellm-cache.workspace = true +litellm-cache-redis.workspace = true +litellm-cache-response.workspace = true +redis = { version = "1.7.0", features = ["tls-rustls"] } +r2d2 = "0.8.10" +serde_json.workspace = true +sha2.workspace = true +tokio.workspace = true + +[dev-dependencies] +redis-test = "1.0.4" +serde_json.workspace = true +tokio.workspace = true diff --git a/litellm-rust/crates/cache-redis-semantic/src/cache.rs b/litellm-rust/crates/cache-redis-semantic/src/cache.rs new file mode 100644 index 00000000000..e0ac31f3630 --- /dev/null +++ b/litellm-rust/crates/cache-redis-semantic/src/cache.rs @@ -0,0 +1,618 @@ +use std::{ + future::Future, + sync::{Arc, OnceLock}, + time::{Duration, SystemTime, UNIX_EPOCH}, +}; + +use litellm_cache::{ + BaseCache, CacheCodec, CacheConnectionResult, CacheConnectionStatus, Error, + SemanticCacheContext, +}; +use litellm_cache_redis::{ + RedisTopology, + connection::{ConnectionRef, Connections}, +}; +use litellm_cache_response::{CacheEntry, ResponseCacheCodec}; +use serde_json::Value; +use sha2::{Digest, Sha256}; + +use crate::prompt::prompt_from_context; + +const CACHE_KEY_FIELD: &str = "litellm_cache_key"; +const VECTOR_FIELD: &str = "prompt_vector"; + +pub trait Embedder: Send + Sync + 'static { + fn embed(&self, prompt: &str, metadata: Option<&Value>) -> Result, Error>; + + fn async_embed( + &self, + prompt: &str, + metadata: Option<&Value>, + ) -> impl Future, Error>> + Send; +} + +#[derive(Clone, Debug)] +pub struct RedisSemanticConfig { + pub index_name: String, + pub similarity_threshold: f32, +} + +struct Inner { + index_name: String, + distance_threshold: f64, + resolved_index: OnceLock, + codec: ResponseCacheCodec, + clock: fn() -> f64, +} + +impl Inner { + fn new(config: RedisSemanticConfig) -> Self { + Self { + index_name: config.index_name, + distance_threshold: 1.0 - f64::from(config.similarity_threshold), + resolved_index: OnceLock::new(), + codec: ResponseCacheCodec, + clock: timestamp, + } + } + + fn ensure_index( + &self, + connection: &mut ConnectionRef<'_>, + dims: usize, + ) -> Result { + if let Some(name) = self.resolved_index.get() { + return Ok(name.clone()); + } + let name = match index_compatible(connection, &self.index_name, dims)? { + Some(true) => self.index_name.clone(), + Some(false) => self.isolated_index(connection, dims)?, + None => match create_index(connection, &self.index_name, dims) { + Ok(()) => self.index_name.clone(), + Err(_) => match index_compatible(connection, &self.index_name, dims)? { + Some(true) => self.index_name.clone(), + Some(false) => self.isolated_index(connection, dims)?, + None => return Err(Error::Unavailable), + }, + }, + }; + let _ = self.resolved_index.set(name.clone()); + Ok(name) + } + + fn isolated_index( + &self, + connection: &mut ConnectionRef<'_>, + dims: usize, + ) -> Result { + let name = format!("{}_isolated", self.index_name); + match index_compatible(connection, &name, dims)? { + Some(true) => Ok(name), + Some(false) => { + redis::cmd("FT.DROPINDEX") + .arg(&name) + .query::<()>(connection) + .map_err(|_| Error::Unavailable)?; + create_index(connection, &name, dims)?; + Ok(name) + } + None => { + create_index(connection, &name, dims)?; + Ok(name) + } + } + } + + fn store( + &self, + connection: &mut ConnectionRef<'_>, + tag: &str, + value: &CacheEntry, + prompt: &str, + vector: &[f32], + ttl: Option, + ) -> Result<(), Error> { + let index = self.ensure_index(connection, vector.len())?; + let entry_id = entry_id(prompt, tag); + let hash_key = format!("{index}:{entry_id}"); + let response = self.codec.encode(value)?; + redis::cmd("HSET") + .arg(&hash_key) + .arg("entry_id") + .arg(&entry_id) + .arg("prompt") + .arg(prompt) + .arg("response") + .arg(response) + .arg(VECTOR_FIELD) + .arg(vector_buffer(vector)) + .arg("inserted_at") + .arg(format!("{}", (self.clock)())) + .arg("updated_at") + .arg(format!("{}", (self.clock)())) + .arg(CACHE_KEY_FIELD) + .arg(tag) + .query::<()>(connection) + .map_err(|_| Error::Unavailable)?; + if let Some(ttl) = ttl { + redis::cmd("EXPIRE") + .arg(&hash_key) + .arg(ttl_seconds(ttl)) + .query::<()>(connection) + .map_err(|_| Error::Unavailable)?; + } + Ok(()) + } + + fn lookup( + &self, + connection: &mut ConnectionRef<'_>, + tag: &str, + vector: &[f32], + ) -> Result, Error> { + let index = self.ensure_index(connection, vector.len())?; + let query = format!( + "(@{CACHE_KEY_FIELD}:{{{}}})=>[KNN 1 @{VECTOR_FIELD} $vector AS vector_distance]", + escape_tag(tag) + ); + let result = redis::cmd("FT.SEARCH") + .arg(&index) + .arg(query) + .arg("RETURN") + .arg(8) + .arg("entry_id") + .arg("prompt") + .arg("response") + .arg("inserted_at") + .arg("updated_at") + .arg("metadata") + .arg(CACHE_KEY_FIELD) + .arg("vector_distance") + .arg("SORTBY") + .arg("vector_distance") + .arg("ASC") + .arg("DIALECT") + .arg(2) + .arg("LIMIT") + .arg(0) + .arg(1) + .arg("PARAMS") + .arg(2) + .arg("vector") + .arg(vector_buffer(vector)) + .query::(connection) + .map_err(|_| Error::Unavailable)?; + let Some(fields) = first_document(&result) else { + return Ok(None); + }; + if string_field(fields, CACHE_KEY_FIELD).as_deref() != Some(tag) { + return Ok(None); + } + if number_field(fields, "vector_distance") + .is_none_or(|distance| distance > self.distance_threshold) + { + return Ok(None); + } + let Some(response) = bytes_field(fields, "response") else { + return Ok(None); + }; + self.codec.decode(&response).map(Some) + } +} + +pub struct RedisSemanticCache { + connections: Arc>, + embedder: E, + inner: Arc, +} + +impl RedisSemanticCache { + pub fn new(url: &str, embedder: E, config: RedisSemanticConfig) -> Result { + Ok(Self { + connections: Arc::new(Connections::open(url, &RedisTopology::Standalone)?), + embedder, + inner: Arc::new(Inner::new(config)), + }) + } +} + +impl RedisSemanticCache { + pub fn with_connection(connection: C, embedder: E, config: RedisSemanticConfig) -> Self { + Self { + connections: Arc::new(Connections::fixed(connection)), + embedder, + inner: Arc::new(Inner::new(config)), + } + } + + pub fn with_clock(self, clock: fn() -> f64) -> Self { + Self { + inner: Arc::new(Inner { + index_name: self.inner.index_name.clone(), + distance_threshold: self.inner.distance_threshold, + resolved_index: OnceLock::new(), + codec: self.inner.codec, + clock, + }), + ..self + } + } + + pub fn embedder(&self) -> &E { + &self.embedder + } + + pub fn index_name(&self) -> &str { + &self.inner.index_name + } + + pub fn similarity_threshold(&self) -> f32 { + (1.0 - self.inner.distance_threshold) as f32 + } + + fn tag<'a>(key: &'a str, context: &'a SemanticCacheContext) -> &'a str { + context.scope.as_deref().unwrap_or(key) + } +} + +impl BaseCache + for RedisSemanticCache +{ + type Value = CacheEntry; + type Context = SemanticCacheContext; + + fn get_ttl(&self, context: &Self::Context) -> Option { + context.ttl + } + + fn set_cache( + &self, + key: &str, + value: Self::Value, + context: &Self::Context, + ) -> Result<(), Error> { + let Some(prompt) = prompt_from_context(context) else { + return Ok(()); + }; + let vector = self.embedder.embed(&prompt, context.metadata.as_ref())?; + let tag = Self::tag(key, context).to_string(); + self.connections.execute(|connection| { + self.inner + .store(connection, &tag, &value, &prompt, &vector, context.ttl) + }) + } + + fn get_cache(&self, key: &str, context: &Self::Context) -> Result, Error> { + let Some(prompt) = prompt_from_context(context) else { + return Ok(None); + }; + let vector = self.embedder.embed(&prompt, context.metadata.as_ref())?; + let tag = Self::tag(key, context).to_string(); + self.connections + .execute(|connection| self.inner.lookup(connection, &tag, &vector)) + } + + async fn async_set_cache( + &self, + key: &str, + value: Self::Value, + context: Self::Context, + ) -> Result<(), Error> { + let Some(prompt) = prompt_from_context(&context) else { + return Ok(()); + }; + let vector = self + .embedder + .async_embed(&prompt, context.metadata.as_ref()) + .await?; + let tag = Self::tag(key, &context).to_string(); + let inner = Arc::clone(&self.inner); + Connections::run_blocking(Arc::clone(&self.connections), move |connection| { + inner.store(connection, &tag, &value, &prompt, &vector, context.ttl) + }) + .await + } + + async fn async_get_cache( + &self, + key: &str, + context: &Self::Context, + ) -> Result, Error> { + let Some(prompt) = prompt_from_context(context) else { + return Ok(None); + }; + let vector = self + .embedder + .async_embed(&prompt, context.metadata.as_ref()) + .await?; + let tag = Self::tag(key, context).to_string(); + let inner = Arc::clone(&self.inner); + Connections::run_blocking(Arc::clone(&self.connections), move |connection| { + inner.lookup(connection, &tag, &vector) + }) + .await + } + + async fn disconnect(&self) -> Result<(), Error> { + Ok(()) + } + + async fn test_connection(&self) -> Result { + match Connections::run_blocking(Arc::clone(&self.connections), |connection| { + Ok(match redis::cmd("PING").query::(connection) { + Ok(_) => CacheConnectionResult { + status: CacheConnectionStatus::Success, + message: "Redis cache connection test successful".into(), + error: None, + }, + Err(error) => CacheConnectionResult { + status: CacheConnectionStatus::Failed, + message: format!("Redis connection failed: {error}"), + error: Some(error.to_string()), + }, + }) + }) + .await + { + Ok(result) => Ok(result), + Err(error) => Ok(CacheConnectionResult { + status: CacheConnectionStatus::Failed, + message: format!("Redis connection failed: {error}"), + error: Some(error.to_string()), + }), + } + } +} + +fn timestamp() -> f64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|duration| duration.as_secs_f64()) + .unwrap_or_default() +} + +fn entry_id(prompt: &str, tag: &str) -> String { + let mut digest = Sha256::new(); + digest.update(prompt.as_bytes()); + digest.update(CACHE_KEY_FIELD.as_bytes()); + digest.update(tag.as_bytes()); + format!("{:x}", digest.finalize()) +} + +fn vector_buffer(vector: &[f32]) -> Vec { + vector + .iter() + .flat_map(|component| component.to_le_bytes()) + .collect() +} + +fn escape_tag(value: &str) -> String { + value + .chars() + .flat_map(|ch| { + if matches!( + ch, + ',' | '.' + | '<' + | '>' + | '{' + | '}' + | '[' + | ']' + | '\\' + | '"' + | '\'' + | ':' + | ';' + | '!' + | '@' + | '#' + | '$' + | '%' + | '^' + | '&' + | '*' + | '(' + | ')' + | '-' + | '+' + | '=' + | '~' + | '|' + | '/' + | ' ' + | '?' + ) { + vec!['\\', ch] + } else { + vec![ch] + } + }) + .collect() +} + +fn create_index(connection: &mut ConnectionRef<'_>, name: &str, dims: usize) -> Result<(), Error> { + redis::cmd("FT.CREATE") + .arg(name) + .arg("ON") + .arg("HASH") + .arg("PREFIX") + .arg(1) + .arg(name) + .arg("SCORE") + .arg(1.0) + .arg("SCHEMA") + .arg("prompt") + .arg("TEXT") + .arg("WEIGHT") + .arg(1) + .arg("response") + .arg("TEXT") + .arg("WEIGHT") + .arg(1) + .arg("inserted_at") + .arg("NUMERIC") + .arg("updated_at") + .arg("NUMERIC") + .arg(VECTOR_FIELD) + .arg("VECTOR") + .arg("FLAT") + .arg(6) + .arg("TYPE") + .arg("FLOAT32") + .arg("DIM") + .arg(dims) + .arg("DISTANCE_METRIC") + .arg("COSINE") + .arg(CACHE_KEY_FIELD) + .arg("TAG") + .arg("SEPARATOR") + .arg(",") + .query::<()>(connection) + .map_err(|_| Error::Unavailable) +} + +fn index_compatible( + connection: &mut ConnectionRef<'_>, + name: &str, + dims: usize, +) -> Result, Error> { + let info = match redis::cmd("FT.INFO") + .arg(name) + .query::(connection) + { + Ok(info) => info, + Err(error) if unknown_index(&error) => return Ok(None), + Err(_) => return Err(Error::Unavailable), + }; + Ok(Some(schema_compatible(&info, dims))) +} + +fn unknown_index(error: &redis::RedisError) -> bool { + let message = error.to_string().to_lowercase(); + message.contains("unknown") && message.contains("index") +} + +fn schema_compatible(info: &redis::Value, dims: usize) -> bool { + let redis::Value::Array(entries) = info else { + return false; + }; + let attributes = entries + .as_chunks::<2>() + .0 + .iter() + .find(|pair| string_value(&pair[0]).as_deref() == Some("attributes")) + .map(|pair| &pair[1]); + let Some(redis::Value::Array(attributes)) = attributes else { + return false; + }; + let fields = attributes + .iter() + .map(|attribute| { + let redis::Value::Array(attribute) = attribute else { + return (None, None, None, None, None); + }; + let mut name = None; + let mut field_type = None; + let mut dim = None; + let mut data_type = None; + let mut distance_metric = None; + for pair in attribute.as_chunks::<2>().0 { + match string_value(&pair[0]).as_deref() { + Some("identifier") => name = string_value(&pair[1]), + Some("type") => field_type = string_value(&pair[1]), + Some("dim") => dim = number_value(&pair[1]), + Some("data_type") => data_type = string_value(&pair[1]), + Some("distance_metric") => distance_metric = string_value(&pair[1]), + _ => {} + } + } + (name, field_type, dim, data_type, distance_metric) + }) + .collect::>(); + let has_field = |name: &str, field_type: &str| { + fields + .iter() + .any(|(n, t, ..)| n.as_deref() == Some(name) && t.as_deref() == Some(field_type)) + }; + has_field("prompt", "TEXT") + && has_field("response", "TEXT") + && has_field("inserted_at", "NUMERIC") + && has_field("updated_at", "NUMERIC") + && has_field(CACHE_KEY_FIELD, "TAG") + && fields.iter().any(|(n, t, d, data, metric)| { + n.as_deref() == Some(VECTOR_FIELD) + && t.as_deref() == Some("VECTOR") + && *d == Some(dims as f64) + && data + .as_deref() + .is_some_and(|data| data.eq_ignore_ascii_case("float32")) + && metric + .as_deref() + .is_some_and(|metric| metric.eq_ignore_ascii_case("cosine")) + }) +} + +fn string_value(value: &redis::Value) -> Option { + match value { + redis::Value::BulkString(bytes) => String::from_utf8(bytes.clone()).ok(), + redis::Value::SimpleString(text) => Some(text.clone()), + redis::Value::VerbatimString { text, .. } => Some(text.clone()), + _ => None, + } +} + +fn number_value(value: &redis::Value) -> Option { + match value { + redis::Value::Int(number) => Some(*number as f64), + redis::Value::Double(number) => Some(*number), + _ => string_value(value).and_then(|text| text.parse().ok()), + } +} + +fn first_document(result: &redis::Value) -> Option<&[redis::Value]> { + let redis::Value::Array(items) = result else { + return None; + }; + let [count, _document_id, fields, ..] = items.as_slice() else { + return None; + }; + if !matches!(count, redis::Value::Int(count) if *count > 0) { + return None; + } + match fields { + redis::Value::Array(fields) => Some(fields.as_slice()), + _ => None, + } +} + +fn field_value<'a>(fields: &'a [redis::Value], name: &str) -> Option<&'a redis::Value> { + fields + .as_chunks::<2>() + .0 + .iter() + .find(|pair| string_value(&pair[0]).as_deref() == Some(name)) + .map(|pair| &pair[1]) +} + +fn string_field(fields: &[redis::Value], name: &str) -> Option { + field_value(fields, name).and_then(string_value) +} + +fn number_field(fields: &[redis::Value], name: &str) -> Option { + field_value(fields, name).and_then(number_value) +} + +fn bytes_field(fields: &[redis::Value], name: &str) -> Option> { + match field_value(fields, name)? { + redis::Value::BulkString(bytes) => Some(bytes.clone()), + redis::Value::SimpleString(text) => Some(text.clone().into_bytes()), + _ => None, + } +} + +fn ttl_seconds(ttl: Duration) -> u64 { + ttl.as_secs() + .saturating_add(u64::from(ttl.subsec_nanos() > 0)) + .max(1) +} diff --git a/litellm-rust/crates/cache-redis-semantic/src/lib.rs b/litellm-rust/crates/cache-redis-semantic/src/lib.rs new file mode 100644 index 00000000000..51d0b4ba5f3 --- /dev/null +++ b/litellm-rust/crates/cache-redis-semantic/src/lib.rs @@ -0,0 +1,5 @@ +mod cache; +mod prompt; + +pub use cache::{Embedder, RedisSemanticCache, RedisSemanticConfig}; +pub use prompt::prompt_from_context; diff --git a/litellm-rust/crates/cache-redis-semantic/src/prompt.rs b/litellm-rust/crates/cache-redis-semantic/src/prompt.rs new file mode 100644 index 00000000000..b9c38e98d77 --- /dev/null +++ b/litellm-rust/crates/cache-redis-semantic/src/prompt.rs @@ -0,0 +1,97 @@ +use litellm_cache::SemanticCacheContext; +use serde_json::Value; + +pub fn prompt_from_context(context: &SemanticCacheContext) -> Option { + if let Some(messages) = context.messages.as_ref().and_then(Value::as_array) + && !messages.is_empty() + { + return Some(messages_text(messages)); + } + let input = context.input.as_ref()?; + let mut parts = Vec::new(); + collect_input_text(input, &mut parts); + let prompt = parts.join("\n").trim().to_string(); + (!prompt.is_empty()).then_some(prompt) +} + +fn messages_text(messages: &[Value]) -> String { + let mut text = String::new(); + for message in messages { + let Some(message) = message.as_object() else { + continue; + }; + match message.get("content") { + Some(Value::String(content)) => text.push_str(content), + Some(Value::Array(parts)) => { + for part in parts { + if let Some(text_content) = part.get("text").and_then(Value::as_str) { + text.push_str(text_content); + } + } + } + _ => {} + } + text.push_str(&search_results_text(message.get("search_results"))); + } + text +} + +fn search_results_text(search_results: Option<&Value>) -> String { + let Some(Value::Array(results)) = search_results else { + return String::new(); + }; + let mut text = String::new(); + for result in results { + let Some(result) = result.as_object() else { + continue; + }; + for key in ["source", "title"] { + if let Some(value) = result.get(key).and_then(Value::as_str) { + text.push_str(value); + } + } + if let Some(Value::Array(content)) = result.get("content") { + for block in content { + if let Some(value) = block.get("text").and_then(Value::as_str) { + text.push_str(value); + } + } + } + if let Some(citations) = result.get("citations") { + text.push_str(&citations.to_string()); + } + } + text +} + +fn collect_input_text(value: &Value, parts: &mut Vec) { + match value { + Value::String(text) => { + let trimmed = text.trim(); + if !trimmed.is_empty() { + parts.push(trimmed.to_string()); + } + } + Value::Array(items) => { + for item in items { + collect_input_text(item, parts); + } + } + Value::Object(map) => { + if let Some(content) = map.get("content").filter(|content| !content.is_null()) { + collect_input_text(content, parts); + return; + } + for key in ["text", "output", "input_text", "output_text"] { + if let Some(Value::String(text)) = map.get(key) { + let trimmed = text.trim(); + if !trimmed.is_empty() { + parts.push(trimmed.to_string()); + return; + } + } + } + } + _ => {} + } +} diff --git a/litellm-rust/crates/cache-redis-semantic/tests/cache.rs b/litellm-rust/crates/cache-redis-semantic/tests/cache.rs new file mode 100644 index 00000000000..233b87ec52f --- /dev/null +++ b/litellm-rust/crates/cache-redis-semantic/tests/cache.rs @@ -0,0 +1,1003 @@ +use std::{ + collections::HashMap, + sync::{Arc, Mutex}, + time::Duration, +}; + +use litellm_cache::{BaseCache, CacheCodec, Error, SemanticCacheContext}; +use litellm_cache_redis_semantic::{Embedder, RedisSemanticCache, RedisSemanticConfig}; +use litellm_cache_response::{CacheEntry, ResponseCacheCodec}; +use redis_test::{MockCmd, MockRedisConnection}; +use serde_json::{Value, json}; +use sha2::{Digest, Sha256}; + +const INDEX: &str = "litellm_semantic_cache_index"; + +struct FakeEmbedder { + vectors: HashMap>, + calls: Arc>>, +} + +impl FakeEmbedder { + fn new(vectors: &[(&str, &[f32])]) -> (Self, Arc>>) { + let calls = Arc::new(Mutex::new(Vec::new())); + ( + Self { + vectors: vectors + .iter() + .map(|(prompt, vector)| (prompt.to_string(), vector.to_vec())) + .collect(), + calls: Arc::clone(&calls), + }, + calls, + ) + } +} + +impl Embedder for FakeEmbedder { + fn embed(&self, prompt: &str, _: Option<&Value>) -> Result, Error> { + self.calls.lock().unwrap().push(prompt.to_string()); + + Ok(self + .vectors + .get(prompt) + .cloned() + .unwrap_or_else(|| vec![0.1, 0.2, 0.3])) + } + + async fn async_embed(&self, prompt: &str, metadata: Option<&Value>) -> Result, Error> { + self.embed(prompt, metadata) + } +} + +fn config() -> RedisSemanticConfig { + RedisSemanticConfig { + index_name: INDEX.into(), + similarity_threshold: 0.9, + } +} + +fn messages_context(messages: Vec) -> SemanticCacheContext { + SemanticCacheContext { + messages: Some(Value::Array(messages)), + ..Default::default() + } +} + +fn entry() -> CacheEntry { + CacheEntry { + timestamp: Some(1.0), + response: json!({"answer": "yes"}), + } +} + +fn encoded(entry: &CacheEntry) -> Vec { + ResponseCacheCodec.encode(entry).unwrap() +} + +fn vector_bytes(vector: &[f32]) -> Vec { + vector + .iter() + .flat_map(|component| component.to_le_bytes()) + .collect() +} + +fn entry_id(prompt: &str, tag: &str) -> String { + let mut digest = Sha256::new(); + digest.update(prompt.as_bytes()); + digest.update(b"litellm_cache_key"); + digest.update(tag.as_bytes()); + format!("{:x}", digest.finalize()) +} + +fn s(value: &str) -> redis::Value { + redis::Value::BulkString(value.as_bytes().to_vec()) +} + +fn unknown_index_error() -> redis::RedisError { + redis::RedisError::from((redis::ErrorKind::Extension, "Unknown index name")) +} + +fn attribute(name: &str, field_type: &str, extra: Vec) -> redis::Value { + let mut parts = vec![ + s("identifier"), + s(name), + s("attribute"), + s(name), + s("type"), + s(field_type), + ]; + parts.extend(extra); + redis::Value::Array(parts) +} + +fn index_info(attributes: Vec) -> redis::Value { + redis::Value::Array(vec![ + s("index_name"), + s(INDEX), + s("attributes"), + redis::Value::Array(attributes), + ]) +} + +fn vector_attribute_with(dims: i64, data_type: &str, distance_metric: &str) -> redis::Value { + attribute( + "prompt_vector", + "VECTOR", + vec![ + s("algorithm"), + s("FLAT"), + s("data_type"), + s(data_type), + s("dim"), + redis::Value::Int(dims), + s("distance_metric"), + s(distance_metric), + ], + ) +} + +fn vector_attribute(dims: i64) -> redis::Value { + vector_attribute_with(dims, "FLOAT32", "COSINE") +} + +fn info_with_vector(vector: redis::Value) -> redis::Value { + index_info(vec![ + attribute("prompt", "TEXT", vec![]), + attribute("response", "TEXT", vec![]), + attribute("inserted_at", "NUMERIC", vec![]), + attribute("updated_at", "NUMERIC", vec![]), + vector, + attribute("litellm_cache_key", "TAG", vec![]), + ]) +} + +fn compatible_info(dims: i64) -> redis::Value { + info_with_vector(vector_attribute(dims)) +} + +fn unscoped_info(dims: i64) -> redis::Value { + index_info(vec![ + attribute("prompt", "TEXT", vec![]), + attribute("response", "TEXT", vec![]), + attribute("inserted_at", "NUMERIC", vec![]), + attribute("updated_at", "NUMERIC", vec![]), + vector_attribute(dims), + ]) +} + +fn create_index_command(name: &str, dims: usize) -> redis::Cmd { + let mut command = redis::cmd("FT.CREATE"); + command + .arg(name) + .arg("ON") + .arg("HASH") + .arg("PREFIX") + .arg(1) + .arg(name) + .arg("SCORE") + .arg(1.0) + .arg("SCHEMA") + .arg("prompt") + .arg("TEXT") + .arg("WEIGHT") + .arg(1) + .arg("response") + .arg("TEXT") + .arg("WEIGHT") + .arg(1) + .arg("inserted_at") + .arg("NUMERIC") + .arg("updated_at") + .arg("NUMERIC") + .arg("prompt_vector") + .arg("VECTOR") + .arg("FLAT") + .arg(6) + .arg("TYPE") + .arg("FLOAT32") + .arg("DIM") + .arg(dims) + .arg("DISTANCE_METRIC") + .arg("COSINE") + .arg("litellm_cache_key") + .arg("TAG") + .arg("SEPARATOR") + .arg(","); + command +} + +fn search_command(index: &str, tag: &str, vector: &[f32]) -> redis::Cmd { + let mut command = redis::cmd("FT.SEARCH"); + command + .arg(index) + .arg(format!( + "(@litellm_cache_key:{{{tag}}})=>[KNN 1 @prompt_vector $vector AS vector_distance]" + )) + .arg("RETURN") + .arg(8) + .arg("entry_id") + .arg("prompt") + .arg("response") + .arg("inserted_at") + .arg("updated_at") + .arg("metadata") + .arg("litellm_cache_key") + .arg("vector_distance") + .arg("SORTBY") + .arg("vector_distance") + .arg("ASC") + .arg("DIALECT") + .arg(2) + .arg("LIMIT") + .arg(0) + .arg(1) + .arg("PARAMS") + .arg(2) + .arg("vector") + .arg(vector_bytes(vector)); + command +} + +fn hit_fields(tag: &str, distance: &str, response: Vec) -> redis::Value { + redis::Value::Array(vec![ + s("entry_id"), + s("stored-id"), + s("prompt"), + s("hello prompt"), + s("response"), + redis::Value::BulkString(response), + s("inserted_at"), + s("1700000000.5"), + s("updated_at"), + s("1700000000.5"), + s("litellm_cache_key"), + s(tag), + s("vector_distance"), + s(distance), + ]) +} + +fn search_result(fields: redis::Value) -> redis::Value { + redis::Value::Array(vec![ + redis::Value::Int(1), + s("litellm_semantic_cache_index:stored-id"), + fields, + ]) +} + +fn empty_result() -> redis::Value { + redis::Value::Array(vec![redis::Value::Int(0)]) +} + +#[test] +fn store_creates_index_and_writes_hash_with_expire() { + let vector = vec![0.1f32, 0.2, 0.3]; + let prompt = "hello prompt"; + let tag = "key1"; + let hash_key = format!("{INDEX}:{}", entry_id(prompt, tag)); + let value = entry(); + let connection = MockRedisConnection::new([ + MockCmd::new( + redis::cmd("FT.INFO").arg(INDEX), + Err::(unknown_index_error()), + ), + MockCmd::new(create_index_command(INDEX, 3), Ok("OK")), + MockCmd::new( + redis::cmd("HSET") + .arg(&hash_key) + .arg("entry_id") + .arg(entry_id(prompt, tag)) + .arg("prompt") + .arg(prompt) + .arg("response") + .arg(encoded(&value)) + .arg("prompt_vector") + .arg(vector_bytes(&vector)) + .arg("inserted_at") + .arg("1700000000.5") + .arg("updated_at") + .arg("1700000000.5") + .arg("litellm_cache_key") + .arg(tag), + Ok(7), + ), + MockCmd::new(redis::cmd("EXPIRE").arg(&hash_key).arg(5), Ok(1)), + ]) + .assert_all_commands_consumed(); + let (embedder, _) = FakeEmbedder::new(&[(prompt, &vector)]); + let cache = RedisSemanticCache::with_connection(connection, embedder, config()) + .with_clock(|| 1700000000.5); + + let context = SemanticCacheContext { + ttl: Some(Duration::from_secs(5)), + ..messages_context(vec![json!({"role": "user", "content": prompt})]) + }; + cache.set_cache(tag, value, &context).unwrap(); +} + +#[test] +fn store_without_ttl_skips_expire() { + let prompt = "hello prompt"; + let connection = MockRedisConnection::new([ + MockCmd::new(redis::cmd("FT.INFO").arg(INDEX), Ok(compatible_info(3))), + MockCmd::new( + redis::cmd("HSET") + .arg(format!("{INDEX}:{}", entry_id(prompt, "key1"))) + .arg("entry_id") + .arg(entry_id(prompt, "key1")) + .arg("prompt") + .arg(prompt) + .arg("response") + .arg(encoded(&entry())) + .arg("prompt_vector") + .arg(vector_bytes(&[0.1f32, 0.2, 0.3])) + .arg("inserted_at") + .arg("1700000000.5") + .arg("updated_at") + .arg("1700000000.5") + .arg("litellm_cache_key") + .arg("key1"), + Ok(7), + ), + ]) + .assert_all_commands_consumed(); + let (embedder, _) = FakeEmbedder::new(&[]); + let cache = RedisSemanticCache::with_connection(connection, embedder, config()) + .with_clock(|| 1700000000.5); + + cache + .set_cache( + "key1", + entry(), + &messages_context(vec![json!({"role": "user", "content": prompt})]), + ) + .unwrap(); +} + +#[test] +fn lookup_returns_hit_below_distance_threshold() { + let vector = vec![0.1f32, 0.2, 0.3]; + let value = entry(); + let connection = MockRedisConnection::new([ + MockCmd::new(redis::cmd("FT.INFO").arg(INDEX), Ok(compatible_info(3))), + MockCmd::new( + search_command(INDEX, "key1", &vector), + Ok(search_result(hit_fields("key1", "0.05", encoded(&value)))), + ), + ]) + .assert_all_commands_consumed(); + let (embedder, _) = FakeEmbedder::new(&[]); + let cache = RedisSemanticCache::with_connection(connection, embedder, config()); + + let hit = cache + .get_cache( + "key1", + &messages_context(vec![json!({"role": "user", "content": "hello prompt"})]), + ) + .unwrap(); + assert_eq!(hit, Some(value)); +} + +#[test] +fn lookup_misses_above_distance_threshold_and_on_tag_mismatch() { + let vector = vec![0.1f32, 0.2, 0.3]; + let connection = MockRedisConnection::new([ + MockCmd::new(redis::cmd("FT.INFO").arg(INDEX), Ok(compatible_info(3))), + MockCmd::new( + search_command(INDEX, "key1", &vector), + Ok(search_result(hit_fields("key1", "0.5", encoded(&entry())))), + ), + MockCmd::new( + search_command(INDEX, "key1", &vector), + Ok(search_result(hit_fields( + "other", + "0.05", + encoded(&entry()), + ))), + ), + ]) + .assert_all_commands_consumed(); + let (embedder, _) = FakeEmbedder::new(&[]); + let cache = RedisSemanticCache::with_connection(connection, embedder, config()); + let context = messages_context(vec![json!({"role": "user", "content": "hello prompt"})]); + + assert_eq!(cache.get_cache("key1", &context).unwrap(), None); + assert_eq!(cache.get_cache("key1", &context).unwrap(), None); +} + +#[test] +fn lookup_returns_invalid_entry_on_malformed_response() { + let vector = vec![0.1f32, 0.2, 0.3]; + let connection = MockRedisConnection::new([ + MockCmd::new(redis::cmd("FT.INFO").arg(INDEX), Ok(compatible_info(3))), + MockCmd::new( + search_command(INDEX, "key1", &vector), + Ok(search_result(hit_fields( + "key1", + "0.05", + b"not json!".to_vec(), + ))), + ), + ]) + .assert_all_commands_consumed(); + let (embedder, _) = FakeEmbedder::new(&[]); + let cache = RedisSemanticCache::with_connection(connection, embedder, config()); + + assert_eq!( + cache + .get_cache( + "key1", + &messages_context(vec![json!({"role": "user", "content": "hello prompt"})]) + ) + .unwrap_err(), + Error::InvalidEntry + ); +} + +#[test] +fn missing_prompt_is_noop_and_never_embeds() { + let connection = MockRedisConnection::new(Vec::::new()).assert_all_commands_consumed(); + let (embedder, calls) = FakeEmbedder::new(&[]); + let cache = RedisSemanticCache::with_connection(connection, embedder, config()); + + let context = SemanticCacheContext::default(); + cache.set_cache("key1", entry(), &context).unwrap(); + assert_eq!(cache.get_cache("key1", &context).unwrap(), None); + assert!(calls.lock().unwrap().is_empty()); +} + +#[test] +fn scope_overrides_key_as_filter_tag() { + let vector = vec![0.1f32, 0.2, 0.3]; + let prompt = "hello prompt"; + let value = entry(); + let connection = MockRedisConnection::new([ + MockCmd::new(redis::cmd("FT.INFO").arg(INDEX), Ok(compatible_info(3))), + MockCmd::new( + redis::cmd("HSET") + .arg(format!("{INDEX}:{}", entry_id(prompt, "scope-a"))) + .arg("entry_id") + .arg(entry_id(prompt, "scope-a")) + .arg("prompt") + .arg(prompt) + .arg("response") + .arg(encoded(&value)) + .arg("prompt_vector") + .arg(vector_bytes(&vector)) + .arg("inserted_at") + .arg("1700000000.5") + .arg("updated_at") + .arg("1700000000.5") + .arg("litellm_cache_key") + .arg("scope-a"), + Ok(7), + ), + MockCmd::new( + search_command(INDEX, "scope\\-a", &vector), + Ok(search_result(hit_fields( + "scope-a", + "0.05", + encoded(&value), + ))), + ), + ]) + .assert_all_commands_consumed(); + let (embedder, _) = FakeEmbedder::new(&[]); + let cache = RedisSemanticCache::with_connection(connection, embedder, config()) + .with_clock(|| 1700000000.5); + let context = SemanticCacheContext { + scope: Some("scope-a".into()), + ..messages_context(vec![json!({"role": "user", "content": prompt})]) + }; + + cache.set_cache("key1", value.clone(), &context).unwrap(); + assert_eq!(cache.get_cache("key1", &context).unwrap(), Some(value)); +} + +#[test] +fn incompatible_schema_falls_back_to_isolated_index() { + let prompt = "hello prompt"; + let tag = "key1"; + let isolated = format!("{INDEX}_isolated"); + let connection = MockRedisConnection::new([ + MockCmd::new(redis::cmd("FT.INFO").arg(INDEX), Ok(unscoped_info(3))), + MockCmd::new( + redis::cmd("FT.INFO").arg(&isolated), + Err::(unknown_index_error()), + ), + MockCmd::new(create_index_command(&isolated, 3), Ok("OK")), + MockCmd::new( + redis::cmd("HSET") + .arg(format!("{isolated}:{}", entry_id(prompt, tag))) + .arg("entry_id") + .arg(entry_id(prompt, tag)) + .arg("prompt") + .arg(prompt) + .arg("response") + .arg(encoded(&entry())) + .arg("prompt_vector") + .arg(vector_bytes(&[0.1f32, 0.2, 0.3])) + .arg("inserted_at") + .arg("1700000000.5") + .arg("updated_at") + .arg("1700000000.5") + .arg("litellm_cache_key") + .arg(tag), + Ok(7), + ), + ]) + .assert_all_commands_consumed(); + let (embedder, _) = FakeEmbedder::new(&[]); + let cache = RedisSemanticCache::with_connection(connection, embedder, config()) + .with_clock(|| 1700000000.5); + + cache + .set_cache( + tag, + entry(), + &messages_context(vec![json!({"role": "user", "content": prompt})]), + ) + .unwrap(); +} + +#[test] +fn create_index_race_rechecks_schema_and_stores() { + let prompt = "hello prompt"; + let tag = "key1"; + let connection = MockRedisConnection::new([ + MockCmd::new( + redis::cmd("FT.INFO").arg(INDEX), + Err::(unknown_index_error()), + ), + MockCmd::new( + create_index_command(INDEX, 3), + Err::<&str, _>(redis::RedisError::from(( + redis::ErrorKind::Extension, + "Index already exists", + ))), + ), + MockCmd::new(redis::cmd("FT.INFO").arg(INDEX), Ok(compatible_info(3))), + MockCmd::new( + redis::cmd("HSET") + .arg(format!("{INDEX}:{}", entry_id(prompt, tag))) + .arg("entry_id") + .arg(entry_id(prompt, tag)) + .arg("prompt") + .arg(prompt) + .arg("response") + .arg(encoded(&entry())) + .arg("prompt_vector") + .arg(vector_bytes(&[0.1f32, 0.2, 0.3])) + .arg("inserted_at") + .arg("1700000000.5") + .arg("updated_at") + .arg("1700000000.5") + .arg("litellm_cache_key") + .arg(tag), + Ok(7), + ), + ]) + .assert_all_commands_consumed(); + let (embedder, _) = FakeEmbedder::new(&[]); + let cache = RedisSemanticCache::with_connection(connection, embedder, config()) + .with_clock(|| 1700000000.5); + + cache + .set_cache( + tag, + entry(), + &messages_context(vec![json!({"role": "user", "content": prompt})]), + ) + .unwrap(); +} + +#[test] +fn wrong_distance_metric_falls_back_to_isolated_index() { + let prompt = "hello prompt"; + let tag = "key1"; + let isolated = format!("{INDEX}_isolated"); + let connection = MockRedisConnection::new([ + MockCmd::new( + redis::cmd("FT.INFO").arg(INDEX), + Ok(info_with_vector(vector_attribute_with(3, "FLOAT32", "L2"))), + ), + MockCmd::new( + redis::cmd("FT.INFO").arg(&isolated), + Err::(unknown_index_error()), + ), + MockCmd::new(create_index_command(&isolated, 3), Ok("OK")), + MockCmd::new( + redis::cmd("HSET") + .arg(format!("{isolated}:{}", entry_id(prompt, tag))) + .arg("entry_id") + .arg(entry_id(prompt, tag)) + .arg("prompt") + .arg(prompt) + .arg("response") + .arg(encoded(&entry())) + .arg("prompt_vector") + .arg(vector_bytes(&[0.1f32, 0.2, 0.3])) + .arg("inserted_at") + .arg("1700000000.5") + .arg("updated_at") + .arg("1700000000.5") + .arg("litellm_cache_key") + .arg(tag), + Ok(7), + ), + ]) + .assert_all_commands_consumed(); + let (embedder, _) = FakeEmbedder::new(&[]); + let cache = RedisSemanticCache::with_connection(connection, embedder, config()) + .with_clock(|| 1700000000.5); + + cache + .set_cache( + tag, + entry(), + &messages_context(vec![json!({"role": "user", "content": prompt})]), + ) + .unwrap(); +} + +#[test] +fn tag_special_characters_are_escaped_in_search_filter() { + let vector = vec![0.1f32, 0.2, 0.3]; + let tag = "a:b, c|d"; + let connection = MockRedisConnection::new([ + MockCmd::new(redis::cmd("FT.INFO").arg(INDEX), Ok(compatible_info(3))), + MockCmd::new( + search_command(INDEX, "a\\:b\\,\\ c\\|d", &vector), + Ok(empty_result()), + ), + ]) + .assert_all_commands_consumed(); + let (embedder, _) = FakeEmbedder::new(&[]); + let cache = RedisSemanticCache::with_connection(connection, embedder, config()); + + assert_eq!( + cache + .get_cache( + tag, + &messages_context(vec![json!({"role": "user", "content": "hello prompt"})]) + ) + .unwrap(), + None + ); +} + +#[test] +fn prompt_extraction_matches_python_message_and_input_shapes() { + let vector = vec![0.1f32, 0.2, 0.3]; + let lookups = 5; + let mut commands = vec![MockCmd::new( + redis::cmd("FT.INFO").arg(INDEX), + Ok(compatible_info(3)), + )]; + for _ in 0..lookups { + commands.push(MockCmd::new( + search_command(INDEX, "key1", &vector), + Ok(empty_result()), + )); + } + let connection = MockRedisConnection::new(commands).assert_all_commands_consumed(); + let (embedder, calls) = FakeEmbedder::new(&[]); + let cache = RedisSemanticCache::with_connection(connection, embedder, config()); + + cache + .get_cache( + "key1", + &messages_context(vec![ + json!({"role": "user", "content": [{"type": "text", "text": "first"}, {"type": "text", "text": "second"}]}), + json!({"role": "assistant", "content": "reply"}), + ]), + ) + .unwrap(); + cache + .get_cache( + "key1", + &SemanticCacheContext { + input: Some(json!(" plain input ")), + ..Default::default() + }, + ) + .unwrap(); + cache + .get_cache( + "key1", + &SemanticCacheContext { + input: Some( + json!([{"content": [{"type": "input_text", "text": "nested"}]}, "tail"]), + ), + ..Default::default() + }, + ) + .unwrap(); + cache + .get_cache( + "key1", + &SemanticCacheContext { + input: Some(json!({"output_text": " result text "})), + ..Default::default() + }, + ) + .unwrap(); + cache + .get_cache( + "key1", + &messages_context(vec![json!({ + "role": "user", + "content": "question", + "search_results": [{"source": "src", "title": "t", "content": [{"text": "found"}], "citations": {"a": 1}}], + })]), + ) + .unwrap(); + + assert_eq!( + *calls.lock().unwrap(), + vec![ + "firstsecondreply", + "plain input", + "nested\ntail", + "result text", + "questionsrctfound{\"a\":1}", + ] + ); +} + +#[test] +fn ttl_passes_through_context_only() { + let (embedder, _) = FakeEmbedder::new(&[]); + let cache = RedisSemanticCache::with_connection( + MockRedisConnection::new(Vec::::new()), + embedder, + config(), + ); + assert_eq!(cache.get_ttl(&SemanticCacheContext::default()), None); + assert_eq!( + cache.get_ttl(&SemanticCacheContext { + ttl: Some(Duration::from_secs(9)), + ..Default::default() + }), + Some(Duration::from_secs(9)) + ); +} + +#[tokio::test] +async fn async_paths_embed_then_run_blocking_redis_work() { + let vector = vec![0.1f32, 0.2, 0.3]; + let prompt = "hello prompt"; + let tag = "key1"; + let hash_key = format!("{INDEX}:{}", entry_id(prompt, tag)); + let value = entry(); + let connection = MockRedisConnection::new([ + MockCmd::new(redis::cmd("FT.INFO").arg(INDEX), Ok(compatible_info(3))), + MockCmd::new( + redis::cmd("HSET") + .arg(&hash_key) + .arg("entry_id") + .arg(entry_id(prompt, tag)) + .arg("prompt") + .arg(prompt) + .arg("response") + .arg(encoded(&value)) + .arg("prompt_vector") + .arg(vector_bytes(&vector)) + .arg("inserted_at") + .arg("1700000000.5") + .arg("updated_at") + .arg("1700000000.5") + .arg("litellm_cache_key") + .arg(tag), + Ok(7), + ), + MockCmd::new( + search_command(INDEX, tag, &vector), + Ok(search_result(hit_fields(tag, "0.05", encoded(&value)))), + ), + ]) + .assert_all_commands_consumed(); + let (embedder, _) = FakeEmbedder::new(&[]); + let cache = RedisSemanticCache::with_connection(connection, embedder, config()) + .with_clock(|| 1700000000.5); + let context = messages_context(vec![json!({"role": "user", "content": prompt})]); + + cache + .async_set_cache(tag, value.clone(), context.clone()) + .await + .unwrap(); + assert_eq!( + cache.async_get_cache(tag, &context).await.unwrap(), + Some(value) + ); +} + +#[test] +fn shared_base_index_across_dimensions_replaces_the_isolated_index() { + // Pins parity with Python's `_isolated` + overwrite=True flow. + let prompt = "shared prompt"; + let tag = "key1"; + let isolated = format!("{INDEX}_isolated"); + let value = entry(); + let context = || messages_context(vec![json!({"role": "user", "content": prompt})]); + let store_hash = |index: &str, vector: &[f32]| { + MockCmd::new( + redis::cmd("HSET") + .arg(format!("{index}:{}", entry_id(prompt, tag))) + .arg("entry_id") + .arg(entry_id(prompt, tag)) + .arg("prompt") + .arg(prompt) + .arg("response") + .arg(encoded(&value)) + .arg("prompt_vector") + .arg(vector_bytes(vector)) + .arg("inserted_at") + .arg("1700000000.5") + .arg("updated_at") + .arg("1700000000.5") + .arg("litellm_cache_key") + .arg(tag), + Ok(7), + ) + }; + + let vector_a = vec![0.1f32; 8]; + let connection_a = MockRedisConnection::new([ + MockCmd::new( + redis::cmd("FT.INFO").arg(INDEX), + Err::(unknown_index_error()), + ), + MockCmd::new(create_index_command(INDEX, 8), Ok("OK")), + store_hash(INDEX, &vector_a), + ]) + .assert_all_commands_consumed(); + let (embedder_a, _) = FakeEmbedder::new(&[(prompt, &vector_a)]); + let worker_a = RedisSemanticCache::with_connection(connection_a, embedder_a, config()) + .with_clock(|| 1700000000.5); + worker_a.set_cache(tag, value.clone(), &context()).unwrap(); + + let vector_b = vec![0.2f32; 4]; + let connection_b = MockRedisConnection::new([ + MockCmd::new(redis::cmd("FT.INFO").arg(INDEX), Ok(compatible_info(8))), + MockCmd::new( + redis::cmd("FT.INFO").arg(&isolated), + Err::(unknown_index_error()), + ), + MockCmd::new(create_index_command(&isolated, 4), Ok("OK")), + store_hash(&isolated, &vector_b), + MockCmd::new( + search_command(&isolated, tag, &vector_b), + Ok(search_result(hit_fields(tag, "0.0", encoded(&value)))), + ), + MockCmd::new( + search_command(&isolated, tag, &vector_b), + Err::(redis::RedisError::from(( + redis::ErrorKind::Extension, + "Vector dimension mismatch", + ))), + ), + ]) + .assert_all_commands_consumed(); + let (embedder_b, _) = FakeEmbedder::new(&[(prompt, &vector_b)]); + let worker_b = RedisSemanticCache::with_connection(connection_b, embedder_b, config()) + .with_clock(|| 1700000000.5); + worker_b.set_cache(tag, value.clone(), &context()).unwrap(); + assert_eq!( + worker_b.get_cache(tag, &context()).unwrap(), + Some(value.clone()) + ); + + let vector_c = vec![0.3f32; 16]; + let connection_c = MockRedisConnection::new([ + MockCmd::new(redis::cmd("FT.INFO").arg(INDEX), Ok(compatible_info(8))), + MockCmd::new(redis::cmd("FT.INFO").arg(&isolated), Ok(compatible_info(4))), + MockCmd::new(redis::cmd("FT.DROPINDEX").arg(&isolated), Ok("OK")), + MockCmd::new(create_index_command(&isolated, 16), Ok("OK")), + store_hash(&isolated, &vector_c), + ]) + .assert_all_commands_consumed(); + let (embedder_c, _) = FakeEmbedder::new(&[(prompt, &vector_c)]); + let worker_c = RedisSemanticCache::with_connection(connection_c, embedder_c, config()) + .with_clock(|| 1700000000.5); + worker_c.set_cache(tag, value.clone(), &context()).unwrap(); + + assert_eq!( + worker_b.get_cache(tag, &context()).unwrap_err(), + Error::Unavailable + ); +} + +#[test] +fn live_shared_index_is_replaced_across_dimensions() { + let Ok(url) = std::env::var("LITELLM_REDIS_STACK_URL") else { + return; + }; + // Pins parity with Python's `_isolated` + overwrite=True flow. + let base = format!("rust_semantic_shared_{}", std::process::id()); + let isolated = format!("{base}_isolated"); + let prompt = "shared live prompt"; + let tag = "key1"; + let context = || messages_context(vec![json!({"role": "user", "content": prompt})]); + let value = entry(); + let worker = |vector: Vec| { + let (embedder, _) = FakeEmbedder::new(&[(prompt, vector.as_slice())]); + RedisSemanticCache::new( + &url, + embedder, + RedisSemanticConfig { + index_name: base.clone(), + similarity_threshold: 0.9, + }, + ) + .unwrap() + }; + + let worker_a = worker(vec![0.1f32; 8]); + worker_a.set_cache(tag, value.clone(), &context()).unwrap(); + + let worker_b = worker(vec![0.2f32; 4]); + worker_b.set_cache(tag, value.clone(), &context()).unwrap(); + assert_eq!( + worker_b.get_cache(tag, &context()).unwrap(), + Some(value.clone()) + ); + + let worker_c = worker(vec![0.3f32; 16]); + worker_c.set_cache(tag, value.clone(), &context()).unwrap(); + + assert_eq!( + worker_b.get_cache(tag, &context()).unwrap_err(), + Error::Unavailable + ); + + let mut connection = redis::Client::open(url).unwrap().get_connection().unwrap(); + for index in [&base, &isolated] { + let _: Result<(), _> = redis::cmd("FT.DROPINDEX") + .arg(index) + .arg("DD") + .query(&mut connection); + } +} + +#[test] +fn live_store_lookup_and_ttl_against_redis_stack() { + let Ok(url) = std::env::var("LITELLM_REDIS_STACK_URL") else { + return; + }; + let vector = vec![0.1f32, 0.2, 0.3, 0.4]; + let prompt = "rust semantic cache live prompt"; + let tag = "live-key"; + let index_name = format!("rust_semantic_test_{}", std::process::id()); + let (embedder, _) = FakeEmbedder::new(&[(prompt, &vector)]); + let cache = RedisSemanticCache::new( + &url, + embedder, + RedisSemanticConfig { + index_name: index_name.clone(), + similarity_threshold: 0.9, + }, + ) + .unwrap(); + let context = SemanticCacheContext { + ttl: Some(Duration::from_secs(120)), + ..messages_context(vec![json!({"role": "user", "content": prompt})]) + }; + let value = entry(); + + cache.set_cache(tag, value.clone(), &context).unwrap(); + assert_eq!(cache.get_cache(tag, &context).unwrap(), Some(value)); + assert_eq!(cache.get_cache("other-key", &context).unwrap(), None); + + let mut connection = redis::Client::open(url).unwrap().get_connection().unwrap(); + let ttl: i64 = redis::Commands::ttl( + &mut connection, + format!("{index_name}:{}", entry_id(prompt, tag)), + ) + .unwrap(); + assert!( + ttl > 0, + "expected stored hash to carry an expiry, got {ttl}" + ); +} diff --git a/litellm-rust/crates/cache-redis/src/cache.rs b/litellm-rust/crates/cache-redis/src/cache.rs index e2e2656fcbb..24399c9b2f9 100644 --- a/litellm-rust/crates/cache-redis/src/cache.rs +++ b/litellm-rust/crates/cache-redis/src/cache.rs @@ -14,7 +14,7 @@ use crate::topology::RedisTopology; mod connection; mod operations; -pub(crate) use connection::ConnectionRef; +pub use connection::ConnectionRef; use connection::{ClusterConnectionManager, ConnectionManager}; pub use operations::{ @@ -40,7 +40,8 @@ const CLAIM_SCRIPT: &str = concat!( ); const CLAIM_ATTEMPTS: usize = 8; -enum Connections { +#[allow(private_interfaces)] +pub enum Connections { Pool(r2d2::Pool), Cluster(r2d2::Pool), Fixed(Mutex), @@ -50,7 +51,7 @@ impl Connections where C: redis::ConnectionLike + Send + 'static, { - fn execute( + pub fn execute( &self, operation: impl FnOnce(&mut ConnectionRef<'_>) -> Result, ) -> Result { @@ -73,6 +74,29 @@ where } } } + + pub async fn run_blocking(connections: Arc, operation: F) -> Result + where + T: Send + 'static, + F: FnOnce(&mut ConnectionRef<'_>) -> Result + Send + 'static, + { + tokio::task::spawn_blocking(move || connections.execute(operation)) + .await + .map_err(|_| Error::Unavailable)? + } + + pub fn fixed(connection: C) -> Self { + Self::Fixed(Mutex::new(connection)) + } + + pub fn open(url: &str, topology: &RedisTopology) -> Result { + match topology { + RedisTopology::Standalone => Ok(Self::Pool(pool(ConnectionManager::open(url)?)?)), + RedisTopology::Cluster { startup_nodes } => Ok(Self::Cluster(pool( + ClusterConnectionManager::open(url, startup_nodes)?, + )?)), + } + } } pub struct RedisCache { @@ -94,12 +118,7 @@ impl RedisCache { default_ttl: Option, codec: S, ) -> Result { - let connections = match topology { - RedisTopology::Standalone => Connections::Pool(pool(ConnectionManager::open(url)?)?), - RedisTopology::Cluster { startup_nodes } => { - Connections::Cluster(pool(ClusterConnectionManager::open(url, startup_nodes)?)?) - } - }; + let connections = Connections::open(url, topology)?; Ok(Self { connections: Arc::new(connections), default_ttl: default_ttl.unwrap_or(DEFAULT_TTL), @@ -127,7 +146,7 @@ where { pub fn with_connection(connection: C, default_ttl: Option, codec: S) -> Self { Self { - connections: Arc::new(Connections::Fixed(Mutex::new(connection))), + connections: Arc::new(Connections::fixed(connection)), default_ttl: default_ttl.unwrap_or(DEFAULT_TTL), codec, namespace: None, @@ -203,16 +222,6 @@ where .saturating_add(u64::from(ttl.subsec_nanos() > 0)) .max(1) } - - async fn run_blocking(connections: Arc>, operation: F) -> Result - where - T: Send + 'static, - F: FnOnce(&mut ConnectionRef<'_>) -> Result + Send + 'static, - { - tokio::task::spawn_blocking(move || connections.execute(operation)) - .await - .map_err(|_| Error::Unavailable)? - } } fn namespaced_key(namespace: Option<&str>, key: &str) -> String { @@ -271,7 +280,7 @@ where let payload = self.codec.encode(&value)?; let key = self.namespaced_key(key); let ttl = Self::ttl_seconds(self.get_ttl(&context).unwrap_or(self.default_ttl)); - Self::run_blocking(Arc::clone(&self.connections), move |connection| { + Connections::run_blocking(Arc::clone(&self.connections), move |connection| { connection .set_ex::<_, _, ()>(key, payload, ttl) .map_err(|_| Error::Unavailable) @@ -285,7 +294,7 @@ where _: &ExactCacheContext, ) -> Result, Error> { let key = self.namespaced_key(key); - let value = Self::run_blocking(Arc::clone(&self.connections), move |connection| { + let value = Connections::run_blocking(Arc::clone(&self.connections), move |connection| { connection .get::<_, redis::Value>(key) .map_err(|_| Error::Unavailable) @@ -311,7 +320,7 @@ where if entries.is_empty() { return Ok(()); } - Self::run_blocking(Arc::clone(&self.connections), move |connection| { + Connections::run_blocking(Arc::clone(&self.connections), move |connection| { let commands = entries .into_iter() .map(|(key, payload)| { @@ -330,7 +339,7 @@ where } async fn test_connection(&self) -> Result { - match Self::run_blocking(Arc::clone(&self.connections), |connection| { + match Connections::run_blocking(Arc::clone(&self.connections), |connection| { Ok(match connection.ping() { Ok(_) => CacheConnectionResult { status: CacheConnectionStatus::Success, @@ -391,7 +400,7 @@ where .iter() .map(|key| self.namespaced_key(key)) .collect::>(); - let values = Self::run_blocking(Arc::clone(&self.connections), move |connection| { + let values = Connections::run_blocking(Arc::clone(&self.connections), move |connection| { redis::cmd("MGET") .arg(keys) .query::>(connection) @@ -418,7 +427,7 @@ where async fn async_delete_cache(&self, key: &str) -> Result<(), Error> { let key = self.namespaced_key(key); - Self::run_blocking(Arc::clone(&self.connections), move |connection| { + Connections::run_blocking(Arc::clone(&self.connections), move |connection| { connection.del::<_, ()>(key).map_err(|_| Error::Unavailable) }) .await @@ -438,7 +447,7 @@ where async fn async_flush_cache(&self) -> Result<(), Error> { let pattern = self.namespaced_pattern()?; - Self::run_blocking(Arc::clone(&self.connections), move |connection| { + Connections::run_blocking(Arc::clone(&self.connections), move |connection| { Self::flush_matching(connection, &pattern) }) .await @@ -470,7 +479,7 @@ where ) -> Result { let key = self.namespaced_key(key); let ttl = Self::ttl_seconds(self.get_ttl(&context).unwrap_or(self.default_ttl)); - Self::run_blocking(Arc::clone(&self.connections), move |connection| { + Connections::run_blocking(Arc::clone(&self.connections), move |connection| { increment(connection, key, amount, ttl) }) .await @@ -581,7 +590,7 @@ where let key = self.namespaced_key(key); let ttl = Self::ttl_seconds(self.get_ttl(&context).unwrap_or(self.default_ttl)); let codec = self.codec.clone(); - Self::run_blocking(Arc::clone(&self.connections), move |connection| { + Connections::run_blocking(Arc::clone(&self.connections), move |connection| { claim(connection, &codec, &key, candidate, &eligible, ttl) }) .await diff --git a/litellm-rust/crates/cache-redis/src/cache/connection.rs b/litellm-rust/crates/cache-redis/src/cache/connection.rs index 1834f1d94e5..013bf055f89 100644 --- a/litellm-rust/crates/cache-redis/src/cache/connection.rs +++ b/litellm-rust/crates/cache-redis/src/cache/connection.rs @@ -12,7 +12,7 @@ use redis::{ use super::REDIS_TIMEOUT; use crate::topology::RedisNode; -pub(super) struct PooledConnection { +pub struct PooledConnection { pub(super) connection: C, pub(super) failed: bool, } @@ -20,7 +20,7 @@ pub(super) struct PooledConnection { /// Pools connections without a checkout PING, which would double every operation's round trips. /// A timed-out command leaves its reply on the socket while redis still reports the connection /// open, so any connection whose operation failed is discarded instead of being reused. -pub(super) struct ConnectionManager(redis::Client); +pub struct ConnectionManager(redis::Client); impl ConnectionManager { pub(super) fn open(url: &str) -> Result { @@ -54,7 +54,7 @@ impl r2d2::ManageConnection for ConnectionManager { } } -pub(super) struct ClusterConnectionManager(ClusterClient); +pub struct ClusterConnectionManager(ClusterClient); impl ClusterConnectionManager { pub(super) fn open(url: &str, startup_nodes: &[RedisNode]) -> Result { @@ -117,7 +117,7 @@ impl r2d2::ManageConnection for ClusterConnectionManager { } } -pub(crate) enum ConnectionRef<'a> { +pub enum ConnectionRef<'a> { Node(&'a mut dyn redis::ConnectionLike), Cluster(&'a mut ClusterConnection), } diff --git a/litellm-rust/crates/cache-redis/src/cache/operations.rs b/litellm-rust/crates/cache-redis/src/cache/operations.rs index 9a7023338bf..4345ee879b3 100644 --- a/litellm-rust/crates/cache-redis/src/cache/operations.rs +++ b/litellm-rust/crates/cache-redis/src/cache/operations.rs @@ -144,7 +144,7 @@ where .into_iter() .map(|key| self.namespaced_key(&key)) .collect::>(); - Self::run_blocking(Arc::clone(&self.connections), move |connection| { + Connections::run_blocking(Arc::clone(&self.connections), move |connection| { connection.del(keys).map_err(|_| Error::Unavailable) }) .await @@ -172,7 +172,7 @@ where .iter() .map(|key| self.namespaced_key(key)) .collect::>(); - let values = Self::run_blocking(Arc::clone(&self.connections), move |connection| { + let values = Connections::run_blocking(Arc::clone(&self.connections), move |connection| { redis::cmd("MGET") .arg(keys) .query::>(connection) @@ -188,7 +188,7 @@ where } pub async fn ping(&self) -> Result { - Self::run_blocking(Arc::clone(&self.connections), |connection| { + Connections::run_blocking(Arc::clone(&self.connections), |connection| { connection.ping().map_err(|_| Error::Unavailable) }) .await @@ -196,7 +196,7 @@ where pub async fn async_get_ttl(&self, key: &str) -> Result, Error> { let key = self.namespaced_key(key); - let ttl = Self::run_blocking(Arc::clone(&self.connections), move |connection| { + let ttl = Connections::run_blocking(Arc::clone(&self.connections), move |connection| { redis::cmd("TTL") .arg(key) .query::(connection) @@ -208,7 +208,7 @@ where pub async fn async_scan_iter(&self, pattern: &str, count: usize) -> Result, Error> { let pattern = format!("{}*", self.namespaced_key(pattern)); - Self::run_blocking(Arc::clone(&self.connections), move |connection| { + Connections::run_blocking(Arc::clone(&self.connections), move |connection| { let mut matches = Vec::new(); connection.scan(&pattern, count, |_, keys| { matches.extend(keys); @@ -231,7 +231,7 @@ where } let key = self.namespaced_key(key); let ttl = Self::ttl_seconds(ttl.unwrap_or(self.default_ttl)); - Self::run_blocking(Arc::clone(&self.connections), move |connection| { + Connections::run_blocking(Arc::clone(&self.connections), move |connection| { let mut sadd = redis::cmd("SADD"); sadd.arg(&key).arg(values); let mut expire = redis::cmd("EXPIRE"); @@ -253,7 +253,7 @@ where return Err(Error::InvalidEntry); } let key = self.namespaced_key(key); - Self::run_blocking(Arc::clone(&self.connections), move |connection| { + Connections::run_blocking(Arc::clone(&self.connections), move |connection| { redis::cmd("RPUSH") .arg(key) .arg(values) @@ -279,7 +279,7 @@ where if operations.is_empty() { return Ok(Vec::new()); } - Self::run_blocking(Arc::clone(&self.connections), move |connection| { + Connections::run_blocking(Arc::clone(&self.connections), move |connection| { let commands = operations .into_iter() .map(|(key, values)| { @@ -304,7 +304,7 @@ where ) -> Result { let key = self.namespaced_key(key); let multiple = count.is_some(); - let value = Self::run_blocking(Arc::clone(&self.connections), move |connection| { + let value = Connections::run_blocking(Arc::clone(&self.connections), move |connection| { let mut command = redis::cmd("LPOP"); command.arg(key); if let Some(count) = count { @@ -333,7 +333,7 @@ where .iter() .map(|(_, count)| count.is_some()) .collect::>(); - let values = Self::run_blocking(Arc::clone(&self.connections), move |connection| { + let values = Connections::run_blocking(Arc::clone(&self.connections), move |connection| { let commands = operations .into_iter() .map(|(key, count)| { @@ -365,7 +365,7 @@ where .into_iter() .map(|key| self.namespaced_key(&key)) .collect::>(); - Self::run_blocking(Arc::clone(&self.connections), move |connection| { + Connections::run_blocking(Arc::clone(&self.connections), move |connection| { redis::cmd("EVAL") .arg(script) .arg(keys.len()) @@ -426,7 +426,7 @@ where if operations.is_empty() { return Ok(Vec::new()); } - Self::run_blocking(Arc::clone(&self.connections), move |connection| { + Connections::run_blocking(Arc::clone(&self.connections), move |connection| { let mut commands = Vec::with_capacity(operations.len() * 2); let mut increments = Vec::with_capacity(operations.len()); for (key, amount, ttl) in operations { @@ -460,7 +460,7 @@ where ) -> Result { let key = self.namespaced_key(key); let ttl = Self::ttl_seconds(ttl); - Self::run_blocking(Arc::clone(&self.connections), move |connection| { + Connections::run_blocking(Arc::clone(&self.connections), move |connection| { increment_with_floor(connection, key, amount, ttl) }) .await @@ -474,7 +474,7 @@ where ) -> Result { let key = self.namespaced_key(key); let ttl = Self::ttl_seconds(ttl.unwrap_or(self.default_ttl)); - Self::run_blocking(Arc::clone(&self.connections), move |connection| { + Connections::run_blocking(Arc::clone(&self.connections), move |connection| { redis::cmd("EVAL") .arg(SET_MAX_SCRIPT) .arg(1) diff --git a/litellm-rust/crates/cache-redis/src/lib.rs b/litellm-rust/crates/cache-redis/src/lib.rs index 98f6bfd8ce5..efb0db931ac 100644 --- a/litellm-rust/crates/cache-redis/src/lib.rs +++ b/litellm-rust/crates/cache-redis/src/lib.rs @@ -1,6 +1,10 @@ mod cache; mod topology; +pub mod connection { + pub use crate::cache::{ConnectionRef, Connections}; +} + pub use cache::{ RedisArg, RedisCache, RedisLpopOperation, RedisLpopResult, RedisRpushOperation, RedisScript, }; diff --git a/litellm-rust/crates/cache-response/README.md b/litellm-rust/crates/cache-response/README.md index 56c1646d343..46e561ddad1 100644 --- a/litellm-rust/crates/cache-response/README.md +++ b/litellm-rust/crates/cache-response/README.md @@ -58,4 +58,4 @@ Verify typed values, TTL precedence, missing entries, serialization failures, na Public SDK, Router, and proxy activation still need constructor parity, stream replay, embedding partial-batch integration, response reconstruction, callback scheduling, and failure-policy integration. This foundation does not switch those request paths -Redis cluster, disk, cloud stores, and semantic caching remain follow-ups. The generic dual cache takes read, write, and remote-failure policies, runs its async operations through the async L2 methods, and provides L2-first counters and atomic affinity claims. Errors propagate by default, and `RemoteFailurePolicy::UseLocal` opts key-value operations and claims into the local tier when L2 is unavailable. Claims compare decoded values, so a pin written by Python still matches. Public Router integration remains follow-up work. Reservations and pubsub still need explicit capabilities owned by their consuming features. Adding a cache backend does not establish those guarantees +Redis cluster, disk, and cloud stores remain follow-ups. Semantic backends plug in through `SemanticCacheContext`, which carries the prompt inputs and metadata alongside the cache TTL. The generic dual cache takes read, write, and remote-failure policies, runs its async operations through the async L2 methods, and provides L2-first counters and atomic affinity claims. Errors propagate by default, and `RemoteFailurePolicy::UseLocal` opts key-value operations and claims into the local tier when L2 is unavailable. Claims compare decoded values, so a pin written by Python still matches. Public Router integration remains follow-up work. Reservations and pubsub still need explicit capabilities owned by their consuming features. Adding a cache backend does not establish those guarantees diff --git a/litellm-rust/crates/cache-response/src/response.rs b/litellm-rust/crates/cache-response/src/response.rs index e50e68cdabb..5088402f125 100644 --- a/litellm-rust/crates/cache-response/src/response.rs +++ b/litellm-rust/crates/cache-response/src/response.rs @@ -1,21 +1,21 @@ use std::{sync::Arc, time::Duration}; use litellm_cache::{ - BaseCache, BatchCache, BatchEntry, CacheConnectionResult, Error, ExactCacheContext, FlushCache, + BaseCache, BatchCache, BatchEntry, CacheConnectionResult, CacheContext, Error, FlushCache, }; use serde_json::Value; use crate::{CacheControls, CacheEntry, CacheKeyInput, PartialHits, cache_key}; #[derive(Clone)] -pub struct ResponseCacheRequest { +pub struct ResponseCacheRequest { pub key: CacheKeyInput, pub controls: CacheControls, - pub context: ExactCacheContext, + pub context: C, pub max_age: Option, } -impl ResponseCacheRequest { +impl ResponseCacheRequest { pub fn new(key: CacheKeyInput) -> Self { Self { key, @@ -26,17 +26,35 @@ impl ResponseCacheRequest { default_on: true, ..Default::default() }, - context: ExactCacheContext::default(), + context: C::default(), max_age: None, } } } -pub struct ResponseCache> { +impl ResponseCacheRequest { + pub fn with_context(self, context: D) -> ResponseCacheRequest { + ResponseCacheRequest { + key: self.key, + controls: self.controls, + context, + max_age: self.max_age, + } + } +} + +pub struct ResponseCache> +where + B::Context: Default + PartialEq, +{ backend: Arc, } -impl> ResponseCache { +impl ResponseCache +where + B: BaseCache, + B::Context: Default + PartialEq, +{ pub fn new(backend: Arc) -> Self { Self { backend } } @@ -45,8 +63,12 @@ impl> ResponseCach &self.backend } + pub fn backend_arc(&self) -> &Arc { + &self.backend + } + pub fn default_ttl(&self) -> Option { - self.backend.get_ttl(&ExactCacheContext::default()) + self.backend.get_ttl(&B::Context::default()) } pub async fn async_flush(&self) -> Result<(), Error> @@ -62,7 +84,7 @@ impl> ResponseCach pub fn lookup( &self, - request: &ResponseCacheRequest, + request: &ResponseCacheRequest, now: Duration, ) -> Result, Error> { if !request.controls.reads() { @@ -81,7 +103,7 @@ impl> ResponseCach pub async fn async_lookup( &self, - request: &ResponseCacheRequest, + request: &ResponseCacheRequest, now: Duration, ) -> Result, Error> { if !request.controls.reads() { @@ -101,7 +123,7 @@ impl> ResponseCach pub fn lookup_batch( &self, - requests: &[ResponseCacheRequest], + requests: &[ResponseCacheRequest], now: Duration, ) -> Result where @@ -126,7 +148,7 @@ impl> ResponseCach pub async fn async_lookup_batch( &self, - requests: &[ResponseCacheRequest], + requests: &[ResponseCacheRequest], now: Duration, ) -> Result where @@ -153,7 +175,7 @@ impl> ResponseCach pub fn store( &self, - request: &ResponseCacheRequest, + request: &ResponseCacheRequest, response: Value, now: Duration, ) -> Result<(), Error> { @@ -172,7 +194,7 @@ impl> ResponseCach pub async fn async_store( &self, - request: &ResponseCacheRequest, + request: &ResponseCacheRequest, response: Value, now: Duration, ) -> Result<(), Error> { @@ -193,7 +215,7 @@ impl> ResponseCach pub async fn async_store_batch( &self, - entries: Vec<(ResponseCacheRequest, Value)>, + entries: Vec<(ResponseCacheRequest, Value)>, now: Duration, ) -> Result<(), Error> { self.async_store_entries( @@ -209,7 +231,7 @@ impl> ResponseCach /// the freshness of its original response. pub async fn async_store_entries( &self, - entries: Vec<(ResponseCacheRequest, Value, Duration)>, + entries: Vec<(ResponseCacheRequest, Value, Duration)>, ) -> Result<(), Error> { let writable = entries .into_iter() @@ -249,8 +271,8 @@ impl> ResponseCach } fn partial_hits( - requests: &[ResponseCacheRequest], - readable: Vec<(usize, &ResponseCacheRequest)>, + requests: &[ResponseCacheRequest], + readable: Vec<(usize, &ResponseCacheRequest)>, entries: Vec>, now: Duration, ) -> Result { diff --git a/litellm-rust/crates/cache-response/tests/response.rs b/litellm-rust/crates/cache-response/tests/response.rs index e4f78dae8b2..dcfc0301148 100644 --- a/litellm-rust/crates/cache-response/tests/response.rs +++ b/litellm-rust/crates/cache-response/tests/response.rs @@ -1,12 +1,15 @@ use std::{ sync::{ - Arc, + Arc, Mutex, atomic::{AtomicU64, Ordering}, }, time::Duration, }; -use litellm_cache::{BaseCache, CacheCodec, Error}; +use litellm_cache::{ + BaseCache, CacheCodec, CacheConnectionResult, CacheConnectionStatus, Error, + SemanticCacheContext, +}; use litellm_cache_memory::InMemoryCache; use litellm_cache_redis::RedisCache; use litellm_cache_response::{ @@ -30,6 +33,82 @@ fn request() -> ResponseCacheRequest { }) } +struct SemanticBackend { + entries: Mutex>, + contexts: Mutex>, +} + +impl BaseCache for SemanticBackend { + type Value = CacheEntry; + type Context = SemanticCacheContext; + + fn get_ttl(&self, _: &Self::Context) -> Option { + None + } + + fn set_cache( + &self, + key: &str, + value: Self::Value, + context: &Self::Context, + ) -> Result<(), Error> { + self.contexts.lock().unwrap().push(context.clone()); + self.entries.lock().unwrap().push((key.to_owned(), value)); + Ok(()) + } + + fn get_cache(&self, key: &str, context: &Self::Context) -> Result, Error> { + self.contexts.lock().unwrap().push(context.clone()); + Ok(self + .entries + .lock() + .unwrap() + .iter() + .find(|(entry_key, _)| entry_key == key) + .map(|(_, entry)| entry.clone())) + } + + async fn disconnect(&self) -> Result<(), Error> { + Ok(()) + } + + async fn test_connection(&self) -> Result { + Ok(CacheConnectionResult { + status: CacheConnectionStatus::Success, + message: "ok".into(), + error: None, + }) + } +} + +#[test] +fn semantic_context_reaches_backend_for_store_and_lookup() { + let backend = Arc::new(SemanticBackend { + entries: Mutex::new(Vec::new()), + contexts: Mutex::new(Vec::new()), + }); + let cache = ResponseCache::new(backend.clone()); + let context = SemanticCacheContext { + messages: Some(json!([{"role": "user", "content": "hello"}])), + ..Default::default() + }; + let request = request().with_context(context.clone()); + let response = json!({"answer": 42}); + + cache + .store(&request, response.clone(), Duration::from_secs(100)) + .unwrap(); + + assert_eq!( + cache.lookup(&request, Duration::from_secs(100)).unwrap(), + Some(response) + ); + assert_eq!( + backend.contexts.lock().unwrap().as_slice(), + &[context.clone(), context] + ); +} + #[tokio::test] async fn sync_and_async_consumers_share_keys_ttls_and_freshness() { let clock = Arc::new(AtomicU64::new(100)); diff --git a/litellm-rust/crates/cache-s3/Cargo.toml b/litellm-rust/crates/cache-s3/Cargo.toml new file mode 100644 index 00000000000..cdc17e732cb --- /dev/null +++ b/litellm-rust/crates/cache-s3/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "litellm-cache-s3" +version = "0.1.0" +edition.workspace = true +license.workspace = true +repository.workspace = true + +[dependencies] +litellm-cache.workspace = true +litellm-auth-aws.workspace = true +aws-sdk-s3 = { version = "1.146.1", default-features = false, features = ["rustls", "rt-tokio"] } +aws-credential-types = "1.3.0" +aws-smithy-types = "1.6.0" +aws-types = "1.6.0" +tokio.workspace = true + +[dev-dependencies] +wiremock = "0.6.5" +serde_json.workspace = true +tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } diff --git a/litellm-rust/crates/cache-s3/src/auth.rs b/litellm-rust/crates/cache-s3/src/auth.rs new file mode 100644 index 00000000000..b7ca722cea3 --- /dev/null +++ b/litellm-rust/crates/cache-s3/src/auth.rs @@ -0,0 +1,101 @@ +use aws_credential_types::{ + Credentials as AwsCredentials, + provider::{ProvideCredentials, error::CredentialsError, future}, +}; +use litellm_auth_aws::{AwsAuthConfig, resolve_credentials}; + +#[derive(Clone)] +pub(crate) struct Credentials { + config: AwsAuthConfig, + env: fn(&str) -> Option, +} + +impl Credentials { + pub(crate) fn new(config: AwsAuthConfig) -> Self { + Self::with_env(config, |name| std::env::var(name).ok()) + } + + pub(crate) fn with_env(config: AwsAuthConfig, env: fn(&str) -> Option) -> Self { + Self { config, env } + } +} + +impl ProvideCredentials for Credentials { + fn provide_credentials<'a>(&'a self) -> future::ProvideCredentials<'a> + where + Self: 'a, + { + future::ProvideCredentials::new(async { + if let (Some(access_key_id), Some(secret_access_key)) = ( + self.config.access_key_id.clone(), + self.config.secret_access_key.clone(), + ) { + return Ok(AwsCredentials::new( + access_key_id, + secret_access_key, + self.config.session_token.clone(), + None, + "litellm-s3-cache", + )); + } + resolve_credentials(self.config.clone(), &self.env) + .await + .map_err(|_| CredentialsError::provider_error("S3 cache authentication failed")) + }) + } +} + +impl std::fmt::Debug for Credentials { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Credentials").finish_non_exhaustive() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn explicit_keys_ignore_an_ambient_session_token() { + let provider = Credentials::with_env( + AwsAuthConfig { + access_key_id: Some("key".to_string()), + secret_access_key: Some("secret".to_string()), + region_name: Some("us-east-1".to_string()), + ..Default::default() + }, + |name| (name == "AWS_SESSION_TOKEN").then(|| "ambient".to_string()), + ); + let credentials = provider.provide_credentials().await.unwrap(); + assert_eq!(credentials.access_key_id(), "key"); + assert_eq!(credentials.secret_access_key(), "secret"); + assert_eq!(credentials.session_token(), None); + } + + #[tokio::test] + async fn explicit_keys_keep_their_session_token() { + let provider = Credentials::new(AwsAuthConfig { + access_key_id: Some("key".to_string()), + secret_access_key: Some("secret".to_string()), + session_token: Some("t".to_string()), + region_name: Some("us-east-1".to_string()), + ..Default::default() + }); + let credentials = provider.provide_credentials().await.unwrap(); + assert_eq!(credentials.session_token(), Some("t")); + } + + #[tokio::test] + async fn environment_keys_resolve_with_their_session_token() { + let provider = Credentials::with_env(AwsAuthConfig::default(), |name| match name { + "AWS_ACCESS_KEY_ID" => Some("env-key".to_string()), + "AWS_SECRET_ACCESS_KEY" => Some("env-secret".to_string()), + "AWS_SESSION_TOKEN" => Some("env-token".to_string()), + _ => None, + }); + let credentials = provider.provide_credentials().await.unwrap(); + assert_eq!(credentials.access_key_id(), "env-key"); + assert_eq!(credentials.secret_access_key(), "env-secret"); + assert_eq!(credentials.session_token(), Some("env-token")); + } +} diff --git a/litellm-rust/crates/cache-s3/src/cache.rs b/litellm-rust/crates/cache-s3/src/cache.rs new file mode 100644 index 00000000000..9c791f42c3b --- /dev/null +++ b/litellm-rust/crates/cache-s3/src/cache.rs @@ -0,0 +1,220 @@ +use std::{ + future::Future, + sync::Arc, + time::{Duration, SystemTime}, +}; + +use aws_sdk_s3::{ + config::{BehaviorVersion, Region, RequestChecksumCalculation, ResponseChecksumValidation}, + error::SdkError, + primitives::ByteStream, +}; +use aws_smithy_types::{DateTime, date_time::Format}; +use litellm_auth_aws::AwsAuthConfig; +use litellm_cache::{ + BaseCache, BatchCache, CacheCodec, CacheConnectionResult, Error, ExactCacheContext, FlushCache, +}; +use tokio::runtime::Handle; + +use crate::auth::Credentials; + +pub struct S3Endpoint { + pub url: String, +} + +pub struct S3CacheConfig { + pub bucket: String, + pub key_prefix: String, + pub region: String, + pub endpoint: Option, + pub auth: AwsAuthConfig, +} + +pub struct S3Cache { + client: aws_sdk_s3::Client, + codec: C, + runtime: Handle, + bucket: Arc, + key_prefix: Arc, + region: Arc, + endpoint: Option>, +} + +impl S3Cache { + pub fn new(config: S3CacheConfig, codec: C, runtime: Handle) -> Self { + let endpoint_url: Option = config.endpoint.map(|endpoint| endpoint.url); + let base = aws_sdk_s3::Config::builder() + .behavior_version(BehaviorVersion::latest()) + .region(Region::new(config.region.clone())) + .credentials_provider(Credentials::new(config.auth)) + .request_checksum_calculation(RequestChecksumCalculation::WhenRequired) + .response_checksum_validation(ResponseChecksumValidation::WhenRequired); + let builder = match &endpoint_url { + Some(url) => base.endpoint_url(url).force_path_style(true), + None => base, + }; + Self { + client: aws_sdk_s3::Client::from_conf(builder.build()), + codec, + runtime, + bucket: config.bucket.into(), + key_prefix: config.key_prefix.into(), + region: config.region.into(), + endpoint: endpoint_url.map(Into::into), + } + } + + pub fn bucket(&self) -> &str { + &self.bucket + } + + pub fn key_prefix(&self) -> &str { + &self.key_prefix + } + + pub fn region(&self) -> &str { + &self.region + } + + pub fn endpoint(&self) -> Option<&str> { + self.endpoint.as_deref() + } + + pub fn to_s3_key(&self, key: &str) -> String { + format!("{}{}", self.key_prefix, key.replace(':', "/")) + } + + fn block_on(&self, future: F) -> F::Output { + if Handle::try_current().is_ok() { + tokio::task::block_in_place(|| self.runtime.block_on(future)) + } else { + self.runtime.block_on(future) + } + } + + async fn put( + &self, + key: &str, + value: C::Value, + context: &ExactCacheContext, + ) -> Result<(), Error> { + let s3_key = self.to_s3_key(key); + let body = self.codec.encode(&value)?; + let request = self + .client + .put_object() + .bucket(self.bucket.as_ref()) + .key(&s3_key) + .body(ByteStream::from(body)) + .content_type("application/json") + .content_language("en") + .content_disposition(format!("inline; filename=\"{s3_key}.json\"")); + let request = match context.ttl { + Some(ttl) => { + let seconds = ttl.as_secs_f64(); + request + .cache_control(format!("immutable, max-age={seconds}, s-maxage={seconds}")) + .expires(DateTime::from(SystemTime::now() + ttl)) + } + None => request.cache_control("immutable, max-age=31536000, s-maxage=31536000"), + }; + request.send().await.map_err(|_| Error::Unavailable)?; + Ok(()) + } + + async fn get(&self, key: &str) -> Result, Error> { + let output = match self + .client + .get_object() + .bucket(self.bucket.as_ref()) + .key(self.to_s3_key(key)) + .send() + .await + { + Ok(output) => output, + Err(error) => { + if let SdkError::ServiceError(service) = &error { + let status = error + .raw_response() + .map(|response| response.status().as_u16()); + let not_found = service.err().is_no_such_key() + || service.err().meta().code() == Some("AccessDenied") + || status == Some(404) + || status == Some(403); + if not_found { + return Ok(None); + } + } + return Err(Error::Unavailable); + } + }; + if let Some(expires) = output.expires_string() + && let Ok(expires) = DateTime::from_str(expires, Format::HttpDate) + && expires < DateTime::from(SystemTime::now()) + { + return Ok(None); + } + let bytes = output + .body + .collect() + .await + .map_err(|_| Error::Unavailable)? + .into_bytes(); + self.codec.decode(&bytes).map(Some) + } +} + +impl BaseCache for S3Cache { + type Value = C::Value; + type Context = ExactCacheContext; + + fn get_ttl(&self, context: &Self::Context) -> Option { + context.ttl + } + + fn set_cache( + &self, + key: &str, + value: Self::Value, + context: &Self::Context, + ) -> Result<(), Error> { + self.block_on(self.put(key, value, context)) + } + + fn get_cache(&self, key: &str, _context: &Self::Context) -> Result, Error> { + self.block_on(self.get(key)) + } + + async fn async_set_cache( + &self, + key: &str, + value: Self::Value, + context: Self::Context, + ) -> Result<(), Error> { + self.put(key, value, &context).await + } + + async fn async_get_cache( + &self, + key: &str, + _context: &Self::Context, + ) -> Result, Error> { + self.get(key).await + } + + async fn disconnect(&self) -> Result<(), Error> { + Ok(()) + } + + async fn test_connection(&self) -> Result { + Err(Error::UnsupportedOperation) + } +} + +impl BatchCache for S3Cache {} + +impl FlushCache for S3Cache { + fn flush_cache(&self) -> Result<(), Error> { + Ok(()) + } +} diff --git a/litellm-rust/crates/cache-s3/src/lib.rs b/litellm-rust/crates/cache-s3/src/lib.rs new file mode 100644 index 00000000000..f6126dfa908 --- /dev/null +++ b/litellm-rust/crates/cache-s3/src/lib.rs @@ -0,0 +1,4 @@ +mod auth; +mod cache; + +pub use cache::{S3Cache, S3CacheConfig, S3Endpoint}; diff --git a/litellm-rust/crates/cache-s3/tests/cache.rs b/litellm-rust/crates/cache-s3/tests/cache.rs new file mode 100644 index 00000000000..9a71656286b --- /dev/null +++ b/litellm-rust/crates/cache-s3/tests/cache.rs @@ -0,0 +1,278 @@ +use std::time::{Duration, SystemTime, UNIX_EPOCH}; + +use litellm_auth_aws::AwsAuthConfig; +use litellm_cache::{ + BaseCache, BatchCache, BatchEntry, Error, ExactCacheContext, FlushCache, JsonCodec, +}; +use litellm_cache_s3::{S3Cache, S3CacheConfig, S3Endpoint}; +use serde_json::{Value, json}; +use tokio::runtime::Handle; +use wiremock::{ + Mock, MockServer, ResponseTemplate, + matchers::{method, path}, +}; + +fn config(endpoint: String) -> S3CacheConfig { + S3CacheConfig { + bucket: "cache-bucket".to_string(), + key_prefix: "team/".to_string(), + region: "us-east-1".to_string(), + endpoint: Some(S3Endpoint { url: endpoint }), + auth: AwsAuthConfig { + access_key_id: Some("key".to_string()), + secret_access_key: Some("secret".to_string()), + region_name: Some("us-east-1".to_string()), + ..Default::default() + }, + } +} + +fn cache(endpoint: &str) -> S3Cache> { + S3Cache::new( + config(endpoint.to_string()), + JsonCodec::::new(), + Handle::current(), + ) +} + +async fn mock_server() -> MockServer { + let server = MockServer::start().await; + Mock::given(method("PUT")) + .respond_with(ResponseTemplate::new(200).insert_header("etag", "\"etag\"")) + .mount(&server) + .await; + server +} + +fn http_date_from(headers: &wiremock::http::HeaderMap, name: &str) -> Option { + use aws_smithy_types::{DateTime, date_time::Format}; + headers + .get(name) + .and_then(|value| DateTime::from_str(value.to_str().ok()?, Format::HttpDate).ok()) + .map(|date| UNIX_EPOCH + Duration::new(date.secs() as u64, date.subsec_nanos())) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn set_writes_python_metadata_with_and_without_ttl() { + let server = mock_server().await; + let cache = cache(&server.uri()); + let context = ExactCacheContext { + ttl: Some(Duration::from_secs(90)), + }; + cache + .set_cache("alpha:beta", json!({"answer": 1}), &context) + .unwrap(); + cache + .set_cache("plain", json!({"answer": 2}), &ExactCacheContext::default()) + .unwrap(); + + let requests = server.received_requests().await.unwrap(); + let ttl_request = requests + .iter() + .find(|request| request.url.path() == "/cache-bucket/team/alpha/beta") + .expect("ttl write should hit the converted S3 key"); + assert_eq!( + ttl_request.headers["cache-control"].to_str().unwrap(), + "immutable, max-age=90, s-maxage=90" + ); + assert_eq!( + ttl_request.headers["content-type"].to_str().unwrap(), + "application/json" + ); + assert_eq!( + ttl_request.headers["content-language"].to_str().unwrap(), + "en" + ); + assert_eq!( + ttl_request.headers["content-disposition"].to_str().unwrap(), + "inline; filename=\"team/alpha/beta.json\"" + ); + let expires = http_date_from(&ttl_request.headers, "expires").expect("ttl write sets Expires"); + let remaining = expires.duration_since(SystemTime::now()).unwrap(); + assert!(remaining > Duration::from_secs(60) && remaining <= Duration::from_secs(91)); + assert_eq!( + serde_json::from_slice::(&ttl_request.body).unwrap(), + json!({"answer": 1}) + ); + + let plain = requests + .iter() + .find(|request| request.url.path() == "/cache-bucket/team/plain") + .expect("no-ttl write should hit the converted S3 key"); + assert_eq!( + plain.headers["cache-control"].to_str().unwrap(), + "immutable, max-age=31536000, s-maxage=31536000" + ); + assert!(plain.headers.get("expires").is_none()); + assert_eq!( + plain.headers["content-disposition"].to_str().unwrap(), + "inline; filename=\"team/plain.json\"" + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn get_hit_miss_expired_and_invalid_entries() { + let server = mock_server().await; + Mock::given(method("GET")) + .and(path("/cache-bucket/team/hit")) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({"answer": 3}))) + .mount(&server) + .await; + Mock::given(method("GET")) + .and(path("/cache-bucket/team/missing")) + .respond_with( + ResponseTemplate::new(404).set_body_string("NoSuchKey"), + ) + .mount(&server) + .await; + Mock::given(method("GET")) + .and(path("/cache-bucket/team/denied")) + .respond_with( + ResponseTemplate::new(403).set_body_string("AccessDenied"), + ) + .mount(&server) + .await; + Mock::given(method("GET")) + .and(path("/cache-bucket/team/expired")) + .respond_with( + ResponseTemplate::new(200) + .insert_header("expires", "Thu, 01 Jan 1970 00:00:00 GMT") + .set_body_json(json!({"answer": 4})), + ) + .mount(&server) + .await; + Mock::given(method("GET")) + .and(path("/cache-bucket/team/malformed")) + .respond_with(ResponseTemplate::new(200).set_body_string("not a cache entry")) + .mount(&server) + .await; + let cache = cache(&server.uri()); + let context = ExactCacheContext::default(); + + assert_eq!( + cache.get_cache("hit", &context).unwrap(), + Some(json!({"answer": 3})) + ); + assert_eq!(cache.get_cache("missing", &context).unwrap(), None); + assert_eq!(cache.get_cache("denied", &context).unwrap(), None); + assert_eq!(cache.get_cache("expired", &context).unwrap(), None); + assert_eq!( + cache.get_cache("malformed", &context), + Err(Error::InvalidEntry) + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn batch_get_preserves_order_with_hits_misses_and_invalid() { + let server = mock_server().await; + for (key, status, body) in [ + ("first", 200, "{\"answer\": 1}"), + ("invalid", 200, "garbage"), + ] { + Mock::given(method("GET")) + .and(path(format!("/cache-bucket/team/{key}"))) + .respond_with(ResponseTemplate::new(status).set_body_string(body)) + .mount(&server) + .await; + } + Mock::given(method("GET")) + .and(path("/cache-bucket/team/miss")) + .respond_with(ResponseTemplate::new(404)) + .mount(&server) + .await; + let cache = cache(&server.uri()); + let context = ExactCacheContext::default(); + let keys = vec![ + "first".to_string(), + "miss".to_string(), + "invalid".to_string(), + ]; + + let entries = cache.batch_get_cache(&keys, &context).unwrap(); + + assert_eq!( + entries, + vec![ + BatchEntry::Hit(json!({"answer": 1})), + BatchEntry::Miss, + BatchEntry::Invalid, + ] + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn unsupported_and_noop_capabilities_match_python() { + let server = mock_server().await; + let cache = cache(&server.uri()); + + assert_eq!( + cache.test_connection().await, + Err(Error::UnsupportedOperation) + ); + cache.flush_cache().unwrap(); + cache.disconnect().await.unwrap(); + assert_eq!(cache.get_ttl(&ExactCacheContext::default()), None); + assert_eq!( + cache.get_ttl(&ExactCacheContext { + ttl: Some(Duration::from_secs(45)), + }), + Some(Duration::from_secs(45)) + ); + assert!(server.received_requests().await.unwrap().is_empty()); +} + +#[test] +fn key_conversion_prefixes_and_splits_colons() { + let runtime = tokio::runtime::Builder::new_multi_thread() + .worker_threads(1) + .enable_all() + .build() + .unwrap(); + let _guard = runtime.enter(); + let cache = S3Cache::new( + S3CacheConfig { + key_prefix: "team/".to_string(), + ..config("http://localhost".to_string()) + }, + JsonCodec::::new(), + runtime.handle().clone(), + ); + + assert_eq!(cache.bucket(), "cache-bucket"); + assert_eq!(cache.key_prefix(), "team/"); + assert_eq!(cache.to_s3_key("a:b:c"), "team/a/b/c"); + assert_eq!(cache.to_s3_key("plain"), "team/plain"); + + let unprefixed = S3Cache::new( + S3CacheConfig { + key_prefix: String::new(), + ..config("http://localhost".to_string()) + }, + JsonCodec::::new(), + runtime.handle().clone(), + ); + assert_eq!(unprefixed.to_s3_key("a:b"), "a/b"); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn sync_methods_block_inside_and_outside_the_runtime() { + let server = mock_server().await; + Mock::given(method("GET")) + .and(path("/cache-bucket/team/key")) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({"answer": 9}))) + .mount(&server) + .await; + let uri = server.uri(); + let cache = tokio::task::spawn_blocking(move || { + let cache = cache(&uri); + let context = ExactCacheContext::default(); + cache + .set_cache("key", json!({"answer": 9}), &context) + .unwrap(); + cache.get_cache("key", &context).unwrap() + }) + .await + .unwrap(); + + assert_eq!(cache, Some(json!({"answer": 9}))); +} diff --git a/litellm-rust/crates/cache-valkey-semantic/Cargo.toml b/litellm-rust/crates/cache-valkey-semantic/Cargo.toml new file mode 100644 index 00000000000..f98bb5a5fa8 --- /dev/null +++ b/litellm-rust/crates/cache-valkey-semantic/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "litellm-cache-valkey-semantic" +version = "0.1.0" +edition.workspace = true +license.workspace = true +repository.workspace = true + +[dependencies] +litellm-cache.workspace = true +litellm-cache-redis.workspace = true +litellm-cache-response.workspace = true +redis = { version = "1.7.0", features = ["tls-rustls"] } +serde_json.workspace = true +sha2.workspace = true +tokio.workspace = true +uuid = { version = "1", features = ["v4"] } + +[dev-dependencies] +redis-test = "1.0.4" +rstest.workspace = true diff --git a/litellm-rust/crates/cache-valkey-semantic/src/lib.rs b/litellm-rust/crates/cache-valkey-semantic/src/lib.rs new file mode 100644 index 00000000000..6062ccc842c --- /dev/null +++ b/litellm-rust/crates/cache-valkey-semantic/src/lib.rs @@ -0,0 +1,1153 @@ +use std::{ + future::Future, + sync::{Arc, Mutex}, + time::Duration, +}; + +use litellm_cache::{BaseCache, CacheCodec, CacheConnectionResult, Error, SemanticCacheContext}; +use litellm_cache_redis::{ + RedisTopology, + connection::{ConnectionRef, Connections}, +}; +use litellm_cache_response::CacheEntry; +use serde_json::Value; +use sha2::{Digest, Sha256}; +use uuid::Uuid; + +pub trait Embedder: Send + Sync + 'static { + fn embed(&self, prompt: &str, metadata: Option<&Value>) -> Result, Error>; + + fn async_embed( + &self, + prompt: &str, + metadata: Option<&Value>, + ) -> impl Future, Error>> + Send; +} + +pub struct PreparedEmbedding(pub Vec); + +impl Embedder for PreparedEmbedding { + fn embed(&self, _prompt: &str, _metadata: Option<&Value>) -> Result, Error> { + Ok(self.0.clone()) + } + + async fn async_embed( + &self, + _prompt: &str, + _metadata: Option<&Value>, + ) -> Result, Error> { + Ok(self.0.clone()) + } +} + +#[derive(Clone, Debug, PartialEq)] +pub struct ValkeySemanticConfig { + pub similarity_threshold: f64, + pub index_name: String, +} + +pub const DEFAULT_INDEX_NAME: &str = "litellm_semantic_cache_index"; + +#[derive(Clone)] +struct IndexState { + name: String, + prefix: String, + dimension: Arc>>, + similarity_threshold: f64, +} + +pub struct ValkeySemanticCache< + E: Embedder, + S: CacheCodec, + C = redis::Connection, +> { + connections: Arc>, + embedder: E, + codec: S, + config: ValkeySemanticConfig, + index_dimension: Arc>>, +} + +impl ValkeySemanticCache +where + E: Embedder, + S: CacheCodec, +{ + pub fn new( + url: &str, + embedder: E, + codec: S, + config: ValkeySemanticConfig, + ) -> Result { + Ok(Self { + connections: Arc::new(Connections::open(url, &RedisTopology::Standalone)?), + embedder, + codec, + config, + index_dimension: Arc::new(Mutex::new(None)), + }) + } +} + +impl ValkeySemanticCache +where + E: Embedder, + S: CacheCodec, + C: redis::ConnectionLike + Send + 'static, +{ + pub fn with_connection( + connection: C, + embedder: E, + codec: S, + config: ValkeySemanticConfig, + ) -> Self { + Self { + connections: Arc::new(Connections::fixed(connection)), + embedder, + codec, + config, + index_dimension: Arc::new(Mutex::new(None)), + } + } + + pub fn similarity_threshold(&self) -> f64 { + self.config.similarity_threshold + } + + pub fn index_name(&self) -> &str { + &self.config.index_name + } + + fn index_state(&self) -> IndexState { + IndexState { + name: self.config.index_name.clone(), + prefix: format!("{}:", self.config.index_name), + dimension: Arc::clone(&self.index_dimension), + similarity_threshold: self.config.similarity_threshold, + } + } +} + +impl ValkeySemanticCache +where + E: Embedder, + S: CacheCodec + Clone, + C: redis::ConnectionLike + Send + 'static, +{ + pub fn with_embedder(&self, embedder: E2) -> ValkeySemanticCache { + ValkeySemanticCache { + connections: Arc::clone(&self.connections), + embedder, + codec: self.codec.clone(), + config: self.config.clone(), + index_dimension: Arc::clone(&self.index_dimension), + } + } +} + +impl BaseCache for ValkeySemanticCache +where + E: Embedder, + S: CacheCodec, + C: redis::ConnectionLike + Send + 'static, +{ + type Value = CacheEntry; + type Context = SemanticCacheContext; + + fn get_ttl(&self, context: &Self::Context) -> Option { + context.ttl + } + + fn set_cache( + &self, + key: &str, + value: Self::Value, + context: &Self::Context, + ) -> Result<(), Error> { + let Some(prompt) = prompt_from_context(context) else { + return Ok(()); + }; + let embedding = self.embedder.embed(&prompt, context.metadata.as_ref())?; + let scope = scope_tag(key); + let response = self.codec.encode(&value)?; + let vector = embedding_bytes(&embedding); + let index = self.index_state(); + self.connections.execute(|connection| { + write_document( + connection, + &index, + &scope, + &prompt, + response, + vector, + self.get_ttl(context), + ) + }) + } + + fn get_cache(&self, key: &str, context: &Self::Context) -> Result, Error> { + let Some(prompt) = prompt_from_context(context) else { + return Ok(None); + }; + let embedding = self.embedder.embed(&prompt, context.metadata.as_ref())?; + let scope = scope_tag(key); + let vector = embedding_bytes(&embedding); + let index = self.index_state(); + let response = self.connections.execute(|connection| { + search_document(connection, &index, &scope, vector, embedding.len()) + })?; + let Some(response) = response else { + return Ok(None); + }; + self.codec.decode(&response).map(Some) + } + + fn async_set_cache( + &self, + key: &str, + value: Self::Value, + context: Self::Context, + ) -> impl Future> + Send { + let key = key.to_owned(); + let prompt = prompt_from_context(&context); + let metadata = context.metadata.clone(); + async move { + let Some(prompt) = prompt else { + return Ok(()); + }; + let embedding = self + .embedder + .async_embed(&prompt, metadata.as_ref()) + .await?; + let connections = Arc::clone(&self.connections); + let index = self.index_state(); + let response = self.codec.encode(&value)?; + let vector = embedding_bytes(&embedding); + let scope = scope_tag(&key); + let ttl = context.ttl; + Connections::run_blocking(connections, move |connection| { + write_document(connection, &index, &scope, &prompt, response, vector, ttl) + }) + .await + } + } + + fn async_get_cache( + &self, + key: &str, + context: &Self::Context, + ) -> impl Future, Error>> + Send { + let key = key.to_owned(); + let prompt = prompt_from_context(context); + let metadata = context.metadata.clone(); + async move { + let Some(prompt) = prompt else { + return Ok(None); + }; + let embedding = self + .embedder + .async_embed(&prompt, metadata.as_ref()) + .await?; + let connections = Arc::clone(&self.connections); + let index = self.index_state(); + Connections::run_blocking(connections, move |connection| { + let scope = scope_tag(&key); + let vector = embedding_bytes(&embedding); + search_document(connection, &index, &scope, vector, embedding.len()) + }) + .await + .and_then(|response| response.map(|bytes| self.codec.decode(&bytes)).transpose()) + } + } + + async fn disconnect(&self) -> Result<(), Error> { + Ok(()) + } + + async fn test_connection(&self) -> Result { + Err(Error::UnsupportedOperation) + } +} + +pub fn prompt_from_context(context: &SemanticCacheContext) -> Option { + if let Some(Value::Array(messages)) = context.messages.as_ref() + && !messages.is_empty() + { + return messages + .iter() + .filter_map(Value::as_object) + .map(message_text) + .collect(); + } + let input = context.input.as_ref()?; + let mut parts = Vec::new(); + collect_input_text(input, &mut parts); + let prompt = parts.join("\n").trim().to_owned(); + (!prompt.is_empty()).then_some(prompt) +} + +fn message_text(message: &serde_json::Map) -> Option { + let content = match message.get("content") { + Some(Value::String(value)) => value.clone(), + Some(Value::Array(parts)) => { + let mut content = String::new(); + for part in parts { + let part = part.as_object()?; + if let Some(text) = part.get("text").and_then(Value::as_str) { + content.push_str(text); + } + } + content + } + _ => String::new(), + }; + Some(format!( + "{content}{}", + search_results_text(message.get("search_results")) + )) +} + +fn search_results_text(value: Option<&Value>) -> String { + let Some(Value::Array(results)) = value else { + return String::new(); + }; + results + .iter() + .filter_map(Value::as_object) + .map(|result| { + let source = result.get("source").and_then(Value::as_str).unwrap_or(""); + let title = result.get("title").and_then(Value::as_str).unwrap_or(""); + let content = result + .get("content") + .and_then(Value::as_array) + .map(|blocks| { + blocks + .iter() + .filter_map(Value::as_object) + .filter_map(|block| block.get("text").and_then(Value::as_str)) + .collect::() + }) + .unwrap_or_default(); + let citations = result + .get("citations") + .filter(|value| !value.is_null()) + .and_then(|value| serde_json::to_string(value).ok()) + .unwrap_or_default(); + format!("{source}{title}{content}{citations}") + }) + .collect() +} + +fn collect_input_text(value: &Value, parts: &mut Vec) { + match value { + Value::String(value) => { + let value = value.trim(); + if !value.is_empty() { + parts.push(value.to_owned()); + } + } + Value::Array(values) => values + .iter() + .for_each(|value| collect_input_text(value, parts)), + Value::Object(object) => { + if let Some(content) = object.get("content").filter(|value| !value.is_null()) { + collect_input_text(content, parts); + return; + } + for key in ["text", "output", "input_text", "output_text"] { + if let Some(Value::String(value)) = object.get(key) { + let value = value.trim(); + if !value.is_empty() { + parts.push(value.to_owned()); + return; + } + } + } + } + _ => {} + } +} + +fn scope_tag(key: &str) -> String { + let digest = Sha256::digest(key.as_bytes()); + digest.iter().map(|byte| format!("{byte:02x}")).collect() +} + +fn embedding_bytes(embedding: &[f32]) -> Vec { + embedding + .iter() + .flat_map(|value| value.to_le_bytes()) + .collect() +} + +fn write_document( + connection: &mut ConnectionRef<'_>, + index: &IndexState, + scope: &str, + prompt: &str, + response: Vec, + vector: Vec, + ttl: Option, +) -> Result<(), Error> { + let dimension = vector.len() / std::mem::size_of::(); + ensure_index( + connection, + &index.name, + &index.prefix, + &index.dimension, + dimension, + )?; + let document = format!("{}{scope}:{}", index.prefix, Uuid::new_v4()); + let mut pipeline = redis::pipe(); + pipeline + .cmd("HSET") + .arg(&document) + .arg("litellm_cache_key") + .arg(scope) + .arg("prompt") + .arg(prompt) + .arg("response") + .arg(response) + .arg("embedding") + .arg(vector) + .ignore(); + if let Some(ttl) = ttl { + pipeline + .cmd("EXPIRE") + .arg(&document) + .arg(ttl.as_secs()) + .ignore(); + } + pipeline + .query::<()>(connection) + .map_err(|_| Error::Unavailable) +} + +fn search_document( + connection: &mut ConnectionRef<'_>, + index: &IndexState, + scope: &str, + vector: Vec, + dimension: usize, +) -> Result>, Error> { + ensure_index( + connection, + &index.name, + &index.prefix, + &index.dimension, + dimension, + )?; + let query = + format!("(@litellm_cache_key:{{{scope}}})=>[KNN 1 @embedding $vec AS vector_distance]"); + let response = redis::cmd("FT.SEARCH") + .arg(&index.name) + .arg(query) + .arg("PARAMS") + .arg(2) + .arg("vec") + .arg(vector) + .arg("RETURN") + .arg(2) + .arg("response") + .arg("vector_distance") + .arg("DIALECT") + .arg(2) + .query::(connection) + .map_err(|_| Error::Unavailable)?; + let Some(fields) = search_fields(response)? else { + return Ok(None); + }; + let response = fields + .iter() + .find_map(|(name, value)| (name == "response").then(|| value.clone())) + .ok_or(Error::InvalidEntry)?; + let distance = fields + .iter() + .find_map(|(name, value)| (name == "vector_distance").then(|| value.clone())) + .ok_or(Error::InvalidEntry)?; + let distance = parse_f64(&distance)?; + if 1.0 - distance < index.similarity_threshold { + return Ok(None); + } + Ok(Some(response)) +} + +fn ensure_index( + connection: &mut ConnectionRef<'_>, + index_name: &str, + prefix: &str, + index_dimension: &Mutex>, + dimension: usize, +) -> Result<(), Error> { + if index_dimension + .lock() + .map_err(|_| Error::Unavailable)? + .is_some_and(|existing| existing == dimension) + { + return Ok(()); + } + let create = redis::cmd("FT.CREATE") + .arg(index_name) + .arg("ON") + .arg("HASH") + .arg("PREFIX") + .arg(1) + .arg(prefix) + .arg("SCHEMA") + .arg("litellm_cache_key") + .arg("TAG") + .arg("embedding") + .arg("VECTOR") + .arg("HNSW") + .arg(6) + .arg("TYPE") + .arg("FLOAT32") + .arg("DIM") + .arg(dimension) + .arg("DISTANCE_METRIC") + .arg("COSINE") + .query::(connection) + .map(|_| ()) + .map_err(|error| error.to_string()); + if let Err(message) = create { + if !message.to_ascii_lowercase().contains("already exists") { + return Err(Error::Unavailable); + } + let info = redis::cmd("FT.INFO") + .arg(index_name) + .query::(connection) + .map_err(|_| Error::Unavailable)?; + let existing = index_dimension_from_info(&info).ok_or(Error::Unavailable)?; + if existing != dimension { + return Err(Error::Unavailable); + } + } + *index_dimension.lock().map_err(|_| Error::Unavailable)? = Some(dimension); + Ok(()) +} + +fn index_dimension_from_info(value: &redis::Value) -> Option { + let redis::Value::Array(values) = value else { + return None; + }; + let attributes = values.windows(2).find_map(|pair| { + (value_text(&pair[0]).as_deref() == Some("attributes")).then_some(&pair[1]) + })?; + let redis::Value::Array(fields) = attributes else { + return None; + }; + fields.iter().find_map(|field| { + let redis::Value::Array(values) = field else { + return None; + }; + let flattened = values.iter().flat_map(|value| match value { + redis::Value::Array(values) => values.as_slice(), + _ => std::slice::from_ref(value), + }); + let values = flattened.collect::>(); + values.windows(2).find_map(|pair| { + if value_text(pair[0]).as_deref() == Some("dimensions") { + return value_text(pair[1]).and_then(|value| value.parse().ok()); + } + None + }) + }) +} + +type SearchFields = Vec<(String, Vec)>; + +fn search_fields(value: redis::Value) -> Result, Error> { + let redis::Value::Array(values) = value else { + return Err(Error::InvalidEntry); + }; + let total = parse_i64(values.first().ok_or(Error::InvalidEntry)?)?; + if total <= 0 || values.len() < 3 { + return Ok(None); + } + let redis::Value::Array(fields) = &values[2] else { + return Err(Error::InvalidEntry); + }; + let (pairs, remainder) = fields.as_chunks::<2>(); + if !remainder.is_empty() { + return Err(Error::InvalidEntry); + } + let pairs = pairs + .iter() + .map(|pair| { + Ok(( + value_text(&pair[0]).ok_or(Error::InvalidEntry)?, + value_bytes(&pair[1])?, + )) + }) + .collect::, Error>>()?; + Ok(Some(pairs)) +} + +fn parse_i64(value: &redis::Value) -> Result { + value_text(value) + .ok_or(Error::InvalidEntry)? + .parse() + .map_err(|_| Error::InvalidEntry) +} + +fn parse_f64(value: &[u8]) -> Result { + std::str::from_utf8(value) + .map_err(|_| Error::InvalidEntry)? + .parse() + .map_err(|_| Error::InvalidEntry) +} + +fn value_text(value: &redis::Value) -> Option { + match value { + redis::Value::BulkString(bytes) => String::from_utf8(bytes.clone()).ok(), + redis::Value::SimpleString(value) => Some(value.clone()), + redis::Value::Int(value) => Some(value.to_string()), + _ => None, + } +} + +fn value_bytes(value: &redis::Value) -> Result, Error> { + match value { + redis::Value::BulkString(bytes) => Ok(bytes.clone()), + redis::Value::SimpleString(value) => Ok(value.as_bytes().to_vec()), + redis::Value::Int(value) => Ok(value.to_string().into_bytes()), + _ => Err(Error::InvalidEntry), + } +} + +#[cfg(test)] +mod tests { + use std::{ + collections::VecDeque, + sync::{Arc, Mutex}, + time::Duration, + }; + + use litellm_cache::{BaseCache, CacheCodec}; + use litellm_cache_response::{ + CacheEntry, CacheKeyInput, ResponseCache, ResponseCacheCodec, ResponseCacheRequest, + }; + use redis_test::MockRedisConnection; + use rstest::rstest; + use serde_json::{Value, json}; + + use super::{ + Embedder, PreparedEmbedding, ValkeySemanticCache, ValkeySemanticConfig, + index_dimension_from_info, prompt_from_context, scope_tag, + }; + + #[derive(Clone)] + struct FixedEmbedder { + vector: Vec, + calls: EmbedderCalls, + } + + type EmbedderCalls = Arc)>>>; + type RecordingCache = + ValkeySemanticCache; + type RecordingSetup = (RecordingCache, Arc>>>, EmbedderCalls); + + impl Embedder for FixedEmbedder { + fn embed(&self, prompt: &str, metadata: Option<&Value>) -> Result, super::Error> { + self.calls + .lock() + .unwrap() + .push((prompt.into(), metadata.cloned())); + Ok(self.vector.clone()) + } + + async fn async_embed( + &self, + prompt: &str, + metadata: Option<&Value>, + ) -> Result, super::Error> { + self.embed(prompt, metadata) + } + } + + struct RecordingConnection { + requests: Arc>>>, + replies: Mutex>>, + } + + impl RecordingConnection { + fn new(replies: impl IntoIterator>) -> Self { + Self { + requests: Arc::default(), + replies: Mutex::new(replies.into_iter().collect()), + } + } + + fn requests(&self) -> Arc>>> { + Arc::clone(&self.requests) + } + + fn reply(&self) -> redis::RedisResult { + self.replies + .lock() + .unwrap() + .pop_front() + .unwrap_or_else(|| Ok(redis::Value::SimpleString("OK".into()))) + } + } + + impl redis::ConnectionLike for RecordingConnection { + fn req_packed_command(&mut self, command: &[u8]) -> redis::RedisResult { + self.requests.lock().unwrap().push(command.to_vec()); + self.reply() + } + + fn req_packed_commands( + &mut self, + command: &[u8], + _offset: usize, + count: usize, + ) -> redis::RedisResult> { + self.requests.lock().unwrap().push(command.to_vec()); + (0..count).map(|_| self.reply()).collect() + } + + fn get_db(&self) -> i64 { + 0 + } + + fn check_connection(&mut self) -> bool { + true + } + + fn is_open(&self) -> bool { + true + } + } + + fn context( + messages: Option, + input: Option, + ) -> litellm_cache::SemanticCacheContext { + litellm_cache::SemanticCacheContext { + messages, + input, + ..Default::default() + } + } + + #[rstest] + #[case(json!([{"content": "hello"}]), None, Some("hello"))] + #[case(json!([{"content": [{"text": "hello"}, {"text": " world"}]}]), None, Some("hello world"))] + #[case(json!([{"content": ["raw", {"text": "hello"}]}]), None, None)] + #[case(json!([{"search_results": [{"source": "s", "title": "t", "content": [{"text": "c"}], "citations": ["x"]}]}]), None, Some(r#"stc["x"]"#))] + #[case(Value::Array(vec![]), Some(json!(" hello ")), Some("hello"))] + #[case(Value::Array(vec![]), Some(json!([{"content": "first"}, {"text": "second"}])), Some("first\nsecond"))] + #[case(Value::Array(vec![]), Some(json!(" ")), None)] + fn prompt_shapes( + #[case] messages: Value, + #[case] input: Option, + #[case] expected: Option<&str>, + ) { + assert_eq!( + prompt_from_context(&context(Some(messages), input)), + expected.map(str::to_owned) + ); + } + + #[test] + fn scope_tags_are_lowercase_sha256() { + assert_eq!( + scope_tag("key"), + "2c70e12b7a0646f92279f427c7b38e7334d8e5389cff167a1dc30e73f826b683" + ); + } + + #[test] + fn existing_index_dimension_is_read_from_attributes() { + let info = redis::Value::Array(vec![ + redis::Value::SimpleString("attributes".into()), + redis::Value::Array(vec![redis::Value::Array(vec![ + redis::Value::SimpleString("identifier".into()), + redis::Value::SimpleString("embedding".into()), + redis::Value::Array(vec![ + redis::Value::SimpleString("dimensions".into()), + redis::Value::SimpleString("2".into()), + ]), + ])]), + ]); + assert_eq!(index_dimension_from_info(&info), Some(2)); + } + + #[tokio::test] + async fn unsupported_connection_test_is_reported() { + let cache = ValkeySemanticCache::with_connection( + MockRedisConnection::new([]).assert_all_commands_consumed(), + FixedEmbedder { + vector: vec![1.0, 0.0], + calls: Arc::default(), + }, + ResponseCacheCodec, + ValkeySemanticConfig { + similarity_threshold: 0.8, + index_name: "test".into(), + }, + ); + assert_eq!( + cache.test_connection().await, + Err(super::Error::UnsupportedOperation) + ); + } + + #[tokio::test] + async fn prepared_embedding_returns_its_vector_for_any_prompt() { + let embedding = PreparedEmbedding(vec![1.0, 2.0]); + assert_eq!( + embedding + .async_embed("different prompt", None) + .await + .unwrap(), + vec![1.0, 2.0] + ); + } + + #[test] + fn with_embedder_shares_index_state_and_connections() { + let entry = CacheEntry { + timestamp: Some(1.0), + response: json!({"answer": "ok"}), + }; + let encoded = ResponseCacheCodec.encode(&entry).unwrap(); + let cache = ValkeySemanticCache::with_connection( + RecordingConnection::new([ok(), ok(), Ok(search_hit(encoded, "0.1"))]), + FixedEmbedder { + vector: vec![1.0, 0.0], + calls: Arc::default(), + }, + ResponseCacheCodec, + ValkeySemanticConfig { + similarity_threshold: 0.8, + index_name: "test".into(), + }, + ); + cache + .set_cache("key", entry.clone(), &semantic_context(None)) + .unwrap(); + let prepared = cache.with_embedder(PreparedEmbedding(vec![1.0, 0.0])); + assert_eq!( + prepared.get_cache("key", &semantic_context(None)).unwrap(), + Some(entry) + ); + } + + #[test] + fn missing_prompt_does_not_touch_redis() { + let cache = ValkeySemanticCache::with_connection( + MockRedisConnection::new([]).assert_all_commands_consumed(), + FixedEmbedder { + vector: vec![1.0, 0.0], + calls: Arc::default(), + }, + ResponseCacheCodec, + ValkeySemanticConfig { + similarity_threshold: 0.8, + index_name: "test".into(), + }, + ); + assert_eq!(cache.get_cache("key", &context(None, None)).unwrap(), None); + assert_eq!(cache.get_ttl(&context(None, None)), None); + } + + fn semantic_context(ttl: Option) -> litellm_cache::SemanticCacheContext { + litellm_cache::SemanticCacheContext { + messages: Some(json!([{"role": "user", "content": "hello"}])), + metadata: Some(json!({"source": "test"})), + ttl, + ..Default::default() + } + } + + fn cache_with_recording( + replies: impl IntoIterator>, + vector: Vec, + threshold: f64, + ) -> RecordingSetup { + let connection = RecordingConnection::new(replies); + let requests = connection.requests(); + let calls: EmbedderCalls = Arc::default(); + let cache = ValkeySemanticCache::with_connection( + connection, + FixedEmbedder { + vector, + calls: Arc::clone(&calls), + }, + ResponseCacheCodec, + ValkeySemanticConfig { + similarity_threshold: threshold, + index_name: "test".into(), + }, + ); + (cache, requests, calls) + } + + fn ok() -> redis::RedisResult { + Ok(redis::Value::SimpleString("OK".into())) + } + + fn already_exists() -> redis::RedisResult { + Err(redis::RedisError::from(( + redis::ErrorKind::Io, + "already exists", + ))) + } + + fn info_dimension(dimension: usize) -> redis::Value { + redis::Value::Array(vec![ + redis::Value::SimpleString("attributes".into()), + redis::Value::Array(vec![redis::Value::Array(vec![ + redis::Value::SimpleString("embedding".into()), + redis::Value::Array(vec![ + redis::Value::SimpleString("dimensions".into()), + redis::Value::Int(dimension as i64), + ]), + ])]), + ]) + } + + fn search_hit(response: Vec, distance: &str) -> redis::Value { + redis::Value::Array(vec![ + redis::Value::Int(1), + redis::Value::BulkString(b"test:document".to_vec()), + redis::Value::Array(vec![ + redis::Value::BulkString(b"response".to_vec()), + redis::Value::BulkString(response), + redis::Value::BulkString(b"vector_distance".to_vec()), + redis::Value::BulkString(distance.as_bytes().to_vec()), + ]), + ]) + } + + fn requests_text(requests: &Arc>>>) -> String { + requests + .lock() + .unwrap() + .iter() + .map(|request| String::from_utf8_lossy(request)) + .collect::>() + .join("\n") + } + + #[test] + fn set_without_ttl_writes_hset_without_expire() { + let (cache, requests, calls) = cache_with_recording([ok()], vec![1.0, 0.0], 0.8); + cache + .set_cache( + "key", + CacheEntry { + timestamp: None, + response: json!({"answer": "ok"}), + }, + &semantic_context(None), + ) + .unwrap(); + let text = requests_text(&requests); + assert!(text.contains("FT.CREATE")); + assert!(text.contains("HSET")); + assert!( + text.contains("test:2c70e12b7a0646f92279f427c7b38e7334d8e5389cff167a1dc30e73f826b683:") + ); + assert!(!text.contains("EXPIRE")); + assert_eq!( + *calls.lock().unwrap(), + vec![("hello".into(), Some(json!({"source": "test"})))] + ); + } + + #[test] + fn set_with_ttl_truncates_expire_seconds() { + let (cache, requests, _) = cache_with_recording([ok()], vec![1.0, 0.0], 0.8); + cache + .set_cache( + "key", + CacheEntry { + timestamp: None, + response: json!({"answer": "ok"}), + }, + &semantic_context(Some(Duration::from_millis(1900))), + ) + .unwrap(); + let text = requests_text(&requests); + assert!(text.contains("EXPIRE")); + assert!(text.contains("\r\n$1\r\n1\r\n")); + } + + #[test] + fn second_set_skips_create_after_dimension_is_cached() { + let (cache, requests, _) = cache_with_recording([ok()], vec![1.0, 0.0], 0.8); + let context = semantic_context(None); + let entry = CacheEntry { + timestamp: None, + response: json!({"answer": "ok"}), + }; + cache.set_cache("key", entry.clone(), &context).unwrap(); + cache.set_cache("key", entry, &context).unwrap(); + let text = requests_text(&requests); + assert_eq!(text.matches("FT.CREATE").count(), 1); + assert_eq!(text.matches("HSET").count(), 2); + } + + #[test] + fn existing_index_dimension_must_match_embedding() { + let (cache, _, _) = cache_with_recording( + [already_exists(), Ok(info_dimension(2))], + vec![1.0, 0.0], + 0.8, + ); + cache + .set_cache( + "key", + CacheEntry { + timestamp: None, + response: json!({"answer": "ok"}), + }, + &semantic_context(None), + ) + .unwrap(); + + let (cache, _, _) = cache_with_recording( + [already_exists(), Ok(info_dimension(3))], + vec![1.0, 0.0], + 0.8, + ); + assert_eq!( + cache.set_cache( + "key", + CacheEntry { + timestamp: None, + response: json!({"answer": "ok"}), + }, + &semantic_context(None), + ), + Err(super::Error::Unavailable) + ); + } + + #[test] + fn get_applies_threshold_and_decodes_entry() { + let entry = CacheEntry { + timestamp: Some(1.0), + response: json!({"answer": "ok"}), + }; + let encoded = ResponseCacheCodec.encode(&entry).unwrap(); + let (cache, _, _) = cache_with_recording( + [ok(), Ok(search_hit(encoded.clone(), "0.1"))], + vec![1.0, 0.0], + 0.8, + ); + assert_eq!( + cache.get_cache("key", &semantic_context(None)).unwrap(), + Some(entry) + ); + + let (cache, _, _) = + cache_with_recording([ok(), Ok(search_hit(encoded, "0.5"))], vec![1.0, 0.0], 0.8); + assert_eq!( + cache.get_cache("key", &semantic_context(None)).unwrap(), + None + ); + } + + #[test] + fn get_zero_docs_is_a_miss() { + let (cache, _, _) = cache_with_recording( + [ok(), Ok(redis::Value::Array(vec![redis::Value::Int(0)]))], + vec![1.0, 0.0], + 0.8, + ); + assert_eq!( + cache.get_cache("key", &semantic_context(None)).unwrap(), + None + ); + } + + #[rstest] + #[case(redis::Value::Array(vec![ + redis::Value::Int(1), + redis::Value::BulkString(b"document".to_vec()), + redis::Value::Array(vec![ + redis::Value::BulkString(b"vector_distance".to_vec()), + redis::Value::BulkString(b"0.1".to_vec()), + ]), + ]))] + #[case(redis::Value::Array(vec![ + redis::Value::Int(1), + redis::Value::BulkString(b"document".to_vec()), + redis::Value::Array(vec![ + redis::Value::BulkString(b"response".to_vec()), + redis::Value::BulkString(b"not-json".to_vec()), + redis::Value::BulkString(b"vector_distance".to_vec()), + redis::Value::BulkString(b"abc".to_vec()), + ]), + ]))] + fn malformed_entries_are_invalid(#[case] search: redis::Value) { + let (cache, _, _) = cache_with_recording([ok(), Ok(search)], vec![1.0, 0.0], 0.8); + assert_eq!( + cache.get_cache("key", &semantic_context(None)), + Err(super::Error::InvalidEntry) + ); + } + + #[test] + fn response_cache_turns_invalid_entries_into_misses() { + let (cache, _, _) = cache_with_recording( + [ + ok(), + Ok(redis::Value::Array(vec![ + redis::Value::Int(1), + redis::Value::BulkString(b"document".to_vec()), + redis::Value::Array(vec![ + redis::Value::BulkString(b"response".to_vec()), + redis::Value::BulkString(b"not-json".to_vec()), + redis::Value::BulkString(b"vector_distance".to_vec()), + redis::Value::BulkString(b"0.1".to_vec()), + ]), + ])), + ], + vec![1.0, 0.0], + 0.8, + ); + let service = ResponseCache::new(Arc::new(cache)); + let request = ResponseCacheRequest { + key: CacheKeyInput { + preset: Some("key".into()), + ..Default::default() + }, + context: semantic_context(None), + ..ResponseCacheRequest::new(CacheKeyInput::default()) + }; + assert_eq!(service.lookup(&request, Duration::ZERO).unwrap(), None); + } + + #[tokio::test] + async fn async_set_and_get_use_shared_document_helpers() { + let entry = CacheEntry { + timestamp: Some(1.0), + response: json!({"answer": "ok"}), + }; + let encoded = ResponseCacheCodec.encode(&entry).unwrap(); + let (cache, requests, calls) = cache_with_recording( + [ok(), ok(), ok(), Ok(search_hit(encoded, "0.1"))], + vec![1.0, 0.0], + 0.8, + ); + let context = semantic_context(Some(Duration::from_millis(1900))); + cache + .async_set_cache("key", entry.clone(), context.clone()) + .await + .unwrap(); + assert_eq!( + cache.async_get_cache("key", &context).await.unwrap(), + Some(entry) + ); + let text = requests_text(&requests); + assert!(text.contains("FT.CREATE")); + assert!(text.contains("HSET")); + assert!(text.contains("EXPIRE")); + assert_eq!(calls.lock().unwrap().len(), 2); + } +} diff --git a/litellm-rust/crates/cache/src/base_cache.rs b/litellm-rust/crates/cache/src/base_cache.rs index 8bd69ba5ad6..5c10e7fd5c3 100644 --- a/litellm-rust/crates/cache/src/base_cache.rs +++ b/litellm-rust/crates/cache/src/base_cache.rs @@ -32,6 +32,28 @@ impl CacheContext for ExactCacheContext { } } +#[derive(Clone, Debug, Default, PartialEq)] +pub struct SemanticCacheContext { + pub input: Option, + pub messages: Option, + pub metadata: Option, + pub scope: Option, + pub ttl: Option, +} + +impl CacheContext for SemanticCacheContext { + fn ttl(&self) -> Option { + self.ttl + } + + fn with_ttl(&self, ttl: Option) -> Self { + Self { + ttl, + ..self.clone() + } + } +} + #[derive(Clone, Copy, Debug, Deserialize, Serialize, PartialEq, Eq)] #[serde(rename_all = "lowercase")] pub enum CacheConnectionStatus { @@ -105,3 +127,31 @@ pub trait BaseCache: Send + Sync { fn test_connection(&self) -> impl Future> + Send; } + +#[cfg(test)] +mod tests { + use std::time::Duration; + + use serde_json::json; + + use super::{CacheContext, SemanticCacheContext}; + + #[test] + fn semantic_context_with_ttl_only_replaces_ttl() { + let context = SemanticCacheContext { + input: Some(json!({"input": "hello"})), + messages: Some(json!([{"role": "user", "content": "hello"}])), + metadata: Some(json!({"tenant": "team"})), + scope: Some("scope".into()), + ttl: Some(Duration::from_secs(10)), + }; + + let updated = context.with_ttl(Some(Duration::from_secs(20))); + + assert_eq!(updated.ttl, Some(Duration::from_secs(20))); + assert_eq!(updated.input, context.input); + assert_eq!(updated.messages, context.messages); + assert_eq!(updated.metadata, context.metadata); + assert_eq!(updated.scope, context.scope); + } +} diff --git a/litellm-rust/crates/cache/src/error.rs b/litellm-rust/crates/cache/src/error.rs index ff3ff6572d4..79ef9cd18b1 100644 --- a/litellm-rust/crates/cache/src/error.rs +++ b/litellm-rust/crates/cache/src/error.rs @@ -6,4 +6,8 @@ pub enum Error { InvalidEntry, #[error("flushing Redis requires an explicit namespace")] UnscopedFlush, + #[error("operation is not supported by this cache")] + UnsupportedOperation, + #[error("semantic cache requires request messages")] + MissingPrompt, } diff --git a/litellm-rust/crates/cache/src/lib.rs b/litellm-rust/crates/cache/src/lib.rs index ce9f93b6dc4..8364c635e3a 100644 --- a/litellm-rust/crates/cache/src/lib.rs +++ b/litellm-rust/crates/cache/src/lib.rs @@ -8,7 +8,7 @@ mod error; pub use base_cache::{ BaseCache, BatchEntry, CacheConnectionResult, CacheConnectionStatus, CacheContext, - ExactCacheContext, + ExactCacheContext, SemanticCacheContext, }; pub use cache_type::CacheType; pub use caching::{Cache, CacheBackend, get_cache, set_cache}; diff --git a/litellm-rust/crates/cache/tests/caching.rs b/litellm-rust/crates/cache/tests/caching.rs index 9180ee9d0dc..36307ac9b33 100644 --- a/litellm-rust/crates/cache/tests/caching.rs +++ b/litellm-rust/crates/cache/tests/caching.rs @@ -1,7 +1,8 @@ use std::{sync::Mutex, time::Duration}; use litellm_cache::{ - BaseCache, CacheConnectionResult, CacheContext, Error, ExactCacheContext, get_cache, + BaseCache, CacheConnectionResult, CacheContext, Error, ExactCacheContext, SemanticCacheContext, + get_cache, }; struct TestCache { @@ -126,6 +127,24 @@ fn associated_context_preserves_backend_specific_lookup_inputs() { ); } +#[test] +fn semantic_context_with_ttl_preserves_lookup_inputs() { + let context = SemanticCacheContext { + input: Some(serde_json::json!("text")), + messages: Some(serde_json::json!([{"role": "user", "content": "hi"}])), + metadata: Some(serde_json::json!({"key": "value"})), + scope: Some("scope".into()), + ttl: None, + }; + let updated = context.with_ttl(Some(Duration::from_secs(30))); + assert_eq!(updated.ttl(), Some(Duration::from_secs(30))); + assert_eq!(updated.input, context.input); + assert_eq!(updated.messages, context.messages); + assert_eq!(updated.metadata, context.metadata); + assert_eq!(updated.scope, context.scope); + assert_eq!(context.with_ttl(None).ttl(), None); +} + #[tokio::test] async fn default_batch_operations_use_async_writes_and_stop_on_failure() { let cache = TestCache { diff --git a/litellm-rust/crates/framer/Cargo.toml b/litellm-rust/crates/framer/Cargo.toml index d22502f871a..62bfcc7da3d 100644 --- a/litellm-rust/crates/framer/Cargo.toml +++ b/litellm-rust/crates/framer/Cargo.toml @@ -11,7 +11,7 @@ aws = ["dep:aws-smithy-eventstream", "dep:aws-smithy-types"] sse = ["dep:sse-stream"] [dependencies] -aws-smithy-eventstream = { version = "=0.61.1", optional = true } +aws-smithy-eventstream = { version = "=0.61.4", optional = true } aws-smithy-types = { version = "1.6.1", optional = true } bytes = "1" futures-util.workspace = true diff --git a/litellm-rust/crates/llms/Cargo.toml b/litellm-rust/crates/llms/Cargo.toml index 0cc7af1836f..f04b78feee1 100644 --- a/litellm-rust/crates/llms/Cargo.toml +++ b/litellm-rust/crates/llms/Cargo.toml @@ -34,7 +34,7 @@ tokio = { workspace = true, features = ["sync"] } url.workspace = true [dev-dependencies] -aws-smithy-eventstream = "=0.61.1" +aws-smithy-eventstream = "=0.61.4" aws-smithy-types = "1.6.1" rstest.workspace = true tokio.workspace = true diff --git a/litellm-rust/crates/python-bridge/Cargo.toml b/litellm-rust/crates/python-bridge/Cargo.toml index 5528e12ee2d..6b4476d897c 100644 --- a/litellm-rust/crates/python-bridge/Cargo.toml +++ b/litellm-rust/crates/python-bridge/Cargo.toml @@ -24,9 +24,17 @@ litellm-cache.workspace = true litellm-cache-azure-blob.workspace = true litellm-cache-memory.workspace = true litellm-cache-redis.workspace = true +litellm-cache-s3.workspace = true +litellm-cache-gcs.workspace = true +litellm-cache-disk.workspace = true +litellm-cache-redis-semantic.workspace = true litellm-cache-response.workspace = true +litellm-cache-qdrant-semantic.workspace = true +qdrant-client.workspace = true +litellm-cache-valkey-semantic = { path = "../cache-valkey-semantic" } serde.workspace = true litellm-auth.workspace = true +litellm-auth-aws.workspace = true litellm-callbacks-legacy-python.workspace = true litellm-core.workspace = true litellm-core-utils.workspace = true @@ -38,8 +46,11 @@ litellm-host-python.workspace = true litellm-token-counter = { path = "../token-counter", default-features = false } pyo3.workspace = true pyo3-async-runtimes.workspace = true +reqwest.workspace = true +redis = { version = "1.7.0", features = ["tls-rustls"] } serde_json.workspace = true -tokio = { workspace = true, features = ["sync"] } +url.workspace = true +tokio = { workspace = true, features = ["rt", "sync"] } [dev-dependencies] serde.workspace = true @@ -47,6 +58,7 @@ serde_with.workspace = true criterion.workspace = true futures-util.workspace = true rstest.workspace = true +sha2.workspace = true tokio-tungstenite.workspace = true [[bench]] diff --git a/litellm-rust/crates/python-bridge/src/cache/binding.rs b/litellm-rust/crates/python-bridge/src/cache/binding.rs index ad64b24d3c1..2ff73238202 100644 --- a/litellm-rust/crates/python-bridge/src/cache/binding.rs +++ b/litellm-rust/crates/python-bridge/src/cache/binding.rs @@ -56,12 +56,7 @@ impl ResolvedCache { CacheBinding::Disabled => ready_none(py)?, CacheBinding::Native(service) => { let request = request(input)?; - let service = service.clone(); - run_async( - py, - async move { service.async_lookup(&request, now()).await }, - cache_error, - )? + service.async_lookup_py(py, request)? } CacheBinding::PythonCallback(callback) => callback.async_lookup(py, kwargs)?, }; @@ -179,12 +174,7 @@ impl ResolvedCache { CacheBinding::Native(service) => { let request = self::request(request)?; let response: Value = from_py(response)?; - let service = service.clone(); - run_async( - py, - async move { service.async_store(&request, response, now()).await }, - cache_error, - ) + service.async_store_py(py, request, response) } CacheBinding::PythonCallback(callback) => { callback.async_store(py, response, callback_kwargs) @@ -241,12 +231,7 @@ impl ResolvedCache { )); } let entries = requests.into_iter().zip(responses).collect(); - let service = service.clone(); - run_async( - py, - async move { service.async_store_batch(entries, now()).await }, - cache_error, - ) + service.async_store_batch_py(py, entries) } CacheBinding::PythonCallback(callback) => { callback.async_store_batch(py, callback_result, callback_kwargs) diff --git a/litellm-rust/crates/python-bridge/src/cache/config.rs b/litellm-rust/crates/python-bridge/src/cache/config.rs index 5c962f2bc7a..766d526cf5f 100644 --- a/litellm-rust/crates/python-bridge/src/cache/config.rs +++ b/litellm-rust/crates/python-bridge/src/cache/config.rs @@ -1,11 +1,14 @@ -use std::time::Duration; +use std::{path::PathBuf, time::Duration}; +use litellm_auth_aws::AwsAuthConfig; use litellm_cache::CacheType; +use litellm_cache_qdrant_semantic::{OpenAiEmbedderConfig, QdrantSemanticConfig, Quantization}; use litellm_cache_redis::{RedisNode, RedisTopology}; +use litellm_cache_s3::{S3CacheConfig, S3Endpoint}; use pyo3::{ - exceptions::{PyTypeError, PyValueError}, + exceptions::{PyAttributeError, PyTypeError, PyValueError}, prelude::*, - types::{PyAny, PyDict, PyList, PyString}, + types::{PyAny, PyBool, PyDict, PyList, PyString}, }; use super::{native::NativeResponseCache, request::duration}; @@ -26,6 +29,10 @@ pub(super) struct MemoryCacheConfig { pub(super) max_entry_bytes: usize, } +pub(super) struct DiskCacheConfig { + pub(super) directory: PathBuf, +} + #[derive(Debug, PartialEq)] pub(super) enum RedisProtocol { Resp2, @@ -75,11 +82,31 @@ pub(super) struct RedisCacheConfig { pub(super) connection: RedisConnectionConfig, } +#[derive(Debug, PartialEq)] +pub(super) struct GcsCacheConfig { + pub(super) bucket_name: String, + pub(super) key_prefix: String, + pub(super) path_service_account: Option, +} + pub(super) struct AzureBlobCacheConfig { pub(super) account_url: String, pub(super) container: String, } +#[allow( + dead_code, + reason = "embedding settings are projected so drift falls back to Python" +)] +pub(super) struct RedisSemanticCacheConfig { + pub(super) redis_url: String, + pub(super) index_name: String, + pub(super) similarity_threshold: f64, + pub(super) embedding_model: String, + pub(super) embedding_max_input_tokens: Option, + pub(super) embedding_timeout: Option, +} + struct RedisClientProjection<'py> { topology: RedisTopology, host: String, @@ -91,10 +118,45 @@ struct RedisClientProjection<'py> { const REDIS_PY_DEFAULT_MAX_CONNECTIONS: usize = 1 << 31; +#[allow(dead_code, reason = "consumed by the cache activation follow-up")] +pub(super) struct ValkeySemanticCacheConfig { + pub(super) similarity_threshold: f64, + pub(super) index_name: String, + pub(super) embedding_model: String, + pub(super) connection: RedisConnectionConfig, +} + +pub(super) struct QdrantSemanticCacheConfig { + pub(super) grpc_url: String, + pub(super) api_key: Option, + pub(super) collection_name: String, + pub(super) similarity_threshold: f64, + pub(super) vector_size: u64, + pub(super) embedding: OpenAiEmbedderConfig, + pub(super) quantization: Quantization, +} + +impl QdrantSemanticCacheConfig { + pub(super) fn to_qdrant_config(&self) -> QdrantSemanticConfig { + QdrantSemanticConfig { + collection_name: self.collection_name.clone(), + similarity_threshold: self.similarity_threshold, + vector_size: self.vector_size, + quantization: self.quantization.clone(), + } + } +} + pub(super) enum CacheBackendConfig { Memory(MemoryCacheConfig), Redis(Box), + S3(Box), + Gcs(GcsCacheConfig), + ValkeySemantic(Box), + Disk(DiskCacheConfig), AzureBlob(AzureBlobCacheConfig), + RedisSemantic(Box), + QdrantSemantic(Box), } #[allow(dead_code, reason = "consumed by the cache activation follow-up")] @@ -109,6 +171,13 @@ pub(super) enum UnsupportedCacheConfig { RedisCredentials, RedisConnection, RedisOption, + S3Client, + S3Credentials, + S3Option, + GcsBucket, + DiskStore, + QdrantEndpoint, + SemanticEmbedding, } impl UnsupportedCacheConfig { @@ -119,6 +188,15 @@ impl UnsupportedCacheConfig { Self::RedisCredentials => "native Redis credentials require Python", Self::RedisConnection => "native Redis connection type is not implemented", Self::RedisOption => "native Redis configuration requires Python", + Self::S3Client => "native S3 client type is not implemented", + Self::S3Credentials => "native S3 credentials require Python", + Self::S3Option => "native S3 configuration requires Python", + Self::GcsBucket => "native GCS cache requires a configured bucket name", + Self::DiskStore => "native disk cache requires the built-in diskcache store", + Self::QdrantEndpoint => { + "native Qdrant requires the default REST port so the gRPC port can be derived" + } + Self::SemanticEmbedding => "native semantic embedding requires Python", } } } @@ -161,21 +239,54 @@ impl NativeCacheConfig { }))), Err(reason) => Ok(CacheConfigProjection::Unsupported(reason)), }, + Some(CacheType::S3) => match project_s3(&backend)? { + Ok(backend) => Ok(CacheConfigProjection::Native(Box::new(Self { + policy, + backend: CacheBackendConfig::S3(Box::new(backend)), + }))), + Err(reason) => Ok(CacheConfigProjection::Unsupported(reason)), + }, + Some(CacheType::Gcs) => match project_gcs(&backend)? { + Ok(backend) => Ok(CacheConfigProjection::Native(Box::new(Self { + policy, + backend: CacheBackendConfig::Gcs(backend), + }))), + Err(reason) => Ok(CacheConfigProjection::Unsupported(reason)), + }, + Some(CacheType::ValkeySemantic) => match project_valkey_semantic(&backend)? { + Ok(backend) => Ok(CacheConfigProjection::Native(Box::new(Self { + policy, + backend: CacheBackendConfig::ValkeySemantic(Box::new(backend)), + }))), + Err(reason) => Ok(CacheConfigProjection::Unsupported(reason)), + }, + Some(CacheType::Disk) => match project_disk(&backend)? { + Ok(backend) => Ok(CacheConfigProjection::Native(Box::new(Self { + policy, + backend: CacheBackendConfig::Disk(backend), + }))), + Err(reason) => Ok(CacheConfigProjection::Unsupported(reason)), + }, + Some(CacheType::QdrantSemantic) => match project_qdrant_semantic(&backend)? { + Ok(backend) => Ok(CacheConfigProjection::Native(Box::new(Self { + policy, + backend: CacheBackendConfig::QdrantSemantic(Box::new(backend)), + }))), + Err(reason) => Ok(CacheConfigProjection::Unsupported(reason)), + }, Some(CacheType::AzureBlob) => project_azure_blob(&backend).map(|backend| { CacheConfigProjection::Native(Box::new(Self { policy, backend: CacheBackendConfig::AzureBlob(backend), })) }), - Some( - CacheType::RedisSemantic - | CacheType::ValkeySemantic - | CacheType::S3 - | CacheType::Disk - | CacheType::QdrantSemantic - | CacheType::Gcs, - ) - | None => Ok(CacheConfigProjection::Unsupported( + Some(CacheType::RedisSemantic) => project_redis_semantic(&backend).map(|backend| { + CacheConfigProjection::Native(Box::new(Self { + policy, + backend: CacheBackendConfig::RedisSemantic(Box::new(backend)), + })) + }), + None => Ok(CacheConfigProjection::Unsupported( UnsupportedCacheConfig::Backend, )), } @@ -185,9 +296,17 @@ impl NativeCacheConfig { let default_ttl = match &self.backend { CacheBackendConfig::Memory(config) => Some(config.default_ttl), CacheBackendConfig::Redis(config) => Some(config.default_ttl), - CacheBackendConfig::AzureBlob(_) => None, + CacheBackendConfig::S3(_) => None, + CacheBackendConfig::ValkeySemantic(_) => Some(Duration::ZERO), + CacheBackendConfig::Disk(_) + | CacheBackendConfig::AzureBlob(_) + | CacheBackendConfig::Gcs(_) + | CacheBackendConfig::RedisSemantic(_) + | CacheBackendConfig::QdrantSemantic(_) => None, }; - if service.default_ttl() != default_ttl { + if !matches!(self.backend, CacheBackendConfig::ValkeySemantic(_)) + && service.default_ttl() != default_ttl + { return Some("facade and native backend default TTLs must match"); } match &self.backend { @@ -212,6 +331,115 @@ impl NativeCacheConfig { CacheBackendConfig::Redis(config) => (service.namespace() != config.namespace.as_deref()) .then_some("facade and native backend namespaces must match"), + CacheBackendConfig::S3(_) if service.kind() != "s3" => { + Some("facade and native backend types must match") + } + CacheBackendConfig::S3(config) if service.bucket() != Some(config.bucket.as_str()) => { + Some("facade and native backend buckets must match") + } + CacheBackendConfig::S3(config) + if service.key_prefix() != Some(config.key_prefix.as_str()) => + { + Some("facade and native backend key prefixes must match") + } + CacheBackendConfig::S3(config) if service.region() != Some(config.region.as_str()) => { + Some("facade and native backend regions must match") + } + CacheBackendConfig::S3(config) + if service.endpoint() + != config + .endpoint + .as_ref() + .map(|endpoint| endpoint.url.as_str()) => + { + Some("facade and native backend endpoints must match") + } + CacheBackendConfig::S3(_) => None, + CacheBackendConfig::Gcs(_) if service.kind() != "gcs" => { + Some("facade and native backend types must match") + } + CacheBackendConfig::Gcs(config) + if service + .gcs_backend() + .is_none_or(|backend| backend.bucket_name() != config.bucket_name) => + { + Some("facade and native backend buckets must match") + } + CacheBackendConfig::Gcs(config) + if service + .gcs_backend() + .is_none_or(|backend| backend.key_prefix() != config.key_prefix) => + { + Some("facade and native backend key prefixes must match") + } + CacheBackendConfig::Gcs(config) + if service.gcs_backend().is_none_or(|backend| { + backend.path_service_account() != config.path_service_account.as_deref() + }) => + { + Some("facade and native backend credentials must match") + } + CacheBackendConfig::Gcs(_) => None, + CacheBackendConfig::ValkeySemantic(config) => { + if service.kind() != "valkey-semantic" { + return Some("facade and native backend types must match"); + } + let Some((threshold, index_name)) = service.semantic_config() else { + return Some("facade and native backend types must match"); + }; + (threshold != config.similarity_threshold || index_name != config.index_name) + .then_some("facade and native semantic settings must match") + } + CacheBackendConfig::Disk(_) if service.kind() != "disk" => { + Some("facade and native backend types must match") + } + CacheBackendConfig::Disk(config) => { + let Some(directory) = service.directory() else { + return Some("facade and native backend types must match"); + }; + let native = std::fs::canonicalize(directory).ok(); + let facade = std::fs::canonicalize(&config.directory).ok(); + (native != facade).then_some("facade and native backend directories must match") + } + CacheBackendConfig::RedisSemantic(_) if service.kind() != "redis_semantic" => { + Some("facade and native backend types must match") + } + CacheBackendConfig::RedisSemantic(config) + if service.index_name() != Some(config.index_name.as_str()) => + { + Some("facade and native backend index names must match") + } + CacheBackendConfig::RedisSemantic(config) + if service.similarity_threshold() + != Some(f64::from(config.similarity_threshold as f32)) => + { + Some("facade and native backend similarity thresholds must match") + } + CacheBackendConfig::RedisSemantic(_) => None, + CacheBackendConfig::QdrantSemantic(config) if service.kind() != "qdrant_semantic" => { + Some("facade and native backend types must match") + } + CacheBackendConfig::QdrantSemantic(config) + if service.collection_name() != Some(config.collection_name.as_str()) => + { + Some("facade and native backend collections must match") + } + CacheBackendConfig::QdrantSemantic(config) + if service.similarity_threshold() != Some(config.similarity_threshold) => + { + Some("facade and native backend similarity thresholds must match") + } + CacheBackendConfig::QdrantSemantic(config) + if service.vector_size() != Some(config.vector_size) => + { + Some("facade and native backend vector sizes must match") + } + CacheBackendConfig::QdrantSemantic(config) + if service.embedding_model() != Some(config.embedding.model.as_str()) => + { + Some("facade and native backend embedding models must match") + } + CacheBackendConfig::QdrantSemantic(_) => None, CacheBackendConfig::AzureBlob(config) => match service.azure_blob_identity() { None => Some("facade and native backend types must match"), Some((account_url, container)) @@ -225,6 +453,105 @@ impl NativeCacheConfig { } } +#[inline(never)] +fn project_qdrant_semantic( + backend: &Bound<'_, PyAny>, +) -> PyResult> { + let rest_url = backend.getattr("qdrant_api_base")?.extract::()?; + let parsed = match url::Url::parse(&rest_url) { + Ok(value) => value, + Err(_) => return Ok(Err(UnsupportedCacheConfig::QdrantEndpoint)), + }; + if !matches!(parsed.scheme(), "http" | "https") + || (!parsed.path().is_empty() && parsed.path() != "/") + || parsed.query().is_some() + || parsed.host_str().is_none() + || parsed.port() != Some(6333) + { + return Ok(Err(UnsupportedCacheConfig::QdrantEndpoint)); + } + let mut grpc_url = parsed; + if grpc_url.set_port(Some(6334)).is_err() { + return Ok(Err(UnsupportedCacheConfig::QdrantEndpoint)); + } + grpc_url.set_path(""); + grpc_url.set_query(None); + + if optional_attribute(backend, "embedding_max_input_tokens")? + .is_some_and(|value| !value.is_none()) + { + return Ok(Err(UnsupportedCacheConfig::SemanticEmbedding)); + } + let configured_model = backend.getattr("embedding_model")?.extract::()?; + let embedding_model = configured_model + .strip_prefix("openai/") + .unwrap_or(&configured_model) + .to_owned(); + if !embedding_model.starts_with("text-embedding-") { + return Ok(Err(UnsupportedCacheConfig::SemanticEmbedding)); + } + let proxy_server = py_sys_module(backend.py())?; + if let Some(proxy_server) = proxy_server { + let router = proxy_server.getattr("llm_router")?; + let model_list = proxy_server.getattr("llm_model_list")?; + let embedding_router = backend.py().import("litellm.caching._embedding_router")?; + if !embedding_router + .getattr("resolve_embedding_router")? + .call1((configured_model.as_str(), router, model_list))? + .is_none() + { + return Ok(Err(UnsupportedCacheConfig::SemanticEmbedding)); + } + } + let litellm = backend.py().import("litellm")?; + for name in ["api_key", "openai_key", "api_base"] { + if !litellm.getattr(name)?.is_none() { + return Ok(Err(UnsupportedCacheConfig::SemanticEmbedding)); + } + } + let Ok(embedding_api_key) = std::env::var("OPENAI_API_KEY") else { + return Ok(Err(UnsupportedCacheConfig::SemanticEmbedding)); + }; + if embedding_api_key.is_empty() { + return Ok(Err(UnsupportedCacheConfig::SemanticEmbedding)); + } + let embedding_api_base = std::env::var("OPENAI_BASE_URL") + .or_else(|_| std::env::var("OPENAI_API_BASE")) + .unwrap_or_else(|_| "https://api.openai.com/v1".to_owned()); + let timeout = optional_attribute(backend, "embedding_timeout")? + .map(|value| value.extract::>()) + .transpose()? + .flatten() + .map(duration) + .transpose()?; + Ok(Ok(QdrantSemanticCacheConfig { + grpc_url: grpc_url.to_string().trim_end_matches('/').to_owned(), + api_key: optional_string(backend.getattr("qdrant_api_key")?)?, + collection_name: backend.getattr("collection_name")?.extract()?, + similarity_threshold: backend.getattr("similarity_threshold")?.extract()?, + vector_size: backend.getattr("vector_size")?.extract::()?, + embedding: OpenAiEmbedderConfig { + api_base: embedding_api_base, + api_key: embedding_api_key, + model: embedding_model, + timeout, + }, + quantization: Quantization::Binary, + })) +} + +fn py_sys_module(py: Python<'_>) -> PyResult>> { + match py + .import("sys")? + .getattr("modules")? + .get_item("litellm.proxy.proxy_server") + { + Ok(module) => Ok(Some(module)), + Err(error) if error.is_instance_of::(py) => Ok(None), + Err(error) => Err(error), + } +} + #[inline(never)] fn project_azure_blob(backend: &Bound<'_, PyAny>) -> PyResult { let client = backend.getattr("container_client")?; @@ -240,6 +567,27 @@ fn project_azure_blob(backend: &Bound<'_, PyAny>) -> PyResult, +) -> PyResult { + Ok(RedisSemanticCacheConfig { + redis_url: backend.getattr("_redis_url")?.extract::()?, + index_name: backend + .getattr("_index_name")? + .extract::>()? + .unwrap_or_else(|| "litellm_semantic_cache_index".into()), + similarity_threshold: backend.getattr("similarity_threshold")?.extract::()?, + embedding_model: backend.getattr("embedding_model")?.extract::()?, + embedding_max_input_tokens: backend + .getattr("embedding_max_input_tokens")? + .extract::>()?, + embedding_timeout: backend + .getattr("embedding_timeout")? + .extract::>()?, + }) +} + #[inline(never)] fn project_memory(backend: &Bound<'_, PyAny>) -> PyResult { let max_size_kib = backend.getattr("max_size_per_item")?.extract::()?; @@ -252,6 +600,38 @@ fn project_memory(backend: &Bound<'_, PyAny>) -> PyResult { }) } +#[inline(never)] +fn project_gcs( + backend: &Bound<'_, PyAny>, +) -> PyResult> { + let bucket_name = match backend.getattr("bucket_name")?.extract::>() { + Ok(Some(bucket_name)) if !bucket_name.is_empty() => bucket_name, + _ => return Ok(Err(UnsupportedCacheConfig::GcsBucket)), + }; + Ok(Ok(GcsCacheConfig { + bucket_name, + key_prefix: backend.getattr("key_prefix")?.extract::()?, + path_service_account: backend + .getattr("path_service_account")? + .extract::>()?, + })) +} + +#[inline(never)] +fn project_disk( + backend: &Bound<'_, PyAny>, +) -> PyResult> { + let store = backend.getattr("disk_cache")?; + if !instance_class_is(&store, "diskcache.core", "Cache")? + || !instance_class_is(&store.getattr("_disk")?, "diskcache.core", "Disk")? + { + return Ok(Err(UnsupportedCacheConfig::DiskStore)); + } + Ok(Ok(DiskCacheConfig { + directory: PathBuf::from(store.getattr("directory")?.extract::()?), + })) +} + #[inline(never)] fn project_redis( backend: &Bound<'_, PyAny>, @@ -346,6 +726,77 @@ fn project_redis( })) } +#[inline(never)] +fn project_s3( + backend: &Bound<'_, PyAny>, +) -> PyResult> { + let client = backend.getattr("s3_client")?; + if !instance_class_is(&client, "botocore.client", "S3")? { + return Ok(Err(UnsupportedCacheConfig::S3Client)); + } + let meta = client.getattr("meta")?; + let Some(region) = optional_string(meta.getattr("region_name")?)? else { + return Ok(Err(UnsupportedCacheConfig::S3Option)); + }; + let Some(endpoint_url) = optional_string(meta.getattr("endpoint_url")?)? else { + return Ok(Err(UnsupportedCacheConfig::S3Option)); + }; + let client_config = meta.getattr("config")?; + for name in ["s3", "proxies", "client_cert"] { + if optional_attribute(&client_config, name)?.is_some_and(|value| !value.is_none()) { + return Ok(Err(UnsupportedCacheConfig::S3Option)); + } + } + let signature = match optional_attribute(&client_config, "signature_version")? { + Some(value) => value.extract::>()?, + None => None, + }; + if signature.as_deref() != Some("s3v4") { + return Ok(Err(UnsupportedCacheConfig::S3Option)); + } + let insecure = endpoint_url.starts_with("http://"); + let verify = optional_attribute_chain(&client, &["_endpoint", "http_session", "_verify"])?; + let verified = verify + .and_then(|value| value.cast::().ok().map(|value| value.is_true())) + .unwrap_or(false); + if !verified && !insecure { + return Ok(Err(UnsupportedCacheConfig::S3Option)); + } + let credentials = optional_attribute_chain(&client, &["_request_signer", "_credentials"])? + .ok_or(UnsupportedCacheConfig::S3Credentials); + let credentials = match credentials { + Ok(credentials) if !credentials.is_none() => credentials, + _ => return Ok(Err(UnsupportedCacheConfig::S3Credentials)), + }; + let auth = if credentials.getattr("method")?.extract::()?.as_str() == "explicit" { + AwsAuthConfig { + access_key_id: credentials + .getattr("access_key")? + .extract::>()?, + secret_access_key: credentials + .getattr("secret_key")? + .extract::>()?, + session_token: credentials.getattr("token")?.extract::>()?, + region_name: Some(region.clone()), + ..Default::default() + } + } else { + AwsAuthConfig { + region_name: Some(region.clone()), + ..Default::default() + } + }; + let default_endpoint = endpoint_url == format!("https://s3.{region}.amazonaws.com") + || (region == "us-east-1" && endpoint_url == "https://s3.amazonaws.com"); + Ok(Ok(S3CacheConfig { + bucket: backend.getattr("bucket_name")?.extract::()?, + key_prefix: backend.getattr("key_prefix")?.extract::()?, + region, + endpoint: (!default_endpoint).then_some(S3Endpoint { url: endpoint_url }), + auth, + })) +} + #[inline(never)] fn project_standalone_client<'py>( client: &Bound<'py, PyAny>, @@ -380,7 +831,7 @@ fn project_standalone_client<'py>( #[inline(never)] fn project_cluster_client<'py>( - source: &Bound<'py, PyDict>, + source: &Bound<'_, PyDict>, client: &Bound<'py, PyAny>, ) -> PyResult, UnsupportedCacheConfig>> { let Some(startup_nodes) = startup_nodes(source)? else { @@ -468,6 +919,71 @@ fn port(value: i64) -> PyResult { u16::try_from(value).map_err(|_| PyValueError::new_err("invalid Redis port")) } +#[inline(never)] +fn project_valkey_semantic( + backend: &Bound<'_, PyAny>, +) -> PyResult> { + let client = backend.getattr("sync_client")?; + let pool = client.getattr("connection_pool")?; + let Ok((resolved, is_tls)) = project_connection_pool(&pool)? else { + return Ok(Err(UnsupportedCacheConfig::RedisConnection)); + }; + for key in ["credential_provider", "redis_connect_func"] { + if has_value(&resolved, key)? { + return Ok(Err(UnsupportedCacheConfig::RedisCredentials)); + } + } + if is_tls { + return Ok(Err(UnsupportedCacheConfig::RedisConnection)); + } + let connection = RedisConnectionConfig { + host: required_string(&resolved, "host")?, + port: u16::try_from(required_i64(&resolved, "port")?) + .map_err(|_| PyValueError::new_err("invalid Redis port"))?, + database: optional_i64(&resolved, "db")?.unwrap_or(0), + username: optional_dict_string(&resolved, "username")?, + password: optional_dict_string(&resolved, "password")?, + protocol: RedisProtocol::Resp2, + pool_size: pool.getattr("max_connections")?.extract::()?, + read_timeout: None, + connect_timeout: None, + socket_keepalive: None, + health_check_interval: Duration::ZERO, + client_name: None, + tls: None, + }; + if connection.host.is_empty() { + return Ok(Err(UnsupportedCacheConfig::RedisConnection)); + } + Ok(Ok(ValkeySemanticCacheConfig { + similarity_threshold: backend.getattr("similarity_threshold")?.extract()?, + index_name: backend.getattr("index_name")?.extract()?, + embedding_model: backend.getattr("embedding_model")?.extract()?, + connection, + })) +} + +#[inline(never)] +fn project_connection_pool<'py>( + pool: &Bound<'py, PyAny>, +) -> PyResult, bool), UnsupportedCacheConfig>> { + if !instance_class_is(pool, "redis.connection", "ConnectionPool")? { + return Ok(Err(UnsupportedCacheConfig::RedisConnection)); + } + let resolved = pool.getattr("connection_kwargs")?.cast_into::()?; + let connection_class = resolved + .get_item("connection_class")? + .unwrap_or(pool.getattr("connection_class")?); + let is_tls = if class_is(&connection_class, "redis.connection", "Connection")? { + false + } else if class_is(&connection_class, "redis.connection", "SSLConnection")? { + true + } else { + return Ok(Err(UnsupportedCacheConfig::RedisConnection)); + }; + Ok(Ok((resolved, is_tls))) +} + #[inline(never)] fn project_tls(values: &Bound<'_, PyDict>) -> PyResult { Ok(RedisTlsConfig { @@ -549,6 +1065,31 @@ fn optional_attribute_string(value: &Bound<'_, PyAny>, name: &str) -> PyResult( + value: &Bound<'py, PyAny>, + name: &str, +) -> PyResult>> { + match value.getattr(name) { + Ok(value) => Ok(Some(value)), + Err(error) if error.is_instance_of::(value.py()) => Ok(None), + Err(error) => Err(error), + } +} + +#[inline(never)] +fn optional_attribute_chain<'py>( + value: &Bound<'py, PyAny>, + names: &[&str], +) -> PyResult>> { + names + .iter() + .try_fold(Some(value.clone()), |current, name| match current { + Some(current) => optional_attribute(¤t, name), + None => Ok(None), + }) +} + #[inline(never)] fn optional_string(value: Bound<'_, PyAny>) -> PyResult> { Ok(value @@ -637,12 +1178,13 @@ mod tests { use pyo3::{prelude::*, types::PyDict}; use litellm_cache_redis::{RedisNode, RedisTopology}; + use litellm_cache_redis_semantic::RedisSemanticConfig; use super::{ - CacheBackendConfig, CacheConfigProjection, CertificateRequirement, NativeCacheConfig, - RedisProtocol, + CacheBackendConfig, CacheConfigProjection, CachePolicy, CertificateRequirement, + GcsCacheConfig, NativeCacheConfig, RedisProtocol, UnsupportedCacheConfig, }; - use crate::cache::native::NativeResponseCache; + use crate::cache::{embedder::PythonEmbedder, native::NativeResponseCache}; fn cluster_facade<'py>(py: Python<'py>, startup_nodes: &str, hook: &str) -> Bound<'py, PyAny> { facade( @@ -715,6 +1257,49 @@ mod tests { }); } + #[test] + fn redis_semantic_service_mismatch_accepts_backend_precision_threshold() { + Python::initialize(); + Python::attach(|py| { + let facade = facade( + py, + "backend = SimpleNamespace(_redis_url='redis://127.0.0.1/', _index_name='semantic_idx', similarity_threshold=0.8, embedding_model='text-embedding-3-small', embedding_max_input_tokens=None, embedding_timeout=None)\n\ + facade = SimpleNamespace(type='redis-semantic', mode='default-on', ttl=None, namespace=None, supported_call_types=None, redis_flush_size=None, semantic_cache_scope='key', cache=backend)", + ); + let backend = facade.getattr("cache").unwrap(); + let embedder = PythonEmbedder::new(backend.clone().unbind()); + let CacheConfigProjection::Native(config) = + NativeCacheConfig::project(&facade).unwrap() + else { + panic!("Redis semantic cache should be supported"); + }; + let CacheBackendConfig::RedisSemantic(config) = config.backend else { + panic!("expected Redis semantic configuration"); + }; + let service = NativeResponseCache::redis_semantic( + &config.redis_url, + embedder, + RedisSemanticConfig { + index_name: config.index_name.clone(), + similarity_threshold: config.similarity_threshold as f32, + }, + ) + .unwrap(); + let matching_config = NativeCacheConfig { + policy: CachePolicy { + mode: "default-on".into(), + ttl: None, + namespace: None, + supported_call_types: None, + redis_flush_size: None, + semantic_cache_scope: "key".into(), + }, + backend: CacheBackendConfig::RedisSemantic(config), + }; + assert_eq!(matching_config.service_mismatch(&service), None); + }); + } + #[test] fn projects_resolved_redis_tls_configuration() { Python::initialize(); @@ -758,6 +1343,87 @@ mod tests { }); } + #[test] + fn projects_valkey_semantic_configuration() { + Python::initialize(); + Python::attach(|py| { + let facade = facade( + py, + "pool = ConnectionPool()\n\ + pool.connection_class = Connection\n\ + pool.max_connections = 12\n\ + pool.connection_kwargs = {'host': 'cache.internal', 'port': 6390, 'db': 2}\n\ + client = SimpleNamespace(connection_pool=pool)\n\ + backend = SimpleNamespace(similarity_threshold=0.85, index_name='semantic_idx', embedding_model='text-embedding-3-small', sync_client=client)\n\ + facade = SimpleNamespace(type='valkey-semantic', mode='default-on', ttl=None, namespace=None, supported_call_types=None, redis_flush_size=None, semantic_cache_scope='key', cache=backend)", + ); + let CacheConfigProjection::Native(config) = + NativeCacheConfig::project(&facade).unwrap() + else { + panic!("Valkey semantic cache should be supported"); + }; + let CacheBackendConfig::ValkeySemantic(valkey) = config.backend else { + panic!("expected Valkey semantic configuration"); + }; + assert_eq!(valkey.similarity_threshold, 0.85); + assert_eq!(valkey.index_name, "semantic_idx"); + assert_eq!(valkey.embedding_model, "text-embedding-3-small"); + assert_eq!(valkey.connection.host, "cache.internal"); + assert_eq!(valkey.connection.port, 6390); + assert_eq!(valkey.connection.database, 2); + assert_eq!(valkey.connection.pool_size, 12); + assert_eq!(valkey.connection.protocol, RedisProtocol::Resp2); + assert!(valkey.connection.tls.is_none()); + }); + } + + #[test] + fn valkey_semantic_tls_stays_on_python() { + Python::initialize(); + Python::attach(|py| { + let facade = facade( + py, + "pool = ConnectionPool()\n\ + pool.connection_class = SSLConnection\n\ + pool.connection_kwargs = {'host': 'cache.internal', 'port': 6390}\n\ + client = SimpleNamespace(connection_pool=pool)\n\ + backend = SimpleNamespace(similarity_threshold=0.85, index_name='semantic_idx', embedding_model='text-embedding-3-small', sync_client=client)\n\ + facade = SimpleNamespace(type='valkey-semantic', mode='default-on', ttl=None, namespace=None, supported_call_types=None, redis_flush_size=None, semantic_cache_scope='key', cache=backend)", + ); + let CacheConfigProjection::Unsupported(reason) = + NativeCacheConfig::project(&facade).unwrap() + else { + panic!("TLS Valkey semantic cache should stay on Python"); + }; + assert_eq!( + reason.message(), + "native Redis connection type is not implemented" + ); + }); + } + + #[test] + fn valkey_semantic_dynamic_auth_stays_on_python() { + Python::initialize(); + Python::attach(|py| { + let facade = facade( + py, + "pool = ConnectionPool()\n\ + pool.connection_class = Connection\n\ + pool.connection_kwargs = {'host': 'cache.internal', 'port': 6390, 'credential_provider': object()}\n\ + client = SimpleNamespace(connection_pool=pool)\n\ + backend = SimpleNamespace(similarity_threshold=0.85, index_name='semantic_idx', embedding_model='text-embedding-3-small', sync_client=client)\n\ + facade = SimpleNamespace(type='valkey-semantic', mode='default-on', ttl=None, namespace=None, supported_call_types=None, redis_flush_size=None, semantic_cache_scope='key', cache=backend)", + ); + let CacheConfigProjection::Unsupported(reason) = + NativeCacheConfig::project(&facade).unwrap() + else { + panic!("dynamic Valkey authentication must stay on Python"); + }; + assert_eq!(reason.message(), "native Redis credentials require Python"); + }); + } + #[test] fn dynamic_redis_auth_stays_on_python() { Python::initialize(); @@ -822,6 +1488,71 @@ mod tests { }); } + #[test] + fn projects_gcs_configuration() { + Python::initialize(); + Python::attach(|py| { + let facade = facade( + py, + "backend = SimpleNamespace(bucket_name='bucket', key_prefix='cache/', path_service_account='credentials.json')\n\ + facade = SimpleNamespace(type='gcs', mode='default-on', ttl=None, namespace=None, supported_call_types=None, redis_flush_size=None, semantic_cache_scope='key', cache=backend)", + ); + let CacheConfigProjection::Native(config) = + NativeCacheConfig::project(&facade).unwrap() + else { + panic!("GCS cache should be supported"); + }; + let CacheBackendConfig::Gcs(gcs) = config.backend else { + panic!("expected GCS configuration"); + }; + assert_eq!( + gcs, + GcsCacheConfig { + bucket_name: "bucket".into(), + key_prefix: "cache/".into(), + path_service_account: Some("credentials.json".into()), + } + ); + let matching = NativeResponseCache::gcs( + litellm_cache_gcs::GcsConfig { + bucket_name: "bucket".into(), + gcs_path: Some("cache/".into()), + path_service_account: Some("credentials.json".into()), + endpoint: litellm_cache_gcs::DEFAULT_ENDPOINT.into(), + }, + Some("token".into()), + ) + .unwrap(); + let matching_config = NativeCacheConfig { + policy: config.policy, + backend: CacheBackendConfig::Gcs(gcs), + }; + assert_eq!(matching_config.service_mismatch(&matching), None); + }); + } + + #[test] + fn rejects_gcs_without_a_bucket_name() { + Python::initialize(); + Python::attach(|py| { + let facade = facade( + py, + "backend = SimpleNamespace(bucket_name=None, key_prefix='', path_service_account=None)\n\ + facade = SimpleNamespace(type='gcs', mode='default-on', ttl=None, namespace=None, supported_call_types=None, redis_flush_size=None, semantic_cache_scope='key', cache=backend)", + ); + let CacheConfigProjection::Unsupported(reason) = + NativeCacheConfig::project(&facade).unwrap() + else { + panic!("GCS cache without a bucket should be unsupported"); + }; + assert!(matches!(&reason, UnsupportedCacheConfig::GcsBucket)); + assert_eq!( + reason.message(), + "native GCS cache requires a configured bucket name" + ); + }); + } + #[test] fn malformed_startup_nodes_and_foreign_connect_hooks_stay_on_python() { Python::initialize(); diff --git a/litellm-rust/crates/python-bridge/src/cache/embedder.rs b/litellm-rust/crates/python-bridge/src/cache/embedder.rs new file mode 100644 index 00000000000..9398e5a862b --- /dev/null +++ b/litellm-rust/crates/python-bridge/src/cache/embedder.rs @@ -0,0 +1,156 @@ +use std::future::Future; + +use litellm_cache::Error; +use litellm_host_python::to_py; +use pyo3::{PyTraverseError, PyVisit, prelude::*, types::PyDict}; +use serde_json::Value; + +tokio::task_local! { + static PREPARED_EMBEDDING: Result, Error>; +} + +pub(super) fn with_prepared_embedding( + vector: Result, Error>, + future: F, +) -> impl Future { + PREPARED_EMBEDDING.scope(vector, future) +} + +pub(super) struct PythonEmbedder(Py); + +impl Clone for PythonEmbedder { + fn clone(&self) -> Self { + Python::attach(|py| Self(self.0.clone_ref(py))) + } +} + +impl PythonEmbedder { + pub(super) fn new(object: Py) -> Self { + Self(object) + } + + pub(super) fn from_backend(backend: &Bound<'_, PyAny>) -> PyResult { + Ok(Self(backend.clone().unbind())) + } + + pub(super) fn object(&self) -> &Py { + &self.0 + } + + pub(super) fn traverse(&self, visit: &PyVisit<'_>) -> Result<(), PyTraverseError> { + visit.call(&self.0) + } + + pub(super) fn async_embed_awaitable<'py>( + &self, + py: Python<'py>, + prompt: &str, + metadata: &Option, + ) -> PyResult> { + let metadata = to_py(py, metadata)?; + self.0 + .bind(py) + .call_method1("_get_async_embedding", (prompt, metadata)) + } + + fn metadata_kwargs<'py>( + py: Python<'py>, + metadata: Option<&Value>, + ) -> PyResult> { + let kwargs = PyDict::new(py); + kwargs.set_item("metadata", to_py(py, &metadata)?)?; + Ok(kwargs) + } + + pub(super) fn async_embedding_coroutine( + &self, + py: Python<'_>, + prompt: &str, + metadata: Option<&Value>, + ) -> PyResult> { + let kwargs = Self::metadata_kwargs(py, metadata)?; + self.0 + .bind(py) + .call_method("_get_async_embedding", (prompt,), Some(&kwargs)) + .map(Bound::unbind) + } + + pub(super) fn extract(vector: Bound<'_, PyAny>) -> PyResult> { + Ok(vector + .extract::>()? + .into_iter() + .map(|value| value as f32) + .collect()) + } +} + +impl litellm_cache_valkey_semantic::Embedder for PythonEmbedder { + fn embed(&self, prompt: &str, metadata: Option<&Value>) -> Result, Error> { + let result = Python::attach(|py| -> PyResult> { + let metadata = to_py(py, &metadata)?; + self.0 + .bind(py) + .call_method1("_get_embedding", (prompt, metadata))? + .extract() + }) + .map_err(|_| Error::Unavailable)?; + Ok(result.into_iter().map(|value| value as f32).collect()) + } + + fn async_embed( + &self, + _prompt: &str, + _metadata: Option<&Value>, + ) -> impl Future, Error>> + Send { + let seeded = PREPARED_EMBEDDING + .try_with(Clone::clone) + .unwrap_or(Err(Error::Unavailable)); + std::future::ready(seeded) + } +} + +impl litellm_cache_redis_semantic::Embedder for PythonEmbedder { + fn embed(&self, prompt: &str, metadata: Option<&Value>) -> Result, Error> { + Python::attach(|py| { + let kwargs = Self::metadata_kwargs(py, metadata)?; + Self::extract(self.0.bind(py).call_method( + "_get_embedding", + (prompt,), + Some(&kwargs), + )?) + }) + .map_err(|_| Error::Unavailable) + } + + fn async_embed( + &self, + _prompt: &str, + _metadata: Option<&Value>, + ) -> impl Future, Error>> + Send { + let seeded = PREPARED_EMBEDDING + .try_with(Clone::clone) + .unwrap_or(Err(Error::Unavailable)); + std::future::ready(seeded) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn async_embed_returns_the_seeded_vector_or_unavailable() { + Python::initialize(); + let object = Python::attach(|py| py.None()); + let embedder = PythonEmbedder::new(object); + let scoped_embedder = embedder.clone(); + let scoped = with_prepared_embedding(Ok(vec![0.25]), async move { + litellm_cache_redis_semantic::Embedder::async_embed(&scoped_embedder, "prompt", None) + .await + }); + assert_eq!(scoped.await, Ok(vec![0.25])); + let unscoped = + litellm_cache_redis_semantic::Embedder::async_embed(&embedder, "prompt", None).await; + assert_eq!(unscoped, Err(Error::Unavailable)); + } +} diff --git a/litellm-rust/crates/python-bridge/src/cache/facade.rs b/litellm-rust/crates/python-bridge/src/cache/facade.rs index 0e20676938f..43b53584943 100644 --- a/litellm-rust/crates/python-bridge/src/cache/facade.rs +++ b/litellm-rust/crates/python-bridge/src/cache/facade.rs @@ -31,9 +31,15 @@ struct RedisPoolGuard { connection_class: Py, connection_kwargs: Py, max_connections: Option, + client_name: &'static str, attributes: RedisPoolAttributes, } +struct DiskStoreGuard { + reference: Py, + directory: String, +} + struct AzureBlobClientGuard { sync_client: Py, async_client: Py, @@ -41,12 +47,18 @@ struct AzureBlobClientGuard { container_name: String, } +struct S3ClientGuard { + reference: Py, +} + enum ConnectionGuard { None, RedisPool(RedisPoolGuard), AzureBlob(AzureBlobClientGuard), + S3(S3ClientGuard), } +#[derive(Clone, Copy)] struct RedisPoolAttributes { pool: &'static str, connection_class: &'static str, @@ -65,9 +77,12 @@ const CLUSTER_POOL: RedisPoolAttributes = RedisPoolAttributes { max_connections: None, }; +const VALKEY_POOL: RedisPoolAttributes = STANDALONE_POOL; + pub(super) struct FacadeGuard { outer: ObjectGuard, backend: ObjectGuard, + disk_store: Option, connection: ConnectionGuard, } @@ -150,7 +165,9 @@ impl ObjectGuard { return Ok(false); } for (name, value) in &expected.attributes { - if instance.contains(name)? || !attributes.get_item(name)?.is(value.bind(py)) { + if (instance.contains(name)? && !self.config_names.contains(&name.as_str())) + || !attributes.get_item(name)?.is(value.bind(py)) + { return Ok(false); } } @@ -171,8 +188,12 @@ impl ObjectGuard { } impl RedisPoolGuard { - fn capture(backend: &Bound<'_, PyAny>, attributes: RedisPoolAttributes) -> PyResult { - let pool = backend.getattr("redis_client")?.getattr(attributes.pool)?; + fn capture( + backend: &Bound<'_, PyAny>, + client_name: &'static str, + attributes: RedisPoolAttributes, + ) -> PyResult { + let pool = backend.getattr(client_name)?.getattr(attributes.pool)?; Ok(Self { reference: pool.clone().unbind(), connection_class: pool.getattr(attributes.connection_class)?.unbind(), @@ -180,31 +201,30 @@ impl RedisPoolGuard { .getattr("connection_kwargs")? .call_method0("copy")? .unbind(), - max_connections: Self::max_connections(&pool, &attributes)?, + max_connections: attributes + .max_connections + .map(|name| pool.getattr(name)?.extract::()) + .transpose()?, + client_name, attributes, }) } - fn max_connections( - pool: &Bound<'_, PyAny>, - attributes: &RedisPoolAttributes, - ) -> PyResult> { - attributes - .max_connections - .map(|name| pool.getattr(name)?.extract::()) - .transpose() - } - fn matches(&self, py: Python<'_>, backend: &Bound<'_, PyAny>) -> PyResult { let pool = backend - .getattr("redis_client")? + .getattr(self.client_name)? .getattr(self.attributes.pool)?; Ok(self.reference.bind(py).is(&pool) && self .connection_class .bind(py) .is(&pool.getattr(self.attributes.connection_class)?) - && self.max_connections == Self::max_connections(&pool, &self.attributes)? + && self.max_connections + == self + .attributes + .max_connections + .map(|name| pool.getattr(name)?.extract::()) + .transpose()? && self .connection_kwargs .bind(py) @@ -218,6 +238,26 @@ impl RedisPoolGuard { } } +impl DiskStoreGuard { + fn capture(backend: &Bound<'_, PyAny>) -> PyResult { + let store = backend.getattr("disk_cache")?; + Ok(Self { + reference: store.clone().unbind(), + directory: store.getattr("directory")?.extract()?, + }) + } + + fn matches(&self, py: Python<'_>, backend: &Bound<'_, PyAny>) -> PyResult { + let store = backend.getattr("disk_cache")?; + Ok(self.reference.bind(py).is(&store) + && self.directory == store.getattr("directory")?.extract::()?) + } + + fn traverse(&self, visit: &PyVisit<'_>) -> Result<(), PyTraverseError> { + visit.call(&self.reference) + } +} + impl AzureBlobClientGuard { fn capture(backend: &Bound<'_, PyAny>) -> PyResult { let sync_client = backend.getattr("container_client")?; @@ -246,12 +286,43 @@ impl AzureBlobClientGuard { } } +impl S3ClientGuard { + fn capture(backend: &Bound<'_, PyAny>) -> PyResult { + Ok(Self { + reference: backend.getattr("s3_client")?.unbind(), + }) + } + + fn matches(&self, py: Python<'_>, backend: &Bound<'_, PyAny>) -> PyResult { + Ok(self.reference.bind(py).is(&backend.getattr("s3_client")?)) + } + + fn traverse(&self, visit: &PyVisit<'_>) -> Result<(), PyTraverseError> { + visit.call(&self.reference) + } +} + impl ConnectionGuard { fn capture(kind: &str, cluster: bool, backend: &Bound<'_, PyAny>) -> PyResult { Ok(match (kind, cluster) { - ("redis", false) => Self::RedisPool(RedisPoolGuard::capture(backend, STANDALONE_POOL)?), - ("redis", true) => Self::RedisPool(RedisPoolGuard::capture(backend, CLUSTER_POOL)?), + ("redis", false) => Self::RedisPool(RedisPoolGuard::capture( + backend, + "redis_client", + STANDALONE_POOL, + )?), + ("redis", true) => Self::RedisPool(RedisPoolGuard::capture( + backend, + "redis_client", + CLUSTER_POOL, + )?), + ("valkey-semantic", _) => Self::RedisPool(RedisPoolGuard::capture( + backend, + "sync_client", + VALKEY_POOL, + )?), + ("disk", _) => Self::None, ("azure-blob", _) => Self::AzureBlob(AzureBlobClientGuard::capture(backend)?), + ("s3", _) => Self::S3(S3ClientGuard::capture(backend)?), _ => Self::None, }) } @@ -261,6 +332,7 @@ impl ConnectionGuard { Self::None => Ok(true), Self::RedisPool(guard) => guard.matches(py, backend), Self::AzureBlob(guard) => guard.matches(py, backend), + Self::S3(guard) => guard.matches(py, backend), } } @@ -269,6 +341,7 @@ impl ConnectionGuard { Self::None => Ok(()), Self::RedisPool(guard) => guard.traverse(visit), Self::AzureBlob(guard) => guard.traverse(visit), + Self::S3(guard) => guard.traverse(visit), } } } @@ -290,16 +363,34 @@ impl FacadeGuard { let (module, name, cache_kind) = match (kind, cluster) { ("memory", _) => ("litellm.caching.in_memory_cache", "InMemoryCache", "local"), ("redis", false) => ("litellm.caching.redis_cache", "RedisCache", "redis"), + ("redis_semantic", _) => ( + "litellm.caching.redis_semantic_cache", + "RedisSemanticCache", + "redis-semantic", + ), + ("qdrant_semantic", _) => ( + "litellm.caching.qdrant_semantic_cache", + "QdrantSemanticCache", + "qdrant-semantic", + ), ("redis", true) => ( "litellm.caching.redis_cluster_cache", "RedisClusterCache", "redis", ), + ("gcs", _) => ("litellm.caching.gcs_cache", "GCSCache", "gcs"), + ("valkey-semantic", false) => ( + "litellm.caching.valkey_semantic_cache", + "ValkeySemanticCache", + "valkey-semantic", + ), + ("disk", _) => ("litellm.caching.disk_cache", "DiskCache", "disk"), ("azure-blob", _) => ( "litellm.caching.azure_blob_cache", "AzureBlobCache", "azure-blob", ), + ("s3", _) => ("litellm.caching.s3_cache", "S3Cache", "s3"), _ => unreachable!(), }; let backend = facade.getattr("cache")?; @@ -319,6 +410,15 @@ impl FacadeGuard { if let Some(message) = config.service_mismatch(service) { return Err(PyTypeError::new_err(message)); } + if kind == "redis_semantic" + && service + .embedder_object() + .is_none_or(|embedder| !backend.is(embedder.bind(py))) + { + return Err(PyTypeError::new_err( + "facade backend must be the native embedder", + )); + } Ok(Self { outer: ObjectGuard::capture( py, @@ -343,8 +443,30 @@ impl FacadeGuard { "max_size_per_item", "redis_kwargs", "redis_flush_size", + "similarity_threshold", + "distance_threshold", + "embedding_model", + "embedding_max_input_tokens", + "embedding_timeout", + "qdrant_api_base", + "qdrant_api_key", + "collection_name", + "vector_size", + "_index_name", + "_redis_url", + "similarity_threshold", + "embedding_model", + "index_name", + "embedding_max_input_tokens", + "embedding_timeout", + "bucket_name", + "key_prefix", + "path_service_account", ], )?, + disk_store: (kind == "disk") + .then(|| DiskStoreGuard::capture(&backend)) + .transpose()?, connection: ConnectionGuard::capture(kind, cluster, &backend)?, }) } @@ -357,12 +479,20 @@ impl FacadeGuard { if !self.backend.matches(py, &backend)? { return Ok(false); } + if let Some(guard) = &self.disk_store + && !guard.matches(py, &backend)? + { + return Ok(false); + } self.connection.matches(py, &backend) } pub(super) fn traverse(&self, visit: PyVisit<'_>) -> Result<(), PyTraverseError> { self.outer.traverse(&visit)?; self.backend.traverse(&visit)?; + if let Some(guard) = &self.disk_store { + guard.traverse(&visit)?; + } self.connection.traverse(&visit) } } diff --git a/litellm-rust/crates/python-bridge/src/cache/handle.rs b/litellm-rust/crates/python-bridge/src/cache/handle.rs index 002cd6b7a33..51e1b02c405 100644 --- a/litellm-rust/crates/python-bridge/src/cache/handle.rs +++ b/litellm-rust/crates/python-bridge/src/cache/handle.rs @@ -1,8 +1,27 @@ +use litellm_auth_aws::AwsAuthConfig; +use litellm_cache_gcs::{DEFAULT_ENDPOINT, GcsConfig}; +use litellm_cache_qdrant_semantic::{OpenAiEmbedderConfig, Quantization}; use litellm_cache_redis::{RedisNode, RedisTopology}; +use litellm_cache_redis_semantic::RedisSemanticConfig; +use litellm_cache_s3::{S3CacheConfig, S3Endpoint}; use litellm_host_python::{release_gil, run_sync_value}; -use pyo3::{PyTraverseError, PyVisit, exceptions::PyRuntimeError, prelude::*}; +use litellm_http::ClientVariant; +use pyo3::{ + PyTraverseError, PyVisit, + exceptions::{PyRuntimeError, PyTypeError}, + prelude::*, + types::PyDict, +}; +use url::Url; -use super::{cache_error, facade::FacadeGuard, native::NativeResponseCache, request::duration}; +use super::{ + cache_error, + config::{QdrantSemanticCacheConfig, project_redis_semantic}, + embedder::PythonEmbedder, + facade::FacadeGuard, + native::NativeResponseCache, + request::duration, +}; #[pyclass(frozen, name = "_CacheTestHandle")] pub(crate) struct CacheTestHandle { @@ -64,6 +83,199 @@ impl CacheTestHandle { }) } + #[staticmethod] + #[allow(clippy::too_many_arguments)] + #[pyo3(signature = (bucket, *, region, endpoint_url=None, key_prefix="", access_key_id=None, secret_access_key=None, session_token=None))] + fn s3( + py: Python<'_>, + bucket: String, + region: String, + endpoint_url: Option, + key_prefix: &str, + access_key_id: Option, + secret_access_key: Option, + session_token: Option, + ) -> PyResult { + let config = S3CacheConfig { + bucket, + key_prefix: key_prefix.to_string(), + region: region.clone(), + endpoint: endpoint_url.map(|url| S3Endpoint { url }), + auth: AwsAuthConfig { + access_key_id, + secret_access_key, + session_token, + region_name: Some(region), + ..Default::default() + }, + }; + let service = run_sync_value(py, async move { Ok(NativeResponseCache::s3(config).await) })?; + Ok(Self { + service, + guard: None, + pid: std::process::id(), + }) + } + + #[staticmethod] + #[pyo3(signature = (bucket_name, *, gcs_path=None, path_service_account=None, endpoint=None, token=None))] + fn gcs( + py: Python<'_>, + bucket_name: String, + gcs_path: Option, + path_service_account: Option, + endpoint: Option, + token: Option, + ) -> PyResult { + let config = GcsConfig { + bucket_name, + gcs_path, + path_service_account, + endpoint: endpoint.unwrap_or_else(|| DEFAULT_ENDPOINT.to_string()), + }; + let service = release_gil(py, move || NativeResponseCache::gcs(config, token)) + .map_err(cache_error)?; + Ok(Self { + service, + guard: None, + pid: std::process::id(), + }) + } + + #[staticmethod] + #[pyo3(signature = (directory))] + fn disk(py: Python<'_>, directory: String) -> PyResult { + let service = + release_gil(py, move || NativeResponseCache::disk(&directory)).map_err(cache_error)?; + Ok(Self { + service, + guard: None, + pid: std::process::id(), + }) + } + + #[staticmethod] + #[pyo3(signature = (url, *, collection_name, similarity_threshold, vector_size, embedding_model="text-embedding-3-small", api_key=None, embedding_api_key=None, embedding_api_base=None, embedding_timeout_seconds=None, quantization="binary"))] + #[expect( + clippy::too_many_arguments, + reason = "the test handle exposes the complete Qdrant constructor" + )] + fn qdrant_semantic( + py: Python<'_>, + url: String, + collection_name: String, + similarity_threshold: f64, + vector_size: u64, + embedding_model: &str, + api_key: Option, + embedding_api_key: Option, + embedding_api_base: Option, + embedding_timeout_seconds: Option, + quantization: &str, + ) -> PyResult { + let parsed = Url::parse(&url).map_err(|_| { + pyo3::exceptions::PyValueError::new_err( + "native Qdrant requires the default REST port so the gRPC port can be derived", + ) + })?; + if !matches!(parsed.scheme(), "http" | "https") + || (!parsed.path().is_empty() && parsed.path() != "/") + || parsed.query().is_some() + || parsed.host_str().is_none() + || parsed.port() != Some(6333) + { + return Err(pyo3::exceptions::PyValueError::new_err( + "native Qdrant requires the default REST port so the gRPC port can be derived", + )); + } + let mut grpc_url = parsed; + grpc_url.set_port(Some(6334)).map_err(|_| { + pyo3::exceptions::PyValueError::new_err( + "native Qdrant requires the default REST port so the gRPC port can be derived", + ) + })?; + grpc_url.set_path(""); + grpc_url.set_query(None); + let embedding_api_key = embedding_api_key + .or_else(|| { + std::env::var("OPENAI_API_KEY") + .ok() + .filter(|value| !value.is_empty()) + }) + .ok_or_else(|| { + pyo3::exceptions::PyValueError::new_err( + "native semantic embedding requires an OpenAI API key", + ) + })?; + let embedding_api_base = embedding_api_base.unwrap_or_else(|| { + std::env::var("OPENAI_BASE_URL") + .or_else(|_| std::env::var("OPENAI_API_BASE")) + .unwrap_or_else(|_| "https://api.openai.com/v1".to_owned()) + }); + let quantization = match quantization { + "binary" => Quantization::Binary, + "scalar" => Quantization::Scalar, + "product" => Quantization::Product, + _ => { + return Err(pyo3::exceptions::PyValueError::new_err( + "unsupported Qdrant quantization", + )); + } + }; + let config = QdrantSemanticCacheConfig { + grpc_url: grpc_url.to_string().trim_end_matches('/').to_owned(), + api_key, + collection_name, + similarity_threshold, + vector_size, + embedding: OpenAiEmbedderConfig { + api_base: embedding_api_base, + api_key: embedding_api_key, + model: embedding_model.to_owned(), + timeout: embedding_timeout_seconds.map(duration).transpose()?, + }, + quantization, + }; + let http_config = crate::http::call_config(py, &PyDict::new(py), true)?; + let client = crate::http::pool() + .client(&http_config, ClientVariant::Provider) + .map_err(crate::http::client_error)?; + let service = run_sync_value(py, async move { + let handle = tokio::runtime::Handle::current(); + NativeResponseCache::qdrant_semantic(config, client, handle) + .await + .map_err(cache_error) + })?; + Ok(Self { + service, + guard: None, + pid: std::process::id(), + }) + } + + #[staticmethod] + #[pyo3(signature = (url, similarity_threshold, index_name, embedder))] + fn valkey_semantic( + url: String, + similarity_threshold: f64, + index_name: String, + embedder: &Bound<'_, PyAny>, + ) -> PyResult { + let python_embedder = PythonEmbedder::from_backend(embedder)?; + let service = NativeResponseCache::valkey_semantic( + &url, + similarity_threshold, + index_name, + python_embedder, + ) + .map_err(cache_error)?; + Ok(Self { + service, + guard: None, + pid: std::process::id(), + }) + } + #[staticmethod] #[pyo3(signature = (account_url, container))] fn azure_blob(py: Python<'_>, account_url: String, container: String) -> PyResult { @@ -79,6 +291,36 @@ impl CacheTestHandle { }) } + #[staticmethod] + fn redis_semantic(py: Python<'_>, backend: Bound<'_, PyAny>) -> PyResult { + let class = py + .import("litellm.caching.redis_semantic_cache")? + .getattr("RedisSemanticCache")?; + if !backend.get_type().is(&class) { + return Err(PyTypeError::new_err( + "native redis-semantic handles require the built-in RedisSemanticCache", + )); + } + let config = project_redis_semantic(&backend)?; + let embedder = PythonEmbedder::new(backend.unbind()); + let service = release_gil(py, move || { + NativeResponseCache::redis_semantic( + &config.redis_url, + embedder, + RedisSemanticConfig { + index_name: config.index_name, + similarity_threshold: config.similarity_threshold as f32, + }, + ) + }) + .map_err(cache_error)?; + Ok(Self { + service, + guard: None, + pid: std::process::id(), + }) + } + #[getter] fn backend(&self) -> &'static str { self.service.kind() @@ -87,11 +329,17 @@ impl CacheTestHandle { fn _bind_facade(&self, py: Python<'_>, facade: &Bound<'_, PyAny>) -> PyResult<()> { let service = self.service()?; let guard = FacadeGuard::capture(py, facade, &service)?; - let service = service.with_redis_flush_size( - facade - .getattr("redis_flush_size")? - .extract::>()?, - ); + let service = service + .with_scope( + facade + .getattr("semantic_cache_scope")? + .extract::()?, + ) + .with_redis_flush_size( + facade + .getattr("redis_flush_size")? + .extract::>()?, + ); let handle = Py::new( py, Self { @@ -104,6 +352,7 @@ impl CacheTestHandle { } fn __traverse__(&self, visit: PyVisit<'_>) -> Result<(), PyTraverseError> { + self.service.traverse(&visit)?; if let Some(guard) = &self.guard { guard.traverse(visit)?; } diff --git a/litellm-rust/crates/python-bridge/src/cache/mod.rs b/litellm-rust/crates/python-bridge/src/cache/mod.rs index aec08610f6e..fa028518559 100644 --- a/litellm-rust/crates/python-bridge/src/cache/mod.rs +++ b/litellm-rust/crates/python-bridge/src/cache/mod.rs @@ -1,16 +1,19 @@ mod binding; mod callback; mod config; +mod embedder; mod facade; mod future; mod handle; mod native; mod request; mod resolver; +mod semantic; +mod semantic_step; use litellm_cache::Error; use pyo3::{ - exceptions::{PyRuntimeError, PyValueError}, + exceptions::{PyNotImplementedError, PyRuntimeError, PyValueError}, prelude::*, }; @@ -21,6 +24,7 @@ pub(crate) use self::{ fn cache_error(error: Error) -> PyErr { match error { Error::InvalidEntry => PyValueError::new_err(error.to_string()), + Error::UnsupportedOperation => PyNotImplementedError::new_err(error.to_string()), _ => PyRuntimeError::new_err(error.to_string()), } } diff --git a/litellm-rust/crates/python-bridge/src/cache/native.rs b/litellm-rust/crates/python-bridge/src/cache/native.rs index 2cd72a7ce14..7fef6f55611 100644 --- a/litellm-rust/crates/python-bridge/src/cache/native.rs +++ b/litellm-rust/crates/python-bridge/src/cache/native.rs @@ -1,14 +1,81 @@ -use std::{sync::Arc, time::Duration}; +use std::{path::Path, sync::Arc, time::Duration}; -use litellm_cache::{CacheCodec, CacheConnectionResult, Error}; -use litellm_cache_azure_blob::AzureBlobCache; -use litellm_cache_memory::InMemoryCache; -use litellm_cache_redis::{RedisCache, RedisTopology}; -use litellm_cache_response::{ - CacheEntry, PartialHits, ResponseCache, ResponseCacheCodec, ResponseCacheRequest, WriteBuffer, +use litellm_cache::{ + CacheCodec, CacheConnectionResult, Error, ExactCacheContext, SemanticCacheContext, }; +use litellm_cache_azure_blob::AzureBlobCache; +use litellm_cache_disk::DiskCache; +use litellm_cache_gcs::{GcsCache, GcsConfig, StaticTokenSource}; +use litellm_cache_memory::InMemoryCache; +use litellm_cache_qdrant_semantic::{Embedder, OpenAiEmbedder, QdrantSemanticCache}; +use litellm_cache_redis::{RedisCache, RedisTopology}; +use litellm_cache_redis_semantic::{RedisSemanticCache, RedisSemanticConfig}; +use litellm_cache_response::{ + CacheEntry, CacheKeyField, PartialHits, ResponseCache, ResponseCacheCodec, + ResponseCacheRequest, WriteBuffer, +}; +use litellm_cache_s3::{S3Cache, S3CacheConfig}; +use litellm_cache_valkey_semantic::{ValkeySemanticCache, ValkeySemanticConfig}; +use pyo3::{PyTraverseError, PyVisit, prelude::*}; use serde_json::Value; +use super::{ + config::QdrantSemanticCacheConfig, + embedder::PythonEmbedder, + request::NativeRequest, + semantic::{SemanticBody, SemanticOperation, drive}, + semantic_step::{SemanticEmbedExecution, drive_semantic}, +}; + +fn semantic_key(request: &NativeRequest, scope: &str) -> litellm_cache_response::CacheKeyInput { + let mut key = request.key.clone(); + if key.preset.is_some() { + return key; + } + key.fields + .retain(|field| !matches!(field.name.as_str(), "messages" | "prompt" | "input")); + const TENANT: [&str; 3] = [ + "user_api_key", + "user_api_key_team_id", + "user_api_key_org_id", + ]; + let end_user = (scope == "end_user").then_some("user_api_key_end_user_id"); + for name in TENANT.into_iter().chain(end_user) { + let sources = [ + request.metadata.as_ref(), + request.litellm_metadata.as_ref(), + request + .litellm_params + .as_ref() + .and_then(|params| params.get("metadata")), + request + .litellm_params + .as_ref() + .and_then(|params| params.get("litellm_metadata")), + ]; + let Some(value) = sources.into_iter().flatten().find_map(|source| { + source + .as_object() + .and_then(|values| values.get(name)) + .filter(|value| !value.is_null()) + }) else { + continue; + }; + let value = match value { + Value::Null => continue, + Value::String(text) => text.clone(), + other => other.to_string(), + }; + key.fields.push(CacheKeyField { + name: name.to_owned(), + value: Some(value), + api_parameter: true, + internal_parameter: false, + }); + } + key +} + #[derive(Clone)] pub(super) enum NativeResponseCache { Memory(Arc>>), @@ -16,6 +83,19 @@ pub(super) enum NativeResponseCache { cache: Arc>>, buffer: Option>, }, + S3(Arc>>), + Gcs(Arc>>), + ValkeySemantic { + cache: Arc>>, + embedder: PythonEmbedder, + scope: String, + }, + RedisSemantic { + cache: Arc>>, + embedder: PythonEmbedder, + }, + QdrantSemantic(Arc>>), + Disk(Arc>>), AzureBlob(Arc>>), } @@ -48,6 +128,91 @@ impl NativeResponseCache { }) } + pub async fn s3(config: S3CacheConfig) -> Self { + let runtime = tokio::runtime::Handle::current(); + Self::S3(Arc::new(ResponseCache::new(Arc::new(S3Cache::new( + config, + ResponseCacheCodec, + runtime, + ))))) + } + + pub fn valkey_semantic( + url: &str, + similarity_threshold: f64, + index_name: String, + embedder: PythonEmbedder, + ) -> Result { + let backend = ValkeySemanticCache::new( + url, + embedder.clone(), + ResponseCacheCodec, + ValkeySemanticConfig { + similarity_threshold, + index_name, + }, + )?; + Ok(Self::ValkeySemantic { + cache: Arc::new(ResponseCache::new(Arc::new(backend))), + embedder, + scope: String::from("key"), + }) + } + + pub fn redis_semantic( + url: &str, + embedder: PythonEmbedder, + config: RedisSemanticConfig, + ) -> Result { + let backend = RedisSemanticCache::new(url, embedder.clone(), config)?; + Ok(Self::RedisSemantic { + cache: Arc::new(ResponseCache::new(Arc::new(backend))), + embedder, + }) + } + + pub async fn qdrant_semantic( + config: QdrantSemanticCacheConfig, + client: reqwest::Client, + runtime: tokio::runtime::Handle, + ) -> Result { + let qdrant = qdrant_client::Qdrant::from_url(&config.grpc_url) + .skip_compatibility_check() + .api_key(config.api_key.as_deref()) + .build() + .map_err(|_| Error::Unavailable)?; + let qdrant_config = config.to_qdrant_config(); + let embedder = OpenAiEmbedder::new(client, config.embedding); + let cache = QdrantSemanticCache::connect( + qdrant, + embedder, + ResponseCacheCodec, + qdrant_config, + runtime, + ) + .await?; + Ok(Self::QdrantSemantic(Arc::new(ResponseCache::new( + Arc::new(cache), + )))) + } + + pub fn disk(directory: &str) -> Result { + let cache = DiskCache::open(directory, ResponseCacheCodec)?; + Ok(Self::Disk(Arc::new(ResponseCache::new(Arc::new(cache))))) + } + + pub fn gcs(config: GcsConfig, token: Option) -> Result { + let backend = match token { + Some(token) => GcsCache::with_token_source( + config, + ResponseCacheCodec, + Arc::new(StaticTokenSource(token)), + )?, + None => GcsCache::new(config, ResponseCacheCodec)?, + }; + Ok(Self::Gcs(Arc::new(ResponseCache::new(Arc::new(backend))))) + } + pub async fn azure_blob(account_url: &str, container: &str) -> Result { let backend = AzureBlobCache::connect( account_url, @@ -67,16 +232,94 @@ impl NativeResponseCache { cache.backend().account_url(), cache.backend().container_name(), )), - Self::Memory(_) | Self::Redis { .. } => None, + Self::Memory(_) + | Self::Redis { .. } + | Self::S3(_) + | Self::ValkeySemantic { .. } + | Self::RedisSemantic { .. } + | Self::QdrantSemantic(_) + | Self::Disk(_) + | Self::Gcs(_) => None, + } + } + + fn exact(request: &NativeRequest) -> ResponseCacheRequest { + ResponseCacheRequest { + key: request.key.clone(), + controls: request.controls, + context: ExactCacheContext { ttl: request.ttl }, + max_age: request.max_age, + } + } + + pub(super) fn semantic_request( + request: &NativeRequest, + ) -> ResponseCacheRequest { + ResponseCacheRequest { + key: request.key.clone(), + controls: request.controls, + context: SemanticCacheContext { + input: request.input.clone(), + messages: request.messages.clone(), + metadata: request.metadata.clone(), + scope: request.scope.clone(), + ttl: request.ttl, + }, + max_age: request.max_age, + } + } + + fn semantic( + request: &NativeRequest, + scope: &str, + ) -> ResponseCacheRequest { + ResponseCacheRequest { + key: semantic_key(request, scope), + controls: request.controls, + context: SemanticCacheContext { + input: request.input.clone(), + messages: request.messages.clone(), + metadata: request.metadata.clone(), + scope: Some(scope.to_owned()), + ttl: request.ttl, + }, + max_age: request.max_age, + } + } + + pub fn with_redis_flush_size(self, flush_size: Option) -> Self { + match self { + Self::Redis { cache, .. } => Self::Redis { + cache, + buffer: flush_size.map(|size| Arc::new(WriteBuffer::new(size))), + }, + value => value, + } + } + + pub fn with_scope(self, scope: String) -> Self { + match self { + Self::ValkeySemantic { + cache, embedder, .. + } => Self::ValkeySemantic { + cache, + embedder, + scope, + }, + value => value, } } -} -impl NativeResponseCache { pub fn kind(&self) -> &'static str { match self { Self::Memory(_) => "memory", Self::Redis { .. } => "redis", + Self::S3(_) => "s3", + Self::Gcs(_) => "gcs", + Self::ValkeySemantic { .. } => "valkey-semantic", + Self::RedisSemantic { .. } => "redis_semantic", + Self::QdrantSemantic(_) => "qdrant_semantic", + Self::Disk(_) => "disk", Self::AzureBlob(_) => "azure-blob", } } @@ -85,20 +328,68 @@ impl NativeResponseCache { match self { Self::Memory(cache) => cache.default_ttl(), Self::Redis { cache, .. } => cache.default_ttl(), + Self::S3(cache) => cache.default_ttl(), + Self::Gcs(cache) => cache.default_ttl(), + Self::ValkeySemantic { cache, .. } => cache.default_ttl(), + Self::RedisSemantic { cache, .. } => cache.default_ttl(), + Self::QdrantSemantic(_) => None, + Self::Disk(cache) => cache.default_ttl(), Self::AzureBlob(cache) => cache.default_ttl(), } } + pub fn bucket(&self) -> Option<&str> { + match self { + Self::S3(cache) => Some(cache.backend().bucket()), + _ => None, + } + } + + pub fn key_prefix(&self) -> Option<&str> { + match self { + Self::S3(cache) => Some(cache.backend().key_prefix()), + _ => None, + } + } + + pub fn region(&self) -> Option<&str> { + match self { + Self::S3(cache) => Some(cache.backend().region()), + _ => None, + } + } + + pub fn endpoint(&self) -> Option<&str> { + match self { + Self::S3(cache) => cache.backend().endpoint(), + _ => None, + } + } + pub fn namespace(&self) -> Option<&str> { match self { - Self::Memory(_) | Self::AzureBlob(_) => None, + Self::Memory(_) + | Self::S3(_) + | Self::ValkeySemantic { .. } + | Self::RedisSemantic { .. } + | Self::QdrantSemantic(_) + | Self::Disk(_) + | Self::AzureBlob(_) + | Self::Gcs(_) => None, Self::Redis { cache, .. } => cache.backend().namespace(), } } pub fn topology(&self) -> Option<&RedisTopology> { match self { - Self::Memory(_) | Self::AzureBlob(_) => None, + Self::Memory(_) + | Self::S3(_) + | Self::ValkeySemantic { .. } + | Self::RedisSemantic { .. } + | Self::QdrantSemantic(_) + | Self::Disk(_) + | Self::AzureBlob(_) + | Self::Gcs(_) => None, Self::Redis { cache, .. } => Some(cache.backend().topology()), } } @@ -106,117 +397,548 @@ impl NativeResponseCache { pub fn capacity(&self) -> Option { match self { Self::Memory(cache) => Some(cache.backend().max_size_in_memory()), - Self::Redis { .. } | Self::AzureBlob(_) => None, + Self::Redis { .. } + | Self::S3(_) + | Self::ValkeySemantic { .. } + | Self::RedisSemantic { .. } + | Self::QdrantSemantic(_) + | Self::Disk(_) + | Self::AzureBlob(_) + | Self::Gcs(_) => None, } } pub fn max_entry_bytes(&self) -> Option { match self { Self::Memory(cache) => cache.backend().max_entry_bytes(), - Self::Redis { .. } | Self::AzureBlob(_) => None, + Self::Redis { .. } + | Self::S3(_) + | Self::ValkeySemantic { .. } + | Self::RedisSemantic { .. } + | Self::QdrantSemantic(_) + | Self::Disk(_) + | Self::AzureBlob(_) + | Self::Gcs(_) => None, } } - pub fn with_redis_flush_size(self, flush_size: Option) -> Self { + pub fn directory(&self) -> Option<&Path> { match self { - Self::Redis { cache, .. } => Self::Redis { - cache, - buffer: flush_size.map(|flush_size| Arc::new(WriteBuffer::new(flush_size))), - }, - other => other, + Self::Disk(cache) => Some(cache.backend().directory()), + Self::Memory(_) + | Self::Redis { .. } + | Self::S3(_) + | Self::ValkeySemantic { .. } + | Self::RedisSemantic { .. } + | Self::QdrantSemantic(_) + | Self::AzureBlob(_) + | Self::Gcs(_) => None, } } - pub fn lookup( - &self, - request: &ResponseCacheRequest, - now: Duration, - ) -> Result, Error> { + pub fn semantic_config(&self) -> Option<(f64, &str)> { match self { - Self::Memory(cache) => cache.lookup(request, now), - Self::Redis { cache, .. } => cache.lookup(request, now), - Self::AzureBlob(cache) => cache.lookup(request, now), + Self::ValkeySemantic { cache, .. } => Some(( + cache.backend().similarity_threshold(), + cache.backend().index_name(), + )), + Self::RedisSemantic { cache, .. } => Some(( + f64::from(cache.backend().similarity_threshold()), + cache.backend().index_name(), + )), + _ => None, + } + } + + pub fn index_name(&self) -> Option<&str> { + match self { + Self::RedisSemantic { cache, .. } => Some(cache.backend().index_name()), + _ => None, + } + } + + pub fn similarity_threshold(&self) -> Option { + match self { + Self::RedisSemantic { cache, .. } => { + Some(f64::from(cache.backend().similarity_threshold())) + } + Self::QdrantSemantic(cache) => Some(cache.backend().similarity_threshold()), + _ => None, + } + } + + pub fn collection_name(&self) -> Option<&str> { + match self { + Self::QdrantSemantic(cache) => Some(cache.backend().collection_name()), + _ => None, + } + } + + pub fn vector_size(&self) -> Option { + match self { + Self::QdrantSemantic(cache) => Some(cache.backend().vector_size()), + _ => None, + } + } + + pub fn embedding_model(&self) -> Option<&str> { + match self { + Self::QdrantSemantic(cache) => Some(cache.backend().embedder().model()), + _ => None, + } + } + + pub fn semantic_embedder(&self) -> Option<&PythonEmbedder> { + match self { + Self::RedisSemantic { embedder, .. } => Some(embedder), + _ => None, + } + } + + pub fn embedder_object(&self) -> Option<&Py> { + match self { + Self::RedisSemantic { embedder, .. } => Some(embedder.object()), + _ => None, + } + } + + pub fn lookup(&self, request: &NativeRequest, now: Duration) -> Result, Error> { + match self { + Self::Memory(cache) => cache.lookup(&Self::exact(request), now), + Self::Redis { cache, .. } => cache.lookup(&Self::exact(request), now), + Self::S3(cache) => cache.lookup(&Self::exact(request), now), + Self::ValkeySemantic { cache, scope, .. } => { + cache.lookup(&Self::semantic(request, scope), now) + } + Self::RedisSemantic { cache, .. } => { + cache.lookup(&Self::semantic_request(request), now) + } + Self::Gcs(cache) => cache.lookup(&Self::exact(request), now), + Self::QdrantSemantic(cache) => cache.lookup(&Self::semantic_request(request), now), + Self::Disk(cache) => cache.lookup(&Self::exact(request), now), + Self::AzureBlob(cache) => cache.lookup(&Self::exact(request), now), } } pub fn store( &self, - request: &ResponseCacheRequest, + request: &NativeRequest, response: Value, now: Duration, ) -> Result<(), Error> { match self { - Self::Memory(cache) => cache.store(request, response, now), - Self::Redis { cache, .. } => cache.store(request, response, now), - Self::AzureBlob(cache) => cache.store(request, response, now), + Self::Memory(cache) => cache.store(&Self::exact(request), response, now), + Self::Redis { cache, .. } => cache.store(&Self::exact(request), response, now), + Self::S3(cache) => cache.store(&Self::exact(request), response, now), + Self::ValkeySemantic { cache, scope, .. } => { + cache.store(&Self::semantic(request, scope), response, now) + } + Self::RedisSemantic { cache, .. } => { + cache.store(&Self::semantic_request(request), response, now) + } + Self::Gcs(cache) => cache.store(&Self::exact(request), response, now), + Self::QdrantSemantic(cache) => { + cache.store(&Self::semantic_request(request), response, now) + } + Self::Disk(cache) => cache.store(&Self::exact(request), response, now), + Self::AzureBlob(cache) => cache.store(&Self::exact(request), response, now), } } pub fn lookup_batch( &self, - requests: &[ResponseCacheRequest], + requests: &[NativeRequest], now: Duration, ) -> Result { match self { - Self::Memory(cache) => cache.lookup_batch(requests, now), - Self::Redis { cache, .. } => cache.lookup_batch(requests, now), - Self::AzureBlob(cache) => cache.lookup_batch(requests, now), + Self::Memory(cache) => { + let requests = requests.iter().map(Self::exact).collect::>(); + cache.lookup_batch(&requests, now) + } + Self::Redis { cache, .. } => { + let requests = requests.iter().map(Self::exact).collect::>(); + cache.lookup_batch(&requests, now) + } + Self::S3(cache) => { + cache.lookup_batch(&requests.iter().map(Self::exact).collect::>(), now) + } + Self::ValkeySemantic { .. } | Self::RedisSemantic { .. } | Self::QdrantSemantic(_) => { + Err(Error::UnsupportedOperation) + } + Self::Gcs(cache) => { + cache.lookup_batch(&requests.iter().map(Self::exact).collect::>(), now) + } + Self::Disk(cache) => { + cache.lookup_batch(&requests.iter().map(Self::exact).collect::>(), now) + } + Self::AzureBlob(cache) => { + cache.lookup_batch(&requests.iter().map(Self::exact).collect::>(), now) + } } } pub async fn async_lookup( &self, - request: &ResponseCacheRequest, + request: &NativeRequest, now: Duration, ) -> Result, Error> { match self { - Self::Memory(cache) => cache.async_lookup(request, now).await, - Self::Redis { cache, .. } => cache.async_lookup(request, now).await, - Self::AzureBlob(cache) => cache.async_lookup(request, now).await, + Self::Memory(cache) => cache.async_lookup(&Self::exact(request), now).await, + Self::Redis { cache, .. } => cache.async_lookup(&Self::exact(request), now).await, + Self::S3(cache) => cache.async_lookup(&Self::exact(request), now).await, + Self::ValkeySemantic { cache, scope, .. } => { + cache + .async_lookup(&Self::semantic(request, scope), now) + .await + } + Self::RedisSemantic { cache, .. } => { + cache + .async_lookup(&Self::semantic_request(request), now) + .await + } + Self::QdrantSemantic(cache) => { + cache + .async_lookup(&Self::semantic_request(request), now) + .await + } + Self::Gcs(cache) => cache.async_lookup(&Self::exact(request), now).await, + Self::Disk(cache) => cache.async_lookup(&Self::exact(request), now).await, + Self::AzureBlob(cache) => cache.async_lookup(&Self::exact(request), now).await, + } + } + + pub(super) fn async_lookup_py<'py>( + &self, + py: Python<'py>, + request: NativeRequest, + ) -> PyResult> { + match self { + Self::Memory(_) + | Self::Redis { .. } + | Self::S3(_) + | Self::Disk(_) + | Self::AzureBlob(_) + | Self::Gcs(_) => { + let service = self.clone(); + litellm_host_python::run_async( + py, + async move { service.async_lookup(&request, super::request::now()).await }, + super::cache_error, + ) + } + Self::ValkeySemantic { + cache, + embedder, + scope, + } => drive_semantic( + py, + SemanticEmbedExecution::lookup( + Arc::clone(cache.backend_arc()), + embedder.clone(), + Self::semantic(&request, scope), + ), + ), + Self::RedisSemantic { .. } => drive( + py, + SemanticBody::new(self.clone(), SemanticOperation::Lookup(request)), + ), + Self::QdrantSemantic(_) => { + let service = self.clone(); + litellm_host_python::run_async( + py, + async move { service.async_lookup(&request, super::request::now()).await }, + super::cache_error, + ) + } } } pub async fn async_store( &self, - request: &ResponseCacheRequest, + request: &NativeRequest, response: Value, now: Duration, ) -> Result<(), Error> { match self { - Self::Memory(cache) => cache.async_store(request, response, now).await, + Self::Memory(cache) => { + cache + .async_store(&Self::exact(request), response, now) + .await + } Self::Redis { cache, buffer: None, - } => cache.async_store(request, response, now).await, + } => { + cache + .async_store(&Self::exact(request), response, now) + .await + } Self::Redis { cache, buffer: Some(buffer), - } => buffer.async_store(cache, request, response, now).await, - Self::AzureBlob(cache) => cache.async_store(request, response, now).await, + } => { + buffer + .async_store(cache, &Self::exact(request), response, now) + .await + } + Self::S3(cache) => { + cache + .async_store(&Self::exact(request), response, now) + .await + } + Self::ValkeySemantic { cache, scope, .. } => { + cache + .async_store(&Self::semantic(request, scope), response, now) + .await + } + Self::RedisSemantic { cache, .. } => { + cache + .async_store(&Self::semantic_request(request), response, now) + .await + } + Self::QdrantSemantic(cache) => { + cache + .async_store(&Self::semantic_request(request), response, now) + .await + } + Self::Gcs(cache) => { + cache + .async_store(&Self::exact(request), response, now) + .await + } + Self::Disk(cache) => { + cache + .async_store(&Self::exact(request), response, now) + .await + } + Self::AzureBlob(cache) => { + cache + .async_store(&Self::exact(request), response, now) + .await + } + } + } + + pub(super) fn async_store_py<'py>( + &self, + py: Python<'py>, + request: NativeRequest, + response: Value, + ) -> PyResult> { + match self { + Self::Memory(_) + | Self::Redis { .. } + | Self::S3(_) + | Self::Disk(_) + | Self::AzureBlob(_) + | Self::Gcs(_) => { + let service = self.clone(); + litellm_host_python::run_async( + py, + async move { + service + .async_store(&request, response, super::request::now()) + .await + }, + super::cache_error, + ) + } + Self::ValkeySemantic { + cache, + embedder, + scope, + } => drive_semantic( + py, + SemanticEmbedExecution::store( + Arc::clone(cache.backend_arc()), + embedder.clone(), + Self::semantic(&request, scope), + response, + ), + ), + Self::RedisSemantic { .. } => drive( + py, + SemanticBody::new(self.clone(), SemanticOperation::Store(request, response)), + ), + Self::QdrantSemantic(_) => { + let service = self.clone(); + litellm_host_python::run_async( + py, + async move { + service + .async_store(&request, response, super::request::now()) + .await + }, + super::cache_error, + ) + } } } pub async fn async_lookup_batch( &self, - requests: &[ResponseCacheRequest], + requests: &[NativeRequest], now: Duration, ) -> Result { match self { - Self::Memory(cache) => cache.async_lookup_batch(requests, now).await, - Self::Redis { cache, .. } => cache.async_lookup_batch(requests, now).await, - Self::AzureBlob(cache) => cache.async_lookup_batch(requests, now).await, + Self::Memory(cache) => { + let requests = requests.iter().map(Self::exact).collect::>(); + cache.async_lookup_batch(&requests, now).await + } + Self::Redis { cache, .. } => { + let requests = requests.iter().map(Self::exact).collect::>(); + cache.async_lookup_batch(&requests, now).await + } + Self::S3(cache) => { + cache + .async_lookup_batch(&requests.iter().map(Self::exact).collect::>(), now) + .await + } + Self::ValkeySemantic { .. } | Self::RedisSemantic { .. } | Self::QdrantSemantic(_) => { + Err(Error::UnsupportedOperation) + } + Self::Gcs(cache) => { + cache + .async_lookup_batch(&requests.iter().map(Self::exact).collect::>(), now) + .await + } + Self::Disk(cache) => { + cache + .async_lookup_batch(&requests.iter().map(Self::exact).collect::>(), now) + .await + } + Self::AzureBlob(cache) => { + cache + .async_lookup_batch(&requests.iter().map(Self::exact).collect::>(), now) + .await + } } } pub async fn async_store_batch( &self, - entries: Vec<(ResponseCacheRequest, Value)>, + entries: Vec<(NativeRequest, Value)>, now: Duration, ) -> Result<(), Error> { match self { - Self::Memory(cache) => cache.async_store_batch(entries, now).await, - Self::Redis { cache, .. } => cache.async_store_batch(entries, now).await, - Self::AzureBlob(cache) => cache.async_store_batch(entries, now).await, + Self::Memory(cache) => { + let entries = entries + .into_iter() + .map(|(request, value)| (Self::exact(&request), value)) + .collect(); + cache.async_store_batch(entries, now).await + } + Self::Redis { cache, .. } => { + let entries = entries + .into_iter() + .map(|(request, value)| (Self::exact(&request), value)) + .collect(); + cache.async_store_batch(entries, now).await + } + Self::S3(cache) => { + let entries = entries + .into_iter() + .map(|(request, value)| (Self::exact(&request), value)) + .collect(); + cache.async_store_batch(entries, now).await + } + Self::ValkeySemantic { cache, scope, .. } => { + let entries = entries + .into_iter() + .map(|(request, value)| (Self::semantic(&request, scope), value)) + .collect(); + cache.async_store_batch(entries, now).await + } + Self::RedisSemantic { .. } => Err(Error::UnsupportedOperation), + Self::QdrantSemantic(cache) => { + let entries = entries + .into_iter() + .map(|(request, value)| (Self::semantic_request(&request), value)) + .collect(); + cache.async_store_batch(entries, now).await + } + Self::Gcs(cache) => { + let entries = entries + .into_iter() + .map(|(request, value)| (Self::exact(&request), value)) + .collect(); + cache.async_store_batch(entries, now).await + } + Self::Disk(cache) => { + let entries = entries + .into_iter() + .map(|(request, value)| (Self::exact(&request), value)) + .collect(); + cache.async_store_batch(entries, now).await + } + Self::AzureBlob(cache) => { + let entries = entries + .into_iter() + .map(|(request, value)| (Self::exact(&request), value)) + .collect(); + cache.async_store_batch(entries, now).await + } + } + } + + pub(super) fn async_store_batch_py<'py>( + &self, + py: Python<'py>, + entries: Vec<(NativeRequest, Value)>, + ) -> PyResult> { + match self { + Self::Memory(_) + | Self::Redis { .. } + | Self::S3(_) + | Self::Disk(_) + | Self::AzureBlob(_) + | Self::Gcs(_) => { + let service = self.clone(); + litellm_host_python::run_async( + py, + async move { + service + .async_store_batch(entries, super::request::now()) + .await + }, + super::cache_error, + ) + } + Self::ValkeySemantic { + cache, + embedder, + scope, + } => { + let (requests, responses): (Vec<_>, Vec<_>) = entries + .into_iter() + .map(|(request, response)| (Self::semantic(&request, scope), response)) + .unzip(); + drive_semantic( + py, + SemanticEmbedExecution::store_batch( + Arc::clone(cache.backend_arc()), + embedder.clone(), + requests, + responses, + ), + ) + } + Self::RedisSemantic { .. } => drive( + py, + SemanticBody::new(self.clone(), SemanticOperation::StoreBatch(entries.into())), + ), + Self::QdrantSemantic(_) => { + let service = self.clone(); + litellm_host_python::run_async( + py, + async move { + service + .async_store_batch(entries, super::request::now()) + .await + }, + super::cache_error, + ) + } } } @@ -229,6 +951,12 @@ impl NativeResponseCache { } cache.async_flush().await } + Self::S3(cache) => cache.async_flush().await, + Self::ValkeySemantic { .. } | Self::RedisSemantic { .. } | Self::QdrantSemantic(_) => { + Err(Error::UnsupportedOperation) + } + Self::Gcs(cache) => cache.async_flush().await, + Self::Disk(cache) => cache.async_flush().await, Self::AzureBlob(cache) => cache.async_flush().await, } } @@ -237,7 +965,107 @@ impl NativeResponseCache { match self { Self::Memory(cache) => cache.test_connection().await, Self::Redis { cache, .. } => cache.test_connection().await, + Self::S3(cache) => cache.test_connection().await, + Self::ValkeySemantic { cache, .. } => cache.test_connection().await, + Self::RedisSemantic { .. } | Self::QdrantSemantic(_) => { + Err(Error::UnsupportedOperation) + } + Self::Gcs(cache) => cache.test_connection().await, + Self::Disk(cache) => cache.test_connection().await, Self::AzureBlob(cache) => cache.test_connection().await, } } + + pub(super) fn traverse(&self, visit: &PyVisit<'_>) -> Result<(), PyTraverseError> { + match self { + Self::ValkeySemantic { embedder, .. } => embedder.traverse(visit)?, + Self::RedisSemantic { embedder, .. } => embedder.traverse(visit)?, + _ => {} + } + Ok(()) + } + + pub fn gcs_backend(&self) -> Option<&GcsCache> { + match self { + Self::Gcs(cache) => Some(cache.backend()), + _ => None, + } + } +} + +#[cfg(test)] +mod tests { + use litellm_cache_response::{CacheControls, CacheKeyInput, cache_key}; + use serde_json::json; + use sha2::{Digest, Sha256}; + + use super::*; + + fn native_request(key: CacheKeyInput, metadata: Value) -> NativeRequest { + NativeRequest { + key, + controls: CacheControls::default(), + ttl: None, + max_age: None, + messages: Some(json!([{"role": "user", "content": "prompt"}])), + input: None, + metadata: Some(metadata), + litellm_metadata: None, + litellm_params: None, + scope: None, + } + } + + #[test] + fn semantic_key_matches_python_scope_material() { + let key = CacheKeyInput { + fields: vec![ + CacheKeyField { + name: "model".to_owned(), + value: Some("gpt-4.1".to_owned()), + api_parameter: true, + internal_parameter: false, + }, + CacheKeyField { + name: "messages".to_owned(), + value: Some("prompt".to_owned()), + api_parameter: true, + internal_parameter: false, + }, + ], + ..Default::default() + }; + let request = native_request( + key, + json!({"user_api_key": "k1", "user_api_key_team_id": null}), + ); + let expected = format!("{:x}", Sha256::digest(b"model: gpt-4.1user_api_key: k1")); + assert_eq!(cache_key(&semantic_key(&request, "key")), expected); + + let end_user_request = native_request( + request.key.clone(), + json!({"user_api_key": "k1", "user_api_key_end_user_id": "u1"}), + ); + let expected = format!( + "{:x}", + Sha256::digest(b"model: gpt-4.1user_api_key: k1user_api_key_end_user_id: u1") + ); + assert_eq!( + cache_key(&semantic_key(&end_user_request, "end_user")), + expected + ); + + let preset_request = native_request( + CacheKeyInput { + preset: Some("preset-key".to_owned()), + ..Default::default() + }, + json!({"user_api_key": "k1"}), + ); + assert_eq!( + semantic_key(&preset_request, "end_user").preset.as_deref(), + Some("preset-key") + ); + assert!(semantic_key(&preset_request, "end_user").fields.is_empty()); + } } diff --git a/litellm-rust/crates/python-bridge/src/cache/request.rs b/litellm-rust/crates/python-bridge/src/cache/request.rs index 0c5343a63d0..3b4b910c1f0 100644 --- a/litellm-rust/crates/python-bridge/src/cache/request.rs +++ b/litellm-rust/crates/python-bridge/src/cache/request.rs @@ -1,9 +1,11 @@ use std::time::{Duration, SystemTime, UNIX_EPOCH}; +use litellm_cache::ExactCacheContext; use litellm_cache_response::{CacheControls, CacheKeyInput, ResponseCacheRequest}; use litellm_host_python::from_py; use pyo3::{exceptions::PyValueError, prelude::*}; use serde::Deserialize; +use serde_json::Value; #[derive(Deserialize)] #[serde(deny_unknown_fields)] @@ -12,24 +14,52 @@ struct RequestInput { controls: Option, ttl_seconds: Option, max_age_seconds: Option, + messages: Option, + input: Option, + metadata: Option, + litellm_metadata: Option, + litellm_params: Option, + scope: Option, } -pub(super) fn request(value: &Bound<'_, PyAny>) -> PyResult { +#[derive(Clone)] +pub(super) struct NativeRequest { + pub(super) key: CacheKeyInput, + pub(super) controls: CacheControls, + pub(super) ttl: Option, + pub(super) max_age: Option, + pub(super) messages: Option, + pub(super) input: Option, + pub(super) metadata: Option, + pub(super) litellm_metadata: Option, + pub(super) litellm_params: Option, + pub(super) scope: Option, +} + +pub(super) fn request(value: &Bound<'_, PyAny>) -> PyResult { let input: RequestInput = from_py(value)?; request_input(input) } -fn request_input(input: RequestInput) -> PyResult { - let mut request = ResponseCacheRequest::new(input.key); - if let Some(controls) = input.controls { - request.controls = controls; - } - request.context.ttl = input.ttl_seconds.map(duration).transpose()?; - request.max_age = input.max_age_seconds.map(duration).transpose()?; - Ok(request) +fn request_input(input: RequestInput) -> PyResult { + let controls = input.controls.unwrap_or_else(|| { + ResponseCacheRequest::::new(input.key.clone()).controls + }); + Ok(NativeRequest { + key: input.key, + controls, + ttl: input.ttl_seconds.map(duration).transpose()?, + max_age: input.max_age_seconds.map(duration).transpose()?, + messages: input.messages, + input: input.input, + metadata: input.metadata, + litellm_metadata: input.litellm_metadata, + litellm_params: input.litellm_params, + scope: input.scope, + }) } -pub(super) fn requests(value: &Bound<'_, PyAny>) -> PyResult> { +pub(super) fn requests(value: &Bound<'_, PyAny>) -> PyResult> { from_py::>(value)? .into_iter() .map(request_input) diff --git a/litellm-rust/crates/python-bridge/src/cache/semantic.rs b/litellm-rust/crates/python-bridge/src/cache/semantic.rs new file mode 100644 index 00000000000..934de01e721 --- /dev/null +++ b/litellm-rust/crates/python-bridge/src/cache/semantic.rs @@ -0,0 +1,175 @@ +use std::collections::VecDeque; + +use litellm_cache::Error; +use litellm_cache_redis_semantic::prompt_from_context; +use litellm_host_python::{Execution, ExecutionBody, ExecutionStep, run_async}; +use pyo3::{ + PyTraverseError, PyVisit, + exceptions::{PyException, PyRuntimeError}, + prelude::*, +}; +use serde_json::Value; + +use super::{ + cache_error, + embedder::{PythonEmbedder, with_prepared_embedding}, + native::NativeResponseCache, + request::{NativeRequest, now}, +}; + +pub(super) enum SemanticOperation { + Lookup(NativeRequest), + Store(NativeRequest, Value), + StoreBatch(VecDeque<(NativeRequest, Value)>), +} + +enum Phase { + Start, + AwaitingEmbedding, + AwaitingBackend, +} + +pub(super) struct SemanticBody { + service: NativeResponseCache, + operation: SemanticOperation, + pending: Option<(NativeRequest, Option)>, + phase: Phase, +} + +impl SemanticBody { + pub(super) fn new(service: NativeResponseCache, operation: SemanticOperation) -> Self { + Self { + service, + operation, + pending: None, + phase: Phase::Start, + } + } + + fn backend_step( + &mut self, + py: Python<'_>, + seed: Result, Error>, + ) -> PyResult { + self.phase = Phase::AwaitingBackend; + let (request, response) = self.pending.take().ok_or_else(|| { + PyRuntimeError::new_err("semantic execution resumed without a pending operation") + })?; + let service = self.service.clone(); + let future = async move { + match response { + None => service.async_lookup(&request, now()).await, + Some(response) => service + .async_store(&request, response, now()) + .await + .map(|_| None), + } + }; + let awaitable = run_async(py, with_prepared_embedding(seed, future), cache_error)?; + Ok(ExecutionStep::Await(awaitable.unbind())) + } +} + +impl ExecutionBody for SemanticBody { + fn resume(&mut self, mut result: Option>>) -> PyResult { + Python::attach(|py| { + loop { + match self.phase { + Phase::Start => { + if result.is_some() { + return Err(PyRuntimeError::new_err( + "semantic execution received a result before starting", + )); + } + if self.pending.is_none() { + match &mut self.operation { + SemanticOperation::Lookup(request) => { + self.pending = Some((request.clone(), None)); + } + SemanticOperation::Store(request, response) => { + let response = std::mem::replace(response, Value::Null); + self.pending = Some((request.clone(), Some(response))); + } + SemanticOperation::StoreBatch(queue) => { + let Some((request, response)) = queue.pop_front() else { + return Ok(ExecutionStep::Return(py.None())); + }; + self.pending = Some((request, Some(response))); + } + } + } + let (request, _) = self.pending.as_ref().ok_or_else(|| { + PyRuntimeError::new_err("semantic execution has no pending operation") + })?; + let semantic = NativeResponseCache::semantic_request(request); + let Some(prompt) = prompt_from_context(&semantic.context) else { + return self.backend_step(py, Err(Error::Unavailable)); + }; + let embedder = self.service.semantic_embedder().ok_or_else(|| { + PyRuntimeError::new_err( + "semantic execution requires a redis-semantic backend", + ) + })?; + let coroutine = embedder.async_embedding_coroutine( + py, + &prompt, + semantic.context.metadata.as_ref(), + )?; + self.phase = Phase::AwaitingEmbedding; + return Ok(ExecutionStep::Await(coroutine)); + } + Phase::AwaitingEmbedding => { + let result = result.take().ok_or_else(|| { + PyRuntimeError::new_err( + "semantic execution expected an embedding result", + ) + })?; + let seed = match result { + Ok(value) => PythonEmbedder::extract(value.into_bound(py)) + .map_err(|_| Error::Unavailable), + Err(error) => { + if !error.is_instance_of::(py) { + return Err(error); + } + Err(Error::Unavailable) + } + }; + return self.backend_step(py, seed); + } + Phase::AwaitingBackend => { + let result = result.take().ok_or_else(|| { + PyRuntimeError::new_err("semantic execution expected a backend result") + })?; + let value = match result { + Ok(value) => value, + Err(error) => return Err(error), + }; + let more = matches!( + &self.operation, + SemanticOperation::StoreBatch(queue) if !queue.is_empty() + ); + if more { + self.phase = Phase::Start; + continue; + } + return Ok(ExecutionStep::Return(value)); + } + } + } + }) + } + + fn traverse(&self, visit: &PyVisit<'_>) -> Result<(), PyTraverseError> { + if let Some(embedder) = self.service.semantic_embedder() { + embedder.traverse(visit)?; + } + Ok(()) + } +} + +pub(super) fn drive(py: Python<'_>, body: SemanticBody) -> PyResult> { + let execution = Py::new(py, Execution::new(body))?; + py.import("litellm.rust_bridge.lifecycle")? + .getattr("drive")? + .call1((execution,)) +} diff --git a/litellm-rust/crates/python-bridge/src/cache/semantic_step.rs b/litellm-rust/crates/python-bridge/src/cache/semantic_step.rs new file mode 100644 index 00000000000..24caf3374d6 --- /dev/null +++ b/litellm-rust/crates/python-bridge/src/cache/semantic_step.rs @@ -0,0 +1,249 @@ +use std::{sync::Arc, time::Duration}; + +use litellm_cache::SemanticCacheContext; +use litellm_cache_response::{ResponseCache, ResponseCacheCodec, ResponseCacheRequest}; +use litellm_cache_valkey_semantic::{PreparedEmbedding, ValkeySemanticCache, prompt_from_context}; +use litellm_host_python::{Execution, ExecutionBody, ExecutionStep, run_async}; +use pyo3::{PyTraverseError, PyVisit, exceptions::PyRuntimeError, prelude::*}; +use serde_json::Value; + +use super::{cache_error, embedder::PythonEmbedder}; + +pub(super) enum Op { + Lookup, + Store(Value), + StoreBatch(Vec), +} + +#[derive(Clone, Copy)] +enum State { + Start, + AwaitingEmbedding, + AwaitingStorage, + Done, +} + +pub(super) struct SemanticEmbedExecution { + backend: Arc>, + embedder: PythonEmbedder, + requests: Vec>, + op: Op, + now: Option, + prepared: Vec>>, + index: usize, + state: State, +} + +impl SemanticEmbedExecution { + pub(super) fn lookup( + backend: Arc>, + embedder: PythonEmbedder, + request: ResponseCacheRequest, + ) -> Self { + Self { + backend, + embedder, + requests: vec![request], + op: Op::Lookup, + now: None, + prepared: vec![None], + index: 0, + state: State::Start, + } + } + + pub(super) fn store( + backend: Arc>, + embedder: PythonEmbedder, + request: ResponseCacheRequest, + response: Value, + ) -> Self { + Self { + backend, + embedder, + requests: vec![request], + op: Op::Store(response), + now: None, + prepared: vec![None], + index: 0, + state: State::Start, + } + } + + pub(super) fn store_batch( + backend: Arc>, + embedder: PythonEmbedder, + requests: Vec>, + responses: Vec, + ) -> Self { + Self { + backend, + embedder, + prepared: vec![None; requests.len()], + requests, + op: Op::StoreBatch(responses), + now: None, + index: 0, + state: State::Start, + } + } + + fn start(&mut self, py: Python<'_>) -> PyResult { + if self.now.is_none() { + self.now = Some(super::request::now()); + } + while self.index < self.requests.len() { + let request = &self.requests[self.index]; + let enabled = match &self.op { + Op::Lookup => request.controls.reads(), + Op::Store(_) | Op::StoreBatch(_) => request.controls.writes(), + }; + if !enabled { + self.index += 1; + continue; + } + let Some(prompt) = prompt_from_context(&request.context) else { + self.index += 1; + continue; + }; + let metadata = request.context.metadata.clone(); + let awaitable = self + .embedder + .async_embed_awaitable(py, &prompt, &metadata)?; + self.state = State::AwaitingEmbedding; + return Ok(ExecutionStep::Await(awaitable.unbind())); + } + self.state = State::AwaitingStorage; + self.storage_step(py) + } + + fn storage_step(&self, py: Python<'_>) -> PyResult { + let requests = self.requests.clone(); + let prepared = self.prepared.clone(); + let backend = Arc::clone(&self.backend); + let now = self + .now + .ok_or_else(|| PyRuntimeError::new_err("semantic cache timestamp is unavailable"))?; + let awaitable = match &self.op { + Op::Lookup => { + let Some(request) = requests.into_iter().next() else { + return Err(PyRuntimeError::new_err( + "semantic lookup requires one request", + )); + }; + match prepared.into_iter().next().flatten() { + Some(values) => { + let backend = backend.with_embedder(PreparedEmbedding(values)); + let cache = Arc::new(ResponseCache::new(Arc::new(backend))); + run_async( + py, + async move { cache.async_lookup(&request, now).await }, + cache_error, + )? + } + None => { + let cache = Arc::new(ResponseCache::new(backend)); + run_async( + py, + async move { cache.async_lookup(&request, now).await }, + cache_error, + )? + } + } + } + Op::Store(response) => { + let Some(request) = requests.into_iter().next() else { + return Err(PyRuntimeError::new_err( + "semantic store requires one request", + )); + }; + let response = response.clone(); + match prepared.into_iter().next().flatten() { + Some(values) => { + let backend = backend.with_embedder(PreparedEmbedding(values)); + let cache = Arc::new(ResponseCache::new(Arc::new(backend))); + run_async( + py, + async move { cache.async_store(&request, response, now).await }, + cache_error, + )? + } + None => { + let cache = Arc::new(ResponseCache::new(backend)); + run_async( + py, + async move { cache.async_store(&request, response, now).await }, + cache_error, + )? + } + } + } + Op::StoreBatch(responses) => { + let responses = responses.clone(); + run_async( + py, + async move { + for ((request, response), prepared) in + requests.into_iter().zip(responses).zip(prepared) + { + let Some(values) = prepared else { + continue; + }; + let backend = backend.with_embedder(PreparedEmbedding(values)); + let cache = ResponseCache::new(Arc::new(backend)); + cache.async_store(&request, response, now).await?; + } + Ok(()) + }, + cache_error, + )? + } + }; + Ok(ExecutionStep::Await(awaitable.unbind())) + } + + fn resume_py( + &mut self, + py: Python<'_>, + result: Option>>, + ) -> PyResult { + match (self.state, result) { + (State::Start, None) => self.start(py), + (State::AwaitingEmbedding, Some(Ok(value))) => { + let values = value.bind(py).extract::>()?; + self.prepared[self.index] = + Some(values.into_iter().map(|value| value as f32).collect()); + self.index += 1; + self.start(py) + } + (State::AwaitingStorage, Some(Ok(value))) => { + self.state = State::Done; + Ok(ExecutionStep::Return(value)) + } + (_, Some(Err(error))) => Err(error), + _ => Err(PyRuntimeError::new_err( + "invalid semantic cache execution state", + )), + } + } +} + +impl ExecutionBody for SemanticEmbedExecution { + fn resume(&mut self, result: Option>>) -> PyResult { + Python::attach(|py| self.resume_py(py, result)) + } + + fn traverse(&self, visit: &PyVisit<'_>) -> Result<(), PyTraverseError> { + self.embedder.traverse(visit) + } +} + +pub(super) fn drive_semantic<'py>( + py: Python<'py>, + body: SemanticEmbedExecution, +) -> PyResult> { + let execution = Py::new(py, Execution::new(body))?; + py.import("litellm.rust_bridge.lifecycle")? + .getattr("drive")? + .call1((execution,)) +} diff --git a/litellm-rust/crates/secrets-hashicorp/Cargo.toml b/litellm-rust/crates/secrets-hashicorp/Cargo.toml new file mode 100644 index 00000000000..c646e02ef09 --- /dev/null +++ b/litellm-rust/crates/secrets-hashicorp/Cargo.toml @@ -0,0 +1,25 @@ +[package] +name = "litellm-secrets-hashicorp" +version = "0.1.0" +edition.workspace = true +license.workspace = true +repository.workspace = true + +[dependencies] +litellm-core-utils.workspace = true +litellm-secrets-types.workspace = true +moka.workspace = true +rustify.workspace = true +rustify_derive.workspace = true +serde.workspace = true +serde_json.workspace = true +thiserror.workspace = true +tokio.workspace = true +vaultrs.workspace = true +veil.workspace = true + +[dev-dependencies] +rstest.workspace = true +tempfile = "3" +tokio.workspace = true +wiremock = "0.6.5" diff --git a/litellm-rust/crates/secrets-hashicorp/src/cert_login.rs b/litellm-rust/crates/secrets-hashicorp/src/cert_login.rs new file mode 100644 index 00000000000..f99df62db12 --- /dev/null +++ b/litellm-rust/crates/secrets-hashicorp/src/cert_login.rs @@ -0,0 +1,22 @@ +#[derive(Debug, rustify_derive::Endpoint)] +#[endpoint(path = "/auth/{self.mount}/login", method = "POST")] +pub struct CertLoginRequest { + #[endpoint(skip)] + pub mount: String, + #[endpoint(raw)] + body: Vec, +} + +impl CertLoginRequest { + pub fn new(name: Option<&str>) -> Self { + let body: Vec = match name { + Some(name) => serde_json::to_vec(&serde_json::json!({ "name": name })) + .expect("json object serialization is infallible"), + None => b"{}".to_vec(), + }; + Self { + mount: "cert".to_owned(), + body, + } + } +} diff --git a/litellm-rust/crates/secrets-hashicorp/src/config.rs b/litellm-rust/crates/secrets-hashicorp/src/config.rs new file mode 100644 index 00000000000..d32491199c4 --- /dev/null +++ b/litellm-rust/crates/secrets-hashicorp/src/config.rs @@ -0,0 +1,161 @@ +use std::{path::PathBuf, time::Duration}; + +use litellm_core_utils::settings::Lookup; +use litellm_secrets_types::SecretValue; + +use crate::Error; + +const DEFAULT_ADDRESS: &str = "http://127.0.0.1:8200"; +const DEFAULT_MOUNT: &str = "secret"; +const DEFAULT_APPROLE_MOUNT_PATH: &str = "approle"; +const DEFAULT_REFRESH_INTERVAL: Duration = Duration::from_secs(86400); +const HCP_VAULT_ADDR: &str = "HCP_VAULT_ADDR"; +const HCP_VAULT_TOKEN: &str = "HCP_VAULT_TOKEN"; +const HCP_VAULT_NAMESPACE: &str = "HCP_VAULT_NAMESPACE"; +const HCP_VAULT_LOGIN_NAMESPACE: &str = "HCP_VAULT_LOGIN_NAMESPACE"; +const HCP_VAULT_SECRET_NAMESPACE: &str = "HCP_VAULT_SECRET_NAMESPACE"; +const HCP_VAULT_MOUNT_NAME: &str = "HCP_VAULT_MOUNT_NAME"; +const HCP_VAULT_PATH_PREFIX: &str = "HCP_VAULT_PATH_PREFIX"; +const HCP_VAULT_APPROLE_ROLE_ID: &str = "HCP_VAULT_APPROLE_ROLE_ID"; +const HCP_VAULT_APPROLE_SECRET_ID: &str = "HCP_VAULT_APPROLE_SECRET_ID"; +const HCP_VAULT_APPROLE_MOUNT_PATH: &str = "HCP_VAULT_APPROLE_MOUNT_PATH"; +const HCP_VAULT_CLIENT_CERT: &str = "HCP_VAULT_CLIENT_CERT"; +const HCP_VAULT_CLIENT_KEY: &str = "HCP_VAULT_CLIENT_KEY"; +const HCP_VAULT_CERT_ROLE: &str = "HCP_VAULT_CERT_ROLE"; +const HCP_VAULT_REFRESH_INTERVAL: &str = "HCP_VAULT_REFRESH_INTERVAL"; +const SECRET_MANAGER_REFRESH_INTERVAL: &str = "SECRET_MANAGER_REFRESH_INTERVAL"; + +#[derive(Clone, Debug)] +pub struct AppRoleAuth { + pub role_id: String, + pub secret_id: SecretValue, + pub mount_path: String, +} + +#[derive(Clone, Debug)] +pub struct TlsCertAuth { + pub cert_path: PathBuf, + pub key_path: PathBuf, + pub role: Option, +} + +#[derive(Clone, Debug)] +pub struct HashicorpVaultConfig { + pub address: String, + pub token: Option, + pub namespace: Option, + pub login_namespace: Option, + pub secret_namespace: Option, + pub mount: String, + pub path_prefix: Option, + pub approle: Option, + pub tls_cert: Option, + pub refresh_interval: Duration, +} + +impl HashicorpVaultConfig { + pub fn from_environment(environment: &dyn Lookup) -> Result { + let address: String = environment + .get(HCP_VAULT_ADDR) + .and_then(|value| nonempty(value.trim())) + .map(|value| value.trim_end_matches('/').to_owned()) + .filter(|value| !value.is_empty()) + .unwrap_or_else(|| DEFAULT_ADDRESS.to_owned()); + let token: Option = environment + .get(HCP_VAULT_TOKEN) + .and_then(nonempty) + .map(SecretValue::new); + let namespace: Option = path_component(environment.get(HCP_VAULT_NAMESPACE)); + let login_namespace: Option = + path_component(environment.get(HCP_VAULT_LOGIN_NAMESPACE)); + let secret_namespace: Option = + path_component(environment.get(HCP_VAULT_SECRET_NAMESPACE)); + let mount: String = path_component(environment.get(HCP_VAULT_MOUNT_NAME)) + .unwrap_or_else(|| DEFAULT_MOUNT.to_owned()); + let path_prefix: Option = path_component(environment.get(HCP_VAULT_PATH_PREFIX)); + let approle: Option = match ( + environment + .get(HCP_VAULT_APPROLE_ROLE_ID) + .and_then(nonempty), + environment + .get(HCP_VAULT_APPROLE_SECRET_ID) + .and_then(nonempty) + .map(SecretValue::new), + ) { + (Some(role_id), Some(secret_id)) => Some(AppRoleAuth { + role_id, + secret_id, + mount_path: path_component(environment.get(HCP_VAULT_APPROLE_MOUNT_PATH)) + .unwrap_or_else(|| DEFAULT_APPROLE_MOUNT_PATH.to_owned()), + }), + _ => None, + }; + let tls_cert: Option = match ( + environment.get(HCP_VAULT_CLIENT_CERT).and_then(nonempty), + environment.get(HCP_VAULT_CLIENT_KEY).and_then(nonempty), + ) { + (Some(cert_path), Some(key_path)) => Some(TlsCertAuth { + cert_path: PathBuf::from(cert_path), + key_path: PathBuf::from(key_path), + role: environment.get(HCP_VAULT_CERT_ROLE).and_then(nonempty), + }), + _ => None, + }; + let refresh_interval: Duration = refresh_interval(environment)?; + Ok(Self { + address, + token, + namespace, + login_namespace, + secret_namespace, + mount, + path_prefix, + approle, + tls_cert, + refresh_interval, + }) + } + + pub fn login_namespace(&self) -> Option<&str> { + self.login_namespace + .as_deref() + .or(self.namespace.as_deref()) + } + + pub fn secret_namespace(&self) -> Option<&str> { + self.secret_namespace + .as_deref() + .or(self.namespace.as_deref()) + } +} + +fn nonempty(value: impl AsRef) -> Option { + let value: &str = value.as_ref(); + (!value.is_empty()).then(|| value.to_owned()) +} + +fn path_component(value: Option) -> Option { + value + .and_then(|value| nonempty(value.trim())) + .map(|value| value.trim_matches('/').to_owned()) + .filter(|value| !value.is_empty()) +} + +fn refresh_interval(environment: &dyn Lookup) -> Result { + let value: Option = environment + .get(HCP_VAULT_REFRESH_INTERVAL) + .and_then(nonempty) + .or_else(|| { + environment + .get(SECRET_MANAGER_REFRESH_INTERVAL) + .and_then(nonempty) + }); + let Some(value) = value else { + return Ok(DEFAULT_REFRESH_INTERVAL); + }; + let seconds: i64 = value.parse().map_err(|_| Error::RefreshInterval)?; + if seconds < 0 { + return Err(Error::RefreshInterval); + } + Ok(Duration::from_secs(seconds as u64)) +} diff --git a/litellm-rust/crates/secrets-hashicorp/src/error.rs b/litellm-rust/crates/secrets-hashicorp/src/error.rs new file mode 100644 index 00000000000..e26033af085 --- /dev/null +++ b/litellm-rust/crates/secrets-hashicorp/src/error.rs @@ -0,0 +1,34 @@ +#[derive(thiserror::Error, veil::Redact)] +pub enum Error { + #[error("HashiCorp Vault requires an enterprise license")] + EnterpriseRequired, + #[error("invalid secret name")] + InvalidSecretName(#[from] litellm_secrets_types::Error), + #[error("HashiCorp Vault client failed")] + Client( + #[from] + #[redact] + vaultrs::error::ClientError, + ), + #[error("HashiCorp Vault client settings are invalid: {message}")] + ClientSettings { message: String }, + #[error("HashiCorp Vault TLS identity could not be configured for {path}: {message}")] + TlsIdentity { + path: std::path::PathBuf, + message: String, + }, + #[error("HashiCorp Vault login returned HTTP {status}")] + LoginStatus { status: u16 }, + #[error("HashiCorp Vault login response is malformed")] + MalformedLogin, + #[error("HashiCorp Vault authentication is not configured")] + NoAuthConfigured, + #[error("HashiCorp Vault returned HTTP {status}")] + Status { status: u16 }, + #[error("HashiCorp Vault response payload is malformed")] + MalformedPayload, + #[error("HashiCorp Vault secret value is not a string")] + NonStringValue, + #[error("invalid HashiCorp Vault refresh interval")] + RefreshInterval, +} diff --git a/litellm-rust/crates/secrets-hashicorp/src/lib.rs b/litellm-rust/crates/secrets-hashicorp/src/lib.rs new file mode 100644 index 00000000000..0c2b05647f8 --- /dev/null +++ b/litellm-rust/crates/secrets-hashicorp/src/lib.rs @@ -0,0 +1,10 @@ +#![forbid(unsafe_code)] + +mod cert_login; +mod config; +mod error; +pub mod secret_manager; + +pub use config::{AppRoleAuth, HashicorpVaultConfig, TlsCertAuth}; +pub use error::Error; +pub use secret_manager::{HashicorpVault, SecretLocation}; diff --git a/litellm-rust/crates/secrets-hashicorp/src/secret_manager.rs b/litellm-rust/crates/secrets-hashicorp/src/secret_manager.rs new file mode 100644 index 00000000000..3ad9a549438 --- /dev/null +++ b/litellm-rust/crates/secrets-hashicorp/src/secret_manager.rs @@ -0,0 +1,359 @@ +use std::{ + collections::HashMap, + fmt, + sync::Arc, + time::{Duration, Instant}, +}; + +use litellm_core_utils::settings::Lookup; +use litellm_secrets_types::{ + BaseSecretManager, SecretValue, async_rotate_secret, validate_secret_name, +}; +use moka::future::Cache; +use rustify::errors::ClientError as RustifyClientError; +use serde_json::Value; +use tokio::sync::Mutex; +use vaultrs::{ + api, + auth::approle, + client::{Identity, VaultClient, VaultClientSettingsBuilder}, + error::ClientError, + kv2, +}; + +use crate::{Error, HashicorpVaultConfig, TlsCertAuth, cert_login::CertLoginRequest}; + +const CACHE_CAPACITY: u64 = 200; + +#[derive(Clone)] +struct CachedClient { + client: Arc, + expires_at: Option, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct SecretLocation { + pub namespace: Option, + pub mount: String, + pub path: String, +} + +#[derive(Clone)] +pub struct HashicorpVault { + config: HashicorpVaultConfig, + cache: Cache, + auth_client: Arc>>, +} + +impl fmt::Debug for HashicorpVault { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("HashicorpVault") + .field("config", &self.config) + .finish_non_exhaustive() + } +} + +impl HashicorpVault { + pub fn new( + environment: Arc, + enterprise_enabled: bool, + ) -> Result { + let config: HashicorpVaultConfig = + HashicorpVaultConfig::from_environment(environment.as_ref())?; + Self::from_config(config, enterprise_enabled) + } + + pub fn from_config( + config: HashicorpVaultConfig, + enterprise_enabled: bool, + ) -> Result { + if !enterprise_enabled { + return Err(Error::EnterpriseRequired); + } + let cache: Cache = Cache::builder() + .max_capacity(CACHE_CAPACITY) + .time_to_live(config.refresh_interval) + .build(); + Ok(Self { + config, + cache, + auth_client: Arc::new(Mutex::new(None)), + }) + } + + pub fn secret_location(&self, secret_name: &str) -> Result { + validate_secret_name(secret_name).map_err(Error::InvalidSecretName)?; + let path: String = [ + self.config.path_prefix.clone(), + Some(secret_name.to_owned()), + ] + .into_iter() + .flatten() + .collect::>() + .join("/"); + Ok(SecretLocation { + namespace: self.config.secret_namespace().map(str::to_owned), + mount: self.config.mount.clone(), + path, + }) + } + + pub fn config(&self) -> &HashicorpVaultConfig { + &self.config + } + + pub async fn async_read_secret(&self, secret_name: &str) -> Result, Error> { + let location: SecretLocation = self.secret_location(secret_name)?; + let cache_key: String = cache_key(&location); + if let Some(value) = self.cache.get(&cache_key).await { + return Ok(Some(value)); + } + let client: Arc = self.vault_client().await?; + let data: HashMap = + match kv2::read(client.as_ref(), &location.mount, &location.path).await { + Ok(data) => data, + Err(error) if api_status(&error) == Some(404) => return Ok(None), + Err(error) => return Err(map_api_error(error, ErrorContext::Read)), + }; + let Some(value) = data.get("key") else { + return Ok(None); + }; + let value: &str = value.as_str().ok_or(Error::NonStringValue)?; + let value: SecretValue = SecretValue::new(value); + self.cache.insert(cache_key, value.clone()).await; + Ok(Some(value)) + } + + pub async fn async_write_secret( + &self, + secret_name: &str, + value: SecretValue, + description: Option<&str>, + ) -> Result { + let location: SecretLocation = self.secret_location(secret_name)?; + let cache_key: String = cache_key(&location); + let data: HashMap = match description { + Some(description) => [ + ("key".to_owned(), Value::String(value.expose().to_owned())), + ( + "description".to_owned(), + Value::String(description.to_owned()), + ), + ] + .into_iter() + .collect(), + None => [("key".to_owned(), Value::String(value.expose().to_owned()))] + .into_iter() + .collect(), + }; + let client: Arc = self.vault_client().await?; + let metadata = kv2::set(client.as_ref(), &location.mount, &location.path, &data) + .await + .map_err(|error| map_api_error(error, ErrorContext::Secret))?; + self.cache.invalidate(&cache_key).await; + serde_json::to_value(metadata) + .map_err(|source| Error::Client(ClientError::JsonParseError { source })) + } + + pub async fn async_delete_secret(&self, secret_name: &str) -> Result<(), Error> { + let location: SecretLocation = self.secret_location(secret_name)?; + let cache_key: String = cache_key(&location); + let client: Arc = self.vault_client().await?; + kv2::delete_latest(client.as_ref(), &location.mount, &location.path) + .await + .map_err(|error| map_api_error(error, ErrorContext::Secret))?; + self.cache.invalidate(&cache_key).await; + Ok(()) + } + + pub async fn async_rotate_secret( + &self, + current_name: &str, + new_name: &str, + value: &SecretValue, + ) -> Result { + async_rotate_secret(self, current_name, new_name, value).await + } + + async fn vault_client(&self) -> Result, Error> { + let mut cached = self.auth_client.lock().await; + if let Some(entry) = cached.as_ref() + && entry + .expires_at + .is_none_or(|expires_at| expires_at > Instant::now()) + { + return Ok(entry.client.clone()); + } + + let (client, expires_at): (VaultClient, Option) = + match (self.config.approle.as_ref(), self.config.tls_cert.as_ref()) { + (Some(approle), _) => { + let login_client: VaultClient = + self.build_client(self.config.login_namespace(), "")?; + let auth = approle::login( + &login_client, + &approle.mount_path, + &approle.role_id, + approle.secret_id.expose(), + ) + .await + .map_err(|error| map_api_error(error, ErrorContext::Login))?; + ( + self.build_client(self.config.secret_namespace(), &auth.client_token)?, + token_expiry(auth.lease_duration), + ) + } + (None, Some(tls)) => { + let login_client: VaultClient = + self.build_client(self.config.login_namespace(), "")?; + let endpoint: CertLoginRequest = CertLoginRequest::new(tls.role.as_deref()); + let auth = api::auth(&login_client, endpoint) + .await + .map_err(|error| map_api_error(error, ErrorContext::Login))?; + ( + self.build_client(self.config.secret_namespace(), &auth.client_token)?, + token_expiry(auth.lease_duration), + ) + } + (None, None) => { + let token: SecretValue = + self.config.token.clone().ok_or(Error::NoAuthConfigured)?; + ( + self.build_client(self.config.secret_namespace(), token.expose())?, + None, + ) + } + }; + let client: Arc = Arc::new(client); + *cached = Some(CachedClient { + client: client.clone(), + expires_at, + }); + Ok(client) + } + + fn build_client(&self, namespace: Option<&str>, token: &str) -> Result { + let settings = VaultClientSettingsBuilder::default() + .address(&self.config.address) + .token(token.to_owned()) + .namespace(namespace.map(str::to_owned)) + .identity(identity_for(self.config.tls_cert.as_ref())?) + .ca_certs(Vec::new()) + .verify(true) + .build() + .map_err(|message| Error::ClientSettings { + message: message.to_string(), + })?; + VaultClient::new(settings).map_err(Error::Client) + } +} + +impl BaseSecretManager for HashicorpVault { + type Error = Error; + type WriteResponse = Value; + type DeleteResponse = (); + + async fn async_read_secret(&self, name: &str) -> Result, Error> { + HashicorpVault::async_read_secret(self, name).await + } + + async fn async_write_secret( + &self, + name: &str, + value: &SecretValue, + description: Option<&str>, + ) -> Result { + HashicorpVault::async_write_secret(self, name, value.clone(), description).await + } + + async fn async_delete_secret( + &self, + name: &str, + _recovery_window_in_days: i64, + ) -> Result<(), Error> { + HashicorpVault::async_delete_secret(self, name).await + } +} + +#[derive(Clone, Copy)] +enum ErrorContext { + Login, + Read, + Secret, +} + +fn cache_key(location: &SecretLocation) -> String { + format!( + "{:?}/{}/{}", + location.namespace, location.mount, location.path + ) +} + +fn identity_for(tls: Option<&TlsCertAuth>) -> Result, Error> { + tls.map(|tls| { + let cert: Vec = std::fs::read(&tls.cert_path).map_err(|source| Error::TlsIdentity { + path: tls.cert_path.clone(), + message: source.to_string(), + })?; + let key: Vec = std::fs::read(&tls.key_path).map_err(|source| Error::TlsIdentity { + path: tls.key_path.clone(), + message: source.to_string(), + })?; + Identity::from_pem(&[cert.as_slice(), key.as_slice()].concat()).map_err(|source| { + Error::TlsIdentity { + path: tls.cert_path.clone(), + message: source.to_string(), + } + }) + }) + .transpose() +} + +fn map_api_error(error: ClientError, context: ErrorContext) -> Error { + match error { + ClientError::APIError { code, .. } => match context { + ErrorContext::Login => Error::LoginStatus { status: code }, + ErrorContext::Read | ErrorContext::Secret => Error::Status { status: code }, + }, + ClientError::JsonParseError { source } => match context { + ErrorContext::Login => Error::MalformedLogin, + ErrorContext::Read => Error::MalformedPayload, + ErrorContext::Secret => Error::Client(ClientError::JsonParseError { source }), + }, + ClientError::ResponseEmptyError | ClientError::ResponseDataEmptyError => { + malformed_response(context) + } + ClientError::RestClientError { source } => match source { + RustifyClientError::ServerResponseError { code, .. } => match context { + ErrorContext::Login => Error::LoginStatus { status: code }, + ErrorContext::Read | ErrorContext::Secret => Error::Status { status: code }, + }, + RustifyClientError::ResponseParseError { .. } => malformed_response(context), + source => Error::Client(ClientError::RestClientError { source }), + }, + error => Error::Client(error), + } +} + +fn api_status(error: &ClientError) -> Option { + match error { + ClientError::APIError { code, .. } => Some(*code), + ClientError::RestClientError { + source: RustifyClientError::ServerResponseError { code, .. }, + } => Some(*code), + _ => None, + } +} + +fn malformed_response(context: ErrorContext) -> Error { + match context { + ErrorContext::Login => Error::MalformedLogin, + ErrorContext::Read => Error::MalformedPayload, + ErrorContext::Secret => Error::Client(ClientError::ResponseDataEmptyError), + } +} + +fn token_expiry(lease_duration: u64) -> Option { + (lease_duration > 0).then(|| Instant::now() + Duration::from_secs(lease_duration)) +} diff --git a/litellm-rust/crates/secrets-hashicorp/tests/secret_manager.rs b/litellm-rust/crates/secrets-hashicorp/tests/secret_manager.rs new file mode 100644 index 00000000000..c52db46e41e --- /dev/null +++ b/litellm-rust/crates/secrets-hashicorp/tests/secret_manager.rs @@ -0,0 +1,602 @@ +use std::{collections::HashMap, sync::Arc, time::Duration}; + +use litellm_core_utils::settings::Lookup; +use litellm_secrets_hashicorp::{Error, HashicorpVault, HashicorpVaultConfig}; +use litellm_secrets_types::SecretValue; +use serde::Deserialize; +use serde_json::json; +use wiremock::{ + Mock, MockServer, ResponseTemplate, + matchers::{body_json, header, method, path}, +}; + +fn config(server: &MockServer, values: &[(&str, &str)]) -> HashicorpVaultConfig { + let mut environment_values: HashMap = values + .iter() + .map(|(name, value)| ((*name).to_owned(), (*value).to_owned())) + .collect(); + environment_values.insert("HCP_VAULT_ADDR".to_owned(), server.uri()); + let environment: Arc = + Arc::new(move |name: &str| environment_values.get(name).cloned()); + HashicorpVaultConfig::from_environment(environment.as_ref()).unwrap() +} + +fn manager(server: &MockServer, values: &[(&str, &str)]) -> HashicorpVault { + HashicorpVault::from_config(config(server, values), true).unwrap() +} + +fn auth_response(token: &str, lease_duration: u64) -> serde_json::Value { + json!({ + "auth": { + "client_token": token, + "accessor": "", + "policies": [], + "token_policies": [], + "metadata": null, + "lease_duration": lease_duration, + "renewable": false, + "entity_id": "", + "token_type": "service", + "orphan": false + }, + "lease_id": "", + "lease_duration": lease_duration, + "renewable": false, + "request_id": "", + "warnings": null, + "wrap_info": null + }) +} + +fn read_response(data: serde_json::Value) -> serde_json::Value { + json!({ + "data": { + "data": data, + "metadata": { + "created_time": "", + "deletion_time": "", + "custom_metadata": null, + "destroyed": false, + "version": 1 + } + }, + "lease_id": "", + "lease_duration": 0, + "renewable": false, + "request_id": "", + "warnings": null, + "wrap_info": null + }) +} + +#[tokio::test] +async fn token_reads_use_vault_headers_and_cache_values() { + let server: MockServer = MockServer::start().await; + Mock::given(method("GET")) + .and(path("/v1/secret/data/name")) + .and(header("X-Vault-Token", "token")) + .respond_with( + ResponseTemplate::new(200).set_body_json(read_response(json!({"key": "value"}))), + ) + .expect(1) + .mount(&server) + .await; + let manager: HashicorpVault = manager(&server, &[("HCP_VAULT_TOKEN", "token")]); + + assert_eq!( + manager + .async_read_secret("name") + .await + .unwrap() + .unwrap() + .expose(), + "value" + ); + let requests = server.received_requests().await.unwrap(); + assert!( + requests + .iter() + .all(|request| !request.headers.contains_key("X-Vault-Namespace")) + ); + assert_eq!( + manager + .async_read_secret("name") + .await + .unwrap() + .unwrap() + .expose(), + "value" + ); +} + +#[tokio::test] +async fn namespace_mount_and_prefix_are_sanitized_in_the_url() { + let server: MockServer = MockServer::start().await; + Mock::given(method("GET")) + .and(path("/v1/kv-prod/data/virtual-keys/name")) + .and(header("X-Vault-Namespace", "team-a")) + .respond_with( + ResponseTemplate::new(200).set_body_json(read_response(json!({"key": "value"}))), + ) + .expect(1) + .mount(&server) + .await; + let manager: HashicorpVault = manager( + &server, + &[ + ("HCP_VAULT_TOKEN", "token"), + ("HCP_VAULT_SECRET_NAMESPACE", " /team-a/ "), + ("HCP_VAULT_MOUNT_NAME", " /kv-prod/ "), + ("HCP_VAULT_PATH_PREFIX", " /virtual-keys/ "), + ], + ); + + let location = manager.secret_location("name").unwrap(); + assert_eq!(location.namespace.as_deref(), Some("team-a")); + assert_eq!(location.mount, "kv-prod"); + assert_eq!(location.path, "virtual-keys/name"); + assert!(manager.async_read_secret("name").await.unwrap().is_some()); +} + +#[test] +fn trailing_address_slashes_are_removed() { + let environment: Arc = Arc::new(|name: &str| match name { + "HCP_VAULT_ADDR" => Some("http://vault.test:8200///".to_owned()), + "HCP_VAULT_TOKEN" => Some("token".to_owned()), + _ => None, + }); + let config: HashicorpVaultConfig = + HashicorpVaultConfig::from_environment(environment.as_ref()).unwrap(); + let manager: HashicorpVault = HashicorpVault::from_config(config, true).unwrap(); + + assert_eq!( + manager.secret_location("name").unwrap(), + litellm_secrets_hashicorp::SecretLocation { + namespace: None, + mount: "secret".to_owned(), + path: "name".to_owned(), + } + ); +} + +#[rstest::rstest] +#[case("-1")] +#[case("not-a-number")] +fn invalid_refresh_intervals_are_rejected(#[case] value: &str) { + let environment: Arc = Arc::new(move |name: &str| match name { + "HCP_VAULT_REFRESH_INTERVAL" => Some(value.to_owned()), + _ => None, + }); + + assert!(matches!( + HashicorpVaultConfig::from_environment(environment.as_ref()), + Err(Error::RefreshInterval) + )); +} + +#[tokio::test] +async fn approle_login_uses_namespace_and_reuses_the_token() { + let server: MockServer = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/v1/auth/custom-approle/login")) + .and(header("X-Vault-Namespace", "login-root")) + .and(body_json(json!({"role_id": "role", "secret_id": "secret"}))) + .respond_with(ResponseTemplate::new(200).set_body_json(auth_response("login-token", 3600))) + .expect(1) + .mount(&server) + .await; + Mock::given(method("GET")) + .and(path("/v1/secret/data/name")) + .and(header("X-Vault-Token", "login-token")) + .and(header("X-Vault-Namespace", "secret-root")) + .respond_with( + ResponseTemplate::new(200).set_body_json(read_response(json!({"key": "value"}))), + ) + .expect(1) + .mount(&server) + .await; + Mock::given(method("GET")) + .and(path("/v1/secret/data/name-2")) + .respond_with(ResponseTemplate::new(404).set_body_json(json!({"errors": ["missing"]}))) + .expect(1) + .mount(&server) + .await; + let manager: HashicorpVault = manager( + &server, + &[ + ("HCP_VAULT_APPROLE_ROLE_ID", "role"), + ("HCP_VAULT_APPROLE_SECRET_ID", "secret"), + ("HCP_VAULT_APPROLE_MOUNT_PATH", "custom-approle"), + ("HCP_VAULT_NAMESPACE", "secret-root"), + ("HCP_VAULT_LOGIN_NAMESPACE", "login-root"), + ], + ); + + assert!(manager.async_read_secret("name").await.unwrap().is_some()); + assert!(manager.async_read_secret("name-2").await.unwrap().is_none()); +} + +#[tokio::test] +async fn approle_tokens_expire_after_the_vault_lease() { + let server: MockServer = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/v1/auth/approle/login")) + .respond_with(ResponseTemplate::new(200).set_body_json(auth_response("login-token", 1))) + .expect(2) + .mount(&server) + .await; + Mock::given(method("GET")) + .respond_with( + ResponseTemplate::new(200).set_body_json(read_response(json!({"key": "value"}))), + ) + .expect(2) + .mount(&server) + .await; + let manager: HashicorpVault = manager( + &server, + &[ + ("HCP_VAULT_APPROLE_ROLE_ID", "role"), + ("HCP_VAULT_APPROLE_SECRET_ID", "secret"), + ("HCP_VAULT_REFRESH_INTERVAL", "0"), + ], + ); + + assert!(manager.async_read_secret("first").await.unwrap().is_some()); + tokio::time::sleep(Duration::from_secs(1) + Duration::from_millis(50)).await; + assert!(manager.async_read_secret("second").await.unwrap().is_some()); +} + +#[tokio::test] +async fn tls_login_posts_the_role_and_uses_the_client_identity() { + let server: MockServer = MockServer::start().await; + let directory: tempfile::TempDir = tempfile::tempdir().unwrap(); + let cert_path = directory.path().join("client.crt"); + let key_path = directory.path().join("client.key"); + std::fs::write(&cert_path, TEST_CERTIFICATE).unwrap(); + std::fs::write(&key_path, TEST_PRIVATE_KEY).unwrap(); + Mock::given(method("POST")) + .and(path("/v1/auth/cert/login")) + .and(header("X-Vault-Namespace", "login-ns")) + .respond_with(ResponseTemplate::new(200).set_body_json(auth_response("cert-token", 0))) + .expect(2) + .mount(&server) + .await; + Mock::given(method("GET")) + .and(path("/v1/secret/data/name")) + .and(header("X-Vault-Token", "cert-token")) + .and(header("X-Vault-Namespace", "secret-ns")) + .respond_with( + ResponseTemplate::new(200).set_body_json(read_response(json!({"key": "value"}))), + ) + .expect(2) + .mount(&server) + .await; + let role_values: HashMap = HashMap::from([ + ("HCP_VAULT_ADDR".to_owned(), server.uri()), + ( + "HCP_VAULT_CLIENT_CERT".to_owned(), + cert_path.to_str().unwrap().to_owned(), + ), + ( + "HCP_VAULT_CLIENT_KEY".to_owned(), + key_path.to_str().unwrap().to_owned(), + ), + ("HCP_VAULT_CERT_ROLE".to_owned(), "vault-role".to_owned()), + ( + "HCP_VAULT_LOGIN_NAMESPACE".to_owned(), + "login-ns".to_owned(), + ), + ( + "HCP_VAULT_SECRET_NAMESPACE".to_owned(), + "secret-ns".to_owned(), + ), + ]); + let role_environment: Arc = + Arc::new(move |name: &str| role_values.get(name).cloned()); + let role_manager: HashicorpVault = HashicorpVault::new(role_environment, true).unwrap(); + assert!( + role_manager + .async_read_secret("name") + .await + .unwrap() + .is_some() + ); + + let no_role_values: HashMap = HashMap::from([ + ("HCP_VAULT_ADDR".to_owned(), server.uri()), + ( + "HCP_VAULT_CLIENT_CERT".to_owned(), + cert_path.to_str().unwrap().to_owned(), + ), + ( + "HCP_VAULT_CLIENT_KEY".to_owned(), + key_path.to_str().unwrap().to_owned(), + ), + ( + "HCP_VAULT_LOGIN_NAMESPACE".to_owned(), + "login-ns".to_owned(), + ), + ( + "HCP_VAULT_SECRET_NAMESPACE".to_owned(), + "secret-ns".to_owned(), + ), + ]); + let no_role_environment: Arc = + Arc::new(move |name: &str| no_role_values.get(name).cloned()); + let no_role_manager: HashicorpVault = HashicorpVault::new(no_role_environment, true).unwrap(); + assert!( + no_role_manager + .async_read_secret("name") + .await + .unwrap() + .is_some() + ); + let login_bodies: Vec = server + .received_requests() + .await + .unwrap() + .iter() + .filter(|request| request.method.as_str() == "POST") + .map(|request| serde_json::from_slice(&request.body).unwrap()) + .collect(); + assert!(login_bodies.contains(&json!({"name": "vault-role"}))); + assert!(login_bodies.contains(&json!({}))); +} + +#[rstest::rstest] +#[case::missing(404, json!({"errors": ["missing"]}), 0)] +#[case::malformed(200, json!({"data": "invalid"}), 1)] +#[case::missing_key(200, json!({}), 0)] +#[case::non_string(200, json!({"key": 1}), 2)] +#[tokio::test] +async fn read_responses_distinguish_absence_and_malformed_payloads( + #[case] status: u16, + #[case] body: serde_json::Value, + #[case] expected: u8, +) { + let server: MockServer = MockServer::start().await; + Mock::given(method("GET")) + .respond_with(ResponseTemplate::new(status).set_body_json( + if status == 200 && expected != 1 { + read_response(body) + } else { + body + }, + )) + .expect(1) + .mount(&server) + .await; + let result: Result, Error> = + manager(&server, &[("HCP_VAULT_TOKEN", "token")]) + .async_read_secret("name") + .await; + match expected { + 0 => assert!(result.unwrap().is_none()), + 1 => assert!(matches!(result, Err(Error::MalformedPayload))), + 2 => assert!(matches!(result, Err(Error::NonStringValue))), + _ => unreachable!(), + } +} + +#[tokio::test] +async fn write_and_delete_invalidate_the_read_cache() { + let server: MockServer = MockServer::start().await; + Mock::given(method("GET")) + .and(path("/v1/secret/data/name")) + .respond_with( + ResponseTemplate::new(200).set_body_json(read_response(json!({"key": "value"}))), + ) + .expect(2) + .mount(&server) + .await; + Mock::given(method("POST")) + .and(path("/v1/secret/data/name")) + .and(body_json( + json!({"data": {"key": "updated", "description": "description"}}), + )) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "data": { + "created_time": "", + "deletion_time": "", + "custom_metadata": null, + "destroyed": false, + "version": 2 + }, + "lease_id": "", + "lease_duration": 0, + "renewable": false, + "request_id": "", + "warnings": null, + "wrap_info": null + }))) + .expect(1) + .mount(&server) + .await; + Mock::given(method("DELETE")) + .and(path("/v1/secret/data/name")) + .respond_with(ResponseTemplate::new(204)) + .expect(1) + .mount(&server) + .await; + let manager: HashicorpVault = manager(&server, &[("HCP_VAULT_TOKEN", "token")]); + + assert!(manager.async_read_secret("name").await.unwrap().is_some()); + assert!( + manager + .async_write_secret("name", SecretValue::new("updated"), Some("description")) + .await + .is_ok() + ); + assert!(manager.async_read_secret("name").await.unwrap().is_some()); + manager.async_delete_secret("name").await.unwrap(); +} + +#[tokio::test] +async fn no_auth_and_invalid_names_fail_without_requests() { + let server: MockServer = MockServer::start().await; + let manager: HashicorpVault = manager(&server, &[]); + + assert!(matches!( + manager.async_read_secret("name").await, + Err(Error::NoAuthConfigured) + )); + assert!(matches!( + manager.async_read_secret("../name").await, + Err(Error::InvalidSecretName(_)) + )); + assert!(server.received_requests().await.unwrap().is_empty()); +} + +#[tokio::test] +async fn debug_output_redacts_authentication_values() { + let server: MockServer = MockServer::start().await; + let manager: HashicorpVault = + HashicorpVault::from_config(config(&server, &[("HCP_VAULT_TOKEN", "token-value")]), true) + .unwrap(); + let debug: String = format!("{manager:?}"); + assert!(!debug.contains("token-value")); + assert!(!debug.contains("secret-id")); +} + +#[derive(Deserialize)] +struct ParityCase { + env: HashMap, + expected_secret_url: String, + expected_login_url: Option, + expected_login_namespace: Option, + expected_secret_namespace: Option, + secret_name: String, +} + +#[test] +fn configuration_matches_python_parity_fixture() { + let cases: Vec = serde_json::from_str(include_str!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/../../../tests/test_litellm/secret_managers/hashicorp_vault_parity.json" + ))) + .unwrap(); + for case in cases { + let values: HashMap = case.env.clone(); + let environment: Arc = + Arc::new(move |name: &str| values.get(name).cloned()); + let config: HashicorpVaultConfig = + HashicorpVaultConfig::from_environment(environment.as_ref()).unwrap(); + let manager: HashicorpVault = HashicorpVault::from_config(config.clone(), true).unwrap(); + let location = manager.secret_location(&case.secret_name).unwrap(); + let namespace = location + .namespace + .as_deref() + .map(|namespace| format!("{namespace}/")) + .unwrap_or_default(); + assert_eq!( + format!( + "{}/v1/{}{}/data/{}", + config.address, namespace, location.mount, location.path + ), + case.expected_secret_url + ); + let login_url = config.approle.as_ref().map_or_else( + || { + config + .tls_cert + .as_ref() + .map(|_| format!("{}/v1/auth/cert/login", config.address)) + }, + |approle| { + Some(format!( + "{}/v1/auth/{}/login", + config.address, approle.mount_path + )) + }, + ); + assert_eq!(login_url, case.expected_login_url); + assert_eq!( + manager.config().login_namespace(), + case.expected_login_namespace.as_deref() + ); + assert_eq!( + manager.config().secret_namespace(), + case.expected_secret_namespace.as_deref() + ); + } +} + +#[tokio::test] +#[ignore] +async fn live_vault_round_trip() { + let environment: Arc = + Arc::new(litellm_core_utils::settings::ProcessEnvironment); + let manager: HashicorpVault = HashicorpVault::new(environment, true).unwrap(); + let name: String = std::env::var("LITELLM_VAULT_LIVE_SECRET_NAME").unwrap(); + let value: SecretValue = SecretValue::new("native-live-value"); + let location = manager.secret_location(&name).unwrap(); + println!( + "native provenance: {} vaultrs {} {:?} {} {}", + module_path!(), + manager.config().address, + location.namespace, + location.mount, + location.path + ); + manager + .async_write_secret(&name, value.clone(), None) + .await + .unwrap(); + assert_eq!( + manager.async_read_secret(&name).await.unwrap().unwrap(), + value + ); + manager.async_delete_secret(&name).await.unwrap(); + assert!(manager.async_read_secret(&name).await.unwrap().is_none()); +} + +const TEST_CERTIFICATE: &str = "-----BEGIN CERTIFICATE----- +MIIDDzCCAfegAwIBAgIUeMzLFLM/mRbPGbNAew5N2UTscocwDQYJKoZIhvcNAQEL +BQAwFzEVMBMGA1UEAwwMbGl0ZWxsbS10ZXN0MB4XDTI2MDkyMTIwMjA1OVoXDTI2 +MDkyMjIwMjA1OVowFzEVMBMGA1UEAwwMbGl0ZWxsbS10ZXN0MIIBIjANBgkqhkiG +9w0BAQEFAAOCAQ8AMIIBCgKCAQEAveYoSUJXybmkHmQsBfhBcv2Ob5Oy8ejZu+B3 +vTnrPumW4ANi1XXKBSazRGB3fEtAgr+3KhKeHaSKEQeBwJkAEBfdmQv0tpXICwHs +1kFNtU0owy54HVW5/ia+LMszsFcPzVIoMnbUOuiKr9RaV7P+IEFzILPBVuV4DoYH +yocjD3+9QNqokWgNL8LK37JijmNEFVaKFz0X6SyL2VRDlfPWTEBK52Gp/pvDgA6G +eTSfyI+kCm9h5ECTYUAtmatk9WPVS8sWOqV1EXVanFyYBU+mDxoywAS1/6CHeIPh +bNmCOZjPoO9qWBJ7ZyGhOconBigXY8qnlXymev+44IPHrx4urwIDAQABo1MwUTAd +BgNVHQ4EFgQUvaZrZ6HKtbr3ekeZmgy4b5Pq95QwHwYDVR0jBBgwFoAUvaZrZ6HK +tbr3ekeZmgy4b5Pq95QwDwYDVR0TAQH/BAUwAwEB/zANBgkqhkiG9w0BAQsFAAOC +AQEAEejrD8d1qDxW55XxQ4IC31rufoEvDV955jyvh2kALPaN/i5oWsBGI+UAQZna +aaoQXwzlmHrtDUBWl0LztVTUamIleUep2+PLLauqqt43vxppxMX8Jn2mnPO20YE/ +hIzGx0jN/LBG8PDyLSvHdlgjP9ofA4Vg4rTQugdXRgOvlCE/epnH/MADcg9KYJtJ +C1RObCIkL3LcdUbjStJRCY/U/FeWcgyncEPz95OFDkbrlNDajb6o6CkYfouqvhTc +8XlgjjAVKIbAbRgbVu3elsquuFM97x2DzWDjkrMNmDt1FJ9ubK36gL6B3o0UMaoQ +00R7x/eqvH+EkWa/2ekW9lpleQ== +-----END CERTIFICATE----- +"; + +const TEST_PRIVATE_KEY: &str = "-----BEGIN PRIVATE KEY----- +MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQC95ihJQlfJuaQe +ZCwF+EFy/Y5vk7Lx6Nm74He9Oes+6ZbgA2LVdcoFJrNEYHd8S0CCv7cqEp4dpIoR +B4HAmQAQF92ZC/S2lcgLAezWQU21TSjDLngdVbn+Jr4syzOwVw/NUigydtQ66Iqv +1FpXs/4gQXMgs8FW5XgOhgfKhyMPf71A2qiRaA0vwsrfsmKOY0QVVooXPRfpLIvZ +VEOV89ZMQErnYan+m8OADoZ5NJ/Ij6QKb2HkQJNhQC2Zq2T1Y9VLyxY6pXURdVqc +XJgFT6YPGjLABLX/oId4g+Fs2YI5mM+g72pYEntnIaE5yicGKBdjyqeVfKZ6/7jg +g8evHi6vAgMBAAECggEAGdJjlP6b8Fa5bdaCM/ebcrbuuNZVJVbb0JPHxGfNSLs7 +pE9hj5QaOdQW2Uviw3h6F61ZCzQH4xD+Iy2po5ZKb2XHYKnDB1bboj+LRGER337T +9aJqe9at2VTMVEv3Rdm40NsEk0QcPLxlK16NQFK90gYEUSSQPDAswJDSG2R/zHn+ +vADI907mW/goEJHeLn8PWGlNlSiR6x+5JJtq+GXCzUzVvJYQSCLGxCSl2x2H+0g7 +NhFI0zPpdzNmO/h+yhzaFb6Rp5U8+ZsnZ3qYjQ/03gw1myTDKJt1YaO9JvArnNYX +hcJQQ8Rt0bHhcrZA16bBOpqZlo5pKCicwI/netgN8QKBgQDcFz7AzdJ26sMSV32V +rwrMgIoggt8qDjO1ARwqW35A1TIge0FoW4M4KpsXQGGfT341uU1esXEcyZ/1L/5X +3ql2gX4DbOYLZLWYzZGR2hq33oi8HkhN98QrEwL9emSH8NqYX3Xxja3PrmCrSYJe +Zbnd9TIm2XkxyMoyXJu6M/QvnwKBgQDc4dzqTbxoGEGa5MuJoGmMwPnqgdG9UM5J +eExVnh7osxc2sOdsiPeRjjQTxs9v2kJwctC359OJoo9yGaaJeSghU4LEWJo1sqnA +fzSCLammYvtVAtniyNv5Mxk/6Uimi4NNDKaAKB+m4K2uSn3U9AmY7KPYMGaSbS9W +XSnobjxm8QKBgC8bPpAvvWs8ZhIn7bY659nLbUT2HeO3dHO6UBf0yzn/J6JyHxbB +93zvCZDZc8uQTRgcmCW7XtVlhjoJUqvl+Wlm39zF0xr/LCsPXKfWAb/2/lcdOCaP +8Emz4QD10EyUTYUtcWYJB/mafhBLRH8F0Nlj4J8WDu2L51MOJTqeYhZLAoGAWffN +icocAbJPlo22sdoa4+/+W5yBF8GAJMDRJtZ+9H1t6SLpQHYRkMIBSETkXUTjZvX9 +Ocs9iIQkNW9pO/mTdO+VBfCo71JUfknR02xR+6m5gYjlws/ZeYlssXGN2/hbhNiw +QOcW7Vv6olFJK6Iy/oz0t6wPO3kpnN3Zogi0paECgYEAwo44M1DdYCtV0snhmYM9 +5u0mPfYt5P2SVLXyUbr+vFTfrTL/WKnXIJgbsnj3Gvf+GIZv9tKcXhSNmEHQCYX4 +X3w9iTPddCHuvZ1fpufi2TyArJh0OkoNtLXJHTKrHjf2N+61AQzFiv5WieJrdE+H +qr32PTUuVGPyO9LyTY4/RL0= +-----END PRIVATE KEY----- +"; diff --git a/litellm-rust/crates/secrets/Cargo.toml b/litellm-rust/crates/secrets/Cargo.toml index 962f66c92d1..e5f30025976 100644 --- a/litellm-rust/crates/secrets/Cargo.toml +++ b/litellm-rust/crates/secrets/Cargo.toml @@ -9,6 +9,7 @@ repository.workspace = true default = [] aws = ["dep:litellm-secrets-aws"] google = ["dep:litellm-secrets-google"] +hashicorp = ["dep:litellm-secrets-hashicorp"] azure = ["dep:litellm-secrets-azure"] cyberark = ["dep:litellm-secrets-cyberark"] @@ -16,6 +17,7 @@ cyberark = ["dep:litellm-secrets-cyberark"] litellm-secrets-types.workspace = true litellm-secrets-aws = { workspace = true, optional = true } litellm-secrets-google = { workspace = true, optional = true } +litellm-secrets-hashicorp = { workspace = true, optional = true } litellm-secrets-azure = { workspace = true, optional = true } litellm-secrets-cyberark = { workspace = true, optional = true } litellm-core-utils.workspace = true diff --git a/litellm-rust/crates/secrets/README.md b/litellm-rust/crates/secrets/README.md index 183a39e15bb..0d333c7d116 100644 --- a/litellm-rust/crates/secrets/README.md +++ b/litellm-rust/crates/secrets/README.md @@ -9,3 +9,5 @@ Backend failures propagate by default. To allow fallback during a backend failur `get_secret` preserves value types. `get_secret_str` accepts a string default and rejects boolean or JSON values with `Error::TypeMismatch`. `get_secret_bool` accepts a boolean default and converts strings containing `true` or `false`, ignoring surrounding whitespace and ASCII case. Other strings and JSON values produce `Error::TypeMismatch`. Conversion failures never activate fallback or replace a found value with the default Provider payloads remain strings unless explicitly selecting a field from an AWS primary JSON secret. Google caches only successfully decoded string payloads, so reads have identical values and types before and after caching. Confirmed absence and failed reads are not cached. AWS resource-not-found responses and Google HTTP 404 responses indicate absence. Other provider errors remain errors, and successful responses without the required payload are malformed responses rather than missing secrets + +The HashiCorp Vault backend is enabled with the `hashicorp` feature and reads KV v2 values from `HCP_VAULT_*` environment variables. It supports static tokens, AppRole authentication, and TLS certificate authentication diff --git a/litellm-rust/crates/secrets/src/error.rs b/litellm-rust/crates/secrets/src/error.rs index 7e03f1f8cbf..1be0adc2bf5 100644 --- a/litellm-rust/crates/secrets/src/error.rs +++ b/litellm-rust/crates/secrets/src/error.rs @@ -30,6 +30,9 @@ pub enum Error { #[cfg(feature = "google")] #[error(transparent)] Google(#[from] litellm_secrets_google::Error), + #[cfg(feature = "hashicorp")] + #[error(transparent)] + Hashicorp(#[from] litellm_secrets_hashicorp::Error), #[cfg(feature = "azure")] #[error(transparent)] Azure(#[from] litellm_secrets_azure::Error), diff --git a/litellm-rust/crates/secrets/src/handler.rs b/litellm-rust/crates/secrets/src/handler.rs index 71214ffc9ce..5ab2caa75d4 100644 --- a/litellm-rust/crates/secrets/src/handler.rs +++ b/litellm-rust/crates/secrets/src/handler.rs @@ -13,6 +13,8 @@ pub enum SecretManager { GoogleKms(crate::google::GoogleKms), #[cfg(feature = "google")] GoogleSecretManager(crate::google::GoogleSecretManager), + #[cfg(feature = "hashicorp")] + HashicorpVault(crate::hashicorp::HashicorpVault), #[cfg(feature = "azure")] AzureKeyVault(crate::azure::AzureKeyVault), #[cfg(feature = "cyberark")] @@ -31,6 +33,8 @@ impl SecretManager { Self::GoogleKms(_) => KeyManagementSystem::GoogleKms, #[cfg(feature = "google")] Self::GoogleSecretManager(_) => KeyManagementSystem::GoogleSecretManager, + #[cfg(feature = "hashicorp")] + Self::HashicorpVault(_) => KeyManagementSystem::HashicorpVault, #[cfg(feature = "azure")] Self::AzureKeyVault(_) => KeyManagementSystem::AzureKeyVault, #[cfg(feature = "cyberark")] @@ -86,6 +90,12 @@ pub async fn get_secret_from_manager( .get_secret_from_google_secret_manager(secret_name) .await .map_err(Error::from), + #[cfg(feature = "hashicorp")] + SecretManager::HashicorpVault(client) => client + .async_read_secret(secret_name) + .await + .map(|value| value.map(Secret::String)) + .map_err(Error::from), #[cfg(feature = "azure")] SecretManager::AzureKeyVault(client) => client .get_secret_from_azure_key_vault(secret_name) diff --git a/litellm-rust/crates/secrets/src/lib.rs b/litellm-rust/crates/secrets/src/lib.rs index dec924abdd0..1acb5269e66 100644 --- a/litellm-rust/crates/secrets/src/lib.rs +++ b/litellm-rust/crates/secrets/src/lib.rs @@ -23,3 +23,5 @@ pub use litellm_secrets_azure as azure; pub use litellm_secrets_cyberark as cyberark; #[cfg(feature = "google")] pub use litellm_secrets_google as google; +#[cfg(feature = "hashicorp")] +pub use litellm_secrets_hashicorp as hashicorp; diff --git a/litellm-rust/crates/secrets/tests/handler.rs b/litellm-rust/crates/secrets/tests/handler.rs index fbe13721491..2a8b7070522 100644 --- a/litellm-rust/crates/secrets/tests/handler.rs +++ b/litellm-rust/crates/secrets/tests/handler.rs @@ -105,6 +105,148 @@ async fn google_handler_requires_canonical_base64_and_preserves_plaintext_whites Err(Error::MissingCiphertext) )); } +#[cfg(feature = "hashicorp")] +#[tokio::test] +async fn hashicorp_handler_resolves_found_missing_and_failed_values() { + use std::sync::Arc; + + use litellm_core_utils::settings::Lookup; + use litellm_secrets::{ + Error, FailurePolicy, KeyManagementSettings, SecretManager, SecretManagerState, + SecretResolver, hashicorp::HashicorpVault, hashicorp::HashicorpVaultConfig, + }; + use wiremock::{ + Mock, MockServer, ResponseTemplate, + matchers::{method, path}, + }; + + let found_server = MockServer::start().await; + Mock::given(method("GET")) + .and(path("/v1/secret/data/KEY")) + .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({ + "data": { + "data": {"key": "remote"}, + "metadata": { + "created_time": "", + "deletion_time": "", + "custom_metadata": null, + "destroyed": false, + "version": 1 + } + }, + "lease_id": "", + "lease_duration": 0, + "renewable": false, + "request_id": "", + "warnings": null, + "wrap_info": null + }))) + .mount(&found_server) + .await; + let found_environment: Arc = Arc::new({ + let address = found_server.uri(); + move |name: &str| match name { + "HCP_VAULT_ADDR" => Some(address.clone()), + "HCP_VAULT_TOKEN" => Some("token".into()), + _ => None, + } + }); + let found_config = HashicorpVaultConfig::from_environment(found_environment.as_ref()).unwrap(); + let found_manager = HashicorpVault::from_config(found_config, true).unwrap(); + let found_resolver = SecretResolver::new( + Arc::new(SecretManagerState::new( + SecretManager::HashicorpVault(found_manager), + KeyManagementSettings { + hosted_keys: Some(vec!["KEY".into()]), + ..Default::default() + }, + )), + Arc::new(|_: &str| None), + litellm_secrets::OidcResolver::default(), + ); + assert_eq!( + found_resolver + .get_secret_str("KEY", None) + .await + .unwrap() + .unwrap() + .expose(), + "remote" + ); + + let missing_server = MockServer::start().await; + Mock::given(method("GET")) + .respond_with( + ResponseTemplate::new(404).set_body_json(serde_json::json!({"errors": ["missing"]})), + ) + .mount(&missing_server) + .await; + let missing_environment: Arc = Arc::new({ + let address = missing_server.uri(); + move |name: &str| match name { + "HCP_VAULT_ADDR" => Some(address.clone()), + "HCP_VAULT_TOKEN" => Some("token".into()), + _ => None, + } + }); + let missing_config = + HashicorpVaultConfig::from_environment(missing_environment.as_ref()).unwrap(); + let missing_manager = HashicorpVault::from_config(missing_config, true).unwrap(); + let missing_state = SecretManagerState::new( + SecretManager::HashicorpVault(missing_manager), + KeyManagementSettings { + hosted_keys: Some(vec!["KEY".into()]), + ..Default::default() + }, + ); + let missing = litellm_secrets::get_secret_from_manager( + missing_state.backend().unwrap(), + "KEY", + missing_state.settings().unwrap(), + &|_: &str| None, + ) + .await + .unwrap(); + assert!(missing.is_none()); + + let failed_server = MockServer::start().await; + Mock::given(method("GET")) + .respond_with( + ResponseTemplate::new(500).set_body_json(serde_json::json!({"errors": ["failed"]})), + ) + .mount(&failed_server) + .await; + let failed_environment: Arc = Arc::new({ + let address = failed_server.uri(); + move |name: &str| match name { + "HCP_VAULT_ADDR" => Some(address.clone()), + "HCP_VAULT_TOKEN" => Some("token".into()), + _ => None, + } + }); + let failed_config = + HashicorpVaultConfig::from_environment(failed_environment.as_ref()).unwrap(); + let failed_manager = HashicorpVault::from_config(failed_config, true).unwrap(); + let failed_state = SecretManagerState::new( + SecretManager::HashicorpVault(failed_manager), + KeyManagementSettings { + hosted_keys: Some(vec!["KEY".into()]), + ..Default::default() + }, + ); + let failed_resolver = SecretResolver::new( + Arc::new(failed_state), + Arc::new(|_: &str| None), + litellm_secrets::OidcResolver::default(), + ) + .with_failure_policy(FailurePolicy::Propagate); + assert!(matches!( + failed_resolver.get_secret_str("KEY", None).await, + Err(Error::Hashicorp( + litellm_secrets::hashicorp::Error::Status { status: 500 } + )) + )); +} #[cfg(feature = "azure")] #[tokio::test] diff --git a/litellm/__init__.py b/litellm/__init__.py index 44515472648..a044676a843 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -2146,6 +2146,10 @@ if TYPE_CHECKING: from .llms.edenai.videos.transformation import ( EdenAIVideoConfig as EdenAIVideoConfig, ) + from .llms.fal_ai.chat.transformation import ( + FalAIChatConfig as FalAIChatConfig, + FalAIError as FalAIError, + ) from .llms.ovhcloud.chat.transformation import ( OVHCloudChatConfig as OVHCloudChatConfig, ) diff --git a/litellm/_lazy_imports_registry.py b/litellm/_lazy_imports_registry.py index db4eb8bdb33..9a53273c9d5 100644 --- a/litellm/_lazy_imports_registry.py +++ b/litellm/_lazy_imports_registry.py @@ -335,6 +335,8 @@ LLM_CONFIG_NAMES: Final = ( "EdenAITextToSpeechConfig", "EdenAIImageGenerationConfig", "EdenAIVideoConfig", + "FalAIChatConfig", + "FalAIError", "OVHCloudChatConfig", "OVHCloudEmbeddingConfig", "CometAPIEmbeddingConfig", @@ -1251,6 +1253,8 @@ _LLM_CONFIGS_IMPORT_MAP: Final = { "EdenAITextToSpeechConfig": (".llms.edenai.text_to_speech.transformation", "EdenAITextToSpeechConfig"), "EdenAIImageGenerationConfig": (".llms.edenai.image_generation.transformation", "EdenAIImageGenerationConfig"), "EdenAIVideoConfig": (".llms.edenai.videos.transformation", "EdenAIVideoConfig"), + "FalAIChatConfig": (".llms.fal_ai.chat.transformation", "FalAIChatConfig"), + "FalAIError": (".llms.fal_ai.chat.transformation", "FalAIError"), "OVHCloudChatConfig": (".llms.ovhcloud.chat.transformation", "OVHCloudChatConfig"), "OVHCloudEmbeddingConfig": ( ".llms.ovhcloud.embedding.transformation", diff --git a/litellm/anthropic_interface/exceptions/exceptions.py b/litellm/anthropic_interface/exceptions/exceptions.py index 91bcf82f455..b48cd2fee6f 100644 --- a/litellm/anthropic_interface/exceptions/exceptions.py +++ b/litellm/anthropic_interface/exceptions/exceptions.py @@ -25,6 +25,7 @@ class AnthropicErrorDetail(TypedDict): type: AnthropicErrorType message: str provider_specific_fields: NotRequired[ReadOnly[Mapping[str, object]]] + litellm_call_id: NotRequired[ReadOnly[str]] class AnthropicErrorResponse(TypedDict, total=False): diff --git a/litellm/caching/qdrant_semantic_cache.py b/litellm/caching/qdrant_semantic_cache.py index 058cc8a1579..b99023c07fd 100644 --- a/litellm/caching/qdrant_semantic_cache.py +++ b/litellm/caching/qdrant_semantic_cache.py @@ -12,6 +12,7 @@ import ast import asyncio import json import os +from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Protocol, cast import litellm @@ -36,6 +37,8 @@ from ._embedding_router import ( ) from .base_cache import BaseCache +_WAIT_FOR_INDEXING: Final = MappingProxyType({"wait": "true"}) + if TYPE_CHECKING: from litellm.router import Router @@ -313,6 +316,7 @@ class QdrantSemanticCache(BaseCache): self.sync_client.put( url=f"{self.qdrant_api_base}/collections/{self.collection_name}/points", headers=self.headers, + params=_WAIT_FOR_INDEXING, json=data, ) @@ -422,6 +426,7 @@ class QdrantSemanticCache(BaseCache): await self.async_client.put( url=f"{self.qdrant_api_base}/collections/{self.collection_name}/points", headers=self.headers, + params=_WAIT_FOR_INDEXING, json=data, ) diff --git a/litellm/constants.py b/litellm/constants.py index 88cc5b04743..1012ca831f2 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -1749,6 +1749,12 @@ SPEND_LOG_CLEANUP_BATCH_FAILURE_BACKOFF_SECONDS: Final = float( SPEND_LOG_CLEANUP_RUN_BUDGET_SECONDS: Final = float(os.getenv("SPEND_LOG_CLEANUP_RUN_BUDGET_SECONDS", "300")) SPEND_LOG_CLEANUP_BATCH_TIMEOUT_SECONDS: Final = float(os.getenv("SPEND_LOG_CLEANUP_BATCH_TIMEOUT_SECONDS", "30")) SPEND_LOG_CLEANUP_REMAINING_COUNT_CAP: Final = int(os.getenv("SPEND_LOG_CLEANUP_REMAINING_COUNT_CAP", "100000")) +SCHEDULED_JOB_SHUTDOWN_FINISH_TIMEOUT_SECONDS: Final = float( + os.getenv("SCHEDULED_JOB_SHUTDOWN_FINISH_TIMEOUT_SECONDS", "5") +) +SCHEDULED_JOB_SHUTDOWN_CANCEL_TIMEOUT_SECONDS: Final = float( + os.getenv("SCHEDULED_JOB_SHUTDOWN_CANCEL_TIMEOUT_SECONDS", "5") +) TOOL_SPEND_TOP_TOOLS: Final = 100 SPEND_LOG_PARTITION_INTERVAL: Final = os.getenv("SPEND_LOG_PARTITION_INTERVAL", "day") SPEND_LOG_PARTITION_PRECREATE_AHEAD: Final = int(os.getenv("SPEND_LOG_PARTITION_PRECREATE_AHEAD", 7)) @@ -1860,6 +1866,12 @@ LITELLM_SETTINGS_SAFE_DB_OVERRIDES: Final = [ "max_ui_session_budget", "budget_rollover", "mcp_tool_search", + "turn_off_message_logging", + "datadog_params", + "datadog_llm_observability_params", + "newrelic_params", + "pointfive_params", + "aws_sqs_callback_params", ] SPECIAL_LITELLM_AUTH_TOKEN: Final = ["ui-token"] DEFAULT_MANAGEMENT_OBJECT_IN_MEMORY_CACHE_TTL = int(os.getenv("DEFAULT_MANAGEMENT_OBJECT_IN_MEMORY_CACHE_TTL", 60)) @@ -2121,3 +2133,6 @@ BATCH_ENQUEUED_TOKEN_LIMIT_METADATA_KEY: Final = "batch_enqueued_token_limit" # Shared read-only empty mapping, for defaulting optional Mapping parameters without # constructing a fresh mutable dict at each call site. EMPTY_MAPPING: Final = MappingProxyType({}) + +# API endpoint for breached password k-anonymity search +HIBP_RANGE_API_BASE: Final = "https://api.pwnedpasswords.com/range" diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index b317e356e1d..37743a9ce33 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -948,7 +948,7 @@ def _extract_service_tier(source: object) -> str | None: return None -def _get_usage_object( +def get_usage_object( completion_response: object, ) -> Usage | None: usage_obj: Final = cast( @@ -1336,7 +1336,7 @@ def completion_cost( cache_creation_input_tokens: int | None = None cache_read_input_tokens: int | None = None audio_transcription_file_duration: float = 0.0 - provider_usage_object: Final = _get_usage_object(completion_response=completion_response) + provider_usage_object: Final = get_usage_object(completion_response=completion_response) cost_per_token_usage_object: Final[Usage | None] = ( _without_provider_stated_cost(provider_usage_object) if custom_pricing else provider_usage_object ) @@ -2033,6 +2033,45 @@ def _cost_map_model_info(model: str, custom_llm_provider: str | None) -> ModelIn return None +def _raw_cost_map_entry(key: str) -> Mapping[str, object] | None: + raw_entry: Final = litellm.model_cost.get(key) + return raw_entry if isinstance(raw_entry, Mapping) else None + + +def pricing_entry_for_cost_calc( + model: str | None, + completion_response: object | None, + custom_llm_provider: str | None, + custom_pricing: bool | None, + base_model: str | None, + router_model_id: str | None, + region_name: str | None, + litellm_logging_obj: LitellmLoggingObject | None, +) -> tuple[str, Mapping[str, object]] | None: + deployment_entry: Final = _deployment_model_info(litellm_logging_obj, custom_pricing, router_model_id) + deployment_key: Final = router_model_id or model + if deployment_entry is not None and deployment_key is not None: + registered_entry: Final = _raw_cost_map_entry(router_model_id) if router_model_id is not None else None + return deployment_key, registered_entry or deployment_entry + selected_model: Final = _select_model_name_for_cost_calc( + model=model, + completion_response=completion_response, + base_model=base_model, + custom_pricing=custom_pricing, + custom_llm_provider=custom_llm_provider, + router_model_id=router_model_id, + region_name=region_name, + ) + candidates: Final = (selected_model, _get_response_model(completion_response), model) + resolved: Final = next( + (info for info in (_cost_map_model_info(name, custom_llm_provider) for name in candidates if name) if info), + None, + ) + if resolved is None: + return None + return resolved["key"], _raw_cost_map_entry(resolved["key"]) or resolved + + def ocr_cost( model: str, custom_llm_provider: str | None, diff --git a/litellm/integrations/prometheus.py b/litellm/integrations/prometheus.py index 37b7344917e..28ac9f5cdae 100644 --- a/litellm/integrations/prometheus.py +++ b/litellm/integrations/prometheus.py @@ -10,6 +10,7 @@ import sys from collections.abc import Awaitable, Callable, Mapping, Sequence from dataclasses import replace from datetime import datetime, timedelta +from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Literal, Protocol, TypeAlias, TypeVar, cast from pydantic import BaseModel @@ -66,6 +67,7 @@ from litellm.types.proxy.carried_budget_state import ( from litellm.types.utils import ( StandardLoggingGuardrailInformation, StandardLoggingPayload, + StandardLoggingZeroCostDiagnostic, ) if TYPE_CHECKING: @@ -713,6 +715,15 @@ class PrometheusLogger(CustomLogger): labelnames=self.get_labels_for_metric("litellm_requests_metric"), ) + self.litellm_zero_cost_requests_total = self._counter_factory( + name="litellm_zero_cost_requests_total", + documentation=( + "Requests that carried usage but were logged at $0 on a model whose pricing entry " + "has a non-zero rate, by reason (missing_pricing_key, pricing_not_applied, cost_calculation_error)" + ), + labelnames=self.get_labels_for_metric("litellm_zero_cost_requests_total"), + ) + # Cache metrics self.litellm_cache_hits_metric = self._counter_factory( name="litellm_cache_hits_metric", @@ -1410,6 +1421,11 @@ class PrometheusLogger(CustomLogger): enum_values=enum_values, label_context=label_context, ) + self._increment_zero_cost_requests_metric( + zero_cost_diagnostic=standard_logging_payload.get("zero_cost_diagnostic"), + enum_values=enum_values, + label_context=label_context, + ) # input, output, total token metrics self._increment_token_metrics( @@ -1983,6 +1999,30 @@ class PrometheusLogger(CustomLogger): amount=float(response_cost), ) + def _increment_zero_cost_requests_metric( + self, + zero_cost_diagnostic: StandardLoggingZeroCostDiagnostic | None, + enum_values: UserAPIKeyLabelValues, + label_context: PrometheusLabelFactoryContext, + ) -> None: + if zero_cost_diagnostic is None: + return + supported_labels: Final = self.get_labels_for_metric("litellm_zero_cost_requests_total") + reason_label: Final = ( + MappingProxyType({ZERO_COST_REASON_LABEL: zero_cost_diagnostic["reason"]}) + if ZERO_COST_REASON_LABEL in supported_labels + else MappingProxyType({}) + ) + labels: Final = MappingProxyType( + { + **prometheus_label_factory( + supported_enum_labels=supported_labels, enum_values=enum_values, label_context=label_context + ), + **reason_label, + } + ) + self.litellm_zero_cost_requests_total.labels(**labels).inc() + @staticmethod def _get_remaining_from_v3_rate_limit_headers( standard_logging_payload: StandardLoggingPayload | None, @@ -2333,6 +2373,8 @@ class PrometheusLogger(CustomLogger): team_alias=user_api_team_alias, user=user_id, model_id=standard_logging_payload.get("model_id", ""), + requested_model=standard_logging_payload.get("model_group"), + api_provider=standard_logging_payload.get("custom_llm_provider"), custom_metadata_labels=get_custom_labels_from_metadata( metadata=_get_combined_custom_metadata_from_standard_logging_payload( standard_logging_payload=standard_logging_payload @@ -2345,6 +2387,11 @@ class PrometheusLogger(CustomLogger): "litellm_llm_api_failed_requests_metric", enum_values, ) + self._increment_zero_cost_requests_metric( + zero_cost_diagnostic=standard_logging_payload.get("zero_cost_diagnostic"), + enum_values=enum_values, + label_context=PrometheusLabelFactoryContext(enum_values), + ) self.set_llm_deployment_failure_metrics(kwargs) await self._set_org_budget_metrics_after_api_request( org_id=user_api_key_org_id, diff --git a/litellm/litellm_core_utils/agentic_followup_kwargs.py b/litellm/litellm_core_utils/agentic_followup_kwargs.py new file mode 100644 index 00000000000..50ec19f62c4 --- /dev/null +++ b/litellm/litellm_core_utils/agentic_followup_kwargs.py @@ -0,0 +1,32 @@ +from collections.abc import Collection, Mapping, Sequence +from itertools import chain +from types import MappingProxyType +from typing import Final + + +def build_agentic_followup_kwargs( + *, + request_kwargs: Mapping[str, object], + patch_kwargs: Mapping[str, object], + request_params: Collection[str], + depth: int, + max_loops: int, + fingerprints: Sequence[str], + fingerprint: str, +) -> Mapping[str, object]: + """Kwargs for an agentic follow-up call: the request's kwargs overlaid by the plan's, never repeating a key already sent as a request param""" + seen: Final = [*fingerprints, fingerprint] # mutable-ok: the chat loop's settings reader only accepts a list + return MappingProxyType( + { + key: value + for key, value in chain( + ((k, v) for k, v in request_kwargs.items() if k not in request_params), + ((k, v) for k, v in patch_kwargs.items() if k not in request_params), + ( + ("_agentic_loop_depth", depth + 1), + ("max_agentic_loops", max_loops), + ("_agentic_loop_fingerprints", seen), + ), + ) + } + ) diff --git a/litellm/litellm_core_utils/chat_completion_agentic_loop.py b/litellm/litellm_core_utils/chat_completion_agentic_loop.py index c8e9e2583ba..e0bd85a7937 100644 --- a/litellm/litellm_core_utils/chat_completion_agentic_loop.py +++ b/litellm/litellm_core_utils/chat_completion_agentic_loop.py @@ -2,10 +2,13 @@ import json from collections.abc import Mapping +from itertools import chain +from types import MappingProxyType from typing import Final, cast from litellm._logging import verbose_logger from litellm.integrations.custom_logger import CustomLogger +from litellm.litellm_core_utils.agentic_followup_kwargs import build_agentic_followup_kwargs from litellm.litellm_core_utils.agentic_loop_settings import ( DEFAULT_MAX_AGENTIC_LOOPS, validated_max_agentic_loops, @@ -117,13 +120,25 @@ def _wrap_response_as_fake_stream( ) -def _add_agentic_loop_metadata(kwargs_for_followup: dict[str, object]) -> None: - metadata = kwargs_for_followup.get("litellm_metadata") - metadata = dict(metadata) if isinstance(metadata, dict) else {} - for key, value in kwargs_for_followup.items(): - if key.startswith("_agentic_loop") or key == "max_agentic_loops" or is_interception_internal_key(key): - metadata[key] = value - kwargs_for_followup["litellm_metadata"] = metadata +def _with_agentic_loop_metadata(kwargs_for_followup: Mapping[str, object]) -> Mapping[str, object]: + metadata: Final = kwargs_for_followup.get("litellm_metadata") + return MappingProxyType( + { + **kwargs_for_followup, + "litellm_metadata": dict( # mutable-ok: the follow-up call's logging and proxy hooks write into litellm_metadata in place + chain( + metadata.items() if isinstance(metadata, dict) else (), + ( + (key, value) + for key, value in kwargs_for_followup.items() + if key.startswith("_agentic_loop") + or key == "max_agentic_loops" + or is_interception_internal_key(key) + ), + ) + ), + } + ) def _filter_followup_kwargs(source: dict[str, object]) -> dict[str, object]: @@ -165,14 +180,17 @@ async def _execute_chat_completion_agentic_plan( if "tool_choice" not in patch.optional_params: optional_params_for_followup.pop("tool_choice", None) - kwargs_for_followup: Final = _filter_followup_kwargs(kwargs) - kwargs_for_followup.update( - {k: v for k, v in _filter_followup_kwargs(patch.kwargs).items() if k not in optional_params_for_followup} + kwargs_for_followup: Final = _with_agentic_loop_metadata( + build_agentic_followup_kwargs( + request_kwargs=_filter_followup_kwargs(kwargs), + patch_kwargs=_filter_followup_kwargs(patch.kwargs), + request_params=frozenset((*optional_params_for_followup, "model", "messages")), + depth=depth, + max_loops=max_loops, + fingerprints=fingerprints, + fingerprint=fingerprint, + ) ) - kwargs_for_followup["_agentic_loop_depth"] = depth + 1 - kwargs_for_followup["max_agentic_loops"] = max_loops - kwargs_for_followup["_agentic_loop_fingerprints"] = fingerprints + [fingerprint] - _add_agentic_loop_metadata(kwargs_for_followup) try: response_followup = await litellm.acompletion( diff --git a/litellm/litellm_core_utils/core_helpers.py b/litellm/litellm_core_utils/core_helpers.py index ce6f77f78a0..5bcde688521 100644 --- a/litellm/litellm_core_utils/core_helpers.py +++ b/litellm/litellm_core_utils/core_helpers.py @@ -359,6 +359,46 @@ def get_litellm_metadata_from_kwargs(kwargs: dict): return {} +def _budget_reservation_on_auth_object(user_api_key_auth: object) -> object: + if isinstance(user_api_key_auth, Mapping): + return user_api_key_auth.get("budget_reservation") + return getattr(user_api_key_auth, "budget_reservation", None) + + +def budget_reservation_from_metadata(metadata: Mapping[str, object]) -> dict | None: + stamped: Final = metadata.get("user_api_key_budget_reservation") + if isinstance(stamped, dict): + return stamped + on_auth_object: Final = _budget_reservation_on_auth_object(metadata.get("user_api_key_auth")) + return on_auth_object if isinstance(on_auth_object, dict) else None + + +def _stamp_budget_reservation_callback_bound(litellm_params: Mapping[str, object], callback_bound: bool) -> None: + for metadata_variable_name in ("metadata", "litellm_metadata"): + metadata = litellm_params.get(metadata_variable_name) + if not isinstance(metadata, Mapping): + continue + budget_reservation = budget_reservation_from_metadata(metadata) + if budget_reservation is not None: + budget_reservation["callback_bound"] = callback_bound + + +def bind_budget_reservation_to_callbacks(litellm_params: Mapping[str, object]) -> None: + """Mark the request's budget reservation as owned by the success callbacks of this call. + + The proxy releases any reservation still unbound when the request ends; one bound here + is left for the cost callback, which may finish after the response has been sent. Bind + only where a success handler is guaranteed to run: a logging object merely existing is + not that, since the proxy builds one for every route before calling anything. + """ + _stamp_budget_reservation_callback_bound(litellm_params, True) + + +def unbind_budget_reservation_from_callbacks(litellm_params: Mapping[str, object]) -> None: + """Hand a failed call's reservation back to the request-end release: failure handlers never settle it.""" + _stamp_budget_reservation_callback_bound(litellm_params, False) + + def reconstruct_model_name( model_name: str, custom_llm_provider: str | None, diff --git a/litellm/litellm_core_utils/get_litellm_params.py b/litellm/litellm_core_utils/get_litellm_params.py index 9b2db9aad18..36fd7fa4e61 100644 --- a/litellm/litellm_core_utils/get_litellm_params.py +++ b/litellm/litellm_core_utils/get_litellm_params.py @@ -46,6 +46,8 @@ OPTIONAL_KWARGS_KEYS: Final = ( "bucket_name", "s3_endpoint_url", "s3_region_name", + "s3_access_key_id", + "s3_secret_access_key", "vertex_credentials", "vertex_project", "vertex_location", diff --git a/litellm/litellm_core_utils/get_llm_provider_logic.py b/litellm/litellm_core_utils/get_llm_provider_logic.py index 5868e79323a..192679957b1 100644 --- a/litellm/litellm_core_utils/get_llm_provider_logic.py +++ b/litellm/litellm_core_utils/get_llm_provider_logic.py @@ -859,6 +859,9 @@ def _get_openai_compatible_provider_info( elif custom_llm_provider == "edenai": api_base = litellm.EdenAIChatConfig.get_api_base(api_base) # rebind-ok: chain resolves in place dynamic_api_key = litellm.EdenAIChatConfig.get_api_key(api_key) # rebind-ok: chain resolves in place + elif custom_llm_provider == "fal_ai": + api_base = litellm.FalAIChatConfig.get_api_base(api_base) # rebind-ok: chain resolves in place + dynamic_api_key = litellm.FalAIChatConfig.get_api_key(api_key) # rebind-ok: chain resolves in place elif custom_llm_provider == "aiml": ( api_base, diff --git a/litellm/litellm_core_utils/get_model_cost_map.py b/litellm/litellm_core_utils/get_model_cost_map.py index 91a22144805..5471fe50d5f 100644 --- a/litellm/litellm_core_utils/get_model_cost_map.py +++ b/litellm/litellm_core_utils/get_model_cost_map.py @@ -17,14 +17,16 @@ import random import sys import threading import time -from collections.abc import Awaitable, Callable +from collections.abc import Awaitable, Callable, Mapping from dataclasses import dataclass, replace from datetime import datetime, timezone from importlib.resources import files from pathlib import Path +from types import MappingProxyType from typing import Final, Protocol import httpx +from pydantic import TypeAdapter from typing_extensions import ReadOnly, TypedDict from litellm import verbose_logger @@ -37,6 +39,7 @@ from litellm.litellm_core_utils.fallback_generalizations import ( ) FALLBACK_GENERALIZATIONS_KEY: Final = "fallback_generalizations" +_CATALOG_ADAPTER: Final = TypeAdapter(dict[str, dict[str, object]]) _CLI_ENTRYPOINT_NAMES: Final = frozenset({"lite", "litellm-proxy"}) @@ -88,6 +91,18 @@ class GetModelCostMap: """Load the local backup model cost map bundled with the package.""" return GetModelCostMap.load_local_model_cost_map_with_revision().model_cost_map + _loaded_catalog: Mapping[str, Mapping[str, object]] = MappingProxyType({}) + + @classmethod + def loaded_model_cost_map(cls) -> Mapping[str, Mapping[str, object]]: + """The catalog as last loaded (bundled or remote), untouched by ``register_model`` or router registrations.""" + return cls._loaded_catalog + + @classmethod + def _snapshot_loaded_catalog(cls, model_cost: Mapping[str, object]) -> None: + raw: Final = _CATALOG_ADAPTER.validate_python(model_cost) + cls._loaded_catalog = MappingProxyType({key: MappingProxyType(entry) for key, entry in raw.items()}) + @classmethod def _get_backup_model_count(cls) -> int: """Return the number of models in the local backup (cached int).""" @@ -533,7 +548,9 @@ def _finalize_model_cost_map(model_cost: dict) -> dict: def _finalize_loaded_model_cost_map(loaded: ModelCostMapReloaded) -> ModelCostMapReloaded: _cost_map_source_info.source_revision = loaded.revision _cost_map_source_info.etag = loaded.etag - return replace(loaded, model_cost_map=_finalize_model_cost_map(loaded.model_cost_map)) + finalized: Final = _finalize_model_cost_map(loaded.model_cost_map) + GetModelCostMap._snapshot_loaded_catalog(finalized) # pyright: ignore[reportPrivateUsage] # same module + return replace(loaded, model_cost_map=finalized) def adopt_model_cost_map( diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index b34f1b3aafd..f5967185af0 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -50,6 +50,8 @@ from litellm.cost_calculator import ( RealtimeAPITokenUsageProcessor, ResponsesWebSocketTokenUsageProcessor, _select_model_name_for_cost_calc, + get_usage_object, + pricing_entry_for_cost_calc, ) from litellm.exceptions import ( BudgetExceededError, @@ -89,6 +91,10 @@ from litellm.litellm_core_utils.llm_cost_calc.tool_call_cost_tracking import ( from litellm.litellm_core_utils.llm_cost_calc.usage_object_transformation import ( InteractionsUsageObjectTransformation, ) +from litellm.litellm_core_utils.llm_cost_calc.zero_cost_diagnostic import ( + diagnose_zero_cost, + zero_cost_warning, +) from litellm.litellm_core_utils.logging_utils import ( truncate_base64_in_messages, truncate_base64_in_messages_async, @@ -157,6 +163,7 @@ from litellm.types.utils import ( StandardLoggingPayloadStatusFields, StandardLoggingPromptManagementMetadata, StandardLoggingVectorStoreRequest, + StandardLoggingZeroCostDiagnostic, TextCompletionResponse, TranscriptionResponse, Usage, @@ -614,6 +621,7 @@ class Logging(LiteLLMLoggingBaseClass): self.truncated_messages_for_logging: str | list | dict | None = None # mutable-ok: logged messages shape ## TIME TO FIRST TOKEN LOGGING ## self.completion_start_time: datetime.datetime | None = None + self.zero_cost_warned: bool = False self._llm_caching_handler: LLMCachingHandler | None = None # INITIAL LITELLM_PARAMS @@ -1757,17 +1765,26 @@ class Logging(LiteLLMLoggingBaseClass): if transformed_result is not None: result = transformed_result - result_hidden_params: Final = getattr(result, "_hidden_params", None) or MappingProxyType({}) - result_additional_headers: Final = ( - result_hidden_params.get("additional_headers") - if isinstance(result_hidden_params, dict) - else getattr(result_hidden_params, "additional_headers", None) + priced_result: Final = ( + result.response + if isinstance(result, (ResponseCompletedEvent, ResponseIncompleteEvent, ResponseFailedEvent)) + else result ) - if isinstance(result, (BaseModel, HttpxBinaryResponseContent)) and hasattr(result, "_hidden_params"): + + result_hidden_params: Final = getattr(priced_result, "_hidden_params", None) or MappingProxyType({}) + if isinstance(priced_result, (BaseModel, HttpxBinaryResponseContent)) and hasattr( + priced_result, "_hidden_params" + ): hidden_params: Final = result_hidden_params if ( "response_cost" in hidden_params and hidden_params["response_cost"] is not None ): # use cost if already calculated + self._record_zero_cost_diagnostic( + priced_result, + hidden_params["response_cost"], + litellm_model_name=litellm_model_name, + router_model_id=router_model_id or hidden_params.get("model_id"), + ) return hidden_params["response_cost"] elif router_model_id is None and "model_id" in hidden_params: # use model_id if not already set router_model_id = hidden_params["model_id"] @@ -1779,18 +1796,7 @@ class Logging(LiteLLMLoggingBaseClass): router_model_id = self.get_router_model_id() ## RESPONSE COST ## - spilled_over: Final = is_spilled_over_ptu_request( - model_info=_deployment_model_info(self.litellm_params if hasattr(self, "litellm_params") else None), - response_headers=self.model_call_details.get("response_headers"), - additional_headers=result_additional_headers, - ) - custom_pricing: Final = ( - False - if spilled_over - else use_custom_pricing_for_model( - litellm_params=(self.litellm_params if hasattr(self, "litellm_params") else None) - ) - ) + custom_pricing: Final = self._custom_pricing_for(priced_result) prompt = self._prompt_for_cost_calculation() @@ -1799,7 +1805,7 @@ class Logging(LiteLLMLoggingBaseClass): try: response_cost_calculator_kwargs: Final = { - "response_object": result, + "response_object": priced_result, "model": litellm_model_name or self.model, "cache_hit": cache_hit, "custom_llm_provider": self.model_call_details.get("custom_llm_provider", None), @@ -1842,9 +1848,18 @@ class Logging(LiteLLMLoggingBaseClass): verbose_logger.debug("response_cost: %s", response_cost) additional_response_cost: Final[object] = self.model_call_details.get("additional_response_cost") - if isinstance(additional_response_cost, (int, float)) and additional_response_cost > 0: - return (response_cost or 0.0) + additional_response_cost - return response_cost + total_response_cost: Final = ( + (response_cost or 0.0) + additional_response_cost + if isinstance(additional_response_cost, (int, float)) and additional_response_cost > 0 + else response_cost + ) + self._record_zero_cost_diagnostic( + priced_result, + total_response_cost, + litellm_model_name=litellm_model_name, + router_model_id=router_model_id, + ) + return total_response_cost except Exception as e: # error calculating cost debug_info = StandardLoggingModelCostFailureDebugInformation( error_str=str(e), @@ -1858,9 +1873,108 @@ class Logging(LiteLLMLoggingBaseClass): ) verbose_logger.debug("response_cost_failure_debug_information: %s", debug_info) self.model_call_details["response_cost_failure_debug_information"] = debug_info + self._record_zero_cost_diagnostic( + priced_result, + None, + calculation_failed=True, + litellm_model_name=litellm_model_name, + router_model_id=router_model_id, + ) return None + def _record_zero_cost_diagnostic( + self, + result: object, + response_cost: float | None, + *, + calculation_failed: bool = False, + litellm_model_name: str | None = None, + router_model_id: str | None = None, + ) -> None: + if response_cost is None and not calculation_failed: + return + if self.model_call_details.get("cache_hit") is True: + self.model_call_details["zero_cost_diagnostic"] = None + return + try: + finding: Final = self._zero_cost_finding( + result, + response_cost, + calculation_failed=calculation_failed, + litellm_model_name=litellm_model_name, + router_model_id=router_model_id, + ) + except Exception as e: # noqa: BLE001 # the pricing helpers raise plain Exception and a diagnostic must never break cost tracking + verbose_logger.debug("zero_cost_diagnostic skipped: %s", e) + return + self.model_call_details["zero_cost_diagnostic"] = finding[0] if finding is not None else None + if finding is None or self.zero_cost_warned: + return + self.zero_cost_warned = True + verbose_logger.warning(finding[1]) + + def _zero_cost_finding( + self, + result: object, + response_cost: float | None, + *, + calculation_failed: bool, + litellm_model_name: str | None, + router_model_id: str | None, + ) -> tuple[StandardLoggingZeroCostDiagnostic, str] | None: + metadata: Final = StandardLoggingPayloadSetup.merge_litellm_metadata(self.litellm_params) + if response_cost or is_unbilled_non_inference_call(self.call_type, metadata, result): + return None + usage: Final = get_usage_object(completion_response=result) + if usage is None: + return None + model: Final = litellm_model_name or self.model + custom_llm_provider: Final = self.model_call_details.get("custom_llm_provider") + pricing: Final = pricing_entry_for_cost_calc( + model=model, + completion_response=result, + custom_llm_provider=custom_llm_provider, + custom_pricing=self._custom_pricing_for(result), + base_model=_get_base_model_from_metadata(model_call_details=self.model_call_details), + router_model_id=router_model_id or self.get_router_model_id(), + region_name=_resolve_mantle_region_for_cost( + custom_llm_provider=custom_llm_provider, + litellm_params=self.model_call_details.get("litellm_params"), + ), + litellm_logging_obj=self, + ) + if pricing is None: + return None + diagnostic: Final = diagnose_zero_cost( + usage=usage, pricing_model=pricing[0], pricing_entry=pricing[1], calculation_failed=calculation_failed + ) + if diagnostic is None: + return None + model_group: Final = metadata.get("model_group") + return diagnostic, zero_cost_warning( + diagnostic, + model_group=model_group if isinstance(model_group, str) else None, + model=model, + custom_llm_provider=custom_llm_provider, + usage=usage, + ) + + def _custom_pricing_for(self, result: object) -> bool: + litellm_params: Final = getattr(self, "litellm_params", None) + result_hidden_params: Final = getattr(result, "_hidden_params", None) or MappingProxyType({}) + additional_headers: Final = ( + result_hidden_params.get("additional_headers") + if isinstance(result_hidden_params, dict) + else getattr(result_hidden_params, "additional_headers", None) + ) + spilled_over: Final = is_spilled_over_ptu_request( + model_info=_deployment_model_info(litellm_params), + response_headers=self.model_call_details.get("response_headers"), + additional_headers=additional_headers, + ) + return False if spilled_over else use_custom_pricing_for_model(litellm_params=litellm_params) + def _prompt_for_cost_calculation(self) -> str: """ The raw input string is only priced directly for text-to-speech, which bills per character. @@ -2205,6 +2319,7 @@ class Logging(LiteLLMLoggingBaseClass): self.model_call_details["response_cost"] = 0.0 elif "response_cost" in hidden_params: self.model_call_details["response_cost"] = hidden_params["response_cost"] + self._record_zero_cost_diagnostic(logging_result, hidden_params["response_cost"]) elif (existing_cost := self.model_call_details.get("response_cost")) is not None and existing_cost != 0: # Preserve response_cost if already calculated (e.g., by pass-through # handlers like Gemini/Vertex which call completion_cost directly). @@ -5378,7 +5493,7 @@ def request_model_access_groups_from_litellm_params(litellm_params: Mapping[str, """Access groups the auth layer stamped onto this request, from whichever metadata field carries them. Detached internal sub-calls only inherit the identity keys, so the auth object is the - fallback there, exactly as _get_budget_reservation_from_metadata does for reservations. + fallback there, exactly as budget_reservation_from_metadata does for reservations. """ for metadata_variable_name in ("metadata", "litellm_metadata"): metadata = litellm_params.get(metadata_variable_name) @@ -6499,6 +6614,7 @@ def get_standard_logging_object_payload( error_str=error_str, error_information=error_information, response_cost_failure_debug_info=kwargs.get("response_cost_failure_debug_information"), + zero_cost_diagnostic=kwargs.get("zero_cost_diagnostic"), guardrail_information=metadata.get("standard_logging_guardrail_information", None), standard_built_in_tools_params=standard_built_in_tools_params, ) @@ -6677,6 +6793,7 @@ def create_dummy_standard_logging_payload() -> StandardLoggingPayload: response_cost=response_cost, autorouter_savings=None, response_cost_failure_debug_info=None, + zero_cost_diagnostic=None, status="success", total_tokens=int(DEFAULT_MOCK_RESPONSE_PROMPT_TOKEN_COUNT + DEFAULT_MOCK_RESPONSE_COMPLETION_TOKEN_COUNT), prompt_tokens=int(DEFAULT_MOCK_RESPONSE_PROMPT_TOKEN_COUNT), diff --git a/litellm/litellm_core_utils/llm_cost_calc/zero_cost_diagnostic.py b/litellm/litellm_core_utils/llm_cost_calc/zero_cost_diagnostic.py new file mode 100644 index 00000000000..6331d815bdc --- /dev/null +++ b/litellm/litellm_core_utils/llm_cost_calc/zero_cost_diagnostic.py @@ -0,0 +1,146 @@ +from collections.abc import Mapping +from functools import reduce +from typing import Final + +from pydantic import TypeAdapter, ValidationError +from typing_extensions import assert_never + +from litellm.types.utils import StandardLoggingZeroCostDiagnostic, Usage + +ZERO_COST_COUNTER_NAME: Final = "litellm_zero_cost_requests_total" + +_TEXT_INPUT_RATE: Final = "input_cost_per_token" +_AUDIO_INPUT_RATE: Final = "input_cost_per_audio_token" +_TEXT_OUTPUT_RATE: Final = "output_cost_per_token" +_AUDIO_OUTPUT_RATE: Final = "output_cost_per_audio_token" +_RATE_KEY_MARKERS: Final = ("cost", "pricing") +_NESTED_PRICING: Final = TypeAdapter(Mapping[str, object] | tuple[object, ...]) +_MAX_PRICING_DEPTH: Final = 4 + + +def _audio_tokens(details: object) -> int: + audio_tokens: Final = getattr(details, "audio_tokens", None) + return audio_tokens if isinstance(audio_tokens, int) and audio_tokens > 0 else 0 + + +def _tokens(value: object) -> int: + return value if isinstance(value, int) and value > 0 else 0 + + +def used_pricing_keys(usage: Usage) -> tuple[str, ...]: + prompt_audio: Final = _audio_tokens(usage.prompt_tokens_details) + completion_audio: Final = _audio_tokens(usage.completion_tokens_details) + prompt_text: Final = _tokens(usage.prompt_tokens) - prompt_audio + completion_text: Final = _tokens(usage.completion_tokens) - completion_audio + components: Final = ( + (_TEXT_INPUT_RATE, prompt_text), + (_AUDIO_INPUT_RATE, prompt_audio), + (_TEXT_OUTPUT_RATE, completion_text), + (_AUDIO_OUTPUT_RATE, completion_audio), + ) + return tuple(key for key, count in components if count > 0) + + +def _nested_pricing(value: object) -> Mapping[str, object] | tuple[object, ...] | None: + try: + return _NESTED_PRICING.validate_python(value) + except ValidationError: + return None + + +def _is_rate_key(key: str) -> bool: + return any(marker in key for marker in _RATE_KEY_MARKERS) + + +def _rate_values(value: object) -> tuple[object, ...]: + nested: Final = _nested_pricing(value) + if isinstance(nested, Mapping): + return tuple(child for key, child in nested.items() if _is_rate_key(key)) + if nested is None: + return (value,) + return nested + + +def _expand_rate_values(values: tuple[object, ...], _depth: int) -> tuple[object, ...]: + return tuple(nested for value in values for nested in _rate_values(value)) + + +def _is_positive_number(value: object) -> bool: + return not isinstance(value, bool) and isinstance(value, (int, float)) and value > 0 + + +def _declares_a_rate(pricing_entry: Mapping[str, object]) -> bool: + leaves: Final = reduce(_expand_rate_values, range(_MAX_PRICING_DEPTH), (pricing_entry,)) + return any(_is_positive_number(leaf) for leaf in leaves) + + +def _is_explicit_zero(value: object) -> bool: + return not isinstance(value, bool) and isinstance(value, (int, float)) and value == 0 + + +def diagnose_zero_cost( + usage: Usage, + pricing_model: str, + pricing_entry: Mapping[str, object], + calculation_failed: bool, +) -> StandardLoggingZeroCostDiagnostic | None: + used_keys: Final = used_pricing_keys(usage) + if not used_keys: + return None + missing_keys: Final = tuple(key for key in used_keys if pricing_entry.get(key) is None) + if not missing_keys and all(_is_explicit_zero(pricing_entry[key]) for key in used_keys): + return None + if not _declares_a_rate(pricing_entry): + return None + if calculation_failed: + return StandardLoggingZeroCostDiagnostic( + reason="cost_calculation_error", pricing_model=pricing_model, missing_pricing_keys=() + ) + if missing_keys: + return StandardLoggingZeroCostDiagnostic( + reason="missing_pricing_key", pricing_model=pricing_model, missing_pricing_keys=missing_keys + ) + return StandardLoggingZeroCostDiagnostic( + reason="pricing_not_applied", pricing_model=pricing_model, missing_pricing_keys=() + ) + + +def _cause(diagnostic: StandardLoggingZeroCostDiagnostic) -> str: + reason: Final = diagnostic["reason"] + match reason: + case "missing_pricing_key": + return ( + f"pricing entry '{diagnostic['pricing_model']}' has no {', '.join(diagnostic['missing_pricing_keys'])}. " + "Set the missing rate in the deployment's model_info or in the model cost map, " + "or set every rate to 0 to mark the model free" + ) + case "pricing_not_applied": + return ( + f"pricing entry '{diagnostic['pricing_model']}' declares non-zero rates for this usage, " + "but the cost calculator returned $0" + ) + case "cost_calculation_error": + return ( + f"cost calculation raised for pricing entry '{diagnostic['pricing_model']}', " + "see response_cost_failure_debug_information" + ) + case _: + return assert_never(reason) + + +def zero_cost_warning( + diagnostic: StandardLoggingZeroCostDiagnostic, + *, + model_group: str | None, + model: str, + custom_llm_provider: str | None, + usage: Usage, +) -> str: + request: Final = ( + f"model_group={model_group or model} model={model} provider={custom_llm_provider or 'unknown'} " + f"prompt_tokens={_tokens(usage.prompt_tokens)} completion_tokens={_tokens(usage.completion_tokens)}" + ) + return ( + f"Billable request priced at $0 and logged as such ({request}): {_cause(diagnostic)}. " + f'Counted in {ZERO_COST_COUNTER_NAME}{{reason="{diagnostic["reason"]}"}}' + ) diff --git a/litellm/litellm_core_utils/logging_worker.py b/litellm/litellm_core_utils/logging_worker.py index 5ccc5632646..2f8e7bdccea 100644 --- a/litellm/litellm_core_utils/logging_worker.py +++ b/litellm/litellm_core_utils/logging_worker.py @@ -484,9 +484,13 @@ class LoggingWorker: so it correctly handles items that have been dequeued but whose callback hasn't finished yet — ``queue.empty()`` would return True in that window and cause us to skip the wait. + + ``start()`` runs first so a queue left behind by a previous event loop + is carried onto this one and drained here instead of joined forever. """ if self._queue is None: return + self.start() await self._queue.join() async def clear_queue(self): diff --git a/litellm/litellm_core_utils/prompt_templates/common_utils.py b/litellm/litellm_core_utils/prompt_templates/common_utils.py index 4ba7c3966c0..1e96e20a03b 100644 --- a/litellm/litellm_core_utils/prompt_templates/common_utils.py +++ b/litellm/litellm_core_utils/prompt_templates/common_utils.py @@ -2261,6 +2261,37 @@ def system_messages_first( ] +def _system_content_as_text_parts(content: object) -> tuple[object, ...]: + if isinstance(content, str): + return (ChatCompletionTextObject(type="text", text=content),) + return tuple(cast(Sequence[object], content)) # cast-ok: non-str system content is a list of content parts + + +def _merge_system_message_run(run: Sequence[AllMessageValues]) -> AllMessageValues: + if len(run) == 1: + return run[0] + contents: Final = tuple(content for content in (message.get("content") for message in run) if content is not None) + if not contents: + return run[0] + if all(isinstance(content, str) for content in contents): + joined_text: Final = "\n\n".join(cast(tuple[str, ...], contents)) # cast-ok: every content is a str + return cast(AllMessageValues, {**run[0], "content": joined_text}) # cast-ok: dict spread keeps message shape + merged_parts: Final = [ # mutable-ok: chat message content must stay a json list + part for content in contents for part in _system_content_as_text_parts(content) + ] + return cast(AllMessageValues, {**run[0], "content": merged_parts}) # cast-ok: dict spread keeps message shape + + +def merge_consecutive_system_messages( + messages: list[AllMessageValues], # mutable-ok: message pipelines type messages as mutable lists +) -> list[AllMessageValues]: # mutable-ok: message pipelines type messages as mutable lists + return [ # mutable-ok: pipelines mutate message lists + merged + for is_system_run, run in groupby(messages, key=lambda message: message.get("role") == "system") + for merged in ((_merge_system_message_run(tuple(run)),) if is_system_run else run) + ] + + def _attempt_json_repair(s: str) -> object | None: """ Attempt to repair truncated JSON produced by LLM tool calls. diff --git a/litellm/llms/anthropic/experimental_pass_through/context_management/editors/compact.py b/litellm/llms/anthropic/experimental_pass_through/context_management/editors/compact.py index ebd0342b10c..f23f2602ba8 100644 --- a/litellm/llms/anthropic/experimental_pass_through/context_management/editors/compact.py +++ b/litellm/llms/anthropic/experimental_pass_through/context_management/editors/compact.py @@ -208,9 +208,12 @@ async def _check_summary_model_access( (``ProxyException`` from ``_can_object_call_model`` / ``can_*_model``). Unexpected errors during an access check fail closed but are logged separately so operators can distinguish them from a real access-denied - response. DB-lookup failures (object missing from cache or DB) skip the - corresponding scope — matching ``common_checks``, which only enforces a - scope when its backing object can be loaded. + response. User and project lookup failures (object missing from cache or + DB) skip the corresponding scope — matching ``common_checks``, which only + enforces a scope when its backing object can be loaded. A failed team + membership read (a database outage) fails closed instead, since a member + whose limits cannot be read must not have the summary model invoked with + those limits dropped. """ if user_api_key_auth is None: return True @@ -346,13 +349,12 @@ async def _check_summary_model_access( proxy_logging_obj=proxy_logging_obj, ) except Exception as e: - verbose_logger.debug( - "compact_20260112: team membership lookup failed for " - "summary_model=%s access check; skipping member-level scope: %s", + verbose_logger.warning( + "compact_20260112: team membership lookup failed for summary_model=%s access check; denying access: %s", summary_model, e, ) - team_membership = None + return False member_allowed_models: Final = ( team_membership.litellm_budget_table.allowed_models if team_membership is not None and team_membership.litellm_budget_table is not None diff --git a/litellm/llms/azure/azure.py b/litellm/llms/azure/azure.py index 449319c1b95..2e7e7bb0c9d 100644 --- a/litellm/llms/azure/azure.py +++ b/litellm/llms/azure/azure.py @@ -481,7 +481,7 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM): additional_args={"complete_input_dict": data}, original_response=str(e), ) - raise AzureOpenAIError(status_code=500, message=str(e)) + raise except Exception as e: message: Final = getattr(e, "message", str(e)) body: Final = getattr(e, "body", None) diff --git a/litellm/llms/bedrock/base_aws_llm.py b/litellm/llms/bedrock/base_aws_llm.py index dd62cdb424a..badb76d00c7 100644 --- a/litellm/llms/bedrock/base_aws_llm.py +++ b/litellm/llms/bedrock/base_aws_llm.py @@ -524,6 +524,17 @@ class BaseAWSLLM(SignsRequestsWithAWS): aws_session_tags=_canonical_aws_session_tags(auth_params.aws_session_tags), ) + def resolve_s3_credentials(self, params: Mapping[str, object], aws_region_name: str | None) -> Credentials: + """S3 signing identity: the s3_* static pair as-is when both are set, otherwise the resolved aws_* params.""" + from botocore.credentials import Credentials + + from litellm.llms.bedrock.common_utils import s3_static_key_pair + + s3_pair: Final = s3_static_key_pair(params) + if s3_pair is None: + return self.resolve_credentials(AwsAuthParams.model_validate(params), aws_region_name) + return Credentials(access_key=s3_pair[0], secret_key=s3_pair[1]) + def _get_aws_region_from_model_arn(self, model: str | None) -> str | None: try: # First check if the string contains the expected prefix diff --git a/litellm/llms/bedrock/common_utils.py b/litellm/llms/bedrock/common_utils.py index f1066643874..f0816566aa7 100644 --- a/litellm/llms/bedrock/common_utils.py +++ b/litellm/llms/bedrock/common_utils.py @@ -112,6 +112,17 @@ def merge_bedrock_aws_request_params( return request_params +def s3_static_key_pair(params: Mapping[str, object]) -> tuple[str, str] | None: + """The s3_access_key_id / s3_secret_access_key pair when both are set, otherwise None.""" + s3_access_key_id: Final = params.get("s3_access_key_id") + s3_secret_access_key: Final = params.get("s3_secret_access_key") + if not isinstance(s3_access_key_id, str) or not s3_access_key_id: + return None + if not isinstance(s3_secret_access_key, str) or not s3_secret_access_key: + return None + return s3_access_key_id, s3_secret_access_key + + # Lazy import cache to avoid circular imports and performance impact _get_model_info = None diff --git a/litellm/llms/bedrock/files/handler.py b/litellm/llms/bedrock/files/handler.py index 0b75474ba1b..3d23b69f846 100644 --- a/litellm/llms/bedrock/files/handler.py +++ b/litellm/llms/bedrock/files/handler.py @@ -11,7 +11,6 @@ from litellm.litellm_core_utils.cloud_storage_security import ( validate_managed_cloud_file_id, ) from litellm.llms.custom_httpx.http_handler import get_async_httpx_client -from litellm.types.llms.bedrock import AwsAuthParams from litellm.types.llms.openai import ( FileContentRequest, HttpxBinaryResponseContent, @@ -103,9 +102,7 @@ class BedrockFilesHandler(BaseAWSLLM): ) aws_region_name: Final = self._get_aws_region_name(optional_params=optional_params, model="") - credentials: Final[Credentials] = self.resolve_credentials( - AwsAuthParams.model_validate(optional_params), aws_region_name - ) + credentials: Final[Credentials] = self.resolve_s3_credentials(optional_params, aws_region_name) # Create S3 client s3_client: Final = boto3.client( diff --git a/litellm/llms/bedrock/files/transformation.py b/litellm/llms/bedrock/files/transformation.py index a7486dd4de0..43faa7d79ea 100644 --- a/litellm/llms/bedrock/files/transformation.py +++ b/litellm/llms/bedrock/files/transformation.py @@ -63,7 +63,11 @@ from litellm.types.utils import ExtractedFileData, LlmProviders, SpecialEnums from litellm.utils import get_llm_provider from ..base_aws_llm import BaseAWSLLM -from ..common_utils import BedrockError, merge_bedrock_aws_request_params, resolve_s3_encryption_key_id +from ..common_utils import ( + BedrockError, + merge_bedrock_aws_request_params, + resolve_s3_encryption_key_id, +) S3_SIGNED_REQUEST_HEADERS_PARAM: Final = "_s3_signed_request_headers" @@ -148,6 +152,8 @@ class _BedrockS3RequestParams(AwsAuthParams): aws_region_name: str | None = None s3_region_name: str | None = None s3_endpoint_url: str | None = None + s3_access_key_id: str | None = None + s3_secret_access_key: str | None = None @dataclass(frozen=True, slots=True) @@ -1147,7 +1153,7 @@ class BedrockFilesConfig(BaseAWSLLM, BaseFilesConfig): raise ImportError("Missing boto3 to call bedrock. Run 'pip install boto3'.") aws_region_name: Final = self._get_aws_region_name(optional_params=optional_params, model="") - credentials: Final = self.resolve_credentials(AwsAuthParams.model_validate(optional_params), aws_region_name) + credentials: Final = self.resolve_s3_credentials(optional_params, aws_region_name) # Calculate SHA256 hash of the content (REQUIRED for S3) content_hash: Final = hashlib.sha256(content.encode("utf-8")).hexdigest() @@ -1494,7 +1500,7 @@ class BedrockFilesConfig(BaseAWSLLM, BaseFilesConfig): except ImportError: raise ImportError("Missing boto3 to call bedrock. Run 'pip install boto3'.") - credentials: Final = self.resolve_credentials(request_params, aws_region_name) + credentials: Final = self.resolve_s3_credentials(request_params.model_dump(exclude_none=True), aws_region_name) empty_body_hash: Final = hashlib.sha256(b"").hexdigest() aws_request: Final = AWSRequest( # any-ok: botocore AWSRequest is untyped diff --git a/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py b/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py index 1f37fafde01..f46edc766c7 100644 --- a/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py +++ b/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py @@ -81,6 +81,10 @@ class AmazonAnthropicClaudeMessagesConfig( def custom_llm_provider(self) -> str | None: return "bedrock" + @property + def beta_headers_provider(self) -> str: + return self.custom_llm_provider or "bedrock" + BEDROCK_INVOKE_ALLOWED_TOP_LEVEL_FIELDS = frozenset(BedrockInvokeAnthropicMessagesRequest.__annotations__.keys()) def get_error_class( @@ -552,7 +556,7 @@ class AmazonAnthropicClaudeMessagesConfig( if "tool-search-tool-2025-10-19" in beta_set: beta_set.add("tool-examples-2025-10-29") - beta_provider: Final = self.custom_llm_provider or "bedrock" + beta_provider: Final = self.beta_headers_provider filtered_betas: Final = sorted( filter_and_transform_beta_headers( beta_headers=list(beta_set), diff --git a/litellm/llms/bedrock/messages/mantle_transformation.py b/litellm/llms/bedrock/messages/mantle_transformation.py index 7c8758960ad..052eb90a833 100644 --- a/litellm/llms/bedrock/messages/mantle_transformation.py +++ b/litellm/llms/bedrock/messages/mantle_transformation.py @@ -2,16 +2,20 @@ Transformation for Bedrock Mantle (Claude Mythos Preview) - /messages endpoint Inherits all Messages API request/response transformations from -AmazonAnthropicClaudeMessagesConfig. Overrides only the URL and model-prefix -stripping that are specific to the bedrock-mantle endpoint. +AmazonAnthropicClaudeMessagesConfig. Overrides the URL, the model-prefix +stripping, and the anthropic-version / anthropic-beta placement (headers, +never the body) that are specific to the bedrock-mantle endpoint. """ -from collections.abc import AsyncIterator +from collections.abc import AsyncIterator, Mapping +from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final import httpx +from pydantic import TypeAdapter from litellm.llms.anthropic.experimental_pass_through.messages.transformation import ( + DEFAULT_ANTHROPIC_API_VERSION, AnthropicMessagesConfig, ) from litellm.llms.bedrock.common_utils import build_mantle_messages_url @@ -31,6 +35,18 @@ if TYPE_CHECKING: else: LiteLLMLoggingObj = Any +_BODY_FIELDS_MANTLE_READS_FROM_HEADERS: Final = frozenset({"anthropic_version", "anthropic_beta"}) +_ANTHROPIC_BETAS: Final = TypeAdapter(tuple[str, ...]) +_MANTLE_REQUEST: Final = TypeAdapter(dict[str, object]) + + +def _move_betas_into_header(request: Mapping[str, object], headers: dict[str, str]) -> None: + betas: Final = _ANTHROPIC_BETAS.validate_python(request.get("anthropic_beta") or ()) + if betas: + headers["anthropic-beta"] = ",".join(betas) # rebind-ok: the handler signs and sends this same dict + return + headers.pop("anthropic-beta", None) # rebind-ok: a caller header Mantle rejects in full must not reach it + class AmazonMantleMessagesConfig(AmazonAnthropicClaudeMessagesConfig): """ @@ -40,6 +56,13 @@ class AmazonMantleMessagesConfig(AmazonAnthropicClaudeMessagesConfig): model ID in the request body (unlike Bedrock Invoke which puts it in the URL). """ + @property + def beta_headers_provider(self) -> str: + return "bedrock_mantle" + + def should_filter_anthropic_beta_headers(self) -> bool: + return False + def get_complete_url( self, api_base: str | None, @@ -66,7 +89,7 @@ class AmazonMantleMessagesConfig(AmazonAnthropicClaudeMessagesConfig): api_key: str | None = None, api_base: str | None = None, ) -> tuple[dict, str | None]: - headers, api_base = super().validate_anthropic_messages_environment( + merged_headers, resolved_api_base = super().validate_anthropic_messages_environment( headers=headers, model=model, messages=messages, @@ -76,9 +99,21 @@ class AmazonMantleMessagesConfig(AmazonAnthropicClaudeMessagesConfig): api_base=api_base, ) project_id: Final = litellm_params.get("aws_bedrock_project_id") - if project_id: - headers["anthropic-workspace"] = project_id - return headers, api_base + has_version: Final = any(name.lower() == "anthropic-version" for name in merged_headers) + mantle_headers: Final = MappingProxyType( + { + name: value + for name, value in ( + ("anthropic-workspace", project_id), + ("anthropic-version", None if has_version else DEFAULT_ANTHROPIC_API_VERSION), + ) + if value + } + ) + return { # mutable-ok: the base class contract returns a dict the handler signs into in place + **merged_headers, + **mantle_headers, + }, resolved_api_base def transform_anthropic_messages_request( self, @@ -88,25 +123,28 @@ class AmazonMantleMessagesConfig(AmazonAnthropicClaudeMessagesConfig): litellm_params: GenericLiteLLMParams, headers: dict, ) -> dict: - # Strip "mantle/" routing prefix to get the real model ID model_id: Final = model.replace("mantle/", "", 1) - - request: Final = super().transform_anthropic_messages_request( - model=model_id, - messages=messages, - anthropic_messages_optional_request_params=anthropic_messages_optional_request_params, - litellm_params=litellm_params, - headers=headers, + request: Final = _MANTLE_REQUEST.validate_python( + super().transform_anthropic_messages_request( + model=model_id, + messages=messages, + anthropic_messages_optional_request_params=anthropic_messages_optional_request_params, + litellm_params=litellm_params, + headers=headers, + ), ) - - # Parent (AmazonAnthropicClaudeMessagesConfig) removes "model" and - # "stream" from the body (Bedrock Invoke puts the model in the URL and - # streams via a dedicated endpoint). The mantle endpoint (Messages API) - # requires both in the request body. - stream_fields: Final[dict[str, bool]] = ( - {"stream": True} if anthropic_messages_optional_request_params.get("stream") is True else {} + _move_betas_into_header(request, headers) + body: Final = MappingProxyType( + {key: value for key, value in request.items() if key not in _BODY_FIELDS_MANTLE_READS_FROM_HEADERS} ) - return {**request, "model": model_id, **stream_fields} + streaming: Final = anthropic_messages_optional_request_params.get("stream") is True + mantle_fields: Final = MappingProxyType( + {key: value for key, value in (("model", model_id), ("stream", streaming)) if value} + ) + return { # mutable-ok: the base class contract returns the dict the handler serializes as the body + **body, + **mantle_fields, + } def transform_anthropic_messages_response( self, diff --git a/litellm/llms/bedrock_mantle/messages/transformation.py b/litellm/llms/bedrock_mantle/messages/transformation.py index 6e975d072ed..78881153e91 100644 --- a/litellm/llms/bedrock_mantle/messages/transformation.py +++ b/litellm/llms/bedrock_mantle/messages/transformation.py @@ -2,11 +2,6 @@ from collections.abc import Mapping from types import MappingProxyType from typing import Final -from pydantic import TypeAdapter - -from litellm.llms.anthropic.experimental_pass_through.messages.transformation import ( - DEFAULT_ANTHROPIC_API_VERSION, -) from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM from litellm.llms.bedrock.common_utils import MANTLE_MESSAGES_PATH from litellm.llms.bedrock.messages.mantle_transformation import AmazonMantleMessagesConfig @@ -17,7 +12,6 @@ from litellm.llms.bedrock_mantle.common_utils import ( ) from litellm.secret_managers.main import get_secret_str from litellm.types.llms.anthropic import ANTHROPIC_BETA_HEADER_VALUES -from litellm.types.router import GenericLiteLLMParams _BASE_SUFFIXES_TO_STRIP: Final = ( MANTLE_MESSAGES_PATH, @@ -27,9 +21,6 @@ _BASE_SUFFIXES_TO_STRIP: Final = ( "/openai/v1", "/v1", ) -_BODY_FIELDS_MANTLE_READS_FROM_HEADERS: Final = frozenset({"anthropic_version", "anthropic_beta"}) -_ANTHROPIC_BETAS: Final = TypeAdapter(tuple[str, ...]) -_MANTLE_REQUEST: Final = TypeAdapter(dict[str, object]) def build_mantle_native_messages_url(api_base: str | None, litellm_params: Mapping[str, object]) -> str: @@ -74,54 +65,3 @@ class BedrockMantleAnthropicMessagesConfig(BedrockMantleAuthMixin, AmazonMantleM stream: bool | None = None, ) -> str: return build_mantle_native_messages_url(api_base=api_base, litellm_params=litellm_params) - - def validate_anthropic_messages_environment( - self, - headers: dict, - model: str, - messages: list[dict], - optional_params: dict, - litellm_params: dict, - api_key: str | None = None, - api_base: str | None = None, - ) -> tuple[dict, str | None]: - merged_headers, resolved_api_base = super().validate_anthropic_messages_environment( - headers=headers, - model=model, - messages=messages, - optional_params=optional_params, - litellm_params=litellm_params, - api_key=api_key, - api_base=api_base, - ) - if any(name.lower() == "anthropic-version" for name in merged_headers): - return merged_headers, resolved_api_base - return { # mutable-ok: the base class contract returns a dict the handler signs into in place - **merged_headers, - "anthropic-version": DEFAULT_ANTHROPIC_API_VERSION, - }, resolved_api_base - - def transform_anthropic_messages_request( - self, - model: str, - messages: list[dict], - anthropic_messages_optional_request_params: dict, - litellm_params: GenericLiteLLMParams, - headers: dict, - ) -> dict: - request: Final = _MANTLE_REQUEST.validate_python( - super().transform_anthropic_messages_request( - model=model, - messages=messages, - anthropic_messages_optional_request_params=anthropic_messages_optional_request_params, - litellm_params=litellm_params, - headers=headers, - ), - ) - betas: Final = request.get("anthropic_beta") - if betas is not None: - header_betas: Final = ",".join(_ANTHROPIC_BETAS.validate_python(betas)) - headers["anthropic-beta"] = header_betas # rebind-ok: the handler signs and sends this same dict - return { # mutable-ok: the base class contract returns the dict the handler serializes as the body - key: value for key, value in request.items() if key not in _BODY_FIELDS_MANTLE_READS_FROM_HEADERS - } diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index db821f42a90..2a105301521 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -4,7 +4,6 @@ import ssl from collections.abc import AsyncGenerator, AsyncIterator, Coroutine, Iterator, Mapping, Sequence from contextlib import asynccontextmanager from functools import lru_cache -from itertools import chain from types import MappingProxyType, ModuleType from typing import ( TYPE_CHECKING, @@ -34,6 +33,7 @@ from litellm._logging import _redact_string, verbose_logger from litellm.anthropic_beta_headers_manager import update_headers_with_filtered_beta from litellm.constants import MAX_FILE_LIST_LIMIT, REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES from litellm.files.types import FileContentStreamingResult +from litellm.litellm_core_utils.agentic_followup_kwargs import build_agentic_followup_kwargs from litellm.litellm_core_utils.agentic_loop_settings import ( DEFAULT_MAX_AGENTIC_LOOPS, validated_max_agentic_loops, @@ -5614,28 +5614,22 @@ class BaseLLMHTTPHandler: } internal_keys: Final = {"litellm_logging_obj"} - kwargs_for_followup: Final = MappingProxyType( - { - key: value - for key, value in chain( - ( - (k, v) - for k, v in kwargs.items() - if not is_interception_internal_key( - k, prefixes=NON_CODE_INTERPRETER_INTERCEPTION_INTERNAL_PREFIXES - ) - and k != "_code_interpreter_interception_converted_stream" - and k not in internal_keys - and k not in optional_params - ), - ((k, v) for k, v in patch.kwargs.items() if k not in optional_params), - ( - ("_agentic_loop_depth", depth + 1), - ("max_agentic_loops", max_loops), - ("_agentic_loop_fingerprints", fingerprints + [fingerprint]), - ), - ) - } + kwargs_for_followup: Final = build_agentic_followup_kwargs( + request_kwargs=MappingProxyType( + { + k: v + for k, v in kwargs.items() + if not is_interception_internal_key(k, prefixes=NON_CODE_INTERPRETER_INTERCEPTION_INTERNAL_PREFIXES) + and k != "_code_interpreter_interception_converted_stream" + and k not in internal_keys + } + ), + patch_kwargs=patch.kwargs, + request_params=frozenset((*optional_params, "model", "input")), + depth=depth, + max_loops=max_loops, + fingerprints=fingerprints, + fingerprint=fingerprint, ) try: @@ -5756,17 +5750,23 @@ class BaseLLMHTTPHandler: "stream_response", "custom_prompt_dict", } - kwargs_for_followup: Final = { - k: v - for k, v in kwargs.items() - if not k.startswith("_websearch_interception") - and not k.startswith("_compression_interception") - and k not in internal_params - } - kwargs_for_followup.update(patch.kwargs) - kwargs_for_followup["_agentic_loop_depth"] = depth + 1 - kwargs_for_followup["max_agentic_loops"] = max_loops - kwargs_for_followup["_agentic_loop_fingerprints"] = fingerprints + [fingerprint] + kwargs_for_followup: Final = build_agentic_followup_kwargs( + request_kwargs=MappingProxyType( + { + k: v + for k, v in kwargs.items() + if not k.startswith("_websearch_interception") + and not k.startswith("_compression_interception") + and k not in internal_params + } + ), + patch_kwargs=patch.kwargs, + request_params=frozenset((*optional_params_for_followup, "model", "messages")), + depth=depth, + max_loops=max_loops, + fingerprints=fingerprints, + fingerprint=fingerprint, + ) return await litellm.acompletion( model=full_model_name, diff --git a/litellm/llms/databricks/chat/transformation.py b/litellm/llms/databricks/chat/transformation.py index dd257cd68b0..30144d29f51 100644 --- a/litellm/llms/databricks/chat/transformation.py +++ b/litellm/llms/databricks/chat/transformation.py @@ -16,6 +16,7 @@ from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response impo ) from litellm.litellm_core_utils.prompt_templates.common_utils import ( _extract_reasoning_content, # pyright: ignore[reportPrivateUsage] # same import as the OpenAI transformation + merge_consecutive_system_messages, strip_litellm_internal_message_fields, strip_name_from_message, ) @@ -465,7 +466,9 @@ class DatabricksConfig(DatabricksBase, OpenAILikeChatConfig, AnthropicConfig): new_messages.append(_message) if "claude" not in model: - new_messages = _split_parallel_tool_calls(cast(list[AllMessageValues], new_messages)) + new_messages = _split_parallel_tool_calls( + merge_consecutive_system_messages(cast(list[AllMessageValues], new_messages)) + ) if is_async: return super()._transform_messages(messages=new_messages, model=model, is_async=cast(Literal[True], True)) diff --git a/litellm/llms/fal_ai/chat/__init__.py b/litellm/llms/fal_ai/chat/__init__.py new file mode 100644 index 00000000000..b2a4a006aef --- /dev/null +++ b/litellm/llms/fal_ai/chat/__init__.py @@ -0,0 +1,3 @@ +from .transformation import FalAIChatConfig, FalAIError + +__all__ = ("FalAIChatConfig", "FalAIError") diff --git a/litellm/llms/fal_ai/chat/transformation.py b/litellm/llms/fal_ai/chat/transformation.py new file mode 100644 index 00000000000..2c5af6538ab --- /dev/null +++ b/litellm/llms/fal_ai/chat/transformation.py @@ -0,0 +1,244 @@ +""" +Support for `/v1/chat/completions` on Fal AI model endpoints, e.g. fal-ai/moondream3-preview/query. + +These endpoints are not OpenAI-compatible: the request body is a flat ``{"prompt", "image_url"}`` +object and the response is ``{"output", "reasoning", "finish_reason", "usage_info"}``. +""" + +import time +from collections.abc import Mapping, Sequence +from typing import TYPE_CHECKING, Final + +import httpx +from pydantic import BaseModel, ConfigDict, TypeAdapter + +from litellm.litellm_core_utils.core_helpers import map_finish_reason +from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException +from litellm.secret_managers.main import get_secret_str +from litellm.types.llms.openai import AllMessageValues +from litellm.types.utils import Message, ModelResponse, Usage + +if TYPE_CHECKING: + import tiktoken + + from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + +DEFAULT_BASE_URL: Final[str] = "https://fal.run" +PROVIDER_PREFIX: Final[str] = "fal_ai/" +PASSTHROUGH_PARAMS: Final[frozenset[str]] = frozenset(("reasoning", "temperature", "top_p")) +REASONING_DISABLED_EFFORTS: Final[frozenset[str]] = frozenset(("none", "minimal")) +REASONING_ENABLED_EFFORTS: Final[frozenset[str]] = frozenset(("low", "medium", "high")) + + +class _FalUsage(BaseModel): + model_config = ConfigDict(frozen=True, extra="ignore") + + input_tokens: int + output_tokens: int + + +class _FalChatResponse(BaseModel): + model_config = ConfigDict(frozen=True, extra="ignore") + + output: str + usage_info: _FalUsage + reasoning: str | None = None + finish_reason: str | None = None + + +_CHAT_RESPONSE: Final = TypeAdapter(_FalChatResponse) + + +class FalAIError(BaseLLMException): + def __init__( + self, + status_code: int, + message: str, + headers: dict | httpx.Headers | None = None, # mutable-ok: BaseLLMException header contract + ) -> None: + super().__init__(status_code=status_code, message=message, headers=headers) + + +def _image_part_url(part: Mapping[str, object]) -> str | None: + image_url: Final = part.get("image_url") + if isinstance(image_url, str): + return image_url + if isinstance(image_url, Mapping): + url: Final = image_url.get("url") + return url if isinstance(url, str) else None + return None + + +def _prompt_and_image(messages: Sequence[AllMessageValues]) -> tuple[str, str]: + if len(messages) != 1 or messages[0].get("role") != "user": + raise FalAIError( + status_code=400, + message="fal_ai chat completions accept exactly one user message; system prompts and multi-turn history are not supported", + ) + content: Final = messages[0].get("content") + if isinstance(content, str): + if not content: + raise FalAIError(status_code=400, message="fal_ai chat completions require text in the user message") + raise FalAIError( + status_code=400, + message="fal_ai chat completions require exactly one image_url content part in the user message", + ) + parts: Final[tuple[Mapping[str, object], ...]] = ( + tuple(part for part in content if isinstance(part, Mapping)) if isinstance(content, Sequence) else () + ) + prompt: Final = "\n".join( + text for part in parts if part.get("type") == "text" and isinstance((text := part.get("text")), str) and text + ) + image_urls: Final = tuple( + url for part in parts if part.get("type") == "image_url" and (url := _image_part_url(part)) is not None + ) + if not prompt: + raise FalAIError(status_code=400, message="fal_ai chat completions require text in the user message") + if len(image_urls) != 1: + raise FalAIError( + status_code=400, + message="fal_ai chat completions require exactly one image_url content part in the user message", + ) + return prompt, image_urls[0] + + +class FalAIChatConfig(BaseConfig): + @staticmethod + def get_api_key(api_key: str | None = None) -> str | None: + return api_key or get_secret_str("FAL_AI_API_KEY") + + @staticmethod + def get_api_base(api_base: str | None = None) -> str: + return (api_base or get_secret_str("FAL_AI_API_BASE") or DEFAULT_BASE_URL).rstrip("/") + + def get_supported_openai_params(self, model: str) -> list: # mutable-ok: inherited contract returns a list + return list(("reasoning_effort", "temperature", "top_p")) # mutable-ok: inherited contract returns a list + + def _map_reasoning_effort(self, value: object, model: str, drop_params: bool) -> bool | None: + if value in REASONING_DISABLED_EFFORTS: + return False + if value in REASONING_ENABLED_EFFORTS: + return True + if drop_params: + return None + raise FalAIError(status_code=400, message=f"Unsupported reasoning_effort '{value}' for {model}") + + def _translate_param(self, param: str, value: object, model: str, drop_params: bool) -> tuple[str, object] | None: + if param in ("temperature", "top_p"): + return param, value + if param == "reasoning_effort": + reasoning: Final = self._map_reasoning_effort(value, model, drop_params) + return ("reasoning", reasoning) if reasoning is not None else None + return None + + def map_openai_params( + self, + non_default_params: dict, # mutable-ok: inherited contract + optional_params: dict, # mutable-ok: inherited contract + model: str, + drop_params: bool, + ) -> dict: # mutable-ok: inherited contract returns a dict + mapped: Final = { # mutable-ok: intermediate translation map, folded into the returned dict + translated[0]: translated[1] + for param, value in non_default_params.items() + if (translated := self._translate_param(param, value, model, drop_params)) is not None + } + return {**optional_params, **mapped} # mutable-ok: inherited contract returns a dict + + def validate_environment( + self, + headers: dict, # mutable-ok: inherited contract + model: str, + messages: list[AllMessageValues], # mutable-ok: inherited contract + optional_params: dict, # mutable-ok: inherited contract + litellm_params: dict, # mutable-ok: inherited contract + api_key: str | None = None, + api_base: str | None = None, + ) -> dict: # mutable-ok: inherited contract returns a dict + final_api_key: Final = self.get_api_key(api_key) + if not final_api_key: + raise ValueError("FAL_AI_API_KEY is not set") + return { # mutable-ok: inherited contract returns a dict + "content-type": "application/json", + **(headers or {}), # mutable-ok: empty default for the inherited contract's headers + "Authorization": f"Key {final_api_key}", + } + + def get_complete_url( + self, + api_base: str | None, + api_key: str | None, + model: str, + optional_params: dict, # mutable-ok: inherited contract + litellm_params: dict, # mutable-ok: inherited contract + stream: bool | None = None, + ) -> str: + return f"{self.get_api_base(api_base)}/{model.removeprefix(PROVIDER_PREFIX)}" + + def transform_request( + self, + model: str, + messages: list[AllMessageValues], # mutable-ok: inherited contract + optional_params: dict, # mutable-ok: inherited contract + litellm_params: dict, # mutable-ok: inherited contract + headers: dict, # mutable-ok: inherited contract + ) -> dict: # mutable-ok: inherited contract returns a dict + if optional_params.get("stream"): + raise FalAIError(status_code=400, message="fal_ai chat completions do not support streaming") + prompt, image_url = _prompt_and_image(messages) + return { # mutable-ok: JSON request body + "prompt": prompt, + "image_url": image_url, + **{ # mutable-ok: JSON request body + key: value for key, value in optional_params.items() if key in PASSTHROUGH_PARAMS and value is not None + }, + } + + def transform_response( + self, + model: str, + raw_response: httpx.Response, + model_response: ModelResponse, + logging_obj: "LiteLLMLoggingObj", + request_data: dict, # mutable-ok: inherited contract + messages: list[AllMessageValues], # mutable-ok: inherited contract + optional_params: dict, # mutable-ok: inherited contract + litellm_params: dict, # mutable-ok: inherited contract + encoding: "tiktoken.Encoding | None", + api_key: str | None = None, + json_mode: bool | None = None, + ) -> ModelResponse: + try: + completion_response: Final = _CHAT_RESPONSE.validate_json(raw_response.content) + except ValueError: + raise FalAIError( + status_code=422, + message=f"fal_ai returned an unexpected response body: {raw_response.text}", + headers=raw_response.headers, + ) + + message: Final = Message( + content=completion_response.output, + role="assistant", + reasoning_content=completion_response.reasoning, + ) + model_response.choices[0].message = message # rebind-ok: ModelResponse populated in place per contract + model_response.choices[0].finish_reason = map_finish_reason( # rebind-ok: same contract + completion_response.finish_reason or "stop" + ) + model_response.created = int(time.time()) # rebind-ok: same contract + model_response.model = model # rebind-ok: same contract + model_response.usage = Usage( # rebind-ok: same contract + prompt_tokens=completion_response.usage_info.input_tokens, + completion_tokens=completion_response.usage_info.output_tokens, + total_tokens=completion_response.usage_info.input_tokens + completion_response.usage_info.output_tokens, + ) + return model_response + + def get_error_class( + self, + error_message: str, + status_code: int, + headers: dict | httpx.Headers, # mutable-ok: inherited contract + ) -> BaseLLMException: + return FalAIError(status_code=status_code, message=error_message, headers=headers) diff --git a/litellm/llms/fal_ai/cost_calculator.py b/litellm/llms/fal_ai/cost_calculator.py index fd7d82d314f..31f0995bf9f 100644 --- a/litellm/llms/fal_ai/cost_calculator.py +++ b/litellm/llms/fal_ai/cost_calculator.py @@ -1,3 +1,4 @@ +import os from collections.abc import Mapping from math import ceil from types import MappingProxyType @@ -9,7 +10,8 @@ import litellm from litellm.types.utils import ImageObject, ImageResponse FAL_KEYED_PRICING_DEFAULT_QUALITY: Final[str] = "high" -FAL_TEXT_TO_IMAGE_DEFAULT_SIZE: Final[str] = "1024-x-768" +_DEFAULT_KEYED_DIMENSIONS: Final[tuple[int, int]] = (1024, 768) +FAL_TEXT_TO_IMAGE_DEFAULT_SIZE: Final[str] = f"{_DEFAULT_KEYED_DIMENSIONS[0]}-x-{_DEFAULT_KEYED_DIMENSIONS[1]}" FAL_PIXELS_PER_MEGAPIXEL: Final[int] = 1_048_576 FAL_NAMED_IMAGE_SIZES: Final[Mapping[str, str]] = MappingProxyType( { @@ -24,6 +26,12 @@ FAL_NAMED_IMAGE_SIZES: Final[Mapping[str, str]] = MappingProxyType( _OBJECT_MAP: Final[TypeAdapter[Mapping[str, object]]] = TypeAdapter(Mapping[str, object]) +FAL_AI_QUEUE_DEFAULT_BASE: Final[str] = "https://queue.fal.run" + + +def fal_ai_queue_base() -> str: + return os.getenv("FAL_AI_QUEUE_API_BASE") or FAL_AI_QUEUE_DEFAULT_BASE + def _keyed_size(optional_params: Mapping[str, object]) -> str | None: image_size: Final = optional_params.get("image_size") @@ -55,37 +63,56 @@ def _image_dimensions(image: object) -> tuple[int, int] | None: return width, height -def _response_size(image: object) -> str | None: - dimensions: Final = _image_dimensions(image) - if dimensions is None: - return None - width, height = dimensions - return f"{width}-x-{height}" - - def _keyed_quality(optional_params: Mapping[str, object]) -> str: raw_quality: Final = optional_params.get("quality") return raw_quality if isinstance(raw_quality, str) and raw_quality != "auto" else FAL_KEYED_PRICING_DEFAULT_QUALITY +def _parse_keyed_dimensions(size: str | None) -> tuple[int, int] | None: + if size is None: + return None + parts: Final = tuple(size.split("-x-")) + if len(parts) != 2: + return None + try: + width, height = (int(part) for part in parts) + except ValueError: + return None + return (width, height) if width > 0 and height > 0 else None + + +def _keyed_rows(model: str, quality: str) -> tuple[tuple[int, int, float], ...]: + prefix: Final = f"fal_ai/{quality}/" + suffix: Final = f"/{model}" + return tuple( + (width, height, float(raw_cost)) + for key in litellm.model_cost + if isinstance(key, str) and key.startswith(prefix) and key.endswith(suffix) + for size in (key[len(prefix) : -len(suffix)],) + for dimensions in (_parse_keyed_dimensions(size),) + if dimensions is not None + for entry in (_entry(key),) + if entry is not None + for raw_cost in (entry.get("output_cost_per_image"),) + if isinstance(raw_cost, (int, float)) + for width, height in (dimensions,) + ) + + def _keyed_cost_per_image( model: str, image: object, optional_params: Mapping[str, object], ) -> float | None: quality: Final = _keyed_quality(optional_params) - request_size: Final = _keyed_size(optional_params) or FAL_TEXT_TO_IMAGE_DEFAULT_SIZE - sizes: Final = (_response_size(image), request_size, FAL_TEXT_TO_IMAGE_DEFAULT_SIZE) - for size in sizes: - if size is None: - continue - keyed_entry = _entry(f"fal_ai/{quality}/{size}/{model}") - if keyed_entry is None: - continue - keyed_cost = keyed_entry.get("output_cost_per_image") - if isinstance(keyed_cost, (int, float)): - return float(keyed_cost) - return None + rows: Final = _keyed_rows(model, quality) + if not rows: + return None + target_dimensions: Final = ( + _image_dimensions(image) or _parse_keyed_dimensions(_keyed_size(optional_params)) or _DEFAULT_KEYED_DIMENSIONS + ) + target_pixels: Final = target_dimensions[0] * target_dimensions[1] + return min(rows, key=lambda row: (abs(row[0] * row[1] - target_pixels), row[0] * row[1]))[2] def _flat_cost_per_image( @@ -108,6 +135,16 @@ def _entry(key: str) -> Mapping[str, object] | None: return _OBJECT_MAP.validate_python(raw_entry) +def fal_ai_passthrough_cost(model: str, request_body: Mapping[str, object]) -> float | None: + entry: Final = _entry(f"{litellm.LlmProviders.FAL_AI.value}/{model}") + if entry is None: + return None + resolution: Final = request_body.get("resolution") + keyed_cost: Final = entry.get(f"output_cost_per_image_{resolution}") if isinstance(resolution, int) else None + cost: Final = keyed_cost if isinstance(keyed_cost, (int, float)) else entry.get("output_cost_per_image") + return float(cost) if isinstance(cost, (int, float)) else None + + def cost_calculator( model: str, image_response: object, @@ -129,7 +166,7 @@ def cost_calculator( ) for image in images ) - if all(cost is not None for cost in keyed_costs): + if not any(cost is None for cost in keyed_costs): return sum(cost for cost in keyed_costs if cost is not None) model_info: Final = litellm.get_model_info( model=normalized_model, @@ -144,10 +181,12 @@ def cost_calculator( float(raw_output_cost_per_pixel) if isinstance(raw_output_cost_per_pixel, (int, float)) else None ) return sum( - _flat_cost_per_image( + keyed_cost + if keyed_cost is not None + else _flat_cost_per_image( image=image, output_cost_per_image=output_cost_per_image, output_cost_per_pixel=output_cost_per_pixel, ) - for image in images + for image, keyed_cost in zip(images, keyed_costs) ) diff --git a/litellm/llms/fal_ai/image_edit/__init__.py b/litellm/llms/fal_ai/image_edit/__init__.py index c2f0f311f8c..60775ef2c8d 100644 --- a/litellm/llms/fal_ai/image_edit/__init__.py +++ b/litellm/llms/fal_ai/image_edit/__init__.py @@ -1,3 +1,24 @@ +from typing import Final + +from litellm.llms.base_llm.image_edit.transformation import BaseImageEditConfig + +from .flux_lora_depth_transformation import FalAIFluxLoraDepthEditConfig from .transformation import FalAIImageEditConfig -__all__ = ("FalAIImageEditConfig",) +__all__ = ("FalAIFluxLoraDepthEditConfig", "FalAIImageEditConfig") + + +def get_fal_ai_image_edit_config(model: str) -> BaseImageEditConfig: + """ + Get the appropriate Fal AI image edit configuration based on the model. + + Args: + model: The Fal AI model name (e.g., "openai/gpt-image-2.5/flare/edit", "fal-ai/flux-lora-depth") + + Returns: + The appropriate configuration class for the specified model + """ + model_lower: Final = model.lower() + if "flux-lora-depth" in model_lower: + return FalAIFluxLoraDepthEditConfig() + return FalAIImageEditConfig() diff --git a/litellm/llms/fal_ai/image_edit/flux_lora_depth_transformation.py b/litellm/llms/fal_ai/image_edit/flux_lora_depth_transformation.py new file mode 100644 index 00000000000..fa469d638d2 --- /dev/null +++ b/litellm/llms/fal_ai/image_edit/flux_lora_depth_transformation.py @@ -0,0 +1,75 @@ +from collections.abc import Mapping +from types import MappingProxyType +from typing import Final + +from httpx._types import RequestFiles + +from litellm.secret_managers.main import get_secret_str +from litellm.types.images.main import ImageEditOptionalRequestParams +from litellm.types.router import GenericLiteLLMParams +from litellm.types.utils import FileTypes + +from .transformation import DEFAULT_BASE_URL, FalAIImageEditConfig, to_data_url + +FLUX_LORA_DEPTH_ENDPOINT: Final[str] = "fal-ai/flux-lora-depth" +SUPPORTED_OPENAI_PARAMS: Final[tuple[str, ...]] = ("n", "size") +PARAM_TRANSLATION: Final[Mapping[str, str]] = MappingProxyType({"n": "num_images", "size": "image_size"}) + + +class FalAIFluxLoraDepthEditConfig(FalAIImageEditConfig): + """ + FLUX.1 [dev] depth LoRA edit endpoint served through Fal AI. + + Unlike the openai gpt-image ``/edit`` endpoints, this endpoint takes a single ``image_url`` + control image and has no ``/edit`` path suffix. + """ + + def get_supported_openai_params(self, model: str) -> list: # mutable-ok: base class contract returns a list + return list(SUPPORTED_OPENAI_PARAMS) # mutable-ok: base class contract returns a list + + def map_openai_params( # mutable-ok: base class contract returns a dict + self, + image_edit_optional_params: ImageEditOptionalRequestParams, + model: str, + drop_params: bool, + ) -> dict: # mutable-ok: base class contract returns a dict + return { # mutable-ok: base class contract returns a dict + PARAM_TRANSLATION.get(key, key): self._translate_value(key, value, model) + for key, value in image_edit_optional_params.items() + if value is not None and key in PARAM_TRANSLATION + } + + def get_complete_url( + self, + model: str, + api_base: str | None, + litellm_params: dict, # mutable-ok: base class contract + ) -> str: + base_url: Final = (api_base or get_secret_str("FAL_AI_API_BASE") or DEFAULT_BASE_URL).rstrip("/") + return f"{base_url}/{FLUX_LORA_DEPTH_ENDPOINT}" + + def transform_image_edit_request( + self, + model: str, + prompt: str | None, + image: FileTypes | None, + image_edit_optional_request_params: dict, # mutable-ok: base class contract + litellm_params: GenericLiteLLMParams, + headers: dict, # mutable-ok: base class contract + ) -> tuple[dict, RequestFiles]: # mutable-ok: base class contract returns a dict + images: Final = tuple(img for img in (image if isinstance(image, list) else (image,)) if img is not None) + if not images: + raise ValueError("Fal AI image edit requires at least one input image") + if len(images) > 1: + raise ValueError(f"{FLUX_LORA_DEPTH_ENDPOINT} accepts exactly one control image") + provider_params: Final[Mapping[str, object]] = MappingProxyType( + { + key: value for key, value in image_edit_optional_request_params.items() if key != "mask" + } # mutable-ok: frozen by MappingProxyType + ) + request_body: Final[dict[str, object]] = { # mutable-ok: base class contract returns a dict + "prompt": prompt, + "image_url": to_data_url(next(iter(images))), + **provider_params, + } + return request_body, () diff --git a/litellm/llms/fal_ai/image_edit/transformation.py b/litellm/llms/fal_ai/image_edit/transformation.py index 70b5d0612f2..6e6a872839a 100644 --- a/litellm/llms/fal_ai/image_edit/transformation.py +++ b/litellm/llms/fal_ai/image_edit/transformation.py @@ -61,7 +61,7 @@ def _read_image_bytes(image: object) -> bytes: raise ValueError(f"Unsupported image type for Fal AI image edit: {type(image).__name__}") -def _to_data_url(image: object) -> str: +def to_data_url(image: object) -> str: if isinstance(image, str): return image image_bytes: Final = _read_image_bytes(image) @@ -143,7 +143,7 @@ class FalAIImageEditConfig(BaseImageEditConfig): raise ValueError("Fal AI image edit requires at least one input image") mask: Final = _first(image_edit_optional_request_params.get("mask")) mask_field: Final[Mapping[str, str]] = ( - MappingProxyType({"mask_url": _to_data_url(mask)}) if mask is not None else MappingProxyType({}) + MappingProxyType({"mask_url": to_data_url(mask)}) if mask is not None else MappingProxyType({}) ) provider_params: Final[Mapping[str, object]] = MappingProxyType( { @@ -152,7 +152,7 @@ class FalAIImageEditConfig(BaseImageEditConfig): ) request_body: Final[dict[str, object]] = { # mutable-ok: base class contract returns a dict "prompt": prompt, - "image_urls": tuple(_to_data_url(img) for img in images), + "image_urls": tuple(to_data_url(img) for img in images), **mask_field, **provider_params, } diff --git a/litellm/llms/openai/responses/guardrail_translation/handler.py b/litellm/llms/openai/responses/guardrail_translation/handler.py index 1ef1011591e..ad8e29ad4ac 100644 --- a/litellm/llms/openai/responses/guardrail_translation/handler.py +++ b/litellm/llms/openai/responses/guardrail_translation/handler.py @@ -238,7 +238,7 @@ _TOOL_CALL_PAYLOAD_EVENT_TYPES: Final = _TOOL_CALL_PAYLOAD_DELTA_EVENT_TYPES | f _OUTPUT_ITEM_EVENT_TYPES: Final = frozenset({"response.output_item.added", "response.output_item.done"}) _OUTPUT_TEXT_EVENT_TYPES: Final = frozenset({"response.output_text.delta", "response.output_text.done"}) _PATCHABLE_ITEM_FIELDS: Final[Mapping[str, str]] = MappingProxyType( - {"function_call_output": "output", "message": "content"} + {"function_call_output": "output", "custom_tool_call_output": "output", "message": "content"} ) _EMPTY_RESPONSES_REQUEST: Final[ResponsesAPIOptionalRequestParams] = {} diff --git a/litellm/llms/openai/videos/guardrail_translation/__init__.py b/litellm/llms/openai/videos/guardrail_translation/__init__.py new file mode 100644 index 00000000000..7bd869612d6 --- /dev/null +++ b/litellm/llms/openai/videos/guardrail_translation/__init__.py @@ -0,0 +1,23 @@ +"""OpenAI Video Generation handler for Unified Guardrails.""" + +from typing import Final + +from litellm.llms.openai.videos.guardrail_translation.handler import ( + OpenAIVideoGenerationHandler, +) +from litellm.types.utils import CallTypes + +guardrail_translation_mappings: Final = { # mutable-ok: discover_guardrail_translation_mappings only accepts isinstance(mappings, dict) + CallTypes.video_generation: OpenAIVideoGenerationHandler, + CallTypes.avideo_generation: OpenAIVideoGenerationHandler, + CallTypes.create_video: OpenAIVideoGenerationHandler, + CallTypes.acreate_video: OpenAIVideoGenerationHandler, + CallTypes.video_remix: OpenAIVideoGenerationHandler, + CallTypes.avideo_remix: OpenAIVideoGenerationHandler, + CallTypes.video_edit: OpenAIVideoGenerationHandler, + CallTypes.avideo_edit: OpenAIVideoGenerationHandler, + CallTypes.video_extension: OpenAIVideoGenerationHandler, + CallTypes.avideo_extension: OpenAIVideoGenerationHandler, +} + +__all__ = ("OpenAIVideoGenerationHandler", "guardrail_translation_mappings") diff --git a/litellm/llms/openai/videos/guardrail_translation/handler.py b/litellm/llms/openai/videos/guardrail_translation/handler.py new file mode 100644 index 00000000000..49a8d05100c --- /dev/null +++ b/litellm/llms/openai/videos/guardrail_translation/handler.py @@ -0,0 +1,48 @@ +from typing import TYPE_CHECKING, Final + +from litellm.llms.base_llm.guardrail_translation.base_translation import BaseTranslation +from litellm.types.utils import GenericGuardrailAPIInputs + +if TYPE_CHECKING: + from litellm.integrations.custom_guardrail import CustomGuardrail + from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + from litellm.proxy._types import UserAPIKeyAuth + + +class OpenAIVideoGenerationHandler(BaseTranslation): + async def process_input_messages( + self, + data: dict[str, object], # mutable-ok: BaseTranslation contract passes the proxy's request dict through + guardrail_to_apply: "CustomGuardrail", + litellm_logging_obj: "LiteLLMLoggingObj | None" = None, + ) -> dict[str, object]: # mutable-ok: BaseTranslation contract returns the proxy's request dict + prompt: Final = data.get("prompt") + if not isinstance(prompt, str): + return data + + model: Final = data.get("model") + texts: Final = [prompt] # mutable-ok: GenericGuardrailAPIInputs.texts is declared list[str] + inputs: Final = ( + GenericGuardrailAPIInputs(texts=texts, model=model) + if isinstance(model, str) + else GenericGuardrailAPIInputs(texts=texts) + ) + guardrailed_inputs: Final = await guardrail_to_apply.apply_guardrail( # pyright: ignore[reportUnknownMemberType] # request_data is a bare dict + inputs=inputs, + request_data=data, + input_type="request", + logging_obj=litellm_logging_obj, + ) + guardrailed_texts: Final = guardrailed_inputs.get("texts") + guardrailed_prompt: Final = guardrailed_texts[0] if guardrailed_texts else prompt + return {**data, "prompt": guardrailed_prompt} # mutable-ok: BaseTranslation contract returns a dict + + async def process_output_response( + self, + response: object, + guardrail_to_apply: "CustomGuardrail", + litellm_logging_obj: "LiteLLMLoggingObj | None" = None, + user_api_key_dict: "UserAPIKeyAuth | None" = None, + request_data: dict[str, object] | None = None, # mutable-ok: BaseTranslation contract + ) -> object: + return response diff --git a/litellm/llms/xai/chat/transformation.py b/litellm/llms/xai/chat/transformation.py index 91bf697487d..33ee727dfab 100644 --- a/litellm/llms/xai/chat/transformation.py +++ b/litellm/llms/xai/chat/transformation.py @@ -118,6 +118,7 @@ class XAIChatConfig(OpenAIGPTConfig): base_openai_params: Final = [ "logit_bias", "logprobs", + "max_completion_tokens", "max_tokens", "n", "parallel_tool_calls", diff --git a/litellm/main.py b/litellm/main.py index 6704358e3ea..2570b93455f 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -3594,6 +3594,37 @@ def _complete_edenai(ctx: _CompletionDispatchContext) -> _CompletionDispatchResu return response +def _complete_fal_ai(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult: + if ctx.stream: + raise litellm.FalAIError( + status_code=400, + message="fal_ai chat completions do not support streaming", + ) + api_base: Final = litellm.FalAIChatConfig.get_api_base(ctx.api_base) + api_key: Final = litellm.FalAIChatConfig.get_api_key(ctx.api_key or litellm.api_key) + response: Final = base_llm_http_handler.completion( + model=ctx.model, + messages=ctx.messages, + api_base=api_base, + custom_llm_provider="fal_ai", + model_response=ctx.model_response, + encoding=_get_encoding(), + logging_obj=ctx.logging, + optional_params=ctx.optional_params, + timeout=ctx.timeout, + litellm_params=ctx.litellm_params, + shared_session=ctx.shared_session, + acompletion=ctx.acompletion, + stream=ctx.stream, + api_key=api_key, + headers=ctx.headers or litellm.headers, + client=_dispatch_client_http(ctx), + provider_config=ctx.provider_config, + ) + ctx.logging.post_call(input=ctx.messages, api_key=api_key, original_response=response) + return response + + def _complete_vertex_ai_beta( ctx: _CompletionDispatchContext, ) -> _CompletionDispatchResult: @@ -5799,6 +5830,8 @@ def completion( response = _complete_hosted_vllm(_dispatch_ctx) elif custom_llm_provider == "edenai": response = _complete_edenai(_dispatch_ctx) # rebind-ok: dispatch chain binds response per branch + elif custom_llm_provider == "fal_ai": + response = _complete_fal_ai(_dispatch_ctx) # rebind-ok: dispatch chain binds response per branch elif ( # A known OpenAI model name only decides the route when nothing else # resolved a provider. get_llm_provider() already maps these names to diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 6b49b1d47a5..70c54b91cbd 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -24966,6 +24966,52 @@ "/v1/images/generations" ] }, + "fal_ai/fal-ai/trellis": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.02, + "source": "https://fal.ai/models/fal-ai/trellis", + "metadata": { + "comment": "image-to-3D, returns a GLB mesh; served through the /fal_ai pass-through route" + } + }, + "fal_ai/fal-ai/trellis-2": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.3, + "output_cost_per_image_512": 0.25, + "output_cost_per_image_1024": 0.3, + "output_cost_per_image_1536": 0.35, + "source": "https://fal.ai/models/fal-ai/trellis-2", + "metadata": { + "comment": "image-to-3D, returns a GLB mesh; priced by the request's resolution field (default 1024); served through the /fal_ai pass-through route" + } + }, + "fal_ai/fal-ai/flux-lora-depth": { + "litellm_provider": "fal_ai", + "metadata": { + "notes": "fal bills fal-ai/flux-lora-depth at $0.035 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries" + }, + "mode": "image_generation", + "output_cost_per_image": 0.035, + "output_cost_per_pixel": 3.337860107421875e-08, + "source": "https://fal.ai/models/fal-ai/flux-lora-depth", + "supported_endpoints": [ + "/v1/images/edits" + ] + }, + "fal_ai/fal-ai/moondream3-preview/query": { + "input_cost_per_token": 4e-07, + "litellm_provider": "fal_ai", + "mode": "chat", + "output_cost_per_token": 3.5e-06, + "source": "https://fal.ai/models/fal-ai/moondream3-preview/query", + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_reasoning": true, + "supports_vision": true + }, "featherless_ai/featherless-ai/Qwerky-72B": { "litellm_provider": "featherless_ai", "max_input_tokens": 32768, @@ -43037,21 +43083,21 @@ "supports_web_search": false }, "openrouter/deepseek/deepseek-v4-pro": { - "input_cost_per_token": 8.95578e-07, + "input_cost_per_token": 9.5526e-07, "input_cost_per_token_cache_hit": 4.4e-08, "litellm_provider": "openrouter", "max_input_tokens": 1048576, "max_output_tokens": 384000, "max_tokens": 384000, "mode": "chat", - "output_cost_per_token": 1.791156e-06, + "output_cost_per_token": 1.91052e-06, "source": "https://openrouter.ai/api/v1/models", "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "cache_read_input_token_cost": 7.46315e-08, + "cache_read_input_token_cost": 7.9605e-08, "supports_audio_input": false, "supports_pdf_input": false, "supports_vision": false, @@ -44812,26 +44858,6 @@ "max_tokens": 128000, "mode": "chat" }, - "openrouter/stealth/union-alpha": { - "deprecation_date": "2098-12-31", - "input_cost_per_token": 0.0, - "output_cost_per_token": 0.0, - "litellm_provider": "openrouter", - "max_input_tokens": 262144, - "max_output_tokens": 131072, - "max_tokens": 131072, - "mode": "chat", - "source": "https://openrouter.ai/api/v1/models", - "supports_audio_input": false, - "supports_function_calling": true, - "supports_pdf_input": false, - "supports_prompt_caching": false, - "supports_reasoning": false, - "supports_tool_choice": true, - "supports_response_schema": true, - "supports_vision": true, - "supports_web_search": false - }, "ovhcloud/DeepSeek-R1-Distill-Llama-70B": { "input_cost_per_token": 6.7e-07, "litellm_provider": "ovhcloud", @@ -54217,7 +54243,7 @@ "max_tokens": 1000000, "mode": "responses", "output_cost_per_token": 2.5e-06, - "source": "https://docs.x.ai/docs/models", + "source": "https://api.x.ai/v1/language-models", "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, @@ -54227,6 +54253,7 @@ "input_cost_per_token_above_200k_tokens": 2.5e-06, "output_cost_per_token_above_200k_tokens": 5e-06, "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_image_token": 1.25e-06, "supports_response_schema": true, "supported_endpoints": [ "/v1/responses" @@ -54241,7 +54268,7 @@ "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, - "source": "https://docs.x.ai/docs/models", + "source": "https://api.x.ai/v1/language-models", "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, @@ -54251,6 +54278,7 @@ "input_cost_per_token_above_200k_tokens": 2.5e-06, "output_cost_per_token_above_200k_tokens": 5e-06, "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_image_token": 1.25e-06, "supports_response_schema": true }, "xai/grok-4.20-0309-reasoning": { @@ -54262,7 +54290,7 @@ "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, - "source": "https://docs.x.ai/docs/models", + "source": "https://api.x.ai/v1/language-models", "supports_function_calling": true, "supports_reasoning": true, "supports_tool_choice": true, @@ -54271,6 +54299,7 @@ "input_cost_per_token_above_200k_tokens": 2.5e-06, "output_cost_per_token_above_200k_tokens": 5e-06, "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_image_token": 1.25e-06, "supports_prompt_caching": true, "supports_response_schema": true }, @@ -54283,7 +54312,7 @@ "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, - "source": "https://docs.x.ai/docs/models", + "source": "https://api.x.ai/v1/language-models", "supports_function_calling": true, "supports_prompt_caching": true, "supports_tool_choice": true, @@ -54292,11 +54321,13 @@ "input_cost_per_token_above_200k_tokens": 2.5e-06, "output_cost_per_token_above_200k_tokens": 5e-06, "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_image_token": 1.25e-06, "supports_response_schema": true }, "xai/grok-4.3": { "cache_read_input_token_cost": 2e-07, "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_image_token": 1.25e-06, "input_cost_per_token": 1.25e-06, "input_cost_per_token_above_200k_tokens": 2.5e-06, "litellm_provider": "xai", @@ -54306,7 +54337,7 @@ "mode": "chat", "output_cost_per_token": 2.5e-06, "output_cost_per_token_above_200k_tokens": 5e-06, - "source": "https://docs.x.ai/docs/models", + "source": "https://api.x.ai/v1/language-models", "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, @@ -54318,6 +54349,7 @@ "xai/grok-4.3-latest": { "cache_read_input_token_cost": 2e-07, "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_image_token": 1.25e-06, "input_cost_per_token": 1.25e-06, "input_cost_per_token_above_200k_tokens": 2.5e-06, "litellm_provider": "xai", @@ -54327,7 +54359,7 @@ "mode": "chat", "output_cost_per_token": 2.5e-06, "output_cost_per_token_above_200k_tokens": 5e-06, - "source": "https://docs.x.ai/docs/models", + "source": "https://api.x.ai/v1/language-models", "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, @@ -54339,6 +54371,7 @@ "xai/grok-4.5": { "cache_read_input_token_cost": 3e-07, "cache_read_input_token_cost_above_200k_tokens": 6e-07, + "input_cost_per_image_token": 2e-06, "input_cost_per_token": 2e-06, "input_cost_per_token_above_200k_tokens": 4e-06, "litellm_provider": "xai", @@ -54348,7 +54381,7 @@ "mode": "chat", "output_cost_per_token": 6e-06, "output_cost_per_token_above_200k_tokens": 1.2e-05, - "source": "https://docs.x.ai/docs/models", + "source": "https://api.x.ai/v1/language-models", "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, @@ -54360,6 +54393,7 @@ "xai/grok-4.5-latest": { "cache_read_input_token_cost": 3e-07, "cache_read_input_token_cost_above_200k_tokens": 6e-07, + "input_cost_per_image_token": 2e-06, "input_cost_per_token": 2e-06, "input_cost_per_token_above_200k_tokens": 4e-06, "litellm_provider": "xai", @@ -54369,7 +54403,7 @@ "mode": "chat", "output_cost_per_token": 6e-06, "output_cost_per_token_above_200k_tokens": 1.2e-05, - "source": "https://docs.x.ai/docs/models", + "source": "https://api.x.ai/v1/language-models", "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, @@ -54381,6 +54415,7 @@ "xai/grok-build-latest": { "cache_read_input_token_cost": 3e-07, "cache_read_input_token_cost_above_200k_tokens": 6e-07, + "input_cost_per_image_token": 2e-06, "input_cost_per_token": 2e-06, "input_cost_per_token_above_200k_tokens": 4e-06, "litellm_provider": "xai", @@ -54390,7 +54425,7 @@ "mode": "chat", "output_cost_per_token": 6e-06, "output_cost_per_token_above_200k_tokens": 1.2e-05, - "source": "https://docs.x.ai/developers/models", + "source": "https://api.x.ai/v1/language-models", "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, @@ -54402,6 +54437,7 @@ "xai/grok-4.6": { "cache_read_input_token_cost": 5e-07, "cache_read_input_token_cost_above_200k_tokens": 1e-06, + "input_cost_per_image_token": 2e-06, "input_cost_per_token": 2e-06, "input_cost_per_token_above_200k_tokens": 4e-06, "litellm_provider": "xai", @@ -54411,7 +54447,7 @@ "mode": "chat", "output_cost_per_token": 6e-06, "output_cost_per_token_above_200k_tokens": 1.2e-05, - "source": "https://docs.x.ai/developers/models", + "source": "https://api.x.ai/v1/language-models", "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, @@ -54423,6 +54459,7 @@ "xai/grok-4.7": { "cache_read_input_token_cost": 5e-07, "cache_read_input_token_cost_above_200k_tokens": 1e-06, + "input_cost_per_image_token": 2e-06, "input_cost_per_token": 2e-06, "input_cost_per_token_above_200k_tokens": 4e-06, "litellm_provider": "xai", @@ -54432,7 +54469,7 @@ "mode": "chat", "output_cost_per_token": 6e-06, "output_cost_per_token_above_200k_tokens": 1.2e-05, - "source": "https://docs.x.ai/developers/models", + "source": "https://api.x.ai/v1/language-models", "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, @@ -54450,7 +54487,7 @@ "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 2e-06, - "source": "https://docs.x.ai/docs/models", + "source": "https://api.x.ai/v1/language-models", "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, @@ -54460,7 +54497,8 @@ "cache_read_input_token_cost_above_200k_tokens": 4e-07, "supports_response_schema": true, "supports_vision": true, - "deprecation_date": "2026-05-15" + "deprecation_date": "2026-05-15", + "input_cost_per_image_token": 1e-06 }, "xai/grok-code-fast-1": { "cache_read_input_token_cost": 2e-07, @@ -54471,7 +54509,7 @@ "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 2e-06, - "source": "https://docs.x.ai/docs/models", + "source": "https://api.x.ai/v1/language-models", "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, @@ -54481,7 +54519,8 @@ "cache_read_input_token_cost_above_200k_tokens": 4e-07, "supports_response_schema": true, "supports_vision": true, - "deprecation_date": "2026-05-15" + "deprecation_date": "2026-05-15", + "input_cost_per_image_token": 1e-06 }, "xai/grok-code-fast-1-0825": { "cache_read_input_token_cost": 2e-07, @@ -54492,7 +54531,7 @@ "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 2e-06, - "source": "https://docs.x.ai/docs/models", + "source": "https://api.x.ai/v1/language-models", "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, @@ -54502,7 +54541,8 @@ "cache_read_input_token_cost_above_200k_tokens": 4e-07, "supports_response_schema": true, "supports_vision": true, - "deprecation_date": "2026-05-15" + "deprecation_date": "2026-05-15", + "input_cost_per_image_token": 1e-06 }, "zai.glm-4.7": { "input_cost_per_token": 6e-07, @@ -61978,7 +62018,7 @@ "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, - "source": "https://docs.x.ai/docs/models", + "source": "https://api.x.ai/v1/language-models", "supports_function_calling": true, "supports_prompt_caching": true, "supports_tool_choice": true, @@ -61987,6 +62027,7 @@ "input_cost_per_token_above_200k_tokens": 2.5e-06, "output_cost_per_token_above_200k_tokens": 5e-06, "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_image_token": 1.25e-06, "supports_response_schema": true }, "xai/grok-4.20-multi-agent-0309": { @@ -61998,7 +62039,7 @@ "max_tokens": 1000000, "mode": "responses", "output_cost_per_token": 2.5e-06, - "source": "https://docs.x.ai/docs/models", + "source": "https://api.x.ai/v1/language-models", "supports_function_calling": false, "supports_prompt_caching": true, "supports_reasoning": true, @@ -62008,6 +62049,7 @@ "input_cost_per_token_above_200k_tokens": 2.5e-06, "output_cost_per_token_above_200k_tokens": 5e-06, "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_image_token": 1.25e-06, "supports_response_schema": true, "supported_endpoints": [ "/v1/responses" @@ -62022,7 +62064,7 @@ "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 2e-06, - "source": "https://docs.x.ai/docs/models", + "source": "https://api.x.ai/v1/language-models", "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, @@ -62030,6 +62072,7 @@ "input_cost_per_token_above_200k_tokens": 2e-06, "output_cost_per_token_above_200k_tokens": 4e-06, "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_image_token": 1e-06, "supports_response_schema": true, "supports_vision": true }, @@ -65225,7 +65268,7 @@ "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, - "source": "https://docs.x.ai/docs/models", + "source": "https://api.x.ai/v1/language-models", "supports_function_calling": true, "supports_reasoning": true, "supports_tool_choice": true, @@ -65234,6 +65277,7 @@ "input_cost_per_token_above_200k_tokens": 2.5e-06, "output_cost_per_token_above_200k_tokens": 5e-06, "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_image_token": 1.25e-06, "supports_prompt_caching": true, "supports_response_schema": true }, @@ -65246,7 +65290,7 @@ "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, - "source": "https://docs.x.ai/docs/models", + "source": "https://api.x.ai/v1/language-models", "supports_function_calling": true, "supports_reasoning": true, "supports_tool_choice": true, @@ -65255,6 +65299,7 @@ "input_cost_per_token_above_200k_tokens": 2.5e-06, "output_cost_per_token_above_200k_tokens": 5e-06, "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_image_token": 1.25e-06, "supports_prompt_caching": true, "supports_response_schema": true }, @@ -65267,7 +65312,7 @@ "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, - "source": "https://docs.x.ai/docs/models", + "source": "https://api.x.ai/v1/language-models", "supports_function_calling": true, "supports_reasoning": true, "supports_tool_choice": true, @@ -65276,6 +65321,7 @@ "input_cost_per_token_above_200k_tokens": 2.5e-06, "output_cost_per_token_above_200k_tokens": 5e-06, "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_image_token": 1.25e-06, "supports_prompt_caching": true, "supports_response_schema": true }, @@ -65519,7 +65565,7 @@ "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, - "source": "https://docs.x.ai/docs/models", + "source": "https://api.x.ai/v1/language-models", "supports_function_calling": true, "supports_prompt_caching": true, "supports_tool_choice": true, @@ -65528,6 +65574,7 @@ "input_cost_per_token_above_200k_tokens": 2.5e-06, "output_cost_per_token_above_200k_tokens": 5e-06, "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_image_token": 1.25e-06, "supports_response_schema": true }, "xai/grok-4.20-non-reasoning-latest": { @@ -65539,7 +65586,7 @@ "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, - "source": "https://docs.x.ai/docs/models", + "source": "https://api.x.ai/v1/language-models", "supports_function_calling": true, "supports_prompt_caching": true, "supports_tool_choice": true, @@ -65548,6 +65595,7 @@ "input_cost_per_token_above_200k_tokens": 2.5e-06, "output_cost_per_token_above_200k_tokens": 5e-06, "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_image_token": 1.25e-06, "supports_response_schema": true }, "xai/grok-4.20-multi-agent": { @@ -65559,7 +65607,7 @@ "max_tokens": 1000000, "mode": "responses", "output_cost_per_token": 2.5e-06, - "source": "https://docs.x.ai/docs/models", + "source": "https://api.x.ai/v1/language-models", "supported_endpoints": [ "/v1/responses" ], @@ -65572,6 +65620,7 @@ "input_cost_per_token_above_200k_tokens": 2.5e-06, "output_cost_per_token_above_200k_tokens": 5e-06, "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_image_token": 1.25e-06, "supports_response_schema": true }, "xai/grok-4.20-multi-agent-latest": { @@ -65583,7 +65632,7 @@ "max_tokens": 1000000, "mode": "responses", "output_cost_per_token": 2.5e-06, - "source": "https://docs.x.ai/docs/models", + "source": "https://api.x.ai/v1/language-models", "supported_endpoints": [ "/v1/responses" ], @@ -65596,6 +65645,7 @@ "input_cost_per_token_above_200k_tokens": 2.5e-06, "output_cost_per_token_above_200k_tokens": 5e-06, "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_image_token": 1.25e-06, "supports_response_schema": true }, "groq/qwen/qwen3.8-27b": { @@ -68212,13 +68262,13 @@ "supports_web_search": false }, "openrouter/z-ai/glm-5.3-flash": { - "input_cost_per_token": 7.5e-08, - "output_cost_per_token": 2.5e-07, - "cache_read_input_token_cost": 2e-08, + "input_cost_per_token": 1.5e-07, + "output_cost_per_token": 5e-07, + "cache_read_input_token_cost": 5e-08, "litellm_provider": "openrouter", "max_input_tokens": 1310720, - "max_output_tokens": 102400, - "max_tokens": 102400, + "max_output_tokens": 943718, + "max_tokens": 943718, "mode": "chat", "source": "https://openrouter.ai/api/v1/models", "supports_audio_input": false, @@ -68252,9 +68302,9 @@ "supports_web_search": false }, "openrouter/z-ai/glm-5.3": { - "input_cost_per_token": 9.1e-07, - "output_cost_per_token": 2.86e-06, - "cache_read_input_token_cost": 1.69e-07, + "input_cost_per_token": 8.4e-07, + "output_cost_per_token": 2.64e-06, + "cache_read_input_token_cost": 1.56e-07, "litellm_provider": "openrouter", "max_input_tokens": 1310720, "max_output_tokens": 131072, @@ -68941,9 +68991,9 @@ "supports_web_search": true }, "openrouter/deepseek/deepseek-v4-flash": { - "input_cost_per_token": 5.544e-08, - "output_cost_per_token": 1.1088e-07, - "cache_read_input_token_cost": 1.1088e-08, + "input_cost_per_token": 8.8606e-08, + "output_cost_per_token": 1.77212e-07, + "cache_read_input_token_cost": 1.77212e-08, "litellm_provider": "openrouter", "max_input_tokens": 1048576, "max_output_tokens": 384000, @@ -70299,8 +70349,8 @@ "supports_web_search": true }, "openrouter/meta-llama/llama-4-maverick": { - "input_cost_per_token": 2e-07, - "output_cost_per_token": 8e-07, + "input_cost_per_token": 1.875e-07, + "output_cost_per_token": 6.525e-07, "litellm_provider": "openrouter", "max_input_tokens": 1048576, "max_output_tokens": 16384, @@ -72854,6 +72904,16 @@ "supports_reasoning": true, "supports_vision": true }, + "openrouter/typesafe/jev-1.13": { + "input_cost_per_token": 4.2e-08, + "litellm_provider": "openrouter", + "max_input_tokens": 32000, + "max_output_tokens": 28800, + "max_tokens": 28800, + "mode": "evaluation", + "output_cost_per_token": 0.0, + "source": "https://openrouter.ai/typesafe/jev-1.13" + }, "typesafe/jev-1.13.0": { "input_cost_per_token": 4.2e-08, "litellm_provider": "typesafe", @@ -73252,14 +73312,14 @@ "supports_web_search": true }, "openrouter/~z-ai/glm-flash-latest": { - "cache_read_input_token_cost": 2e-08, - "input_cost_per_token": 7.5e-08, + "cache_read_input_token_cost": 5e-08, + "input_cost_per_token": 1.5e-07, "litellm_provider": "openrouter", "max_input_tokens": 1310720, - "max_output_tokens": 102400, - "max_tokens": 102400, + "max_output_tokens": 943718, + "max_tokens": 943718, "mode": "chat", - "output_cost_per_token": 2.5e-07, + "output_cost_per_token": 5e-07, "source": "https://openrouter.ai/api/v1/models", "supports_audio_input": false, "supports_function_calling": true, @@ -73272,14 +73332,14 @@ "supports_web_search": false }, "openrouter/~z-ai/glm-latest": { - "cache_read_input_token_cost": 1.69e-07, - "input_cost_per_token": 9.1e-07, + "cache_read_input_token_cost": 1.56e-07, + "input_cost_per_token": 8.4e-07, "litellm_provider": "openrouter", "max_input_tokens": 1310720, "max_output_tokens": 131072, "max_tokens": 131072, "mode": "chat", - "output_cost_per_token": 2.86e-06, + "output_cost_per_token": 2.64e-06, "source": "https://openrouter.ai/api/v1/models", "supports_audio_input": false, "supports_function_calling": true, @@ -73794,6 +73854,7 @@ "supports_web_search": false }, "openrouter/bytedance-seed/seed-1.6": { + "deprecation_date": "2026-11-11", "input_cost_per_token": 2.5e-07, "input_cost_per_token_above_128k_tokens": 5e-07, "litellm_provider": "openrouter", @@ -73815,6 +73876,7 @@ "supports_web_search": false }, "openrouter/bytedance-seed/seed-1.6-flash": { + "deprecation_date": "2026-11-11", "input_cost_per_token": 7.5e-08, "input_cost_per_token_above_128k_tokens": 1e-07, "litellm_provider": "openrouter", @@ -73855,6 +73917,7 @@ "supports_web_search": false }, "openrouter/bytedance-seed/seed-2.0-code": { + "deprecation_date": "2026-11-11", "input_cost_per_token": 5e-07, "input_cost_per_token_above_128k_tokens": 1e-06, "litellm_provider": "openrouter", @@ -76876,6 +76939,26 @@ "supports_vision": true, "supports_web_search": true }, + "moonshotai.kimi-k3": { + "cache_creation_input_token_cost": 3.75e-06, + "cache_read_input_token_cost": 3e-07, + "input_cost_per_token": 3e-06, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 1000000, + "max_output_tokens": 131072, + "max_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 1.5e-05, + "source": "https://aws.amazon.com/bedrock/pricing/", + "supports_audio_input": false, + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true + }, "global.moonshotai.kimi-k3": { "cache_creation_input_token_cost": 3.75e-06, "cache_read_input_token_cost": 3e-07, @@ -76975,5 +77058,473 @@ "supports_tool_choice": true, "supports_vision": true, "supports_web_search": false + }, + "xiaomi_mimo/mimo-v2.6-pro": { + "cache_read_input_token_cost": 3.6e-09, + "input_cost_per_token": 4.35e-07, + "litellm_provider": "xiaomi_mimo", + "max_input_tokens": 1048576, + "max_output_tokens": 131072, + "max_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 8.7e-07, + "source": "https://platform.xiaomimimo.com/static/docs/price/pay-as-you-go.md", + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_audio_input": true, + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_video_input": true, + "supports_vision": true + }, + "xiaomi_mimo/mimo-v2.6-flash": { + "cache_read_input_token_cost": 2.8e-09, + "input_cost_per_token": 1.4e-07, + "litellm_provider": "xiaomi_mimo", + "max_input_tokens": 1048576, + "max_output_tokens": 131072, + "max_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 2.8e-07, + "source": "https://platform.xiaomimimo.com/static/docs/price/pay-as-you-go.md", + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_audio_input": true, + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_video_input": true, + "supports_vision": true + }, + "xai/grok-4.20-0309": { + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 1.25e-06, + "litellm_provider": "xai", + "max_input_tokens": 1000000, + "max_output_tokens": 1000000, + "max_tokens": 1000000, + "mode": "chat", + "output_cost_per_token": 2.5e-06, + "source": "https://api.x.ai/v1/language-models", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true, + "input_cost_per_token_above_200k_tokens": 2.5e-06, + "output_cost_per_token_above_200k_tokens": 5e-06, + "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_image_token": 1.25e-06, + "supports_prompt_caching": true, + "supports_response_schema": true + }, + "xai/grok-4.20-beta": { + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 1.25e-06, + "litellm_provider": "xai", + "max_input_tokens": 1000000, + "max_output_tokens": 1000000, + "max_tokens": 1000000, + "mode": "chat", + "output_cost_per_token": 2.5e-06, + "source": "https://api.x.ai/v1/language-models", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true, + "input_cost_per_token_above_200k_tokens": 2.5e-06, + "output_cost_per_token_above_200k_tokens": 5e-06, + "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_image_token": 1.25e-06, + "supports_prompt_caching": true, + "supports_response_schema": true + }, + "xai/grok-4.20-beta-0309": { + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 1.25e-06, + "litellm_provider": "xai", + "max_input_tokens": 1000000, + "max_output_tokens": 1000000, + "max_tokens": 1000000, + "mode": "chat", + "output_cost_per_token": 2.5e-06, + "source": "https://api.x.ai/v1/language-models", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true, + "input_cost_per_token_above_200k_tokens": 2.5e-06, + "output_cost_per_token_above_200k_tokens": 5e-06, + "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_image_token": 1.25e-06, + "supports_prompt_caching": true, + "supports_response_schema": true + }, + "xai/grok-4.20-beta-latest": { + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 1.25e-06, + "litellm_provider": "xai", + "max_input_tokens": 1000000, + "max_output_tokens": 1000000, + "max_tokens": 1000000, + "mode": "chat", + "output_cost_per_token": 2.5e-06, + "source": "https://api.x.ai/v1/language-models", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true, + "input_cost_per_token_above_200k_tokens": 2.5e-06, + "output_cost_per_token_above_200k_tokens": 5e-06, + "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_image_token": 1.25e-06, + "supports_prompt_caching": true, + "supports_response_schema": true + }, + "xai/grok-4.20-beta-latest-non-reasoning": { + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 1.25e-06, + "litellm_provider": "xai", + "max_input_tokens": 1000000, + "max_output_tokens": 1000000, + "max_tokens": 1000000, + "mode": "chat", + "output_cost_per_token": 2.5e-06, + "source": "https://api.x.ai/v1/language-models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true, + "input_cost_per_token_above_200k_tokens": 2.5e-06, + "output_cost_per_token_above_200k_tokens": 5e-06, + "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_image_token": 1.25e-06, + "supports_response_schema": true + }, + "xai/grok-4.20-beta-latest-reasoning": { + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 1.25e-06, + "litellm_provider": "xai", + "max_input_tokens": 1000000, + "max_output_tokens": 1000000, + "max_tokens": 1000000, + "mode": "chat", + "output_cost_per_token": 2.5e-06, + "source": "https://api.x.ai/v1/language-models", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true, + "input_cost_per_token_above_200k_tokens": 2.5e-06, + "output_cost_per_token_above_200k_tokens": 5e-06, + "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_image_token": 1.25e-06, + "supports_prompt_caching": true, + "supports_response_schema": true + }, + "xai/grok-4.20-beta-non-reasoning": { + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 1.25e-06, + "litellm_provider": "xai", + "max_input_tokens": 1000000, + "max_output_tokens": 1000000, + "max_tokens": 1000000, + "mode": "chat", + "output_cost_per_token": 2.5e-06, + "source": "https://api.x.ai/v1/language-models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true, + "input_cost_per_token_above_200k_tokens": 2.5e-06, + "output_cost_per_token_above_200k_tokens": 5e-06, + "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_image_token": 1.25e-06, + "supports_response_schema": true + }, + "xai/grok-4.20-beta-reasoning": { + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 1.25e-06, + "litellm_provider": "xai", + "max_input_tokens": 1000000, + "max_output_tokens": 1000000, + "max_tokens": 1000000, + "mode": "chat", + "output_cost_per_token": 2.5e-06, + "source": "https://api.x.ai/v1/language-models", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true, + "input_cost_per_token_above_200k_tokens": 2.5e-06, + "output_cost_per_token_above_200k_tokens": 5e-06, + "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_image_token": 1.25e-06, + "supports_prompt_caching": true, + "supports_response_schema": true + }, + "xai/grok-4.20-experimental-beta-0304": { + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 1.25e-06, + "litellm_provider": "xai", + "max_input_tokens": 1000000, + "max_output_tokens": 1000000, + "max_tokens": 1000000, + "mode": "chat", + "output_cost_per_token": 2.5e-06, + "source": "https://api.x.ai/v1/language-models", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true, + "input_cost_per_token_above_200k_tokens": 2.5e-06, + "output_cost_per_token_above_200k_tokens": 5e-06, + "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_image_token": 1.25e-06, + "supports_prompt_caching": true, + "supports_response_schema": true + }, + "xai/grok-4.20-experimental-beta-0304-non-reasoning": { + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 1.25e-06, + "litellm_provider": "xai", + "max_input_tokens": 1000000, + "max_output_tokens": 1000000, + "max_tokens": 1000000, + "mode": "chat", + "output_cost_per_token": 2.5e-06, + "source": "https://api.x.ai/v1/language-models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true, + "input_cost_per_token_above_200k_tokens": 2.5e-06, + "output_cost_per_token_above_200k_tokens": 5e-06, + "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_image_token": 1.25e-06, + "supports_response_schema": true + }, + "xai/grok-4.20-experimental-beta-0304-reasoning": { + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 1.25e-06, + "litellm_provider": "xai", + "max_input_tokens": 1000000, + "max_output_tokens": 1000000, + "max_tokens": 1000000, + "mode": "chat", + "output_cost_per_token": 2.5e-06, + "source": "https://api.x.ai/v1/language-models", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true, + "input_cost_per_token_above_200k_tokens": 2.5e-06, + "output_cost_per_token_above_200k_tokens": 5e-06, + "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_image_token": 1.25e-06, + "supports_prompt_caching": true, + "supports_response_schema": true + }, + "xai/grok-4.20-experimental-beta-latest": { + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 1.25e-06, + "litellm_provider": "xai", + "max_input_tokens": 1000000, + "max_output_tokens": 1000000, + "max_tokens": 1000000, + "mode": "chat", + "output_cost_per_token": 2.5e-06, + "source": "https://api.x.ai/v1/language-models", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true, + "input_cost_per_token_above_200k_tokens": 2.5e-06, + "output_cost_per_token_above_200k_tokens": 5e-06, + "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_image_token": 1.25e-06, + "supports_prompt_caching": true, + "supports_response_schema": true + }, + "xai/grok-4.20-experimental-beta-non-reasoning-latest": { + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 1.25e-06, + "litellm_provider": "xai", + "max_input_tokens": 1000000, + "max_output_tokens": 1000000, + "max_tokens": 1000000, + "mode": "chat", + "output_cost_per_token": 2.5e-06, + "source": "https://api.x.ai/v1/language-models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true, + "input_cost_per_token_above_200k_tokens": 2.5e-06, + "output_cost_per_token_above_200k_tokens": 5e-06, + "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_image_token": 1.25e-06, + "supports_response_schema": true + }, + "xai/grok-4.20-experimental-beta-reasoning-latest": { + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 1.25e-06, + "litellm_provider": "xai", + "max_input_tokens": 1000000, + "max_output_tokens": 1000000, + "max_tokens": 1000000, + "mode": "chat", + "output_cost_per_token": 2.5e-06, + "source": "https://api.x.ai/v1/language-models", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true, + "input_cost_per_token_above_200k_tokens": 2.5e-06, + "output_cost_per_token_above_200k_tokens": 5e-06, + "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_image_token": 1.25e-06, + "supports_prompt_caching": true, + "supports_response_schema": true + }, + "xai/grok-4.20-multi-agent-beta-latest": { + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 1.25e-06, + "litellm_provider": "xai", + "max_input_tokens": 1000000, + "max_output_tokens": 1000000, + "max_tokens": 1000000, + "mode": "responses", + "output_cost_per_token": 2.5e-06, + "source": "https://api.x.ai/v1/language-models", + "supports_function_calling": false, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": false, + "supports_vision": true, + "supports_web_search": true, + "input_cost_per_token_above_200k_tokens": 2.5e-06, + "output_cost_per_token_above_200k_tokens": 5e-06, + "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_image_token": 1.25e-06, + "supports_response_schema": true, + "supported_endpoints": [ + "/v1/responses" + ] + }, + "xai/grok-4.20-multi-agent-experimental-beta-0304": { + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 1.25e-06, + "litellm_provider": "xai", + "max_input_tokens": 1000000, + "max_output_tokens": 1000000, + "max_tokens": 1000000, + "mode": "responses", + "output_cost_per_token": 2.5e-06, + "source": "https://api.x.ai/v1/language-models", + "supports_function_calling": false, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": false, + "supports_vision": true, + "supports_web_search": true, + "input_cost_per_token_above_200k_tokens": 2.5e-06, + "output_cost_per_token_above_200k_tokens": 5e-06, + "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_image_token": 1.25e-06, + "supports_response_schema": true, + "supported_endpoints": [ + "/v1/responses" + ] + }, + "xai/grok-4.20-multi-agent-experimental-beta-latest": { + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 1.25e-06, + "litellm_provider": "xai", + "max_input_tokens": 1000000, + "max_output_tokens": 1000000, + "max_tokens": 1000000, + "mode": "responses", + "output_cost_per_token": 2.5e-06, + "source": "https://api.x.ai/v1/language-models", + "supports_function_calling": false, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": false, + "supports_vision": true, + "supports_web_search": true, + "input_cost_per_token_above_200k_tokens": 2.5e-06, + "output_cost_per_token_above_200k_tokens": 5e-06, + "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_image_token": 1.25e-06, + "supports_response_schema": true, + "supported_endpoints": [ + "/v1/responses" + ] + }, + "xai/grok-4.20-non-reasoning-gv2": { + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 1.25e-06, + "litellm_provider": "xai", + "max_input_tokens": 1000000, + "max_output_tokens": 1000000, + "max_tokens": 1000000, + "mode": "chat", + "output_cost_per_token": 2.5e-06, + "source": "https://api.x.ai/v1/language-models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true, + "input_cost_per_token_above_200k_tokens": 2.5e-06, + "output_cost_per_token_above_200k_tokens": 5e-06, + "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_image_token": 1.25e-06, + "supports_response_schema": true + }, + "xai/grok-4.20-reasoning-gv2": { + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 1.25e-06, + "litellm_provider": "xai", + "max_input_tokens": 1000000, + "max_output_tokens": 1000000, + "max_tokens": 1000000, + "mode": "chat", + "output_cost_per_token": 2.5e-06, + "source": "https://api.x.ai/v1/language-models", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true, + "input_cost_per_token_above_200k_tokens": 2.5e-06, + "output_cost_per_token_above_200k_tokens": 5e-06, + "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_image_token": 1.25e-06, + "supports_prompt_caching": true, + "supports_response_schema": true } } diff --git a/litellm/models/user.py b/litellm/models/user.py index 82f78c28078..92aca87d303 100644 --- a/litellm/models/user.py +++ b/litellm/models/user.py @@ -24,6 +24,8 @@ class LiteLLM_UserTable(LiteLLMPydanticObjectBase): organization_id: str | None = None object_permission_id: str | None = None password: str | None = Field(default=None, exclude=True) + password_reset_required: bool | None = None + last_breach_check_at: datetime | None = None teams: list[str] = [] user_role: str | None = None max_budget: float | None = None diff --git a/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py b/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py index b0640e4f0dd..2e5cad29b0a 100644 --- a/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py +++ b/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py @@ -44,6 +44,11 @@ from litellm.proxy._types import ( UserAPIKeyAuth, user_api_key_has_admin_view, ) +from litellm.proxy.agent_endpoints.auth.agent_access_groups import ( + CeilingResolver, + resolve_agent_access_group_ceiling, +) +from litellm.proxy.agent_endpoints.auth.agent_caller import agent_caller_auth from litellm.proxy.auth.ip_address_utils import IPAddressUtils from litellm.proxy.auth.user_api_key_auth import ( _get_bearer_token_or_received_api_key, # pyright: ignore[reportPrivateUsage] # shared x-litellm-api-key parser lives with user_api_key_auth @@ -184,6 +189,21 @@ def _has_client_supplied_mcp_auth( return bool(mcp_auth_header) or bool(mcp_server_auth_headers) +def _agent_capped_servers( + allowed_mcp_servers: Sequence[str], + agent_servers: Sequence[str], + agent_access_group_servers: frozenset[str] | None, +) -> tuple[str, ...] | None: + if not agent_servers and agent_access_group_servers is None: + return None + return tuple( + s + for s in allowed_mcp_servers + if (not agent_servers or s in agent_servers) + and (agent_access_group_servers is None or s in agent_access_group_servers) + ) + + def _is_mcp_admitted_user_subject(user_api_key_auth: UserAPIKeyAuth | None) -> bool: """True when this auth is a keyless subject admitted by the gateway session / bridge user path, as opposed to a JWT or other keyless auth that merely lacks a ``team_id``. @@ -1546,25 +1566,33 @@ class MCPRequestHandler: # Check agent permissions if agent_id is set on the key ######################################################### if user_api_key_auth and user_api_key_auth.agent_id: - allowed_mcp_servers_for_agent: Final = await MCPRequestHandler._get_allowed_mcp_servers_for_agent( - user_api_key_auth + agent_capped: Final = _agent_capped_servers( + allowed_mcp_servers, + await MCPRequestHandler._get_allowed_mcp_servers_for_agent(user_api_key_auth), + await MCPRequestHandler._get_agent_access_group_server_ceiling(user_api_key_auth), ) - if len(allowed_mcp_servers_for_agent) > 0: + if agent_capped is not None: has_lower_level_mcp_restrictions = True - # Intersect: agent can only use servers allowed by BOTH key/team AND agent config - allowed_mcp_servers = [s for s in allowed_mcp_servers if s in allowed_mcp_servers_for_agent] + allowed_mcp_servers = list(agent_capped) verbose_logger.debug( "Applied agent intersection filter. Final allowed servers: %s", allowed_mcp_servers ) + ######################################################### + # Cap an agent key at what the user and team that invoked the agent may reach + ######################################################### + caller_capped, caller_restricts = await MCPRequestHandler._apply_agent_caller_ceiling( + allowed_mcp_servers, user_api_key_auth + ) + ######################################################### # Apply the internal user's own ceiling (the entitlement attached to the human) ######################################################### capped, user_restricts = await MCPRequestHandler._apply_user_server_ceiling( - allowed_mcp_servers, user_api_key_auth, keyless_source=keyless_source + caller_capped, user_api_key_auth, keyless_source=keyless_source ) allowed_mcp_servers = list(capped) - has_lower_level_mcp_restrictions = has_lower_level_mcp_restrictions or user_restricts + has_lower_level_mcp_restrictions = has_lower_level_mcp_restrictions or caller_restricts or user_restricts ######################################################### # Apply org-level ceiling if org_id is set @@ -2907,6 +2935,28 @@ class MCPRequestHandler: verbose_logger.debug("Applied user ceiling filter. Final allowed servers: %s", capped) return capped, True + @staticmethod + async def _apply_agent_caller_ceiling( + allowed_mcp_servers: Sequence[str], + user_api_key_auth: UserAPIKeyAuth | None = None, + ) -> tuple[tuple[str, ...], bool]: + """Narrow an agent key's servers to those the invoking user and team (echoed back by the agent + as ``x-litellm-user-id`` / ``x-litellm-team-id``) may reach: the echoed team's grants when it + names any, then the echoed user's own entitlement. Raises like the user ceiling when that + entitlement is known but unreadable, so the resolver denies rather than widens.""" + caller_auth: Final = agent_caller_auth(user_api_key_auth) if user_api_key_auth else None + if caller_auth is None: + return tuple(allowed_mcp_servers), False + team_servers: Final = frozenset(await MCPRequestHandler._get_allowed_mcp_servers_for_team(caller_auth)) + team_capped: Final = ( + tuple(server for server in allowed_mcp_servers if server in team_servers) + if team_servers + else tuple(allowed_mcp_servers) + ) + user_capped, user_restricts = await MCPRequestHandler._apply_user_server_ceiling(team_capped, caller_auth) + verbose_logger.debug("Applied agent caller ceiling. Final allowed servers: %s", user_capped) + return user_capped, bool(team_servers) or user_restricts + @staticmethod async def _user_places_mcp_ceiling(user_api_key_auth: UserAPIKeyAuth | None = None) -> bool: """Whether this human's own entitlement bounds their MCP access at all. @@ -3137,6 +3187,27 @@ class MCPRequestHandler: verbose_logger.warning("Failed to get allowed MCP servers for agent: %s", e) return [] + @staticmethod + async def _get_agent_access_group_server_ceiling( + user_api_key_auth: UserAPIKeyAuth, + resolve_ceiling: CeilingResolver = resolve_agent_access_group_ceiling, + ) -> frozenset[str] | None: + """ + Server IDs the agent's attached unified access groups (``LiteLLM_AgentsTable.access_group_ids``) + allow, or None when the agent has none attached. Unlike the object_permission path above, an + attached group set that names no servers is an empty ceiling and denies every server. + """ + from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + global_mcp_server_manager, + ) + + if not user_api_key_auth.agent_id: + return None + ceiling: Final = await resolve_ceiling(user_api_key_auth.agent_id) + if ceiling is None: + return None + return frozenset(global_mcp_server_manager.expand_permission_list(sorted(ceiling.mcp_server_ids))) + @staticmethod async def _get_agent_tool_permissions_for_server( server_id: str, diff --git a/litellm/proxy/_lazy_features.py b/litellm/proxy/_lazy_features.py index 17c85bbdbca..c53f7625550 100644 --- a/litellm/proxy/_lazy_features.py +++ b/litellm/proxy/_lazy_features.py @@ -203,6 +203,7 @@ LAZY_FEATURES: Final[tuple[LazyFeature, ...]] = ( "/cursor/", "/deepgram/", "/eu.assemblyai/", + "/fal_ai/", "/gemini/", "/gigachat/", "/milvus/", @@ -212,6 +213,7 @@ LAZY_FEATURES: Final[tuple[LazyFeature, ...]] = ( "/openai_passthrough/", "/transcribe", "/typesafe/", + "/openrouter/", "/vertex-ai/", "/vertex_ai/", "/vllm/", diff --git a/litellm/proxy/_lazy_openapi_snapshot.json b/litellm/proxy/_lazy_openapi_snapshot.json index 121cffa4cb9..1e05dde3202 100644 --- a/litellm/proxy/_lazy_openapi_snapshot.json +++ b/litellm/proxy/_lazy_openapi_snapshot.json @@ -2357,6 +2357,20 @@ }, "AgentConfig": { "properties": { + "access_group_ids": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Access Group Ids" + }, "agent_card_params": { "$ref": "#/components/schemas/AgentCard" }, @@ -2683,6 +2697,20 @@ }, "AgentResponse": { "properties": { + "access_group_ids": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Access Group Ids" + }, "agent_card_params": { "additionalProperties": true, "title": "Agent Card Params", @@ -3506,6 +3534,20 @@ }, "PatchAgentRequest": { "properties": { + "access_group_ids": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Access Group Ids" + }, "agent_card_params": { "$ref": "#/components/schemas/AgentCard" }, @@ -15382,14 +15424,31 @@ "title": "Jwt Issuer" }, "key": { - "title": "Key", - "type": "string" + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Key" + }, + "token": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Token" } }, "required": [ "jwt_claim_name", - "jwt_claim_value", - "key" + "jwt_claim_value" ], "title": "CreateJWTKeyMappingRequest", "type": "object" @@ -15553,6 +15612,17 @@ } ], "title": "Key" + }, + "token": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Token" } }, "required": [ @@ -18448,6 +18518,223 @@ ] } }, + "/fal_ai/{endpoint}": { + "delete": { + "operationId": "fal_ai_proxy_route_fal_ai__endpoint__delete", + "parameters": [ + { + "in": "path", + "name": "endpoint", + "required": true, + "schema": { + "title": "Endpoint", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Fal Ai Proxy Route", + "tags": [ + "llm_passthrough" + ] + }, + "get": { + "operationId": "fal_ai_proxy_route_fal_ai__endpoint__get", + "parameters": [ + { + "in": "path", + "name": "endpoint", + "required": true, + "schema": { + "title": "Endpoint", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Fal Ai Proxy Route", + "tags": [ + "llm_passthrough" + ] + }, + "patch": { + "operationId": "fal_ai_proxy_route_fal_ai__endpoint__patch", + "parameters": [ + { + "in": "path", + "name": "endpoint", + "required": true, + "schema": { + "title": "Endpoint", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Fal Ai Proxy Route", + "tags": [ + "llm_passthrough" + ] + }, + "post": { + "operationId": "fal_ai_proxy_route_fal_ai__endpoint__post", + "parameters": [ + { + "in": "path", + "name": "endpoint", + "required": true, + "schema": { + "title": "Endpoint", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Fal Ai Proxy Route", + "tags": [ + "llm_passthrough" + ] + }, + "put": { + "operationId": "fal_ai_proxy_route_fal_ai__endpoint__put", + "parameters": [ + { + "in": "path", + "name": "endpoint", + "required": true, + "schema": { + "title": "Endpoint", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Fal Ai Proxy Route", + "tags": [ + "llm_passthrough" + ] + } + }, "/gemini/{endpoint}": { "delete": { "description": "[Docs](https://docs.litellm.ai/docs/pass_through/google_ai_studio)", @@ -20659,6 +20946,223 @@ ] } }, + "/openrouter/{endpoint}": { + "delete": { + "operationId": "openrouter_proxy_route_openrouter__endpoint__delete", + "parameters": [ + { + "in": "path", + "name": "endpoint", + "required": true, + "schema": { + "title": "Endpoint", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Openrouter Proxy Route", + "tags": [ + "llm_passthrough" + ] + }, + "get": { + "operationId": "openrouter_proxy_route_openrouter__endpoint__get", + "parameters": [ + { + "in": "path", + "name": "endpoint", + "required": true, + "schema": { + "title": "Endpoint", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Openrouter Proxy Route", + "tags": [ + "llm_passthrough" + ] + }, + "patch": { + "operationId": "openrouter_proxy_route_openrouter__endpoint__patch", + "parameters": [ + { + "in": "path", + "name": "endpoint", + "required": true, + "schema": { + "title": "Endpoint", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Openrouter Proxy Route", + "tags": [ + "llm_passthrough" + ] + }, + "post": { + "operationId": "openrouter_proxy_route_openrouter__endpoint__post", + "parameters": [ + { + "in": "path", + "name": "endpoint", + "required": true, + "schema": { + "title": "Endpoint", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Openrouter Proxy Route", + "tags": [ + "llm_passthrough" + ] + }, + "put": { + "operationId": "openrouter_proxy_route_openrouter__endpoint__put", + "parameters": [ + { + "in": "path", + "name": "endpoint", + "required": true, + "schema": { + "title": "Endpoint", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Openrouter Proxy Route", + "tags": [ + "llm_passthrough" + ] + } + }, "/transcribe": { "post": { "description": "AWS-SDK-shaped pass-through for Amazon Transcribe: point the SDK's `endpoint_url`\nat `/transcribe` and the operation is read from the `X-Amz-Target` header, per the\nAWS JSON 1.1 protocol.\n\n[Docs](https://docs.litellm.ai/docs/pass_through/transcribe)", diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 370820be68a..de4bd2e161f 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -27,6 +27,7 @@ from litellm.litellm_core_utils.initialize_dynamic_callback_params import ( validate_langfuse_span_scope_value, validate_no_callback_env_reference, ) +from litellm.types.agents import AgentCaller from litellm.types.integrations.compression_interception import ( CompressionSavingsMetadata, ) @@ -493,11 +494,13 @@ class LiteLLMRoutes(enum.Enum): "/vllm", "/mistral", "/typesafe", + "/openrouter", "/milvus", "/gigachat", "/watsonx", "/nvidia_nim", "/deepgram", + "/fal_ai", ] ######################################################### @@ -907,6 +910,7 @@ class LiteLLMRoutes(enum.Enum): "/claude_code_gateway/v1/traces", "/user/list", # org admins checked in endpoint; non-admins get 403 "/management/v1/users/bulk_delete", # proxy admins delete anyone, org admins only their orgs' users; others 403 + "/user/password/change", # endpoint only ever writes the caller's own row "/model/{model_id}/update", "/prompt/list", "/prompt/info", @@ -1867,6 +1871,17 @@ class NewUserRequest(GenerateRequestBase): send_invite_email: bool | None = None sso_user_id: str | None = None organizations: list[str] | None = None + password: str | None = None + + @field_validator("password") + @classmethod + def password_not_supported(cls, value: str | None) -> str | None: + if value is not None: + raise ValueError( + "password cannot be set via /user/new. Users set their own password through an " + "invitation link (POST /invitation/new)." + ) + return value class NewUserResponse(GenerateKeyResponse): @@ -1889,7 +1904,8 @@ class NewUserResponse(GenerateKeyResponse): class UpdateUserRequestNoUserIDorEmail(GenerateRequestBase): # shared with BulkUpdateUserRequest - password: str | None = None + # repr=False keeps the plaintext out of management-endpoint alerts, which str() the request model + password: str | None = Field(default=None, repr=False) spend: float | None = None metadata: dict | None = None user_alias: str | None = None @@ -1919,6 +1935,16 @@ class UpdateUserRequest(UpdateUserRequestNoUserIDorEmail): return values +class ChangePasswordRequest(LiteLLMPydanticObjectBase): + current_password: str = Field(repr=False) + new_password: str = Field(repr=False) + + +class ChangePasswordResponse(LiteLLMPydanticObjectBase): + user_id: str + message: str + + class DeleteUserRequest(LiteLLMPydanticObjectBase): user_ids: list[str] # required @@ -2626,6 +2652,10 @@ class ConfigGeneralSettings(LiteLLMPydanticObjectBase): None, description="opt-in to RFC 8628 verification_uri_complete for the CLI SSO device flow, pre-filling the user_code in the browser. Off by default; intended for same-host clients where the device that starts the flow and the browser run on the same machine", ) + include_call_id_in_error_body: bool | None = Field( + None, + description="opt-in to copy the x-litellm-call-id response header's value into JSON error bodies, as error.litellm_call_id on the OpenAI-shaped and /v1/messages routes and as a top-level litellm_call_id on pass-through routes, so an error a client prints names the request to look up. Off by default", + ) enable_claude_code_gateway: bool | None = Field( None, description="serve the Claude Code gateway protocol (https://code.claude.com/docs/en/claude-apps-gateway) under /claude_code_gateway: OAuth device-flow sign-in reusing proxy SSO, plus managed settings and OTLP telemetry ingestion. Off by default", @@ -3253,6 +3283,15 @@ class UserAPIKeyAuth(LiteLLM_VerificationTokenView): # the expected response ob "user id." ), ) + agent_caller: AgentCaller | None = Field( + default=None, + exclude=True, + description=( + "Set per request from the x-litellm-user-id / x-litellm-team-id headers an agent echoes back on " + "calls made with its own key. Every check treats it as a ceiling, so a forged value can only " + "narrow the agent's access." + ), + ) budget_reservation: dict[str, Any] | None = Field(default=None, exclude=True) team_budget_snapshot: TeamBudgetSnapshot | None = Field(default=None, exclude=True) user_budget_snapshot: UserBudgetSnapshot | None = Field(default=None, exclude=True) @@ -3285,6 +3324,7 @@ class UserAPIKeyAuth(LiteLLM_VerificationTokenView): # the expected response ob values.pop("mcp_session_resource_server_id", None) values.pop("mcp_toolset_id", None) values.pop("via_virtual_key", None) + values.pop("agent_caller", None) if values.get("api_key") is not None: values.update({"token": cls._safe_hash_litellm_api_key(values.get("api_key"))}) if isinstance(values.get("api_key"), str): @@ -3942,6 +3982,12 @@ class AllCallbacks(LiteLLMPydanticObjectBase): ) +class HTTPExceptionErrorDetail(TypedDict): + """The `{"error": }` shape most proxy endpoints raise as `HTTPException.detail`.""" + + error: ReadOnly[str] + + class SpendLogsRouterMetadata(TypedDict): """ Router provenance stamped on spend logs for deployments flagged with @@ -4234,6 +4280,11 @@ class ProxyErrorTypes(str, enum.Enum): Project does not have access to the model """ + agent_model_access_denied = "agent_model_access_denied" + """ + The agent behind the key does not have access to the model + """ + model_cost_map_missing = "model_cost_map_missing" expired_key = "expired_key" @@ -4308,7 +4359,7 @@ class ProxyErrorTypes(str, enum.Enum): @classmethod def get_model_access_error_type_for_object( - cls, object_type: Literal["key", "user", "team", "org", "project"] + cls, object_type: Literal["key", "user", "team", "org", "project", "agent"] ) -> "ProxyErrorTypes": """ Get the model access error type for object_type @@ -4323,6 +4374,8 @@ class ProxyErrorTypes(str, enum.Enum): return cls.org_model_access_denied elif object_type == "project": return cls.project_model_access_denied + elif object_type == "agent": + return cls.agent_model_access_denied @classmethod def get_vector_store_access_error_type_for_object( @@ -4688,7 +4741,8 @@ class KeyHealthResponse(TypedDict, total=False): class CreateJWTKeyMappingRequest(LiteLLMPydanticObjectBase): jwt_claim_name: str jwt_claim_value: str - key: str + key: str | None = None + token: str | None = None jwt_issuer: str | None = None description: str | None = None @@ -4696,6 +4750,7 @@ class CreateJWTKeyMappingRequest(LiteLLMPydanticObjectBase): class UpdateJWTKeyMappingRequest(LiteLLMPydanticObjectBase): id: str key: str | None = None + token: str | None = None jwt_issuer: str | None = None description: str | None = None is_active: bool | None = None diff --git a/litellm/proxy/agent_endpoints/a2a_endpoints.py b/litellm/proxy/agent_endpoints/a2a_endpoints.py index 834c16ba6dc..2a189a76545 100644 --- a/litellm/proxy/agent_endpoints/a2a_endpoints.py +++ b/litellm/proxy/agent_endpoints/a2a_endpoints.py @@ -146,12 +146,17 @@ def _validate_push_notification_url(url: str) -> None: def _caller_identity_headers(user_api_key_dict: UserAPIKeyAuth) -> Mapping[str, str]: + """The human behind this call. An agent key acting for an invoking user forwards that user, not + itself, so a chain of agents stays capped at what the original caller may reach.""" + caller: Final = user_api_key_dict.agent_caller + user_id: Final = caller.user_id if caller is not None else user_api_key_dict.user_id + team_id: Final = caller.team_id if caller is not None else user_api_key_dict.team_id return MappingProxyType( { name: value for name, value in ( - ("X-LiteLLM-User-Id", user_api_key_dict.user_id), - ("X-LiteLLM-Team-Id", user_api_key_dict.team_id), + ("X-LiteLLM-User-Id", user_id), + ("X-LiteLLM-Team-Id", team_id), ) if value } diff --git a/litellm/proxy/agent_endpoints/agent_registry.py b/litellm/proxy/agent_endpoints/agent_registry.py index c7b6bca72cf..d6b12e830e1 100644 --- a/litellm/proxy/agent_endpoints/agent_registry.py +++ b/litellm/proxy/agent_endpoints/agent_registry.py @@ -7,6 +7,7 @@ from types import MappingProxyType from typing import TYPE_CHECKING, Final, NamedTuple, Protocol, TypedDict from pydantic import TypeAdapter, ValidationError +from typing_extensions import ReadOnly import litellm from litellm.constants import REDACTED_BY_LITELM_STRING @@ -37,6 +38,7 @@ class AgentRecordDump(TypedDict): agent_card_params: dict[str, object] static_headers: dict[str, str] | None extra_headers: list[str] | None + access_group_ids: ReadOnly[Sequence[str] | None] object_permission: dict[str, object] | None spend: float tpm_limit: int | None @@ -65,6 +67,9 @@ class AgentRecord(Protocol): @property def object_permission(self) -> AgentObjectPermissionRecord | None: ... + @property + def access_group_ids(self) -> Sequence[str] | None: ... + @property def spend(self) -> float: ... @@ -284,6 +289,12 @@ def _resolved_agent_param_value( return _MISSING_AGENT_PARAM +def _patched_access_group_ids(agent: PatchAgentRequest) -> Mapping[str, object]: + if "access_group_ids" not in agent: + return MappingProxyType({}) + return MappingProxyType({"access_group_ids": tuple(dict.fromkeys(agent.get("access_group_ids") or ()))}) + + def _restore_redacted_litellm_params( incoming: Mapping[str, object], existing: Mapping[str, object], @@ -516,6 +527,7 @@ class AgentRegistry: static_headers_val: Final[str | None] = safe_dumps(dict(static_headers_obj)) if static_headers_obj else None extra_headers_val: Final = agent.get("extra_headers") + access_group_ids_val: Final = agent.get("access_group_ids") create_data: Final[dict[str, object]] = { "agent_name": agent_name, @@ -532,6 +544,8 @@ class AgentRegistry: create_data["static_headers"] = static_headers_val if extra_headers_val is not None: create_data["extra_headers"] = extra_headers_val + if access_group_ids_val is not None: + create_data["access_group_ids"] = tuple(dict.fromkeys(access_group_ids_val)) if object_permission_id is not None: create_data["object_permission_id"] = object_permission_id @@ -601,7 +615,7 @@ class AgentRegistry: existing_agent: Final[Mapping[str, object]] = dict(existing_record) augment_agent: Final = {**existing_agent, **agent} - update_data: Final[dict[str, object]] = {} + update_data: Final[dict[str, object]] = {**_patched_access_group_ids(agent)} if augment_agent.get("agent_name"): update_data["agent_name"] = augment_agent.get("agent_name") if "litellm_params" in agent: @@ -703,6 +717,7 @@ class AgentRegistry: safe_dumps(dict(static_headers_obj_u)) if static_headers_obj_u is not None else safe_dumps({}) ) extra_headers_val_u: Final = agent.get("extra_headers") or [] + access_group_ids_val_u: Final = tuple(dict.fromkeys(agent.get("access_group_ids") or ())) update_data: Final[dict[str, object]] = { "agent_name": agent_name, @@ -710,6 +725,7 @@ class AgentRegistry: "agent_card_params": agent_card_params, "static_headers": static_headers_val_u, "extra_headers": extra_headers_val_u, + "access_group_ids": access_group_ids_val_u, "updated_by": updated_by, "updated_at": datetime.now(timezone.utc), } diff --git a/litellm/proxy/agent_endpoints/auth/agent_access_groups.py b/litellm/proxy/agent_endpoints/auth/agent_access_groups.py new file mode 100644 index 00000000000..49e5407ff88 --- /dev/null +++ b/litellm/proxy/agent_endpoints/auth/agent_access_groups.py @@ -0,0 +1,75 @@ +import asyncio +from collections.abc import Awaitable, Callable +from dataclasses import dataclass +from typing import Final, TypeAlias + +from fastapi import HTTPException + +from litellm._logging import verbose_proxy_logger +from litellm.proxy._types import LiteLLM_AccessGroupTable + +AccessGroupIds: TypeAlias = tuple[str, ...] +AccessGroupIdsLoader: TypeAlias = Callable[[str], Awaitable[AccessGroupIds]] # mutable-ok: Callable params +LoadedAccessGroup: TypeAlias = LiteLLM_AccessGroupTable | None +AccessGroupLoader: TypeAlias = Callable[[str], Awaitable[LoadedAccessGroup]] # mutable-ok: Callable parameter syntax + + +@dataclass(frozen=True, slots=True) +class AgentAccessGroupCeiling: + """Everything the agent's attached access groups allow. An empty set denies that resource kind.""" + + access_group_ids: AccessGroupIds + models: frozenset[str] + mcp_server_ids: frozenset[str] + agent_ids: frozenset[str] + + +CeilingResolver: TypeAlias = Callable[[str], Awaitable[AgentAccessGroupCeiling | None]] # mutable-ok: Callable params + + +async def _registry_access_group_ids(agent_id: str) -> AccessGroupIds: + from litellm.proxy.common_utils.registry_read_through import get_agent_with_read_through + + agent: Final = await get_agent_with_read_through(agent_id) + return tuple(agent.access_group_ids or ()) if agent is not None else () + + +async def _load_access_group(access_group_id: str) -> LoadedAccessGroup: + from litellm.proxy.auth.auth_checks import get_access_object + from litellm.proxy.proxy_server import prisma_client, proxy_logging_obj, user_api_key_cache + + if prisma_client is None: + verbose_proxy_logger.warning("Agent access group %s cannot be loaded without a DB", access_group_id) + return None + try: + return await get_access_object( + access_group_id=access_group_id, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + proxy_logging_obj=proxy_logging_obj, + ) + except HTTPException as e: + verbose_proxy_logger.warning( + "Agent access group %s could not be loaded, treating it as empty: %s", access_group_id, e.detail + ) + return None + + +async def resolve_agent_access_group_ceiling( + agent_id: str, + load_access_group_ids: AccessGroupIdsLoader = _registry_access_group_ids, + load_access_group: AccessGroupLoader = _load_access_group, +) -> AgentAccessGroupCeiling | None: + """``None`` when the agent has no access groups attached, so nothing is capped.""" + access_group_ids: Final = await load_access_group_ids(agent_id) + if not access_group_ids: + return None + + loaded: Final = await asyncio.gather(*(load_access_group(group_id) for group_id in access_group_ids)) + groups: Final = tuple(group for group in loaded if group is not None) + return AgentAccessGroupCeiling( + access_group_ids=access_group_ids, + models=frozenset(model for group in groups for model in group.access_model_names), + mcp_server_ids=frozenset(server_id for group in groups for server_id in group.access_mcp_server_ids), + agent_ids=frozenset(target_id for group in groups for target_id in group.access_agent_ids), + ) diff --git a/litellm/proxy/agent_endpoints/auth/agent_caller.py b/litellm/proxy/agent_endpoints/auth/agent_caller.py new file mode 100644 index 00000000000..47d43e8f71b --- /dev/null +++ b/litellm/proxy/agent_endpoints/auth/agent_caller.py @@ -0,0 +1,87 @@ +"""The human behind an agent's own proxy calls. + +``/a2a/{agent}`` forwards the invoking key's ``X-LiteLLM-User-Id`` / ``X-LiteLLM-Team-Id`` to the +agent backend. When the agent echoes them back on requests made with its own key, the proxy caps +that key at what the invoking user and team may reach. The cap is intersected with, never +substituted for, the agent key's own grants and the agent's access group ceiling, so the headers +can only narrow access and need no trust. +""" + +from collections.abc import Mapping +from typing import Final + +from litellm._logging import verbose_proxy_logger +from litellm.proxy._types import LiteLLM_TeamTable, LiteLLM_UserTable, UserAPIKeyAuth +from litellm.types.agents import ( + AGENT_CALLER_TEAM_ID_HEADER, + AGENT_CALLER_USER_ID_HEADER, + AgentCaller, +) + + +def _header(headers: Mapping[str, str], name: str) -> str | None: + value: Final = next((raw for key, raw in headers.items() if key.lower() == name), None) + return value.strip() or None if value is not None else None + + +def agent_caller_from_headers(headers: Mapping[str, str], user_api_key_auth: UserAPIKeyAuth) -> AgentCaller | None: + """The caller an agent key is acting for, or ``None`` when the key is not an agent's or no id was echoed.""" + if not user_api_key_auth.agent_id: + return None + user_id: Final = _header(headers, AGENT_CALLER_USER_ID_HEADER) + team_id: Final = _header(headers, AGENT_CALLER_TEAM_ID_HEADER) + if user_id is None and team_id is None: + return None + return AgentCaller(user_id=user_id, team_id=team_id) + + +def agent_caller_auth(user_api_key_auth: UserAPIKeyAuth) -> UserAPIKeyAuth | None: + """A minimal auth context standing for the invoking user and team, so the shared key/team/user + resolvers can be reused unchanged to compute what the caller may reach.""" + caller: Final = user_api_key_auth.agent_caller + if caller is None: + return None + return UserAPIKeyAuth( + user_id=caller.user_id, + team_id=caller.team_id, + parent_otel_span=user_api_key_auth.parent_otel_span, + ) + + +async def load_agent_caller_team(user_api_key_auth: UserAPIKeyAuth) -> LiteLLM_TeamTable | None: + """The invoking team's row, or ``None`` when no team id was echoed. Raises when the id names a team + that cannot be loaded, since a caller we cannot resolve must not be treated as unrestricted.""" + from litellm.proxy.auth.auth_checks import get_team_object + from litellm.proxy.proxy_server import prisma_client, proxy_logging_obj, user_api_key_cache + + caller: Final = user_api_key_auth.agent_caller + if caller is None or caller.team_id is None: + return None + return await get_team_object( + team_id=caller.team_id, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + parent_otel_span=user_api_key_auth.parent_otel_span, + proxy_logging_obj=proxy_logging_obj, + ) + + +async def load_agent_caller_user(user_api_key_auth: UserAPIKeyAuth) -> LiteLLM_UserTable | None: + """The invoking user's row, or ``None`` when no user id was echoed or the row does not exist.""" + from litellm.proxy.auth.auth_checks import get_user_object + from litellm.proxy.proxy_server import prisma_client, proxy_logging_obj, user_api_key_cache + + caller: Final = user_api_key_auth.agent_caller + if caller is None or caller.user_id is None: + return None + user_object: Final = await get_user_object( + user_id=caller.user_id, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + user_id_upsert=False, + parent_otel_span=user_api_key_auth.parent_otel_span, + proxy_logging_obj=proxy_logging_obj, + ) + if user_object is None: + verbose_proxy_logger.debug("agent caller user %r not found; no user ceiling applied", caller.user_id) + return user_object diff --git a/litellm/proxy/agent_endpoints/auth/agent_permission_handler.py b/litellm/proxy/agent_endpoints/auth/agent_permission_handler.py index e4dd77e2f82..9fe74bfee3f 100644 --- a/litellm/proxy/agent_endpoints/auth/agent_permission_handler.py +++ b/litellm/proxy/agent_endpoints/auth/agent_permission_handler.py @@ -19,6 +19,11 @@ from litellm.proxy._types import ( LitellmUserRoles, UserAPIKeyAuth, ) +from litellm.proxy.agent_endpoints.auth.agent_access_groups import ( + CeilingResolver, + resolve_agent_access_group_ceiling, +) +from litellm.proxy.agent_endpoints.auth.agent_caller import agent_caller_auth from litellm.repositories.table_repositories import AgentsRepository from litellm.types.agents import AgentResponse @@ -44,6 +49,22 @@ def _to_stable_ids(agent_ids: frozenset[str]) -> frozenset[str]: return frozenset(global_agent_registry.stable_agent_id(agent_id) for agent_id in agent_ids) +def _restricted_ids(access: AgentAccess) -> frozenset[str] | None: + if isinstance(access, UnrestrictedAgentAccess): + return None + return _to_stable_ids(access.agent_ids) + + +def _intersect_agent_access(key_access: AgentAccess, team_access: AgentAccess) -> AgentAccess: + key_ids: Final = _restricted_ids(key_access) + team_ids: Final = _restricted_ids(team_access) + if key_ids is None: + return UnrestrictedAgentAccess() if team_ids is None else RestrictedAgentAccess(team_ids) + if team_ids is None: + return RestrictedAgentAccess(key_ids) + return RestrictedAgentAccess(key_ids & team_ids) + + class AgentRequestHandler: """ Class to handle agent permission checking, including: @@ -61,35 +82,56 @@ class AgentRequestHandler: @staticmethod async def resolve_agent_access( user_api_key_auth: UserAPIKeyAuth | None = None, + resolve_ceiling: CeilingResolver = resolve_agent_access_group_ceiling, ) -> AgentAccess: - """ - Resolve the agents the given user/key may reach. + """Agents the key may reach: key and team grants, intersected with the agent's access group ceiling + and, for an agent key acting on behalf of an invoking user, with that user's team grants.""" + key_team_access: Final = await AgentRequestHandler._resolve_key_team_agent_access(user_api_key_auth) + caller_access: Final = await AgentRequestHandler._agent_caller_access(user_api_key_auth) + own_access: Final = _intersect_agent_access(key_team_access, caller_access) + agent_ceiling: Final = await AgentRequestHandler._agent_access_group_ceiling(user_api_key_auth, resolve_ceiling) + if agent_ceiling is None: + return own_access + if isinstance(own_access, UnrestrictedAgentAccess): + return RestrictedAgentAccess(agent_ceiling) + return RestrictedAgentAccess(own_access.agent_ids & agent_ceiling) - ``UnrestrictedAgentAccess`` is only returned when neither the key nor its team - carries any grant. Grants that intersect to nothing stay restricted, so - narrowing a caller can never widen what it reaches. - """ + @staticmethod + async def _agent_caller_access(user_api_key_auth: UserAPIKeyAuth | None) -> AgentAccess: + caller_auth: Final = agent_caller_auth(user_api_key_auth) if user_api_key_auth else None + if caller_auth is None: + return UnrestrictedAgentAccess() + return await AgentRequestHandler._get_allowed_agents_for_team(caller_auth) + + @staticmethod + async def _resolve_key_team_agent_access( + user_api_key_auth: UserAPIKeyAuth | None, + ) -> AgentAccess: try: key_access: Final = await AgentRequestHandler._get_allowed_agents_for_key(user_api_key_auth) team_access: Final = await AgentRequestHandler._get_allowed_agents_for_team(user_api_key_auth) - - match (key_access, team_access): - case (UnrestrictedAgentAccess(), UnrestrictedAgentAccess()): - return UnrestrictedAgentAccess() - case (UnrestrictedAgentAccess(), RestrictedAgentAccess(team_ids)): - return RestrictedAgentAccess(_to_stable_ids(team_ids)) - case (RestrictedAgentAccess(key_ids), UnrestrictedAgentAccess()): - return RestrictedAgentAccess(_to_stable_ids(key_ids)) - case (RestrictedAgentAccess(key_ids), RestrictedAgentAccess(team_ids)): - return RestrictedAgentAccess(_to_stable_ids(key_ids) & _to_stable_ids(team_ids)) except Exception as e: verbose_logger.warning("Failed to get allowed agents: %s", e) return UnrestrictedAgentAccess() + return _intersect_agent_access(key_access, team_access) + + @staticmethod + async def _agent_access_group_ceiling( + user_api_key_auth: UserAPIKeyAuth | None, + resolve_ceiling: CeilingResolver, + ) -> frozenset[str] | None: + if user_api_key_auth is None or not user_api_key_auth.agent_id: + return None + ceiling: Final = await resolve_ceiling(user_api_key_auth.agent_id) + if ceiling is None: + return None + return _to_stable_ids(ceiling.agent_ids) @staticmethod async def is_agent_allowed( agent_id: str, user_api_key_auth: UserAPIKeyAuth | None = None, + resolve_ceiling: CeilingResolver = resolve_agent_access_group_ceiling, ) -> bool: """ Check if a specific agent is allowed for the given user/key. @@ -103,7 +145,7 @@ class AgentRequestHandler: """ from litellm.proxy.agent_endpoints.agent_registry import global_agent_registry - match await AgentRequestHandler.resolve_agent_access(user_api_key_auth): + match await AgentRequestHandler.resolve_agent_access(user_api_key_auth, resolve_ceiling): case UnrestrictedAgentAccess(): return True case RestrictedAgentAccess(allowed_agent_ids): diff --git a/litellm/proxy/anthropic_endpoints/endpoints.py b/litellm/proxy/anthropic_endpoints/endpoints.py index 644778bcb9f..d9558b86e95 100644 --- a/litellm/proxy/anthropic_endpoints/endpoints.py +++ b/litellm/proxy/anthropic_endpoints/endpoints.py @@ -8,7 +8,11 @@ from fastapi import APIRouter, Depends, HTTPException, Request, Response from fastapi.responses import JSONResponse import litellm -from litellm.anthropic_interface.exceptions import AnthropicErrorResponse, AnthropicExceptionMapping +from litellm.anthropic_interface.exceptions import ( + AnthropicErrorDetail, + AnthropicErrorResponse, + AnthropicExceptionMapping, +) from litellm.integrations.custom_guardrail import ModifyResponseException from litellm.llms.anthropic.experimental_pass_through.context_management import ( AnthropicContextManagementError, @@ -25,8 +29,10 @@ from litellm.proxy.common_request_processing import ( proxy_exception_from_http_exception, resolve_litellm_call_id, ) +from litellm.proxy.common_utils.error_body_call_id import error_body_call_id from litellm.proxy.common_utils.http_parsing_utils import _read_request_body from litellm.proxy.common_utils.openai_error_payload import ( + LITELLM_CALL_ID_HEADER, error_status_code, openai_error_param, openai_error_type, @@ -37,9 +43,29 @@ from litellm.types.utils import TokenCountResponse router: Final = APIRouter() +def _with_provider_specific_fields(exc: ProxyException, detail: AnthropicErrorDetail) -> AnthropicErrorDetail: + if not exc.provider_specific_fields: + return detail + with_fields: Final[AnthropicErrorDetail] = {**detail, "provider_specific_fields": exc.provider_specific_fields} + return with_fields + + +def _anthropic_error_detail( + exc: ProxyException, detail: AnthropicErrorDetail, call_id: str | None +) -> AnthropicErrorDetail: + if call_id is None: + return _with_provider_specific_fields(exc, detail) + with_call_id: Final[AnthropicErrorDetail] = { + **_with_provider_specific_fields(exc, detail), + "litellm_call_id": call_id, + } + return with_call_id + + def _anthropic_error_json_response(exc: ProxyException, request: Request) -> JSONResponse: from litellm.proxy.proxy_server import ( _close_dangling_otel_server_span, # pyright: ignore[reportPrivateUsage] # proxy_server keeps the span-close helper private; error JSONResponses returned by the route must stamp the OTel server span like the global ProxyException handler does + general_settings_view, ) status_code: Final = int(exc.code) if exc.code is not None and exc.code.isdigit() else 500 @@ -49,11 +75,10 @@ def _anthropic_error_json_response(exc: ProxyException, request: Request) -> JSO raw_message=exc.message, request_id=request.headers.get("x-request-id"), ) - if not exc.provider_specific_fields: - return JSONResponse(status_code=status_code, content=envelope, headers=exc.headers) + body_call_id: Final = error_body_call_id(general_settings_view(), exc.headers.get(LITELLM_CALL_ID_HEADER)) content: Final[AnthropicErrorResponse] = { **envelope, - "error": {**envelope["error"], "provider_specific_fields": exc.provider_specific_fields}, + "error": _anthropic_error_detail(exc, envelope["error"], body_call_id), } return JSONResponse(status_code=status_code, content=content, headers=exc.headers) diff --git a/litellm/proxy/auth/auth_checks.py b/litellm/proxy/auth/auth_checks.py index c2279fb2fe1..b3efde7f05f 100644 --- a/litellm/proxy/auth/auth_checks.py +++ b/litellm/proxy/auth/auth_checks.py @@ -15,7 +15,7 @@ import re import time from collections.abc import Awaitable, Callable, Iterator, Mapping, Sequence from types import MappingProxyType -from typing import TYPE_CHECKING, Any, Final, Literal, Optional, Protocol +from typing import TYPE_CHECKING, Any, Final, Literal, Optional, Protocol, TypeAlias from fastapi import HTTPException, Request, status from pydantic import BaseModel, TypeAdapter @@ -68,6 +68,15 @@ from litellm.proxy._types import ( SpecialModelNames, UserAPIKeyAuth, ) +from litellm.proxy.agent_endpoints.auth.agent_access_groups import ( + CeilingResolver, + resolve_agent_access_group_ceiling, +) +from litellm.proxy.agent_endpoints.auth.agent_caller import ( + agent_caller_auth, + load_agent_caller_team, + load_agent_caller_user, +) from litellm.proxy.auth.budget_throttle import ( budget_throttle_percentage, should_throttle_budget_exceeded, @@ -1006,6 +1015,16 @@ async def common_checks( code=status.HTTP_400_BAD_REQUEST, ) + await _check_agent_access_group_model_access(model=_model, valid_token=valid_token, llm_router=llm_router) + await _check_agent_caller_model_access( + model=_model, + valid_token=valid_token, + llm_router=llm_router, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + proxy_logging_obj=proxy_logging_obj, + ) + ## 2.1 If user can call model (if personal key) if _model and team_object is None and user_object is not None: with tracer.trace("litellm.proxy.auth.common_checks.can_user_call_model"): @@ -4251,7 +4270,7 @@ def _can_object_call_model( models: list[str], team_model_aliases: dict[str, str] | None = None, team_id: str | None = None, - object_type: Literal["user", "team", "key", "org", "project"] = "user", + object_type: Literal["user", "team", "key", "org", "project", "agent"] = "user", fallback_depth: int = 0, ) -> Literal[True]: """ @@ -4317,6 +4336,82 @@ def _can_object_call_model( ) +async def _check_agent_access_group_model_access( + model: str | list[str] | None, # mutable-ok: _can_object_call_model and the client message helper take list[str] + valid_token: UserAPIKeyAuth | None, + llm_router: Router | None, + resolve_ceiling: CeilingResolver = resolve_agent_access_group_ceiling, +) -> Literal[True]: + """Attached groups naming no model deny every model; the empty allowlist in ``_can_object_call_model`` allows.""" + if not model or valid_token is None or not valid_token.agent_id: + return True + ceiling: Final = await resolve_ceiling(valid_token.agent_id) + if ceiling is None: + return True + if not ceiling.models: + raise ModelAccessDeniedProxyException( + message=model_access_denied_client_message(model=model), + internal_message=f"agent {valid_token.agent_id} access groups {ceiling.access_group_ids} grant no models", + type=ProxyErrorTypes.agent_model_access_denied, + param="model", + code=status.HTTP_403_FORBIDDEN, + ) + return _can_object_call_model( + model=model, + llm_router=llm_router, + models=sorted(ceiling.models), + team_id=valid_token.team_id, + object_type="agent", + ) + + +LoadedCallerTeam: TypeAlias = LiteLLM_TeamTable | None +LoadedCallerUser: TypeAlias = LiteLLM_UserTable | None +CallerTeamLoader: TypeAlias = Callable[[UserAPIKeyAuth], Awaitable[LoadedCallerTeam]] # mutable-ok: Callable params +CallerUserLoader: TypeAlias = Callable[[UserAPIKeyAuth], Awaitable[LoadedCallerUser]] # mutable-ok: Callable params + + +async def _check_agent_caller_model_access( + model: str | list[str] | None, # mutable-ok: the model checks it delegates to take list[str] + valid_token: UserAPIKeyAuth | None, + llm_router: Router | None, + prisma_client: Optional["PrismaClient"], + user_api_key_cache: UserApiKeyCache, + proxy_logging_obj: ProxyLogging, + load_team: CallerTeamLoader = load_agent_caller_team, + load_user: CallerUserLoader = load_agent_caller_user, +) -> None: + """An agent key acting for an invoking user may call only what that user's own key could: the + invoking team's models (and per-member scope) when a team was echoed, else the user's models.""" + if not model or valid_token is None: + return + caller_auth: Final = agent_caller_auth(valid_token) + if caller_auth is None: + return + caller_team: Final = await load_team(valid_token) + if caller_team is not None: + await can_team_access_model( + model=model, + team_object=caller_team, + llm_router=llm_router, + prisma_client=prisma_client, + ) + await _check_team_member_model_access( + model=model, + team_object=caller_team, + valid_token=caller_auth, + llm_router=llm_router, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + proxy_logging_obj=proxy_logging_obj, + ) + return + caller_user: Final = await load_user(valid_token) + if caller_user is None: + return + await can_user_call_model(model=model, llm_router=llm_router, user_object=caller_user) + + def _model_in_team_aliases(model: str, team_model_aliases: dict[str, str] | None = None) -> bool: """ Returns True if `model` being accessed is an alias of a team model diff --git a/litellm/proxy/auth/handle_jwt.py b/litellm/proxy/auth/handle_jwt.py index 996911cdfaa..07d8d00d202 100644 --- a/litellm/proxy/auth/handle_jwt.py +++ b/litellm/proxy/auth/handle_jwt.py @@ -14,7 +14,7 @@ import hashlib import os import re import time -from collections.abc import Awaitable, Callable, Mapping, Sequence +from collections.abc import Awaitable, Callable, Collection, Mapping, Sequence from dataclasses import dataclass from typing import Any, Final, Literal, NoReturn, Protocol, TypeVar, cast @@ -1602,6 +1602,7 @@ class JWTAuthManager: team_object: LiteLLM_TeamTable | None, route: str, request_method: str | None = None, + team_allowed_routes: Collection[str] = (), ) -> bool: normalized_request_method: Final = request_method.upper() if isinstance(request_method, str) else None if not RouteChecks.is_auth_enforced_pass_through_route( @@ -1610,8 +1611,11 @@ class JWTAuthManager: ): return True + if RouteChecks.jwt_team_routes_grant_pass_through(route=route, team_allowed_routes=team_allowed_routes): + return True + # JWT team selection is team-scoped; key metadata is not available here, - # so passthrough access is granted only by the selected team's metadata. + # so beyond the JWT config grant above, only the selected team's metadata grants access. return RouteChecks.check_passthrough_route_access( route=route, user_api_key_dict=UserAPIKeyAuth(team_metadata=(team_object.metadata or {}) if team_object else {}), @@ -1689,6 +1693,7 @@ class JWTAuthManager: team_object=team_object, route=route, request_method=request_method, + team_allowed_routes=jwt_handler.litellm_jwtauth.team_allowed_routes, ): is_allowed = False denied_auth_enforced_pass_through_route = True @@ -2094,8 +2099,11 @@ class JWTAuthManager: spend / metadata can be attributed correctly. Returns (team_id, team_object, team_membership_object). - Any DB error is debug-logged and the tuple is (None, None, None) — no - exception ever propagates from this helper. + A team that cannot be loaded (HTTPException from get_team_object) is + debug-logged and the tuple is (None, None, None), the same as the DB + team fallback. A failed membership read propagates, so a database + outage surfaces as the 503 the rest of auth answers with instead of + serving the request with the member's limits dropped. """ if user_object is None or not user_object.teams or len(user_object.teams) != 1: return None, None, None @@ -2110,28 +2118,28 @@ class JWTAuthManager: proxy_logging_obj=proxy_logging_obj, team_id_upsert=team_id_upsert, ) - if team_row is None: - return None, None, None - - if not user_id: - return _tid, team_row, None - - team_membership: Final = await get_team_membership( - user_id=user_id, - team_id=_tid, - prisma_client=prisma_client, - user_api_key_cache=user_api_key_cache, - parent_otel_span=parent_otel_span, - proxy_logging_obj=proxy_logging_obj, - ) - return _tid, team_row, team_membership - except Exception: + except HTTPException: verbose_proxy_logger.debug( - "JWT single-team fallback error, skipping. team_id=%s", + "JWT single-team fallback: team could not be loaded, skipping. team_id=%s", _tid, exc_info=True, ) return None, None, None + if team_row is None: + return None, None, None + + if not user_id: + return _tid, team_row, None + + team_membership: Final = await get_team_membership( + user_id=user_id, + team_id=_tid, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + parent_otel_span=parent_otel_span, + proxy_logging_obj=proxy_logging_obj, + ) + return _tid, team_row, team_membership @staticmethod async def _resolve_db_team_fallback( @@ -2584,6 +2592,7 @@ class JWTAuthManager: team_object=team_object, route=route, request_method=request_method, + team_allowed_routes=handler.litellm_jwtauth.team_allowed_routes, ): JWTAuthManager._raise_team_passthrough_route_denial(route=route) @@ -2653,6 +2662,7 @@ class JWTAuthManager: team_object=team_object, route=route, request_method=request_method, + team_allowed_routes=handler.litellm_jwtauth.team_allowed_routes, ): JWTAuthManager._raise_team_passthrough_route_denial(route=route) elif team_id is None: diff --git a/litellm/proxy/auth/login_utils.py b/litellm/proxy/auth/login_utils.py index e0d599b0017..4c2b5d3d0fe 100644 --- a/litellm/proxy/auth/login_utils.py +++ b/litellm/proxy/auth/login_utils.py @@ -10,14 +10,16 @@ import secrets from collections.abc import Mapping from datetime import datetime, timedelta, timezone from types import MappingProxyType -from typing import Final, Literal, cast +from typing import TYPE_CHECKING, Final, Literal, cast import jwt from fastapi import HTTPException import litellm +from litellm._logging import verbose_proxy_logger from litellm.constants import LITELLM_PROXY_ADMIN_NAME, LITELLM_UI_SESSION_DURATION from litellm.litellm_core_utils.duration_parser import duration_in_seconds +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler from litellm.proxy._types import ( LiteLLM_UserTable, LitellmUserRoles, @@ -28,6 +30,7 @@ from litellm.proxy._types import ( ) from litellm.proxy.auth.auth_utils import is_sso_provider_fully_configured from litellm.proxy.auth.login_throttle import LoginAttempt, LoginThrottle +from litellm.proxy.auth.password_policy import is_breach_check_enabled, is_password_breached from litellm.proxy.management_endpoints.internal_user_endpoints import user_update from litellm.proxy.management_endpoints.key_management_endpoints import ( generate_key_helper_fn, @@ -50,6 +53,57 @@ INVALID_UI_CREDENTIALS_MESSAGE: Final = ( ) INVALID_USER_PASSWORD_MESSAGE: Final = "Invalid credentials used to access UI. Check the password set for your user" +if TYPE_CHECKING: + from prisma import types as prisma_types + +BREACH_RECHECK_INTERVAL: Final = timedelta(hours=24) +PASSWORD_RESET_ALLOWED_ROUTES: Final = ("/user/password/change",) +PASSWORD_SESSION_METADATA: Final = MappingProxyType({"login_method": "username_password"}) + + +def _breach_recheck_due(last_breach_check_at: datetime | None) -> bool: + if last_breach_check_at is None: + return True + last_checked_utc: Final = ( + last_breach_check_at + if last_breach_check_at.tzinfo is not None + else last_breach_check_at.replace(tzinfo=timezone.utc) + ) + return datetime.now(timezone.utc) - last_checked_utc >= BREACH_RECHECK_INTERVAL + + +async def screen_login_password_for_breach( + user_id: str, + password: str, + last_breach_check_at: datetime | None, + general_settings: Mapping[str, object], + prisma_client: PrismaClient, + client: AsyncHTTPHandler | None = None, +) -> bool: + """Screens a successfully verified login password against HIBP, stamps + ``password_reset_required`` when breached, and returns whether a breach was + found so the login it runs in can restrict the session it is about to mint. + Fails open (HIBP or DB trouble never fails the login) and rechecks a given + user at most once per ``BREACH_RECHECK_INTERVAL``.""" + if not is_breach_check_enabled(general_settings): + return False + if not _breach_recheck_due(last_breach_check_at): + return False + breached: Final = await is_password_breached(password, general_settings, client) + checked_at: Final = datetime.now(timezone.utc) + breached_update: Final[prisma_types.LiteLLM_UserTableUpdateInput] = { + "last_breach_check_at": checked_at, + "password_reset_required": True, + } + recheck_update: Final[prisma_types.LiteLLM_UserTableUpdateInput] = {"last_breach_check_at": checked_at} + update_data: Final = breached_update if breached else recheck_update + find_user: Final[prisma_types.LiteLLM_UserTableWhereInput] = {"user_id": user_id} + try: + await UserRepository(prisma_client).table.update(where=find_user, data=update_data) + except Exception as e: # noqa: BLE001 # a failed stamp must never surface into the login + verbose_proxy_logger.warning("Login-time breach screening could not update user %s: %s", user_id, e) + return breached + async def _rehash_password_if_needed(user_id: str, password: str, stored: str) -> None: """Rehash legacy password (SHA256) to scrypt on successful login.""" @@ -137,6 +191,7 @@ class LoginResult: user_email: str | None user_role: str login_method: Literal["sso", "username_password"] + password_reset_required: bool def __init__( self, @@ -145,12 +200,14 @@ class LoginResult: user_email: str | None, user_role: str, login_method: Literal["sso", "username_password"] = "username_password", + password_reset_required: bool = False, ): self.user_id = user_id self.key = key self.user_email = user_email self.user_role = user_role self.login_method = login_method + self.password_reset_required = password_reset_required async def authenticate_user( @@ -356,20 +413,28 @@ async def _sign_in( if verify_password(password, _password): await _rehash_password_if_needed(_user_row.user_id, password, _password) + breached_now: Final = prisma_client is not None and await screen_login_password_for_breach( + user_id=_user_row.user_id, + password=password, + last_breach_check_at=getattr(_user_row, "last_breach_check_at", None), + general_settings=general_settings, + prisma_client=prisma_client, + ) + password_reset_required: Final = breached_now or getattr(_user_row, "password_reset_required", None) is True if os.getenv("DATABASE_URL") is not None: response = await generate_key_helper_fn( llm_router=None, request_type="key", - **{ - "user_role": user_role, - "duration": LITELLM_UI_SESSION_DURATION, - "key_max_budget": litellm.max_ui_session_budget, - "models": [], - "aliases": {}, - "config": {}, - "spend": 0, - "user_id": user_id, - "team_id": "litellm-dashboard", + user_role=user_role, + duration=LITELLM_UI_SESSION_DURATION, + key_max_budget=litellm.max_ui_session_budget, + spend=0, + user_id=user_id, + team_id="litellm-dashboard", + allowed_routes=list(PASSWORD_RESET_ALLOWED_ROUTES) if password_reset_required else None, + metadata={ + **PASSWORD_SESSION_METADATA, + **({"password_reset_required": True} if password_reset_required else {}), }, ) else: @@ -390,6 +455,7 @@ async def _sign_in( user_email=user_email, user_role=cast(str, user_role), login_method="username_password", + password_reset_required=password_reset_required, ) else: await attempt.failed() @@ -460,4 +526,5 @@ def create_ui_token_object( auth_header_name=general_settings.get("litellm_key_header_name", "Authorization"), disabled_non_admin_personal_key_creation=disabled_non_admin_personal_key_creation, server_root_path=get_server_root_path(), + password_reset_required=login_result.password_reset_required, ) diff --git a/litellm/proxy/auth/password_policy.py b/litellm/proxy/auth/password_policy.py index ab7a565894a..7f06a0993d3 100644 --- a/litellm/proxy/auth/password_policy.py +++ b/litellm/proxy/auth/password_policy.py @@ -4,13 +4,28 @@ Applied at every path that persists a new or changed password for a DB-backed user (``/user/update``, ``/user/bulk_update``, and the invitation onboarding claim flow), so the strength bar is configured in one place instead of per-endpoint. + +Also screens new passwords against known data breaches via the +haveibeenpwned.com (HIBP) k-anonymity range API: only the first 5 characters +of the password's SHA-1 hash ever leave the proxy, and the check fails open +(allows the password) when HIBP is unreachable. """ -from collections.abc import Mapping +import asyncio +import hashlib +from collections.abc import Mapping, Sequence from dataclasses import dataclass +from types import MappingProxyType from typing import Final +from litellm._logging import verbose_proxy_logger +from litellm._version import version +from litellm.constants import HIBP_RANGE_API_BASE +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, get_async_httpx_client from litellm.proxy._types import ProxyErrorTypes, ProxyException +from litellm.types.llms.custom_http import httpxSpecialProvider + +HIBP_TIMEOUT_SECONDS: Final = 5.0 DEFAULT_MIN_LENGTH: Final = 12 MIN_ALLOWED_LENGTH: Final = 8 @@ -90,3 +105,114 @@ def validate_password_policy(password: str, general_settings: Mapping[str, objec param="password", code=400, ) + + +def _hibp_client() -> AsyncHTTPHandler: + return get_async_httpx_client( + llm_provider=httpxSpecialProvider.PasswordBreachCheck, + params={"timeout": HIBP_TIMEOUT_SECONDS}, # mutable-ok: callee takes a bare dict (PEP 589) + ) + + +def _is_suffix_in_range_response(response_body: str, hash_suffix: str) -> bool: + for line in response_body.upper().splitlines(): + entry_suffix, _, count = line.strip().partition(":") + if entry_suffix == hash_suffix: + return int(count.strip() or "0") > 0 + return False + + +async def _is_password_breached(password: str, client: AsyncHTTPHandler) -> bool: + # usedforsecurity=False: SHA-1 is only a lookup key into the HIBP dataset, so no security property rests on it + sha1_hex: Final = hashlib.sha1(password.encode("utf-8"), usedforsecurity=False).hexdigest().upper() + headers: Final = { # mutable-ok: callee takes a bare dict (PEP 589) + "Add-Padding": "true", + "User-Agent": f"litellm-proxy/{version}", + } + try: + response: Final = await client.get( + f"{HIBP_RANGE_API_BASE}/{sha1_hex[:5]}", + headers=headers, + ) + response.raise_for_status() + breached: Final = _is_suffix_in_range_response(response.text, sha1_hex[5:]) + except Exception as e: # noqa: BLE001 # fail-open: any HIBP failure skips the check, never breaks the caller + verbose_proxy_logger.warning("Breached-password check skipped, HIBP lookup failed: %s", e) + return False + return breached + + +def is_breach_check_enabled(general_settings: Mapping[str, object]) -> bool: + return general_settings.get("password_policy_check_breached_passwords", True) is not False + + +async def is_password_breached( + password: str, + general_settings: Mapping[str, object], + client: AsyncHTTPHandler | None = None, +) -> bool: + """False when the check is disabled, the password is absent from the HIBP + corpus, or HIBP is unreachable (fail open).""" + if not is_breach_check_enabled(general_settings): + return False + return await _is_password_breached(password, client if client is not None else _hibp_client()) + + +def breached_password_error() -> ProxyException: + return ProxyException( + message=( + "This password appears in known data breaches and cannot be used. Please choose a different password." + ), + type=ProxyErrorTypes.validation_error, + param="password", + code=400, + ) + + +async def validate_password_not_breached( + password: str, + general_settings: Mapping[str, object], + client: AsyncHTTPHandler | None = None, +) -> None: + """Raise ``ProxyException`` (400) if ``password`` appears in a known data breach. + + Fails open: an unreachable or misbehaving HIBP allows the password.""" + if not await is_password_breached(password, general_settings, client): + return + raise breached_password_error() + + +def _strength_verdict(password: str, general_settings: Mapping[str, object]) -> ProxyException | None: + try: + validate_password_policy(password, general_settings) + except ProxyException as e: + return e + return None + + +async def validate_passwords_bulk( + passwords: Sequence[str], + general_settings: Mapping[str, object], + client: AsyncHTTPHandler | None = None, +) -> Mapping[str, ProxyException | None]: + """Per-unique-password policy verdicts for a batch: the ProxyException to + surface, or None when the password is acceptable. + + Deduplicates first, then issues every needed HIBP lookup concurrently, so a + batch caller pays one HIBP timeout window in the worst case instead of one + per password (each lookup still fails open independently).""" + unique_passwords: Final = tuple(dict.fromkeys(passwords)) + strength_verdicts: Final[Mapping[str, ProxyException | None]] = MappingProxyType( + {password: _strength_verdict(password, general_settings) for password in unique_passwords} + ) + to_screen: Final = tuple(password for password in unique_passwords if strength_verdicts[password] is None) + breached_flags: Final = await asyncio.gather( + *(is_password_breached(password, general_settings, client) for password in to_screen) + ) + breached_passwords: Final = frozenset(password for password, breached in zip(to_screen, breached_flags) if breached) + return MappingProxyType( + { + password: breached_password_error() if password in breached_passwords else strength_verdicts[password] + for password in unique_passwords + } + ) diff --git a/litellm/proxy/auth/route_checks.py b/litellm/proxy/auth/route_checks.py index 1b9fd7c42bf..38189a2d07b 100644 --- a/litellm/proxy/auth/route_checks.py +++ b/litellm/proxy/auth/route_checks.py @@ -194,6 +194,16 @@ class RouteChecks: if denied_auth_enforced_pass_through_route: raise RouteChecks._auth_pass_through_denied_exception(route=route) + if valid_token.metadata.get("password_reset_required") is True: + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail=( + "This account's password must be changed before the session can be used: " + "it was either found in a known data breach or set by an admin. " + "Change it via POST /user/password/change (UI: /ui/change-password), then log in again." + ), + ) + raise HTTPException( status_code=status.HTTP_403_FORBIDDEN, detail=f"Virtual key is not allowed to call this route. Only allowed to call routes: {valid_token.allowed_routes}. Tried to call route: {route}", @@ -268,7 +278,11 @@ class RouteChecks: route=route, method=RouteChecks._get_request_method(request=request), ): - RouteChecks._require_auth_pass_through_access(route=route, valid_token=valid_token) + RouteChecks._require_auth_pass_through_access( + route=route, + valid_token=valid_token, + jwt_team_allowed_routes=RouteChecks._jwt_team_allowed_routes(valid_token=valid_token), + ) elif RouteChecks.is_llm_api_route(route=route): pass elif RouteChecks.is_info_route(route=route): @@ -679,16 +693,43 @@ class RouteChecks: ), ) + @staticmethod + def jwt_team_routes_grant_pass_through(route: str, team_allowed_routes: Collection[str]) -> bool: + """ + Explicit paths and trailing-wildcard prefixes grant auth=true pass-through. Blanket grants never do: + a named route group like ``openai_routes`` is only ever compared as a path, and an entry that names + no path segment (``*``, ``/*``) is skipped. + """ + return any( + RouteChecks.route_matches_wildcard_pattern(route=route, pattern=allowed_route) + for allowed_route in team_allowed_routes + if allowed_route.rstrip("*").strip("/") + ) + + @staticmethod + def _jwt_team_allowed_routes(valid_token: UserAPIKeyAuth) -> Collection[str]: + """``team_allowed_routes`` for team tokens built by JWT auth; JWT-mapped virtual keys stay key-scoped.""" + if valid_token.jwt_claims is None or valid_token.token is not None or valid_token.team_id is None: + return () + + from litellm.proxy.proxy_server import jwt_handler + + return jwt_handler.litellm_jwtauth.team_allowed_routes + @staticmethod def _require_auth_pass_through_access( route: str, valid_token: UserAPIKeyAuth, + jwt_team_allowed_routes: Collection[str] = (), ) -> None: """ - Require an explicit ``allowed_passthrough_routes`` match for auth=true pass-through. + Require an explicit grant for auth=true pass-through: ``allowed_passthrough_routes`` on the + key or team, or an explicit JWT ``team_allowed_routes`` entry. """ if RouteChecks.check_passthrough_route_access(route=route, user_api_key_dict=valid_token): return + if RouteChecks.jwt_team_routes_grant_pass_through(route=route, team_allowed_routes=jwt_team_allowed_routes): + return raise RouteChecks._auth_pass_through_denied_exception(route=route) @staticmethod @@ -812,7 +853,8 @@ class RouteChecks: in the codebase is automatically readable by Admin Viewer without needing to remember to add it to an allowlist. 3. Unsafe HTTP method (POST/PUT/PATCH/DELETE): - - Allow `/user/update` only when restricted to user_email/password. + - Allow `/user/update` only when restricted to user_email. + - Allow `/user/password/change` (endpoint only writes the caller's own row). - Block all explicit writes in `_ADMIN_VIEWER_BLOCKED_WRITE_ROUTES`. - Otherwise allow only if the route is in admin_viewer_routes / global_spend_tracking_routes (legacy explicit-allow set). @@ -832,10 +874,10 @@ class RouteChecks: if request_data is not None and isinstance(request_data, dict): _params_updated: Final = request_data.keys() for param in _params_updated: - if param not in ["user_email", "password"]: + if param != "user_email": raise HTTPException( status_code=status.HTTP_403_FORBIDDEN, - detail=f"user not allowed to access this route, role= {_user_role}. Trying to access: {route} and updating invalid param: {param}. only user_email and password can be updated", + detail=f"user not allowed to access this route, role= {_user_role}. Trying to access: {route} and updating invalid param: {param}. only user_email can be updated", ) elif RouteChecks.check_route_access(route=route, allowed_routes=_PROXY_ADMIN_VIEW_ONLY_BLOCKED_ROUTES) or ( route.startswith("/key/") and route.endswith(_PROXY_ADMIN_VIEW_ONLY_BLOCKED_KEY_SUFFIXES) @@ -854,21 +896,25 @@ class RouteChecks: return # ── Unsafe HTTP method: explicit checks ────────────────────────── - # Allow `/user/update` for self-service email / password change. + # Allow `/user/update` for self-service email change. if route == "/user/update": if request_data is not None and isinstance(request_data, dict): for param in request_data: - if param not in ["user_email", "password"]: + if param != "user_email": raise HTTPException( status_code=status.HTTP_403_FORBIDDEN, detail=( f"user not allowed to access this route, role= {_user_role}. " f"Trying to access: {route} and updating invalid param: {param}. " - "only user_email and password can be updated" + "only user_email can be updated" ), ) return + # Self-service password change; the endpoint only writes the caller's own row. + if route == "/user/password/change": + return + # Hard-block known write routes regardless of HTTP method (defensive # — these are POSTs in practice, but pinning them here protects # against future GET-shaped writes). diff --git a/litellm/proxy/auth/user_api_key_auth.py b/litellm/proxy/auth/user_api_key_auth.py index 4371ce4fda8..a6c0792a86f 100644 --- a/litellm/proxy/auth/user_api_key_auth.py +++ b/litellm/proxy/auth/user_api_key_auth.py @@ -39,6 +39,7 @@ from litellm.integrations.otel.runtime import phase_span, seed_request_identity from litellm.litellm_core_utils.dd_tracing import tracer from litellm.litellm_core_utils.dot_notation_indexing import get_nested_value from litellm.proxy._types import * +from litellm.proxy.agent_endpoints.auth.agent_caller import agent_caller_from_headers from litellm.proxy.auth.auth_checks import ( ExperimentalUIJWTToken, TeamNotFoundError, @@ -649,6 +650,7 @@ async def user_api_key_auth_websocket_for_model(websocket: WebSocket, model: str "type": "http", "headers": scope_headers, "path": ws_scope.get("path", ""), + "state": ws_scope.setdefault("state", {}), # mutable-ok: Starlette's socket state, shared with the request } for key in ("root_path", "app_root_path"): if key in ws_scope: @@ -3085,31 +3087,30 @@ async def _reserve_budget_after_common_checks( request: Request | None = None, ) -> None: user_api_key_auth_obj.budget_reservation = None - if skip_budget_checks: - return - if general_settings.get("disable_budget_reservation") is True: - return + if not skip_budget_checks and general_settings.get("disable_budget_reservation") is not True: + from litellm.proxy.spend_tracking.budget_reservation import ( + reserve_budget_for_request, + ) - from litellm.proxy.spend_tracking.budget_reservation import ( - reserve_budget_for_request, - ) - - user_api_key_auth_obj.budget_reservation = await reserve_budget_for_request( - request_body=request_data, - route=route, - llm_router=llm_router, - valid_token=user_api_key_auth_obj, - team_object=team_object, - user_object=user_object, - prisma_client=prisma_client, - user_api_key_cache=user_api_key_cache, - proxy_logging_obj=proxy_logging_obj, - end_user_id=end_user_id, - end_user_object=end_user_object, - apply_user_budget_to_team_keys=general_settings.get("apply_user_budget_to_team_keys") is True, - fail_closed_budget_enforcement=general_settings.get("fail_closed_budget_enforcement") is True, - raw_body=await read_raw_json_body(request=request), - ) + user_api_key_auth_obj.budget_reservation = await reserve_budget_for_request( + request_body=request_data, + route=route, + llm_router=llm_router, + valid_token=user_api_key_auth_obj, + team_object=team_object, + user_object=user_object, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + proxy_logging_obj=proxy_logging_obj, + end_user_id=end_user_id, + end_user_object=end_user_object, + apply_user_budget_to_team_keys=general_settings.get("apply_user_budget_to_team_keys") is True, + fail_closed_budget_enforcement=general_settings.get("fail_closed_budget_enforcement") is True, + raw_body=await read_raw_json_body(request=request), + ) + if request is not None: + reservation: Final = user_api_key_auth_obj.budget_reservation + request.state.budget_reservation = reservation # rebind-ok: read by the release middleware def _should_skip_budget_checks( @@ -3330,6 +3331,9 @@ async def user_api_key_auth( raise body_parse_exception raise user_api_key_auth_obj.budget_reservation = None + user_api_key_auth_obj.agent_caller = agent_caller_from_headers( + _safe_get_request_headers(request), user_api_key_auth_obj + ) _seed_request_destinations(user_api_key_auth_obj, request) # A body that never parsed is authenticated (so the trace carries identity diff --git a/litellm/proxy/client/cli/commands/claude_settings.py b/litellm/proxy/client/cli/commands/claude_settings.py index f4bebc4a4cb..b3fdc4695cb 100644 --- a/litellm/proxy/client/cli/commands/claude_settings.py +++ b/litellm/proxy/client/cli/commands/claude_settings.py @@ -22,8 +22,12 @@ from pathlib import Path from types import MappingProxyType from typing import Final, TypeAlias +import click +from filelock import FileLock +from packaging.version import InvalidVersion, Version from pydantic import BaseModel, ConfigDict, JsonValue, TypeAdapter, ValidationError +from litellm._version import version as litellm_version from litellm.litellm_core_utils.private_json import ( commit_staged_json, discard_staged_json, @@ -75,6 +79,7 @@ BACKUP_PATH: Final = Path.home() / ".litellm" / "claude_settings_backup.json" AUTOROUTE_BACKUP_PATH: Final = Path.home() / ".litellm" / "autorouter" / "claude_settings_backup.json" CONFIGURE_STATE_PATH: Final = Path.home() / ".litellm" / "claude_configure_state.json" STATUSLINE_SCRIPT_PATH: Final = Path.home() / ".litellm" / "statusline.py" +STATUSLINE_VERSION_PREFIX: Final = b"# litellm-statusline-version: " @dataclass(frozen=True, slots=True) @@ -305,11 +310,54 @@ def statusline_command(script_path: Path, platform: str = sys.platform) -> str: return " ".join(quote(token) for token in (sys.executable, str(script_path))) -def install_statusline_script(script_path: Path | None = None) -> str: +def _statusline_version(value: str) -> Version | None: + try: + return Version(value) + except InvalidVersion: + return None + + +def _installed_statusline_version(target: Path) -> Version | None: + try: + with target.open("rb") as script: + header: Final = script.readline(256) + except FileNotFoundError: + return None + if not header.startswith(STATUSLINE_VERSION_PREFIX): + return None + try: + return _statusline_version(header.removeprefix(STATUSLINE_VERSION_PREFIX).decode("ascii").strip()) + except UnicodeDecodeError: + return None + + +def install_statusline_script( + script_path: Path | None = None, + *, + package_version: str = litellm_version, + write: Callable[[str, bytes], None] = write_private_bytes, +) -> str: target: Final = script_path or STATUSLINE_SCRIPT_PATH try: ensure_private_dir(target.parent) - write_private_bytes(str(target), Path(statusline_script.__file__).read_bytes()) + bundled_version: Final = _statusline_version(package_version) + with FileLock(str(target) + ".lock", timeout=10, mode=0o600): + installed_version: Final = _installed_statusline_version(target) + if installed_version is not None and (bundled_version is None or installed_version > bundled_version): + cli_version: Final = str(bundled_version) if bundled_version is not None else "unknown" + click.echo( + f"Keeping the status line from LiteLLM {installed_version}; this CLI is {cli_version}. " + "Upgrade the CLI to refresh it.", + err=True, + ) + return statusline_command(target) + source: Final = Path(statusline_script.__file__).read_bytes() + header: Final = ( + STATUSLINE_VERSION_PREFIX + str(bundled_version).encode("ascii") + b"\n" + if bundled_version is not None + else b"" + ) + write(str(target), header + source) except OSError as e: raise ClaudeSettingsError(f"Could not install the status line script at {target}: {e}") from e return statusline_command(target) diff --git a/litellm/proxy/client/cli/commands/statusline_script.py b/litellm/proxy/client/cli/commands/statusline_script.py index 09dd062c888..d16160b1ab8 100644 --- a/litellm/proxy/client/cli/commands/statusline_script.py +++ b/litellm/proxy/client/cli/commands/statusline_script.py @@ -1,7 +1,7 @@ """Claude Code status line and Codex Stop hook for auto-routed sessions. -`lite` copies this file verbatim to ~/.litellm/statusline.py and registers it as Claude -Code's `statusLine` command and as Codex's `[[hooks.Stop]]` command, so it must stay +`lite` copies this file to ~/.litellm/statusline.py with a CLI version header when known and registers +it as Claude Code's `statusLine` command and as Codex's `[[hooks.Stop]]` command, so it must stay standard-library only and must never import litellm. Claude Code re-runs it on every status refresh (about every 300ms while typing), so the proxy is asked at most once per TTL per session and every other refresh is served from a small on-disk cache that holds diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index 9484fd7c723..f6e0d56127f 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -75,11 +75,13 @@ from litellm.proxy.common_utils.callback_utils import ( get_logging_caching_headers, get_remaining_tokens_and_requests_from_request_data, ) +from litellm.proxy.common_utils.error_body_call_id import JSON_OBJECT, error_body_call_id, with_call_id from litellm.proxy.common_utils.http_parsing_utils import ( get_client_requested_model, get_tags_from_request_body, ) from litellm.proxy.common_utils.openai_error_payload import ( + LITELLM_CALL_ID_HEADER, attribute_of, error_status_code, openai_error_param, @@ -946,6 +948,9 @@ async def _resolve_stream_headers( return headers +_NO_GENERAL_SETTINGS: Final[Mapping[str, object]] = MappingProxyType({}) + + async def create_response( generator: AsyncGenerator[str, None], media_type: str, @@ -953,6 +958,7 @@ async def create_response( default_status_code: int = status.HTTP_200_OK, request: Request | None = None, refresh_headers: Callable[[], Awaitable[Mapping[str, str]]] | None = None, + general_settings: Mapping[str, object] = _NO_GENERAL_SETTINGS, ) -> StreamingResponse | JSONResponse: """ Create streaming response, checking if the first chunk is an error. @@ -960,7 +966,8 @@ async def create_response( Otherwise, return StreamingResponse and stream all content. ``refresh_headers`` is consulted once the first chunk has been buffered, for - callers whose headers can only be known then. + callers whose headers can only be known then. ``general_settings`` decides whether + the first-chunk error body also carries the ``x-litellm-call-id`` header's value. """ first_chunk_value: str | None = None final_status_code = default_status_code @@ -987,7 +994,10 @@ async def create_response( ) # Parse error content - error_dict: Final = _extract_error_from_sse_chunk(first_chunk_value) + error_dict: Final = with_call_id( + JSON_OBJECT.validate_python(_extract_error_from_sse_chunk(first_chunk_value)), + error_body_call_id(general_settings, resolved_headers.get(LITELLM_CALL_ID_HEADER)), + ) # Consume and close generator (avoid resource leak) try: @@ -2738,6 +2748,7 @@ class ProxyBaseLLMRequestProcessing: headers=custom_headers, request=request, refresh_headers=refresh_stream_headers, + general_settings=general_settings, ) ### CALL HOOKS ### - modify outgoing data diff --git a/litellm/proxy/common_utils/error_body_call_id.py b/litellm/proxy/common_utils/error_body_call_id.py new file mode 100644 index 00000000000..f50be5df509 --- /dev/null +++ b/litellm/proxy/common_utils/error_body_call_id.py @@ -0,0 +1,20 @@ +from collections.abc import Mapping +from typing import Final + +from pydantic import TypeAdapter + +INCLUDE_CALL_ID_IN_ERROR_BODY_SETTING: Final = "include_call_id_in_error_body" +LITELLM_CALL_ID_BODY_KEY: Final = "litellm_call_id" +JSON_OBJECT: Final[TypeAdapter[dict[str, object]]] = TypeAdapter(dict[str, object]) # mutable-ok: JSONResponse input + + +def error_body_call_id(general_settings: Mapping[str, object], call_id: str | None) -> str | None: + if general_settings.get(INCLUDE_CALL_ID_IN_ERROR_BODY_SETTING) is not True: + return None + return call_id if call_id else None + + +def with_call_id(error: dict[str, object], call_id: str | None) -> dict[str, object]: # mutable-ok: JSONResponse input + if call_id is None: + return error + return {**error, LITELLM_CALL_ID_BODY_KEY: call_id} # mutable-ok: JSONResponse input diff --git a/litellm/proxy/config_resolvers/__init__.py b/litellm/proxy/config_resolvers/__init__.py index eee760df458..f031a8d74b9 100644 --- a/litellm/proxy/config_resolvers/__init__.py +++ b/litellm/proxy/config_resolvers/__init__.py @@ -5,6 +5,17 @@ from litellm.proxy.config_resolvers._descriptors import ( FieldSource, resolve_fields, ) -from litellm.proxy.config_resolvers.settings_store import SettingsStore, config_ownership_message +from litellm.proxy.config_resolvers.settings_store import ( + SettingsStore, + config_ownership_message, + source_for, +) -__all__ = ("FieldDescriptor", "FieldSource", "SettingsStore", "config_ownership_message", "resolve_fields") +__all__ = ( + "FieldDescriptor", + "FieldSource", + "SettingsStore", + "config_ownership_message", + "resolve_fields", + "source_for", +) diff --git a/litellm/proxy/config_resolvers/settings_store.py b/litellm/proxy/config_resolvers/settings_store.py index 291000b3b6a..f05af3de03a 100644 --- a/litellm/proxy/config_resolvers/settings_store.py +++ b/litellm/proxy/config_resolvers/settings_store.py @@ -176,3 +176,10 @@ class SettingsStore(MutableMapping[str, JsonValue]): def _resolution_for(self, key: str) -> Resolved: yaml_value: Final[SettingValue] = self._yaml_values.get(key, ABSENT) return resolve(yaml_value, self._db_value(key)) + + +def source_for(settings: SettingsStore, key: str, default: object = None) -> FieldSource: + source: Final = settings.source(key) + if source == "unset": + return "default" if default is not None else "unset" + return source diff --git a/litellm/proxy/db/autorouter_session_rollup.py b/litellm/proxy/db/autorouter_session_rollup.py index 0d812ee812a..dd08cfd1bef 100644 --- a/litellm/proxy/db/autorouter_session_rollup.py +++ b/litellm/proxy/db/autorouter_session_rollup.py @@ -4,7 +4,7 @@ Per-session auto-router benchmarks rollup. At request time the spend writer builds one AutoRouterTurnTransaction per successful auto-routed request (a request whose metadata carries a routing_decision) and queues it on the prisma client. The spend-log flush job drains the queue into -LiteLLM_AutoRouterSession with one conditional upsert per turn: the statement classifies +key and user session rollups with one atomic statement per turn: each upsert classifies the turn (same model, first visit, return to a model the session already used, out of order) against the row's own columns, so nothing is read before the write and concurrent pods compose. The benchmarks endpoint aggregates these rows and never touches @@ -35,10 +35,27 @@ if TYPE_CHECKING: CACHE_TTL_5M_SECONDS: Final = 300 CACHE_TTL_1H_SECONDS: Final = 3600 -AUTOROUTER_BENCHMARKS_SQL: Final = """ +_SESSION_COLUMNS: Final = """ + api_key, session_id, router_name, router_type, first_turn_at, last_turn_at, + last_model, models, turns, unordered_turns, covered_turns, cache_hits, + same_model_turns, same_model_hits, first_visit_turns, first_visit_hits, + return_turns, return_hits, return_expired_misses, return_within_ttl_misses, + ttl_5m_turns, ttl_1h_turns, total_tokens, spend, saved_spend, classifier_cost, classifier_cost_recorded_turns, tier_turns, + baseline_models, savings_estimated_turns, savings_estimated_actual_spend, savings_estimated_saved_spend, + savings_estimated_baseline_models +""" + +AUTOROUTER_BENCHMARKS_SQL: Final = f""" WITH windowed AS ( - SELECT * FROM "LiteLLM_AutoRouterSession" - WHERE last_turn_at >= $1::timestamp + SELECT {_SESSION_COLUMNS} FROM "LiteLLM_AutoRouterSession" + WHERE $4::text IS NULL + AND last_turn_at >= $1::timestamp + AND first_turn_at < $2::timestamp + AND ($3::text IS NULL OR api_key = $3::text) + UNION ALL + SELECT {_SESSION_COLUMNS} FROM "LiteLLM_AutoRouterUserSession" + WHERE (($4::text IS NOT NULL AND user_id = $4::text) OR ($4::text IS NULL AND api_key = '')) + AND last_turn_at >= $1::timestamp AND first_turn_at < $2::timestamp AND ($3::text IS NULL OR api_key = $3::text) ), @@ -53,7 +70,7 @@ tier_maps AS ( ) SELECT agg.*, - COALESCE(tier_maps.tier_turns, '{}'::jsonb) AS tier_turns + COALESCE(tier_maps.tier_turns, '{{}}'::jsonb) AS tier_turns FROM ( SELECT router_name, @@ -111,6 +128,7 @@ class AutoRouterTurnTransaction: savings_estimated_turns: int = 0 savings_estimated_actual_spend: float = 0.0 savings_estimated_saved_spend: float = 0.0 + user_id: str = "" class TurnCacheFacts(NamedTuple): @@ -214,10 +232,11 @@ def build_autorouter_turn_transaction( if not isinstance(routing_decision, Mapping) or not routing_decision: return None router_name: Final = routing_decision.get("router_model_name") or payload.get("model_group") - api_key: Final = payload.get("api_key") + api_key: Final = payload.get("api_key") or "" + user_id: Final = payload.get("user") or "" session_id: Final = payload.get("session_id") model: Final = payload.get("model") - if not (isinstance(router_name, str) and router_name and api_key and session_id and model): + if not (isinstance(router_name, str) and router_name and (api_key or user_id) and session_id and model): return None turn_at: Final = _turn_time_utc(str(payload.get("startTime") or "")) if turn_at is None: @@ -236,6 +255,7 @@ def build_autorouter_turn_transaction( estimated_savings: Final = recorded_estimated_autorouter_savings(metadata) return AutoRouterTurnTransaction( api_key=api_key, + user_id=user_id, session_id=bounded_session_id(session_id), router_name=router_name, router_type=str(routing_decision.get("router_type") or "unknown"), @@ -293,18 +313,18 @@ _RETURN_MISS: Final = ( _IDLE_SECONDS: Final = f"EXTRACT(EPOCH FROM {_TURN_AT}::timestamp) - (t.models -> {_MODEL} ->> 'at')::float8" _CACHE_TOUCHED: Final = f"{_TOUCHED}::int = 1" -UPSERT_AUTOROUTER_SESSION_SQL: Final = f""" -INSERT INTO "LiteLLM_AutoRouterSession" AS t ( - api_key, session_id, router_name, router_type, first_turn_at, last_turn_at, - last_model, models, turns, unordered_turns, covered_turns, cache_hits, - same_model_turns, same_model_hits, first_visit_turns, first_visit_hits, - return_turns, return_hits, return_expired_misses, return_within_ttl_misses, - ttl_5m_turns, ttl_1h_turns, total_tokens, spend, saved_spend, classifier_cost, classifier_cost_recorded_turns, tier_turns, - baseline_models, savings_estimated_turns, savings_estimated_actual_spend, savings_estimated_saved_spend, - savings_estimated_baseline_models + +def _session_upsert_sql(*, user_scoped: bool) -> str: + table_name: Final = "LiteLLM_AutoRouterUserSession" if user_scoped else "LiteLLM_AutoRouterSession" + user_column: Final = "user_id, " if user_scoped else "" + user_value: Final = f"{_p('user_id')}::text, " if user_scoped else "" + required_identity: Final = _p("user_id" if user_scoped else "api_key") + return f""" +INSERT INTO "{table_name}" AS t ( + {user_column}{_SESSION_COLUMNS} ) -VALUES ( - {_p("api_key")}, {_p("session_id")}, {_p("router_name")}, {_p("router_type")}, {_TURN_AT}::timestamp, {_TURN_AT}::timestamp, +SELECT + {user_value}{_p("api_key")}, {_p("session_id")}, {_p("router_name")}, {_p("router_type")}, {_TURN_AT}::timestamp, {_TURN_AT}::timestamp, {_MODEL}, jsonb_build_object({_MODEL}, jsonb_build_object('at', EXTRACT(EPOCH FROM {_TURN_AT}::timestamp), 'ttl', {_CACHE_TTL}::int)), 1, 0, {_COVERED}::int, {_CACHE_HIT}::int, 0, 0, 1, {_CACHE_HIT}::int, @@ -315,8 +335,8 @@ VALUES ( {_p("classifier_cost")}::float8, 1, {_TIER_DELTA}, {_BASELINE_DELTA}, {_p("savings_estimated_turns")}::int, {_p("savings_estimated_actual_spend")}::float8, {_p("savings_estimated_saved_spend")}::float8, {_ESTIMATED_BASELINE_DELTA} -) -ON CONFLICT (api_key, session_id, router_name) DO UPDATE SET +WHERE {required_identity}::text <> '' +ON CONFLICT ({user_column}api_key, session_id, router_name) DO UPDATE SET turns = t.turns + 1, total_tokens = t.total_tokens + EXCLUDED.total_tokens, spend = t.spend + EXCLUDED.spend, @@ -365,6 +385,17 @@ ON CONFLICT (api_key, session_id, router_name) DO UPDATE SET """ +UPSERT_AUTOROUTER_SESSION_SQL: Final = f""" +WITH key_rollup AS ( + {_session_upsert_sql(user_scoped=False)} + RETURNING 1 +) +{_session_upsert_sql(user_scoped=True)} +""" + +UPSERT_AUTOROUTER_USER_SESSION_SQL: Final = _session_upsert_sql(user_scoped=True) + + def _as_sql_param(value: str | float | bool | datetime | None) -> str | float | None: if isinstance(value, bool): return int(value) @@ -377,18 +408,23 @@ def _upsert_params(transaction: AutoRouterTurnTransaction) -> tuple[str | float return tuple(_as_sql_param(getattr(transaction, name)) for name in _UPSERT_PARAM_FIELDS) -async def write_autorouter_turn(db: SupportsExecuteRaw, transaction: AutoRouterTurnTransaction) -> None: - await db.execute_raw(UPSERT_AUTOROUTER_SESSION_SQL, *_upsert_params(transaction)) +async def write_autorouter_turn( + db: SupportsExecuteRaw, + transaction: AutoRouterTurnTransaction, + statement: str = UPSERT_AUTOROUTER_SESSION_SQL, +) -> None: + await db.execute_raw(statement, *_upsert_params(transaction)) async def _upsert_turn_with_retry( prisma_client: PrismaClient, transaction: AutoRouterTurnTransaction, n_retry_times: int, + statement: str, ) -> None: for attempt in range(n_retry_times + 1): try: - await write_autorouter_turn(prisma_client.db, transaction) + await write_autorouter_turn(prisma_client.db, transaction, statement) except DB_RETRY_SAFE_ERROR_TYPES: if attempt >= n_retry_times: raise @@ -397,6 +433,58 @@ async def _upsert_turn_with_retry( return +def _session_partition(transaction: AutoRouterTurnTransaction) -> tuple[str, str, str, str]: + identity: Final = ("key", transaction.api_key) if transaction.api_key else ("user", transaction.user_id) + return (*identity, transaction.session_id, transaction.router_name) + + +async def _drain_session_partition( + prisma_client: PrismaClient, + transactions: tuple[AutoRouterTurnTransaction, ...], + n_retry_times: int, + statement: str, +) -> tuple[AutoRouterTurnTransaction, ...]: + for position, transaction in enumerate(transactions): + try: + await _upsert_turn_with_retry(prisma_client, transaction, n_retry_times, statement) + except Exception as flush_err: # noqa: BLE001 # stop dependent turns without retrying an ambiguous write + verbose_proxy_logger.error( + "Spend tracking - auto-router session rollup flush failed for router %s; " + "%s of %s turn writes stopped in this partition: %s", + transaction.router_name, + len(transactions) - position, + len(transactions), + flush_err, + ) + return transactions[position:] + return () + + +async def _flush_session_partition( + prisma_client: PrismaClient, + transactions: tuple[AutoRouterTurnTransaction, ...], + n_retry_times: int, +) -> None: + failed_suffix: Final = await _drain_session_partition( + prisma_client, transactions, n_retry_times, UPSERT_AUTOROUTER_SESSION_SQL + ) + if not failed_suffix or not failed_suffix[0].api_key: + return + failed_user: Final = failed_suffix[0].user_id + other_users: Final = sorted( + ( + transaction + for transaction in failed_suffix[1:] + if transaction.user_id and transaction.user_id != failed_user + ), + key=lambda transaction: transaction.user_id, + ) + for _, user_turns in groupby(other_users, key=lambda transaction: transaction.user_id): + await _drain_session_partition( + prisma_client, tuple(user_turns), n_retry_times, UPSERT_AUTOROUTER_USER_SESSION_SQL + ) + + async def flush_autorouter_turn_transactions( prisma_client: PrismaClient, transactions: Sequence[AutoRouterTurnTransaction], @@ -407,38 +495,20 @@ async def flush_autorouter_turn_transactions( Statements run sequentially in per-session event order: a turn's classification depends on the turns before it, and Postgres rejects one multi-row INSERT touching the same key twice. Only ConnectError is retried, per statement, because it proves - that statement never reached the database. Any other failure drops the remaining - turns of THAT session only, with an error log, and the flush continues with the - next session: sessions are independent state machines, so one poisoned statement - must not discard unrelated sessions, and a repeated increment is worse than an - undercount. Callers must not add their own retry around this function. + that statement never reached the database. A failed write stops its key and user + histories for this batch. Other users sharing that key can still advance their + independent user histories, with the key projection disabled and the real key + identity preserved. The failed turn is never replayed. Callers must not add their + own retry around this function. """ if not transactions: return ordered: Final = sorted( transactions, - key=lambda transaction: ( - transaction.api_key, - transaction.session_id, - transaction.router_name, - transaction.turn_at, - ), + key=lambda transaction: (*_session_partition(transaction), transaction.turn_at), ) - for session_key, session_group in groupby( + for _, session_group in groupby( ordered, - key=lambda transaction: (transaction.api_key, transaction.session_id, transaction.router_name), + key=_session_partition, ): - session_turns = tuple(session_group) - for position, transaction in enumerate(session_turns): - try: - await _upsert_turn_with_retry(prisma_client, transaction, n_retry_times) - except Exception as flush_err: # noqa: BLE001 # a statement failure drops only its session's remainder by design - verbose_proxy_logger.error( - "Spend tracking - auto-router session rollup flush failed for router %s; " - "%s of %s turn transactions dropped for one session: %s", - session_key[2], - len(session_turns) - position, - len(session_turns), - flush_err, - ) - break + await _flush_session_partition(prisma_client, tuple(session_group), n_retry_times) diff --git a/litellm/proxy/db/baseline_accounting.py b/litellm/proxy/db/baseline_accounting.py index 8622cb9e481..4219102d9aa 100644 --- a/litellm/proxy/db/baseline_accounting.py +++ b/litellm/proxy/db/baseline_accounting.py @@ -171,6 +171,7 @@ class _Change(BaseModel): request_id: str publication: BaselinePublication api_key: str + user_id: str = "" session_id: str router_name: str baseline_model: str @@ -256,42 +257,54 @@ SET publication = x.publication::text FROM jsonb_to_recordset($1::jsonb) AS x(request_id text, publication jsonb) WHERE observations.request_id = x.request_id """ -_UPDATE_SESSIONS: Final = """ + + +def _session_correction_sql(*, user_scoped: bool) -> str: + table_name: Final = "LiteLLM_AutoRouterUserSession" if user_scoped else "LiteLLM_AutoRouterSession" + identity_columns: Final = ("user_id, " if user_scoped else "") + "api_key, session_id, router_name" + user_filter: Final = "WHERE user_id <> ''" if user_scoped else "" + user_match: Final = "session.user_id = totals.user_id AND " if user_scoped else "" + return f""" WITH changes AS ( SELECT * FROM jsonb_to_recordset($1::jsonb) AS x( - api_key text, session_id text, router_name text, baseline_model text, + user_id text, api_key text, session_id text, router_name text, baseline_model text, covered_delta int, actual_delta float8, savings_delta float8 ) + {user_filter} ), totals AS ( - SELECT api_key, session_id, router_name, SUM(covered_delta)::int AS covered_delta, + SELECT {identity_columns}, SUM(covered_delta)::int AS covered_delta, SUM(actual_delta) AS actual_delta, SUM(savings_delta) AS savings_delta - FROM changes GROUP BY api_key, session_id, router_name + FROM changes GROUP BY {identity_columns} ), models AS ( - SELECT api_key, session_id, router_name, jsonb_object_agg(baseline_model, delta) AS deltas + SELECT {identity_columns}, jsonb_object_agg(baseline_model, delta) AS deltas FROM ( - SELECT api_key, session_id, router_name, baseline_model, SUM(covered_delta)::int AS delta - FROM changes GROUP BY api_key, session_id, router_name, baseline_model - ) grouped GROUP BY api_key, session_id, router_name + SELECT {identity_columns}, baseline_model, SUM(covered_delta)::int AS delta + FROM changes GROUP BY {identity_columns}, baseline_model + ) grouped GROUP BY {identity_columns} ) -UPDATE "LiteLLM_AutoRouterSession" AS session +UPDATE "{table_name}" AS session SET saved_spend = session.saved_spend + totals.savings_delta, savings_estimated_turns = session.savings_estimated_turns + totals.covered_delta, savings_estimated_actual_spend = session.savings_estimated_actual_spend + totals.actual_delta, savings_estimated_saved_spend = session.savings_estimated_saved_spend + totals.savings_delta, savings_estimated_baseline_models = ( - SELECT COALESCE(jsonb_object_agg(key, value), '{}'::jsonb) FROM ( + SELECT COALESCE(jsonb_object_agg(key, value), '{{}}'::jsonb) FROM ( SELECT key, SUM(value::int)::int AS value FROM ( SELECT * FROM jsonb_each_text(session.savings_estimated_baseline_models) UNION ALL SELECT * FROM jsonb_each_text(models.deltas) ) combined GROUP BY key HAVING SUM(value::int) > 0 ) counts ) -FROM totals JOIN models USING (api_key, session_id, router_name) -WHERE session.api_key = totals.api_key AND session.session_id = totals.session_id +FROM totals JOIN models USING ({identity_columns}) +WHERE {user_match}session.api_key = totals.api_key AND session.session_id = totals.session_id AND session.router_name = totals.router_name """ +_UPDATE_SESSIONS: Final = _session_correction_sql(user_scoped=False) +_UPDATE_USER_SESSIONS: Final = _session_correction_sql(user_scoped=True) + + def _primary_transaction(client: PrismaClient) -> _TransactionManager: primary: Final = cast(_TransactionalDatabase, writer_wrapper(client.db)) return primary.tx(timeout=_TRANSACTION_TIMEOUT) @@ -308,6 +321,7 @@ def _change(record: BaselineAccountingRecord, old: BaselinePublication | None, n request_id=record.observation.request_id, publication=new, api_key=record.api_key, + user_id=record.turn.user_id if record.turn is not None else "", session_id=record.session_id, router_name=record.router_name, baseline_model=record.baseline_model, @@ -357,6 +371,8 @@ async def _publish(db: SupportsRawQueries, changes: Sequence[_Change]) -> None: serialized: Final = json.dumps(tuple(change.model_dump(mode="json") for change in changes), separators=(",", ":")) await db.execute_raw(_UPDATE_LOGS, serialized) await db.execute_raw(_UPDATE_SESSIONS, serialized) + if any(change.user_id for change in changes): + await db.execute_raw(_UPDATE_USER_SESSIONS, serialized) for entity, table in DAILY_SPEND_TABLES.items(): if adjustments := tuple( change.daily.adjustment(target, change.savings_delta, change.request_id) diff --git a/litellm/proxy/db/db_transaction_queue/spend_log_cleanup.py b/litellm/proxy/db/db_transaction_queue/spend_log_cleanup.py index b28a653c9aa..85e19fa8a32 100644 --- a/litellm/proxy/db/db_transaction_queue/spend_log_cleanup.py +++ b/litellm/proxy/db/db_transaction_queue/spend_log_cleanup.py @@ -1,5 +1,6 @@ import asyncio import time +from contextvars import ContextVar from dataclasses import dataclass from datetime import datetime, timedelta, timezone from typing import Final, Literal, TypeAlias @@ -40,6 +41,28 @@ class TableCleanupResult: stop_reason: StopReason +class _RunProgress: + """How far one cleanup run has got, reported if that run is cancelled""" + + def __init__(self) -> None: + self.rows_deleted: int = 0 + self.batches: int = 0 + + def record_batch(self, rows_deleted: int) -> None: + self.rows_deleted += rows_deleted + self.batches += 1 + + +_run_progress: ContextVar[_RunProgress] = ContextVar("spend_log_cleanup_run_progress") + + +def _record_run_batch(rows_deleted: int) -> None: + """Count a batch towards the run in progress, if a run is what issued it""" + progress: Final = _run_progress.get(None) + if progress is not None: + progress.record_batch(rows_deleted) + + class _RemainingRow(BaseModel): """One row of the capped outstanding-rows probe, validated out of prisma's untyped result.""" @@ -422,6 +445,7 @@ class SpendLogCleanup: total_deleted += deleted_count run_count += 1 + _record_run_batch(deleted_count) # Add a small sleep to prevent overwhelming the database await asyncio.sleep(0.1) @@ -492,6 +516,18 @@ class SpendLogCleanup: deadline=deadline, ) + async def _delete_old_autorouter_user_session_rows( + self, prisma_client: PrismaClient, cutoff_date: datetime, deadline: float + ) -> TableCleanupResult: + return await self._delete_old_rows_batched( + prisma_client, + cutoff_date, + table_name="LiteLLM_AutoRouterUserSession", + key_columns=("user_id", "api_key", "session_id", "router_name"), + time_column="last_turn_at", + deadline=deadline, + ) + async def _delete_old_health_check_rows( self, prisma_client: PrismaClient, cutoff_date: datetime, deadline: float ) -> TableCleanupResult: @@ -560,9 +596,17 @@ class SpendLogCleanup: ) except Exception: # noqa: BLE001 # retained observations are retried by the next cleanup job verbose_proxy_logger.warning("Auto-router baseline retention remains pending") - sessions_result: Final = await self._delete_old_autorouter_session_rows(prisma_client, session_cutoff, deadline) + sessions_result: Final = await self._delete_old_autorouter_session_rows( + prisma_client, session_cutoff, self._group_deadline(deadline, 2) + ) verbose_proxy_logger.info("Deleted %s expired auto-router session rollup rows", sessions_result.rows_deleted) - return (sessions_result,) + user_sessions_result: Final = await self._delete_old_autorouter_user_session_rows( + prisma_client, session_cutoff, deadline + ) + verbose_proxy_logger.info( + "Deleted %s expired auto-router user session rollup rows", user_sessions_result.rows_deleted + ) + return (sessions_result, user_sessions_result) async def _clean_health_checks( self, prisma_client: PrismaClient, retention_seconds: int, deadline: float @@ -601,6 +645,9 @@ class SpendLogCleanup: If no pod_lock_manager, runs cleanup without distributed locking. """ lock_acquired = False + run_started_at: Final = time.monotonic() + progress: Final = _RunProgress() + progress_token: Final = _run_progress.set(progress) try: verbose_proxy_logger.info("Cleanup job triggered at %s", datetime.now()) self._refresh_bounds() @@ -681,6 +728,15 @@ class SpendLogCleanup: self._run_outcome(spend_log_results + session_results + health_check_results) ) + except asyncio.CancelledError: + verbose_proxy_logger.error( + "Spend log cleanup cancelled after %.2fs (rows_deleted=%d, batches=%d); the next run resumes from here", + time.monotonic() - run_started_at, + progress.rows_deleted, + progress.batches, + ) + SpendLogCleanupMetrics.record_run("aborted") + raise except Exception as e: # .exception() captures the traceback; str(e) alone on a Prisma/DB # timeout is often empty and gives operators no signal to diagnose. @@ -692,6 +748,7 @@ class SpendLogCleanup: SpendLogCleanupMetrics.record_run("aborted") return # Return after error handling finally: + _run_progress.reset(progress_token) # Only release the lock if it was actually acquired if lock_acquired and self.pod_lock_manager and self.pod_lock_manager.redis_cache: await self.pod_lock_manager.release_lock(cronjob_id=SPEND_LOG_CLEANUP_JOB_NAME) diff --git a/litellm/proxy/hooks/proxy_track_cost_callback.py b/litellm/proxy/hooks/proxy_track_cost_callback.py index 08e8e4f8c10..81894a5ff12 100644 --- a/litellm/proxy/hooks/proxy_track_cost_callback.py +++ b/litellm/proxy/hooks/proxy_track_cost_callback.py @@ -11,6 +11,7 @@ from litellm.constants import BACKGROUND_INTERACTION_COST_POLLING_ENABLED from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.core_helpers import ( _get_parent_otel_span_from_kwargs, + budget_reservation_from_metadata, get_litellm_metadata_from_kwargs, ) from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup @@ -630,17 +631,7 @@ def _metadata_keys(metadata: object) -> tuple[str, ...]: def _get_budget_reservation_from_metadata(metadata: dict) -> dict | None: - metadata_budget_reservation: Final = metadata.get("user_api_key_budget_reservation") - if isinstance(metadata_budget_reservation, dict): - return metadata_budget_reservation - - user_api_key_auth_obj: Final = metadata.get("user_api_key_auth") - if user_api_key_auth_obj is None: - return None - if isinstance(user_api_key_auth_obj, dict): - budget_reservation: Final = user_api_key_auth_obj.get("budget_reservation") - return budget_reservation if isinstance(budget_reservation, dict) else None - return getattr(user_api_key_auth_obj, "budget_reservation", None) + return budget_reservation_from_metadata(metadata) def _get_request_tags_for_cost_tracking( diff --git a/litellm/proxy/management_endpoints/access_group_endpoints.py b/litellm/proxy/management_endpoints/access_group_endpoints.py index a6cc5140b15..b4923b0a2dc 100644 --- a/litellm/proxy/management_endpoints/access_group_endpoints.py +++ b/litellm/proxy/management_endpoints/access_group_endpoints.py @@ -5,6 +5,7 @@ from types import MappingProxyType from typing import Final, Protocol from fastapi import APIRouter, Depends, HTTPException, status +from typing_extensions import ReadOnly, TypedDict from litellm._logging import verbose_proxy_logger from litellm.proxy._experimental.mcp_server.mcp_server_manager import global_mcp_server_manager @@ -109,6 +110,36 @@ class _KeyTable(Protocol): async def update(self, where: Mapping[str, object], data: Mapping[str, object]) -> object: ... +class _AgentRecord(Protocol): + @property + def agent_id(self) -> str: ... + + @property + def access_group_ids(self) -> Sequence[str] | None: ... + + +class _AgentTable(Protocol): + async def find_many(self, where: Mapping[str, object]) -> Sequence[_AgentRecord]: ... + + async def update(self, where: Mapping[str, object], data: Mapping[str, object]) -> object: ... + + +class _HasSomeFilter(TypedDict): + hasSome: ReadOnly[Sequence[str]] + + +class _AgentAccessGroupsWhere(TypedDict): + access_group_ids: ReadOnly[_HasSomeFilter] + + +class _AgentIdWhere(TypedDict): + agent_id: ReadOnly[str] + + +class _AgentAccessGroupsData(TypedDict): + access_group_ids: ReadOnly[Sequence[str]] + + class _AccessGroupTx(Protocol): @property def litellm_accessgrouptable(self) -> _AccessGroupTable: ... @@ -119,6 +150,9 @@ class _AccessGroupTx(Protocol): @property def litellm_verificationtoken(self) -> _KeyTable: ... + @property + def litellm_agentstable(self) -> _AgentTable: ... + def _require_proxy_admin(user_api_key_dict: UserAPIKeyAuth) -> None: if user_api_key_dict.user_role != LitellmUserRoles.PROXY_ADMIN: @@ -324,6 +358,41 @@ async def _sync_remove_access_group_from_keys(tx: _AccessGroupTx, key_tokens: li ) +def _without_access_group(access_group_ids: Sequence[str] | None, access_group_id: str) -> tuple[str, ...]: + return tuple(ag for ag in (access_group_ids or ()) if ag != access_group_id) + + +async def _detach_access_group_from_agents(tx: _AccessGroupTx, access_group_id: str) -> tuple[str, ...]: + agents_with_group: Final = await tx.litellm_agentstable.find_many( + where=_AgentAccessGroupsWhere(access_group_ids=_HasSomeFilter(hasSome=(access_group_id,))) + ) + for agent in agents_with_group: + await tx.litellm_agentstable.update( + where=_AgentIdWhere(agent_id=agent.agent_id), + data=_AgentAccessGroupsData( + access_group_ids=_without_access_group(agent.access_group_ids, access_group_id) + ), + ) + return tuple(agent.agent_id for agent in agents_with_group) + + +def _detach_access_group_from_agent_registry(agent_ids: Sequence[str], access_group_id: str) -> None: + registered: Final = tuple( + agent + for agent in (global_agent_registry.get_agent_by_id(agent_id) for agent_id in agent_ids) + if agent is not None + ) + for agent in registered: + global_agent_registry.deregister_agent(agent_name=agent.agent_name) + global_agent_registry.register_agent( + agent_config=agent.model_copy( + update=_AgentAccessGroupsData( + access_group_ids=_without_access_group(agent.access_group_ids, access_group_id) + ) + ) + ) + + # --------------------------------------------------------------------------- # Cache patch helpers # --------------------------------------------------------------------------- @@ -705,11 +774,14 @@ async def delete_access_group( out_of_sync_key_tokens: Final = set(existing.assigned_key_ids or []) - {k.token for k in keys_with_group} await _sync_remove_access_group_from_keys(tx, list(out_of_sync_key_tokens), access_group_id) + detached_agent_ids: Final = await _detach_access_group_from_agents(tx, access_group_id) + await tx.litellm_accessgrouptable.delete(where={"access_group_id": access_group_id}) from litellm.proxy.proxy_server import proxy_logging_obj, user_api_key_cache await invalidate_access_group_cache(access_group_id) + _detach_access_group_from_agent_registry(detached_agent_ids, access_group_id) await _patch_team_caches_remove_access_group( affected_team_ids, access_group_id, user_api_key_cache, proxy_logging_obj ) diff --git a/litellm/proxy/management_endpoints/auto_router_endpoints.py b/litellm/proxy/management_endpoints/auto_router_endpoints.py index d8054a8dc4a..03fc58622ce 100644 --- a/litellm/proxy/management_endpoints/auto_router_endpoints.py +++ b/litellm/proxy/management_endpoints/auto_router_endpoints.py @@ -789,14 +789,18 @@ async def get_auto_router_benchmarks( ] = None, end_date: Annotated[str | None, Query(description="YYYY-MM-DD UTC, inclusive (defaults to today)")] = None, api_key: Annotated[str | None, Query(description="Filter to one virtual key token hash")] = None, + user_id: Annotated[ + str | None, Query(min_length=1, description="Filter to one canonical internal user recorded on each turn") + ] = None, ) -> AutoRouterBenchmarksResponse: """ Benchmarks for the auto-router dashboard: session shape, savings against the configured baseline, and prompt-caching behaviour bucketed by what the router did. - Reads the LiteLLM_AutoRouterSession rollup, folded once per request at spend-write time, - so this endpoint never scans LiteLLM_SpendLogs. A session is in the window when it - overlaps it: its last turn is on or after start_date and its first turn is on or before + Reads session rollups folded once per request at spend-write time, so this endpoint + never scans LiteLLM_SpendLogs. A user filter selects only turns attributed to that + internal user when written; older key-only history remains outside user views. A session + is in the window when it overlaps it: its last turn is on or after start_date and its first turn is on or before end_date. Overall hit rate is over telemetry-bearing turns; each bucket's hit rate is over that bucket's turns. @@ -826,6 +830,7 @@ async def get_auto_router_benchmarks( start_day.isoformat(), (end_day + timedelta(days=1)).isoformat(), api_key, + user_id, ) rows: Final = _SESSION_AGG_ROWS.validate_python(raw_rows or ()) groups: Final = ( diff --git a/litellm/proxy/management_endpoints/internal_user_endpoints.py b/litellm/proxy/management_endpoints/internal_user_endpoints.py index dc1f1c53601..6d4838ae052 100644 --- a/litellm/proxy/management_endpoints/internal_user_endpoints.py +++ b/litellm/proxy/management_endpoints/internal_user_endpoints.py @@ -28,6 +28,7 @@ from typing_extensions import ReadOnly, TypedDict import litellm from litellm._logging import verbose_proxy_logger from litellm._uuid import uuid +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler from litellm.proxy._types import * from litellm.proxy.auth.auth_checks import ( delete_cache_key_objects, @@ -35,7 +36,11 @@ from litellm.proxy.auth.auth_checks import ( get_team_object, get_user_object, ) -from litellm.proxy.auth.password_policy import validate_password_policy +from litellm.proxy.auth.password_policy import ( + validate_password_not_breached, + validate_password_policy, + validate_passwords_bulk, +) from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.common_utils.auth_cache_invalidation_pubsub import evict_and_broadcast from litellm.proxy.common_utils.user_api_key_cache import ( @@ -173,11 +178,23 @@ def _team_membership_table( return team_membership_table -def _hash_password_in_dict(data: dict, general_settings: Mapping[str, object]) -> None: - """Validate and hash password field in-place if present.""" +async def _hash_password_in_dict( + data: dict, general_settings: Mapping[str, object], password_prevalidated: bool = False +) -> None: + """Validate and hash password field in-place if present. + + ``password_prevalidated`` skips the policy checks for callers that already + validated the password (the bulk path screens its whole batch upfront). + + An admin-set password is known to whoever set it, so the user is also + flagged for a forced password change at next login.""" if "password" in data and data["password"] is not None: - validate_password_policy(data["password"], general_settings) + if not password_prevalidated: + validate_password_policy(data["password"], general_settings) + await validate_password_not_breached(data["password"], general_settings) data["password"] = hash_password(data["password"]) + data["password_reset_required"] = True + data["last_breach_check_at"] = None def _strip_password_from_response(response) -> None: @@ -505,6 +522,7 @@ async def new_user( - prompts: Optional[List[str]] - List of allowed prompts for the user. If specified, the user will only be able to use these specific prompts. - organizations: List[str] - List of organization id's the user is a member of - budget_limits: Optional[list] - List of concurrent budget windows for the user. Each window specifies a budget_limit, time_period, and optional budget_duration. Example - [{"budget_limit": 10.0, "time_period": "1d"}, {"budget_limit": 50.0, "time_period": "7d"}]. + - password: Optional[str] - Not supported; any value is rejected with a 422. Users set their own password through an invitation link (POST /invitation/new). Returns: - key: (str) The generated api key for the user - expires: (datetime) Datetime object for when key expires. @@ -524,7 +542,7 @@ async def new_user( ``` """ try: - from litellm.proxy.proxy_server import _license_check, general_settings, prisma_client + from litellm.proxy.proxy_server import _license_check, prisma_client if prisma_client is None: raise HTTPException(status_code=400, detail=CommonProxyErrors.db_not_connected_error.value) @@ -572,7 +590,7 @@ async def new_user( # generate_key_helper_fn only forwards object_permission_id, so without this the entitlement # the caller sent would be dropped on the floor. data_json = await _set_object_permission(data_json=data_json, prisma_client=prisma_client) - _hash_password_in_dict(data_json, general_settings) + data_json.pop("password", None) teams = data.teams if teams is None: teams = check_if_default_team_set() @@ -1438,6 +1456,7 @@ async def _update_single_user_helper( user_request: UpdateUserRequest, user_api_key_dict: UserAPIKeyAuth, litellm_changed_by: str | None = None, + password_prevalidated: bool = False, ) -> dict[str, Any]: """ Helper function to update a single user. @@ -1460,7 +1479,7 @@ async def _update_single_user_helper( data_json: Final[dict] = user_request.model_dump(exclude_unset=True) non_default_values = _update_internal_user_params(data_json=data_json, data=user_request) - _hash_password_in_dict(non_default_values, general_settings) + await _hash_password_in_dict(non_default_values, general_settings, password_prevalidated=password_prevalidated) existing_user_row: BaseModel | None = None if user_request.user_id: @@ -1641,7 +1660,7 @@ async def user_update( Parameters: - user_id: Optional[str] - Specify a user id. If not set, a unique id will be generated. - user_email: Optional[str] - Specify a user email. - - password: Optional[str] - Specify a user password. + - password: Optional[str] - Set the user's password (admin only). Must satisfy the configured password policy. The user is required to change it at their next login. Users change their own password with POST /user/password/change. - user_alias: Optional[str] - A descriptive name for you to know who this user id refers to. - teams: Optional[list] - specify a list of team id's a user belongs to. - send_invite_email: Optional[bool] - Specify if an invite email should be sent. @@ -1709,19 +1728,38 @@ async def bulk_update_processed_users( users_to_update: list[UpdateUserRequest], user_api_key_dict: UserAPIKeyAuth, litellm_changed_by: str | None = None, + hibp_client: AsyncHTTPHandler | None = None, ) -> BulkUpdateUserResponse: + from litellm.proxy.proxy_server import general_settings + results: Final[list[UserUpdateResult]] = [] successful_updates = 0 failed_updates = 0 + # Screen the batch's passwords upfront and concurrently: done per-user + # inside the loop below, each HIBP lookup would be awaited serially and a + # degraded-slow HIBP could stretch a full batch to minutes, timing out the + # request after some updates already persisted. + password_verdicts: Final = await validate_passwords_bulk( + tuple(u.password for u in users_to_update if u.password is not None), + general_settings, + client=hibp_client, + ) + # Process each user update independently try: for user_request in users_to_update: try: + if ( + user_request.password is not None + and (password_error := password_verdicts.get(user_request.password)) is not None + ): + raise password_error response = await _update_single_user_helper( user_request=user_request, user_api_key_dict=user_api_key_dict, litellm_changed_by=litellm_changed_by, + password_prevalidated=True, ) # Record success results.append( @@ -1859,6 +1897,14 @@ async def bulk_user_update( status_code=403, detail="Only proxy admins can update all users at once.", ) + if data.user_updates.password is not None: + bulk_password_error: Final[HTTPExceptionErrorDetail] = { + "error": ( + "Setting one password for all users is not supported. " + "Use per-user updates via the 'users' list instead." + ) + } + raise HTTPException(status_code=400, detail=bulk_password_error) # Optimized path for updating all users directly in database all_users_in_db: Final = await _user_table(prisma_client).find_many(order={"created_at": "desc"}) diff --git a/litellm/proxy/management_endpoints/jwt_key_mapping_endpoints.py b/litellm/proxy/management_endpoints/jwt_key_mapping_endpoints.py index 07234883062..292cec1346d 100644 --- a/litellm/proxy/management_endpoints/jwt_key_mapping_endpoints.py +++ b/litellm/proxy/management_endpoints/jwt_key_mapping_endpoints.py @@ -1,3 +1,4 @@ +import re from collections.abc import Mapping, Sequence from datetime import datetime from typing import Final, Protocol @@ -21,6 +22,51 @@ from litellm.repositories.table_repositories import JWTKeyMappingRepository router: Final = APIRouter() +_TOKEN_HASH_PATTERN: Final = re.compile(r"[0-9a-f]{64}") + + +def _validated_token_hash(token: str) -> str: + """Guards a plaintext key from being stored as a hash of a hash, which would never match.""" + if _TOKEN_HASH_PATTERN.fullmatch(token) is None: + raise HTTPException( + status_code=400, + detail=( + "`token` must be the SHA-256 hash of a virtual key " + "(64 lowercase hex characters). Pass the plaintext as `key` instead." + ), + ) + return token + + +_EXACTLY_ONE_IDENTIFIER: Final = ( + "Provide exactly one of `key` (the plaintext virtual key) or `token` (its SHA-256 hash)." +) +_AT_MOST_ONE_IDENTIFIER: Final = ( + "Provide at most one of `key` (the plaintext virtual key) or `token` (its SHA-256 hash)." +) + + +def _token_hash_for_create(data: CreateJWTKeyMappingRequest) -> str: + """Resolve the token hash to store, from either the plaintext key or its hash.""" + if data.key is not None and data.token is not None: + raise HTTPException(status_code=400, detail=_EXACTLY_ONE_IDENTIFIER) + if data.token is not None: + return _validated_token_hash(data.token) + if data.key is not None: + return hash_token(data.key) + raise HTTPException(status_code=400, detail=_EXACTLY_ONE_IDENTIFIER) + + +def _token_hash_for_update(data: UpdateJWTKeyMappingRequest) -> str | None: + """Resolve the token hash to store, or None to leave the mapped key alone.""" + if data.key is not None and data.token is not None: + raise HTTPException(status_code=400, detail=_AT_MOST_ONE_IDENTIFIER) + if data.token is not None: + return _validated_token_hash(data.token) + if data.key is not None: + return hash_token(data.key) + return None + class _JWTKeyMappingRecord(Protocol): """A ``LiteLLM_JWTKeyMapping`` row, viewed through the columns these endpoints read.""" @@ -111,7 +157,7 @@ async def create_jwt_key_mapping( raise HTTPException(status_code=500, detail="Database not connected") try: - hashed_key: Final = hash_token(data.key) + hashed_key: Final = _token_hash_for_create(data) create_data: Final = { "jwt_issuer": data.jwt_issuer or "", "jwt_claim_name": data.jwt_claim_name, @@ -166,9 +212,10 @@ async def update_jwt_key_mapping( if prisma_client is None: raise HTTPException(status_code=500, detail="Database not connected") - update_data: Final = data.model_dump(exclude_unset=True, exclude={"id", "key"}) - if data.key is not None: - update_data["token"] = hash_token(data.key) + update_data: Final = data.model_dump(exclude_unset=True, exclude={"id", "key", "token"}) + token_hash: Final = _token_hash_for_update(data) + if token_hash is not None: + update_data["token"] = token_hash if "jwt_issuer" in update_data: # DB column is NOT NULL (see schema.prisma); "" is the global/unscoped sentinel. update_data["jwt_issuer"] = update_data["jwt_issuer"] or "" diff --git a/litellm/proxy/management_endpoints/mcp_management_endpoints.py b/litellm/proxy/management_endpoints/mcp_management_endpoints.py index c1388e8bb81..9ad78876043 100644 --- a/litellm/proxy/management_endpoints/mcp_management_endpoints.py +++ b/litellm/proxy/management_endpoints/mcp_management_endpoints.py @@ -983,7 +983,7 @@ if MCP_AVAILABLE: mcp_server_auth_headers=None, ) tools: Final = listing.tools - dumped_tools: Final = [dict(tool) for tool in tools] + dumped_tools: Final = [tool.model_dump(by_alias=True) for tool in tools] return {"tools": dumped_tools} @@ -1078,6 +1078,9 @@ if MCP_AVAILABLE: return {"servers": registry_servers} ## FastAPI Routes + def _mcp_server_display_order(server: LiteLLM_MCPServerTable) -> tuple[str, str]: + return ((server.server_name or server.alias or server.server_id).lower(), server.server_id) + def _get_user_mcp_management_mode() -> UserMCPManagementMode: from litellm.proxy.proxy_server import ( general_settings as proxy_general_settings, @@ -1228,10 +1231,12 @@ if MCP_AVAILABLE: detail="You do not have permission to view MCP servers for this team.", ) - redacted_mcp_servers = await _get_team_scoped_mcp_server_list(sanitized_team_id) + redacted_mcp_servers = sorted( + await _get_team_scoped_mcp_server_list(sanitized_team_id), key=_mcp_server_display_order + ) else: servers: Final = await _resolve_accessible_mcp_servers(user_api_key_dict) - redacted_mcp_servers = _redact_mcp_credentials_list(servers) + redacted_mcp_servers = sorted(_redact_mcp_credentials_list(servers), key=_mcp_server_display_order) if connected_app_view is True and is_ui_session_credential(user_api_key_dict): reachable_ids: Final = await _connected_app_reachable_server_ids(user_api_key_dict) diff --git a/litellm/proxy/management_endpoints/model_management_endpoints.py b/litellm/proxy/management_endpoints/model_management_endpoints.py index 10a0a2f3104..fcadcfe2cae 100644 --- a/litellm/proxy/management_endpoints/model_management_endpoints.py +++ b/litellm/proxy/management_endpoints/model_management_endpoints.py @@ -14,12 +14,12 @@ import asyncio import datetime import json from collections.abc import AsyncGenerator, Awaitable, Callable, Mapping, Sequence -from contextlib import AbstractAsyncContextManager, asynccontextmanager +from contextlib import AbstractAsyncContextManager, asynccontextmanager, suppress from dataclasses import dataclass from fnmatch import fnmatchcase from json import JSONDecodeError from types import MappingProxyType -from typing import TYPE_CHECKING, Annotated, Final, Literal, Protocol, TypeVar, cast, runtime_checkable +from typing import TYPE_CHECKING, Annotated, Final, Literal, Protocol, TypeAlias, TypeVar, cast, runtime_checkable from fastapi import APIRouter, Depends, Header, HTTPException, Request, status from pydantic import BaseModel, ConfigDict, Field, TypeAdapter, ValidationError, field_validator @@ -29,6 +29,7 @@ from litellm._logging import verbose_proxy_logger from litellm._uuid import uuid from litellm.constants import LITELLM_PROXY_ADMIN_NAME from litellm.litellm_core_utils.credential_accessor import CredentialAccessor +from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap from litellm.litellm_core_utils.ptu_pricing import ( CUSTOM_PRICING_FIELDS, PTU_EMPTIED_PRICING_FIELDS, @@ -139,7 +140,12 @@ from litellm.types.router import ( updateDeployment, updateLiteLLMParams, ) -from litellm.types.utils import echoed_cost_map_pricing_fields, without_server_derived_pricing +from litellm.types.utils import ( + COST_MAP_LOOKUP_KEY, + echoed_cost_map_fields, + echoed_cost_map_pricing_fields, + without_server_derived_pricing, +) from litellm.utils import get_utc_datetime if TYPE_CHECKING: @@ -928,7 +934,33 @@ def _ptu_priced_deployment(model_params: Deployment) -> Deployment: ) -def update_db_model(db_model: Deployment, updated_patch: updateDeployment) -> PrismaCompatibleUpdateDBModel: +def _cost_map_entry(db_model: Deployment, incoming_model_info: Mapping[str, object]) -> Mapping[str, object]: + base_model: Final = incoming_model_info.get("base_model") + lookup: Final = base_model if isinstance(base_model, str) else _decrypted_model(db_model.litellm_params.model) + if lookup is None: + return MappingProxyType({}) + with suppress(Exception): + return MappingProxyType(dict(litellm.get_model_info(model=lookup))) + return MappingProxyType({}) + + +LoadedCatalog: TypeAlias = Callable[[], Mapping[str, Mapping[str, object]]] # mutable-ok: Callable parameter syntax + + +def _loaded_catalog_entry( + incoming_model_info: Mapping[str, object], loaded_catalog: LoadedCatalog +) -> Mapping[str, object]: + catalog_key: Final = incoming_model_info.get(COST_MAP_LOOKUP_KEY) + if not isinstance(catalog_key, str): + return MappingProxyType({}) + return loaded_catalog().get(catalog_key, MappingProxyType({})) + + +def update_db_model( + db_model: Deployment, + updated_patch: updateDeployment, + loaded_catalog: LoadedCatalog = GetModelCostMap.loaded_model_cost_map, +) -> PrismaCompatibleUpdateDBModel: if updated_patch.model_info is not None: _raise_if_ptu_cost_attribution_disabled(updated_patch.model_info.model_dump(exclude_none=True)) merged_model_name: Final = updated_patch.model_name or db_model.model_name @@ -955,7 +987,24 @@ def update_db_model(db_model: Deployment, updated_patch: updateDeployment) -> Pr # update model info if updated_patch.model_info: - merged_model_info.update(without_server_derived_pricing(updated_patch.model_info.model_dump(exclude_none=True))) + incoming_model_info: Final = updated_patch.model_info.model_dump(exclude_none=True) + echoed_fields: Final = echoed_cost_map_fields( + incoming_model_info, + _cost_map_entry(db_model, incoming_model_info), + _loaded_catalog_entry(incoming_model_info, loaded_catalog), + ) + merged_model_info.update( + MappingProxyType( + dict( + (k, v) + for k, v in without_server_derived_pricing(incoming_model_info).items() + if k not in echoed_fields + ) + ) + ) + for k in echoed_fields: + if k in merged_model_info and merged_model_info[k] != incoming_model_info[k]: + del merged_model_info[k] # Honor explicit-null clears LAST, after both merges, so a model_info blob a client # passes through cannot silently undo a litellm_params clear via .update(). diff --git a/litellm/proxy/management_endpoints/password_endpoints.py b/litellm/proxy/management_endpoints/password_endpoints.py new file mode 100644 index 00000000000..03a8b4c4010 --- /dev/null +++ b/litellm/proxy/management_endpoints/password_endpoints.py @@ -0,0 +1,154 @@ +""" +Self-service password management. + +/user/password/change + +Deliberately NOT wrapped in `management_endpoint_wrapper`: the wrapper emits +request kwargs to OTEL spans, which would log plaintext passwords. The audit +signal is emitted by hand below, with field names only, never values. +""" + +from typing import TYPE_CHECKING, Annotated, Final + +from fastapi import APIRouter, Depends, HTTPException +from pydantic import TypeAdapter + +from litellm._logging import verbose_proxy_logger +from litellm.proxy._types import ( + UI_TEAM_ID, + ChangePasswordRequest, + ChangePasswordResponse, + CommonProxyErrors, + HTTPExceptionErrorDetail, + LitellmTableNames, + UserAPIKeyAuth, +) +from litellm.proxy.auth.login_utils import PASSWORD_SESSION_METADATA +from litellm.proxy.auth.password_policy import validate_password_not_breached, validate_password_policy +from litellm.proxy.auth.user_api_key_auth import user_api_key_auth +from litellm.proxy.management_helpers.audit_logs import create_object_audit_log +from litellm.proxy.utils import hash_password, verify_password +from litellm.repositories.prisma_protocols import TableActions +from litellm.repositories.user_repository import UserRepository + +if TYPE_CHECKING: + from prisma import models as prisma_models + from prisma import types as prisma_types + + from litellm.proxy.utils import PrismaClient + +router: Final = APIRouter() + +_PASSWORD_CHANGED_AUDIT_VALUES: Final = '{"fields_changed": ["password"]}' +_KEY_METADATA: Final = TypeAdapter(dict[str, object]) + + +def _error_detail(message: str) -> HTTPExceptionErrorDetail: + detail: Final[HTTPExceptionErrorDetail] = {"error": message} + return detail + + +def _is_password_login_session(user_api_key_dict: UserAPIKeyAuth) -> bool: + if user_api_key_dict.team_id != UI_TEAM_ID: + return False + key_metadata: Final = _KEY_METADATA.validate_python(user_api_key_dict.metadata) + return all(key_metadata.get(k) == v for k, v in PASSWORD_SESSION_METADATA.items()) + + +def _user_table( + prisma_client: "PrismaClient | None", +) -> "TableActions[prisma_models.LiteLLM_UserTable]": + user_table: Final[TableActions[prisma_models.LiteLLM_UserTable]] = UserRepository(prisma_client).table + return user_table + + +@router.post( + "/user/password/change", + tags=("Internal User management",), + dependencies=(Depends(user_api_key_auth),), +) +async def change_password( + data: ChangePasswordRequest, + user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], +) -> ChangePasswordResponse: + """ + Change the calling user's own password. + + Only callable with the dashboard session issued by a username/password + login; SSO sessions and virtual keys are rejected with 403. Requires the + current password. The new password must differ from the + current one and satisfy the configured password policy + (`general_settings.password_policy_*`: minimum length, character classes, + and, when enabled, breached-password screening via haveibeenpwned.com). + A successful change lifts any pending forced password reset + (`password_reset_required`) on the account. + + Parameters: + - current_password: str - The user's current password. + - new_password: str - The password to change to. + """ + from litellm.proxy.proxy_server import general_settings, litellm_proxy_admin_name, prisma_client + + if prisma_client is None: + raise HTTPException( + status_code=500, + detail=_error_detail(CommonProxyErrors.db_not_connected_error.value), + ) + + if not _is_password_login_session(user_api_key_dict): + raise HTTPException( + status_code=403, + detail=_error_detail( + "Passwords can only be changed from a dashboard session created by logging in with a password." + ), + ) + + user_id: Final = user_api_key_dict.user_id + if user_id is None: + raise HTTPException( + status_code=400, + detail=_error_detail("No user is associated with this session, so there is no password to change."), + ) + + find_user: Final[prisma_types.LiteLLM_UserTableWhereInput] = {"user_id": user_id} + user_row: Final = await _user_table(prisma_client).find_first(where=find_user) + stored_password: Final = user_row.password if user_row is not None else None + if stored_password is None: + raise HTTPException( + status_code=400, + detail=_error_detail( + "This account has no password set, so there is no password to change. " + "Passwords are set through an invitation link (POST /invitation/new)." + ), + ) + + if not verify_password(data.current_password, stored_password): + raise HTTPException(status_code=400, detail=_error_detail("Current password is incorrect.")) + + if data.new_password == data.current_password: + raise HTTPException( + status_code=400, + detail=_error_detail("New password must be different from the current password."), + ) + + validate_password_policy(data.new_password, general_settings) + await validate_password_not_breached(data.new_password, general_settings) + + password_update: Final[prisma_types.LiteLLM_UserTableUpdateInput] = { + "password": hash_password(data.new_password), + "password_reset_required": False, + "last_breach_check_at": None, + } + await _user_table(prisma_client).update(where=find_user, data=password_update) + + verbose_proxy_logger.info("Password changed via /user/password/change for user_id=%s", user_id) + await create_object_audit_log( + object_id=user_id, + action="updated", + litellm_changed_by=None, + user_api_key_dict=user_api_key_dict, + litellm_proxy_admin_name=litellm_proxy_admin_name, + table_name=LitellmTableNames.USER_TABLE_NAME, + after_value=_PASSWORD_CHANGED_AUDIT_VALUES, + ) + return ChangePasswordResponse(user_id=user_id, message="Password updated successfully.") diff --git a/litellm/proxy/management_endpoints/router_settings_endpoints.py b/litellm/proxy/management_endpoints/router_settings_endpoints.py index fc000b1638b..d6d74ada35a 100644 --- a/litellm/proxy/management_endpoints/router_settings_endpoints.py +++ b/litellm/proxy/management_endpoints/router_settings_endpoints.py @@ -8,6 +8,8 @@ GET /router/fields - Get router settings field definitions without values (for U """ import inspect +from collections.abc import Mapping +from types import MappingProxyType from typing import Any, Final, get_args from fastapi import APIRouter, Depends @@ -16,6 +18,7 @@ from pydantic import BaseModel, Field from litellm._logging import verbose_proxy_logger from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.auth.user_api_key_auth import user_api_key_auth +from litellm.proxy.config_resolvers import FieldSource, SettingsStore, source_for from litellm.router import Router from litellm.types.management_endpoints import ( ROUTER_SETTINGS_FIELDS, @@ -30,6 +33,7 @@ class RouterSettingsResponse(BaseModel): fields: list[RouterSettingsField] = Field(description="List of all configurable router settings with metadata") current_values: dict[str, Any] = Field(description="Current values of router settings") routing_strategy_descriptions: dict[str, str] = Field(description="Descriptions for each routing strategy option") + source: dict[str, FieldSource] = Field(description="Source of each current router setting") class RouterFieldsResponse(BaseModel): @@ -39,6 +43,18 @@ class RouterFieldsResponse(BaseModel): routing_strategy_descriptions: dict[str, str] = Field(description="Descriptions for each routing strategy option") +def _router_setting_source( + settings: SettingsStore, + key: str, + current_value: object, + field_default: object, +) -> FieldSource: + source: Final = source_for(settings, key, field_default) + if source != "unset": + return source + return "default" if current_value is not None else "unset" + + def _get_routing_strategies_from_router_class() -> list[str]: """ Dynamically extract routing strategies from the Router class __init__ method. @@ -109,15 +125,29 @@ async def get_router_settings( # Merge with config values (config takes precedence) current_values.update(router_settings_from_config) - # Update field values with current values for field in router_fields: if field.field_name in current_values: field.field_value = current_values[field.field_name] + field_defaults: Final[Mapping[str, object]] = MappingProxyType( + {field.field_name: field.field_default for field in router_fields} + ) + source: Final[Mapping[str, FieldSource]] = MappingProxyType( + { + key: _router_setting_source( + proxy_config.router_settings, + key, + current_values[key], + field_defaults.get(key), + ) + for key in current_values + } + ) return RouterSettingsResponse( fields=router_fields, current_values=current_values, routing_strategy_descriptions=ROUTING_STRATEGY_DESCRIPTIONS, + source=source, ) except Exception as e: verbose_proxy_logger.error("Error fetching router settings: %s", e) diff --git a/litellm/proxy/management_endpoints/ui_sso.py b/litellm/proxy/management_endpoints/ui_sso.py index 00cf357d89d..7859c678c07 100644 --- a/litellm/proxy/management_endpoints/ui_sso.py +++ b/litellm/proxy/management_endpoints/ui_sso.py @@ -3665,6 +3665,7 @@ class SSOAuthenticationHandler: auth_header_name=general_settings.get("litellm_key_header_name", "Authorization"), disabled_non_admin_personal_key_creation=disabled_non_admin_personal_key_creation, server_root_path=get_server_root_path(), + password_reset_required=False, ) from litellm.proxy.auth.login_utils import encode_ui_session_jwt diff --git a/litellm/proxy/middleware/budget_reservation_release_middleware.py b/litellm/proxy/middleware/budget_reservation_release_middleware.py new file mode 100644 index 00000000000..f7ac885274e --- /dev/null +++ b/litellm/proxy/middleware/budget_reservation_release_middleware.py @@ -0,0 +1,33 @@ +from collections.abc import Awaitable, Callable, Mapping +from typing import Final + +from starlette.types import ASGIApp, Receive, Scope, Send + +_SCOPES_AUTH_STAMPS: Final = frozenset({"http", "websocket"}) + + +class BudgetReservationReleaseMiddleware: + """Releases the budget reservation auth made for a request once no callback owns it. + + Auth stamps the reservation on the request or socket state; a call that starts + claims it for the cost callbacks, which settle it on success or failure. When the + response has been sent or the socket has closed and the reservation is still + unclaimed, nothing else ever would, so it is released here instead of pinning the + spend counter until its TTL. + """ + + def __init__(self, app: ASGIApp, release: Callable[[Mapping[str, object]], Awaitable[None]]) -> None: + self.app = app + self.release = release + + async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: + if scope["type"] not in _SCOPES_AUTH_STAMPS: + await self.app(scope, receive, send) + return + try: + await self.app(scope, receive, send) + finally: + state: Final = scope.get("state") + budget_reservation: Final = state.get("budget_reservation") if isinstance(state, Mapping) else None + if isinstance(budget_reservation, Mapping): + await self.release(budget_reservation) diff --git a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py index 44d9f11360d..74caa1050bb 100644 --- a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py @@ -52,6 +52,7 @@ from litellm.llms.deepgram.common_utils import ( deepgram_listen_requested_model, deepgram_listen_websocket_target, ) +from litellm.llms.fal_ai.cost_calculator import fal_ai_queue_base from litellm.llms.nvidia_nim.passthrough.transformation import nvidia_nim_model_group_in_path from litellm.llms.vertex_ai.vertex_llm_base import VertexBase from litellm.passthrough.main import AsyncPassthroughStreamingResponse @@ -421,6 +422,56 @@ async def cohere_proxy_route( return received_value +def _fal_target(endpoint: str) -> httpx.URL: + base_target_url: Final = fal_ai_queue_base() + encoded_endpoint: Final = httpx.URL(endpoint).path + normalized_endpoint: Final = encoded_endpoint if encoded_endpoint.startswith("/") else f"/{encoded_endpoint}" + base_url: Final = httpx.URL(base_target_url) + return base_url.copy_with( + path=HttpPassThroughEndpointHelpers.join_base_and_endpoint_path(base_url, normalized_endpoint), + ) + + +@router.api_route( + "/fal_ai/{endpoint:path}", + methods=["GET", "POST", "PUT", "DELETE", "PATCH"], # mutable-ok: FastAPI route metadata requires a list + tags=["Fal AI Pass-through", "pass-through"], # mutable-ok: FastAPI route metadata requires a list +) +async def fal_ai_proxy_route( + endpoint: str, + request: Request, + fastapi_response: Response, + user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], +): + updated_url: Final = _fal_target(endpoint) + fal_ai_api_key: Final = passthrough_endpoint_router.get_credentials( + custom_llm_provider="fal_ai", + region_name=None, + ) + if fal_ai_api_key is None: + raise HTTPException( + status_code=401, + detail="FAL_AI_API_KEY is not set and no fal_ai pass-through deployment credentials are configured", + ) + if "/requests/" not in endpoint: + priced_model: Final = f"fal_ai/{endpoint}" + if priced_model not in (litellm.model_cost or {}): + raise HTTPException( + status_code=400, + detail=f"{priced_model} has no pricing entry; only priced Fal endpoints can be submitted through /fal_ai", + ) + endpoint_func: Final = create_pass_through_route( + endpoint=endpoint, + target=str(updated_url), + custom_headers={ + "Authorization": f"Key {fal_ai_api_key}" + }, # mutable-ok: pass-through request headers require a mutable mapping + custom_llm_provider="fal_ai", + is_streaming_request=False, + ) + return await endpoint_func(request, fastapi_response, user_api_key_dict) + + @router.api_route( "/vllm/{endpoint:path}", methods=["GET", "POST", "PUT", "DELETE", "PATCH"], @@ -579,6 +630,42 @@ async def typesafe_proxy_route( return await endpoint_func(request, fastapi_response, user_api_key_dict) +@router.api_route( + "/openrouter/{endpoint:path}", + methods=["GET", "POST", "PUT", "DELETE", "PATCH"], # mutable-ok: FastAPI route metadata requires a list + tags=["OpenRouter Pass-through", "pass-through"], # mutable-ok: FastAPI route metadata requires a list +) +async def openrouter_proxy_route( + endpoint: str, + request: Request, + fastapi_response: Response, + user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], +): + base_target_url: Final = get_secret_str("OPENROUTER_API_BASE") or "https://openrouter.ai/api/v1" + api_root: Final = base_target_url.removesuffix("/").removesuffix("/v1") + encoded_endpoint: Final = httpx.URL(endpoint).path + normalized_endpoint: Final = encoded_endpoint if encoded_endpoint.startswith("/") else f"/{encoded_endpoint}" + base_url: Final = httpx.URL(api_root) + updated_url: Final = base_url.copy_with( + path=HttpPassThroughEndpointHelpers.join_base_and_endpoint_path(base_url, normalized_endpoint), + ) + openrouter_api_key: Final = passthrough_endpoint_router.get_credentials( + custom_llm_provider="openrouter", + region_name=None, + ) + endpoint_func: Final = create_pass_through_route( + endpoint=endpoint, + target=str(updated_url), + custom_headers={ # mutable-ok: pass-through request headers require a mutable mapping + "Authorization": f"Bearer {openrouter_api_key}", + "Content-Type": "application/json", + }, + custom_llm_provider="openrouter", + is_streaming_request=False, + ) + return await endpoint_func(request, fastapi_response, user_api_key_dict) + + @router.api_route( "/milvus/{endpoint:path}", methods=["GET", "POST", "PUT", "DELETE", "PATCH"], diff --git a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/fal_ai_passthrough_logging_handler.py b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/fal_ai_passthrough_logging_handler.py new file mode 100644 index 00000000000..3d1fad90e03 --- /dev/null +++ b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/fal_ai_passthrough_logging_handler.py @@ -0,0 +1,71 @@ +from collections.abc import Mapping, Sequence +from typing import Final +from urllib.parse import urlparse + +import httpx + +import litellm +from litellm._logging import verbose_proxy_logger +from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.llms.fal_ai.cost_calculator import fal_ai_passthrough_cost, fal_ai_queue_base +from litellm.proxy._types import PassThroughEndpointLoggingTypedDict +from litellm.types.utils import ImageObject, ImageResponse + +FAL_AI_PROVIDER: Final[str] = litellm.LlmProviders.FAL_AI.value + + +def _url_parts(value: object) -> tuple[Mapping[str, object], ...]: + if isinstance(value, Mapping): + return (value,) if isinstance(value.get("url"), str) else () + if isinstance(value, Sequence) and not isinstance(value, str): + return tuple(item for item in value if isinstance(item, Mapping) and isinstance(item.get("url"), str)) + return () + + +class FalAIPassthroughLoggingHandler: + @staticmethod + def is_fal_ai_route(url_route: str, custom_llm_provider: str | None) -> bool: + return custom_llm_provider == FAL_AI_PROVIDER + + def fal_ai_passthrough_handler( + self, + response_body: Mapping[str, object], + request_body: Mapping[str, object], + logging_obj: LiteLLMLoggingObj, + url_route: str, + kwargs: Mapping[str, object], + ) -> PassThroughEndpointLoggingTypedDict: + base_path: Final = httpx.URL(fal_ai_queue_base()).path.strip("/") + raw_path: Final = urlparse(url_route).path.strip("/") + upstream_path: Final = raw_path.removeprefix(f"{base_path}/") if base_path else raw_path + model: Final = upstream_path.partition("/requests/")[0] + is_submit: Final = "/requests/" not in upstream_path + response: Final = ImageResponse( + data=tuple( + ImageObject(url=url) + for value in response_body.values() + for part in _url_parts(value) + if isinstance((url := part.get("url")), str) + ) + ) + response_cost: Final = fal_ai_passthrough_cost(model, request_body) if is_submit else None + response._hidden_params["response_cost"] = response_cost # pyright: ignore[reportPrivateUsage] # the logger reads a precomputed cost off the response's hidden params + logging_obj.model = model # rebind-ok: the spend logger reads model and cost off the shared logging object + logging_obj.model_call_details["model"] = model # rebind-ok: same shared logging object + logging_obj.model_call_details["custom_llm_provider"] = FAL_AI_PROVIDER # rebind-ok: same shared logging object + logging_obj.model_call_details["response_cost"] = response_cost # rebind-ok: same shared logging object + verbose_proxy_logger.debug( + "Fal AI passthrough cost tracking: model %s, cost %s", + model, + response_cost, + ) + logging_result: Final[PassThroughEndpointLoggingTypedDict] = { + "result": response, + "kwargs": { + **kwargs, + "model": model, + "custom_llm_provider": FAL_AI_PROVIDER, + "response_cost": response_cost, + }, + } + return logging_result diff --git a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/typesafe_passthrough_logging_handler.py b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/typesafe_passthrough_logging_handler.py index 9b196660c2c..887d17a7a20 100644 --- a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/typesafe_passthrough_logging_handler.py +++ b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/typesafe_passthrough_logging_handler.py @@ -65,6 +65,7 @@ class TypeSafePassthroughLoggingHandler: end_time: datetime, cache_hit: bool, request_body: Mapping[str, object], + custom_llm_provider: str, **kwargs: object, ) -> PassThroughEndpointLoggingTypedDict: response: Final = _parse_typesafe_response(response_body) @@ -72,12 +73,12 @@ class TypeSafePassthroughLoggingHandler: request_model_value: Final = request_body.get("model") request_model: Final = request_model_value if isinstance(request_model_value, str) else None logged_model: Final = response_model or request_model or "unknown" - model_name: Final = f"typesafe/{logged_model}" + model_name: Final = f"{custom_llm_provider}/{logged_model}" usage: Final = response.usage or _TypeSafeUsage() input_tokens: Final = usage.input_tokens output_tokens: Final = usage.output_tokens candidate_model_keys: Final = tuple( - f"typesafe/{model}" for model in (response_model, request_model) if model is not None + f"{custom_llm_provider}/{model}" for model in (response_model, request_model) if model is not None ) pricing: Final = _pricing_for(candidate_model_keys) response_cost: Final = ( @@ -91,13 +92,13 @@ class TypeSafePassthroughLoggingHandler: updated_kwargs: Final = { # mutable-ok: pass-through logging contract requires mutable kwargs **kwargs, "model": model_name, - "custom_llm_provider": "typesafe", + "custom_llm_provider": custom_llm_provider, "response_cost": response_cost, "combined_usage_object": usage_object, } logging_obj.model_call_details.update( model=model_name, - custom_llm_provider="typesafe", + custom_llm_provider=custom_llm_provider, response_cost=response_cost, ) standard_logging_object: Final = get_standard_logging_object_payload( diff --git a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py index 79a328f5199..8902e599788 100644 --- a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py @@ -47,6 +47,7 @@ from litellm.constants import ( from litellm.integrations.custom_guardrail import CustomGuardrail from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.core_helpers import ( + bind_budget_reservation_to_callbacks, get_metadata_variable_name_from_kwargs, get_or_create_metadata_bucket, ) @@ -78,11 +79,13 @@ from litellm.proxy.common_request_processing import ( open_sse_before_first_byte, resolve_litellm_call_id, ) +from litellm.proxy.common_utils.error_body_call_id import JSON_OBJECT, error_body_call_id, with_call_id from litellm.proxy.common_utils.http_parsing_utils import ( _read_request_body, _safe_get_request_headers, ) from litellm.proxy.common_utils.openai_error_payload import ( + LITELLM_CALL_ID_HEADER, error_status_code, litellm_call_id_headers, openai_error_param, @@ -1127,6 +1130,9 @@ async def pass_through_request( from litellm.proxy.proxy_server import ( general_settings as proxy_general_settings, ) + from litellm.proxy.proxy_server import ( + general_settings_view, + ) _managed_id_provider: Final = resolve_passthrough_managed_id_provider(custom_llm_provider) @@ -1632,6 +1638,7 @@ async def pass_through_request( **kwargs, ) ) + bind_budget_reservation_to_callbacks(logging_obj.litellm_params) ## CUSTOM HEADERS - `x-litellm-*` custom_headers = ProxyBaseLLMRequestProcessing.get_custom_headers( @@ -1656,11 +1663,24 @@ async def pass_through_request( headers=response.headers, custom_headers=custom_headers, ) + emitted_call_id: Final = ( + JSON_OBJECT.validate_python(response_headers).get(LITELLM_CALL_ID_HEADER) + if response.status_code >= 400 + else None + ) + error_call_id: Final = ( + error_body_call_id(general_settings_view(), emitted_call_id) if isinstance(emitted_call_id, str) else None + ) + relayed_content: Final = ( + json.dumps(with_call_id(JSON_OBJECT.validate_python(response_body), error_call_id)).encode("utf-8") + if error_call_id is not None and isinstance(response_body, dict) + else content + ) if _content_modified: response_headers.pop("content-length", None) return Response( - content=content, + content=relayed_content, status_code=response.status_code, headers=response_headers, ) @@ -2543,6 +2563,7 @@ async def websocket_passthrough_request( **success_kwargs, ) ) + bind_budget_reservation_to_callbacks(logging_obj.litellm_params) # Call the proxy logging success hook if proxy_logging_obj: @@ -2714,6 +2735,7 @@ async def _relay_passthrough_response_bytes( **success_handler_kwargs, ) ) + bind_budget_reservation_to_callbacks(logging_obj.litellm_params) def _extract_model_from_vertex_ai_setup(setup_response: Mapping[str, object]) -> str | None: diff --git a/litellm/proxy/pass_through_endpoints/streaming_handler.py b/litellm/proxy/pass_through_endpoints/streaming_handler.py index fe9e104789b..fae26b5a72d 100644 --- a/litellm/proxy/pass_through_endpoints/streaming_handler.py +++ b/litellm/proxy/pass_through_endpoints/streaming_handler.py @@ -9,6 +9,7 @@ import httpx import litellm from litellm._logging import verbose_proxy_logger from litellm.litellm_core_utils.asyncify import asyncify +from litellm.litellm_core_utils.core_helpers import bind_budget_reservation_to_callbacks from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER from litellm.proxy._types import PassThroughEndpointLoggingResultValues @@ -218,6 +219,7 @@ class PassThroughStreamingHandler: and response.status_code < 400 ): logging_scheduled = True + bind_budget_reservation_to_callbacks(litellm_logging_obj.litellm_params) litellm_logging_obj._deferred_stream_complete_args = (_build_logging_coroutine(),) except Exception as e: verbose_proxy_logger.error("Error in chunk_processor: %s", e) @@ -250,6 +252,8 @@ class PassThroughStreamingHandler: GLOBAL_LOGGING_WORKER.ensure_initialized_and_enqueue(async_coroutine=_build_logging_coroutine()) except Exception as e: verbose_proxy_logger.error("Error scheduling chunk_processor logging: %s", e) + else: + bind_budget_reservation_to_callbacks(litellm_logging_obj.litellm_params) @staticmethod async def _route_streaming_logging_to_handler( diff --git a/litellm/proxy/pass_through_endpoints/success_handler.py b/litellm/proxy/pass_through_endpoints/success_handler.py index de1a8ae1d93..d1e4da2e47c 100644 --- a/litellm/proxy/pass_through_endpoints/success_handler.py +++ b/litellm/proxy/pass_through_endpoints/success_handler.py @@ -29,6 +29,9 @@ from .llm_provider_handlers.cursor_passthrough_logging_handler import ( from .llm_provider_handlers.deepgram_listen_passthrough_logging_handler import ( DeepgramListenPassthroughLoggingHandler, ) +from .llm_provider_handlers.fal_ai_passthrough_logging_handler import ( + FalAIPassthroughLoggingHandler, +) from .llm_provider_handlers.gemini_passthrough_logging_handler import ( GeminiPassthroughLoggingHandler, ) @@ -311,7 +314,9 @@ class PassThroughEndpointLogging: ) standard_logging_response_object = transcribe_handler_result["result"] # rebind-ok: elif-chain kwargs = transcribe_handler_result["kwargs"] # rebind-ok: elif-chain contract - elif self.is_typesafe_route(custom_llm_provider): + elif self.is_typesafe_route(custom_llm_provider) or self.is_openrouter_decisions_route( + url_route, custom_llm_provider + ): from .llm_provider_handlers.typesafe_passthrough_logging_handler import ( TypeSafePassthroughLoggingHandler, ) @@ -326,6 +331,7 @@ class PassThroughEndpointLogging: end_time=end_time, cache_hit=cache_hit, request_body=request_body, + custom_llm_provider=custom_llm_provider or "", **kwargs, ) standard_logging_response_object = typesafe_handler_result["result"] @@ -367,6 +373,16 @@ class PassThroughEndpointLogging: ) standard_logging_response_object = deepgram_handler_result["result"] # rebind-ok: elif-chain kwargs = deepgram_handler_result["kwargs"] # rebind-ok: elif-chain contract + elif FalAIPassthroughLoggingHandler.is_fal_ai_route(url_route, custom_llm_provider): + fal_ai_handler_result: Final = FalAIPassthroughLoggingHandler().fal_ai_passthrough_handler( + response_body=response_body if isinstance(response_body, dict) else MappingProxyType({}), + request_body=request_body, + logging_obj=logging_obj, + url_route=url_route, + kwargs=kwargs, + ) + standard_logging_response_object = fal_ai_handler_result["result"] # rebind-ok: elif-chain + kwargs = fal_ai_handler_result["kwargs"] # rebind-ok: elif-chain contract return_dict["standard_logging_response_object"] = standard_logging_response_object return_dict["kwargs"] = kwargs @@ -505,6 +521,9 @@ class PassThroughEndpointLogging: def is_typesafe_route(self, custom_llm_provider: str | None) -> bool: return custom_llm_provider == "typesafe" + def is_openrouter_decisions_route(self, url_route: str, custom_llm_provider: str | None) -> bool: + return custom_llm_provider == "openrouter" and urlparse(url_route).path.endswith("/alpha/decisions") + def is_langfuse_route(self, url_route: str): parsed_url: Final = urlparse(url_route) for route in self.TRACKED_LANGFUSE_ROUTES: diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index dac1a8dd001..6fa7a7d761f 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -50,6 +50,7 @@ import anyio import websockets import websockets.exceptions from pydantic import BaseModel, Json, JsonValue, TypeAdapter, ValidationError +from pydantic.fields import FieldInfo, PydanticUndefined from typing_extensions import NotRequired, ReadOnly, assert_never from litellm._uuid import uuid @@ -359,7 +360,7 @@ from litellm.proxy.auth.model_checks import ( get_mcp_server_ids, get_team_models, ) -from litellm.proxy.auth.password_policy import validate_password_policy +from litellm.proxy.auth.password_policy import validate_password_not_breached, validate_password_policy from litellm.proxy.auth.user_api_key_auth import ( _fetch_global_spend_with_event_coordination, user_api_key_auth, @@ -392,6 +393,7 @@ from litellm.proxy.common_utils.encrypt_decrypt_utils import ( decrypt_value_helper, encrypt_value_helper, ) +from litellm.proxy.common_utils.error_body_call_id import JSON_OBJECT, error_body_call_id, with_call_id from litellm.proxy.common_utils.healthy_model_filter import ( get_hidden_unhealthy_model_names, is_healthy_only_listing_default, @@ -417,6 +419,7 @@ from litellm.proxy.common_utils.openai_endpoint_utils import ( remove_sensitive_info_from_deployment, ) from litellm.proxy.common_utils.openai_error_payload import ( + LITELLM_CALL_ID_HEADER, headers_with_litellm_call_id, litellm_call_id_headers, with_litellm_call_id, @@ -456,7 +459,13 @@ from litellm.proxy.common_utils.user_api_key_cache import ( project_spend_counter_key, tag_cache_key, ) -from litellm.proxy.config_resolvers import SettingsStore, config_ownership_message, resolve_fields +from litellm.proxy.config_resolvers import ( + FieldSource, + SettingsStore, + config_ownership_message, + resolve_fields, + source_for, +) from litellm.proxy.config_resolvers.alerting import ( EMAIL_DESCRIPTORS, MS_TEAMS_DESCRIPTORS, @@ -601,6 +610,9 @@ from litellm.proxy.management_endpoints.model_management_endpoints import ( from litellm.proxy.management_endpoints.organization_endpoints import ( router as organization_router, ) +from litellm.proxy.management_endpoints.password_endpoints import ( + router as password_management_router, +) from litellm.proxy.management_endpoints.prompt_caching_requests import ( router as prompt_caching_requests_router, ) @@ -642,6 +654,9 @@ from litellm.proxy.middleware.billable_request_metrics_middleware import ( BillableRequestMetricsMiddleware, BillingRecorder, ) +from litellm.proxy.middleware.budget_reservation_release_middleware import ( + BudgetReservationReleaseMiddleware, +) from litellm.proxy.plugin_routes import ( register_plugins_from_config, ) @@ -712,7 +727,15 @@ from litellm.proxy.route_llm_request import route_request from litellm.proxy.route_priority import hot_routes_first from litellm.proxy.search_endpoints.endpoints import router as search_router from litellm.proxy.shutdown.graceful_shutdown_manager import GracefulShutdownManager -from litellm.proxy.spend_tracking.budget_reservation import get_budget_window_start +from litellm.proxy.shutdown.scheduled_jobs import ( + AwaitableAsyncIOExecutor, + pause_scheduled_jobs, + stop_in_flight_scheduler_jobs, +) +from litellm.proxy.spend_tracking.budget_reservation import ( + get_budget_window_start, + release_unbound_budget_reservation, +) from litellm.proxy.spend_tracking.daily_global_spend_rollup import ( run_scheduled_daily_global_spend_reconcile, ) @@ -1485,6 +1508,10 @@ async def proxy_startup_event(app: FastAPI) -> AsyncGenerator[None, None]: if model_info_scheduler is not scheduler: model_info_scheduler.shutdown(wait=False) + # Shutdown event - stop starting scheduled jobs; the ones already running keep the drain window + if scheduler is not None: + pause_scheduled_jobs(scheduler) + # Shutdown event - drain in-flight requests before tearing down dependencies # so SIGTERM (rolling update, scale-down, liveness kill) doesn't drop them. GracefulShutdownManager.start_shutdown() @@ -1524,6 +1551,13 @@ async def proxy_startup_event(app: FastAPI) -> AsyncGenerator[None, None]: await _drain_spend_event_producer_on_shutdown() + # Shutdown event - finish or cancel in-flight scheduled jobs before the shutdown flushes and the DB disconnect + if scheduler is not None and scheduler_executor is not None: + try: + await stop_in_flight_scheduler_jobs(scheduler, scheduler_executor) + except Exception as e: + verbose_proxy_logger.error("Error stopping in-flight scheduled jobs: %s", e) + await flush_spend_counters_on_shutdown() await _flush_spend_logs_queue_on_shutdown() @@ -1787,7 +1821,10 @@ async def openai_exception_handler(request: Request, exc: ProxyException): # NOTE: DO NOT MODIFY THIS, its crucial to map to Openai exceptions _log_model_access_denial(exc) headers: Final = exc.headers - error_dict: Final = exc.to_dict() + error_dict: Final = with_call_id( + JSON_OBJECT.validate_python(exc.to_dict()), + error_body_call_id(general_settings_view(), headers.get(LITELLM_CALL_ID_HEADER)), + ) status_code: Final = int(exc.code) if exc.code else status.HTTP_500_INTERNAL_SERVER_ERROR _close_dangling_otel_server_span(request, status_code, exc=exc) return JSONResponse( @@ -2327,6 +2364,7 @@ app.add_middleware( # it sees prisma_client as of the first request rather than import time. sink_factory=lambda: gateway_request_accumulator if prisma_client is not None else None, ) +app.add_middleware(BudgetReservationReleaseMiddleware, release=release_unbound_budget_reservation) app.add_middleware(InFlightRequestsMiddleware) app.add_middleware(SecurityHeadersMiddleware) @@ -2451,6 +2489,13 @@ heuristic_v1_tuning_baselines: Mapping[str, str] | None = None # second ProxyConfig instance must not get its own independent lock over it. MODEL_RECONCILE_LOCK: Final = asyncio.Lock() general_settings: dict = {} +_GENERAL_SETTINGS_VIEW: Final[TypeAdapter[Mapping[str, object]]] = TypeAdapter(Mapping[str, object]) + + +def general_settings_view() -> Mapping[str, object]: + return _GENERAL_SETTINGS_VIEW.validate_python(general_settings) + + config_passthrough_endpoints: list[dict[str, Any]] | None = None log_file: Final = "api_log.json" worker_config: Final = None @@ -2526,6 +2571,7 @@ celery_app_conn: Final = None celery_fn: Final = None # Redis Queue for handling requests scheduler = None +scheduler_executor: AwaitableAsyncIOExecutor | None = None # rebind-ok: bound once the scheduler is built at startup # Global variable for anthropic beta headers reload scheduling last_anthropic_beta_headers_reload = None @@ -4948,6 +4994,12 @@ def _as_settings_mapping(value: object) -> Mapping[str, SettingsJsonValue]: return _SETTINGS_MAPPING.validate_python(value) +def _get_field_default(field_info: FieldInfo) -> JsonValue: + if field_info.default is PydanticUndefined: + return None + return cast(JsonValue, field_info.default) # cast-ok: Pydantic field defaults are JSON values at runtime + + def _bind_general_settings_store(settings: SettingsStore) -> None: global general_settings general_settings = settings # pyright: ignore[reportAssignmentType] # legacy global accepts mappings @@ -10078,7 +10130,7 @@ class ProxyStartupEvent: proxy_logging_obj: ProxyLogging, ) -> ProxyWorkerHeartbeat: """Initializes scheduled background jobs""" - global heuristic_v1_tuning_baselines, store_model_in_db, scheduler # rebind-ok: startup publishes the one read-only baseline snapshot + global heuristic_v1_tuning_baselines, store_model_in_db, scheduler, scheduler_executor # rebind-ok: startup publishes the one read-only baseline snapshot # MEMORY LEAK FIX: Configure scheduler with optimized settings # Memray analysis showed APScheduler's normalize() and _apply_jitter() causing @@ -10087,9 +10139,9 @@ class ProxyStartupEvent: # 1. Remove/minimize jitter to avoid normalize() memory explosion # 2. Use larger misfire_grace_time to prevent backlog calculations # 3. Set replace_existing=True to avoid duplicate jobs - from apscheduler.executors.asyncio import AsyncIOExecutor from apscheduler.jobstores.memory import MemoryJobStore + scheduler_executor = AwaitableAsyncIOExecutor() # rebind-ok: shutdown awaits the jobs this executor runs scheduler = AsyncIOScheduler( job_defaults={ "coalesce": APSCHEDULER_COALESCE, @@ -10102,7 +10154,7 @@ class ProxyStartupEvent: jobstores={"default": MemoryJobStore()}, # explicitly use memory job store # Use simple executor to minimize overhead executors={ - "default": AsyncIOExecutor(), + "default": scheduler_executor, }, # Disable timezone awareness to reduce computation timezone=None, @@ -16003,6 +16055,22 @@ async def model_settings(): #### ALERTING MANAGEMENT ENDPOINTS #### +def _nested_setting_source( + settings: SettingsStore, + db_values: Mapping[str, JsonValue], + parent_key: str, + field_name: str, + field_default: JsonValue, +) -> FieldSource: + unset_source: Final[FieldSource] = "default" if field_default is not None else "unset" + parent_value: Final = settings.config_value(parent_key) + if isinstance(parent_value, Mapping) and field_name in parent_value: + return "config" + if settings.owned_by_config(parent_key): + return unset_source + return "db" if field_name in db_values else unset_source + + @router.get( "/alerting/settings", description="Return the configurable alerting param, description, and current value", @@ -16040,17 +16108,20 @@ async def alerting_settings( where={"param_name": "general_settings"} ) - if db_general_settings is not None and db_general_settings.param_value is not None: - db_general_settings_dict: Final = dict(db_general_settings.param_value) - alerting_args_dict: dict = cast( # cast-ok: ConfigGeneralSettings validates alerting_args as a dict on write - dict[str, JsonValue], db_general_settings_dict.get("alerting_args", {}) - ) - alerting_values: list | None = cast( # cast-ok: ConfigGeneralSettings validates alerting as a list on write - list[JsonValue] | None, db_general_settings_dict.get("alerting") - ) - else: - alerting_args_dict = {} - alerting_values = None + db_general_settings_dict: Final[Mapping[str, JsonValue]] = MappingProxyType( + dict(db_general_settings.param_value) # mutable-ok: Prisma returns the JSON column as a plain dict + if db_general_settings is not None and db_general_settings.param_value is not None + else {} + ) + alerting_args_value: Final = db_general_settings_dict.get("alerting_args") + alerting_args_dict: Final[Mapping[str, JsonValue]] = MappingProxyType( + alerting_args_value if isinstance(alerting_args_value, dict) else {} + ) + alerting_values: Final = cast( # cast-ok: alerting is stored as a JSON list when present + list[JsonValue] | None, db_general_settings_dict.get("alerting") + ) + + settings: Final = proxy_config.settings allowed_args: Final = MappingProxyType( { @@ -16079,9 +16150,9 @@ async def alerting_settings( is_slack_enabled = False - if general_settings.get("alerting") and isinstance(general_settings["alerting"], list): - if "slack" in general_settings["alerting"]: - is_slack_enabled = True + alerting: Final = settings.get("alerting") + if isinstance(alerting, list) and "slack" in alerting: + is_slack_enabled = True _response_obj = ConfigList( field_name="slack_alerting", @@ -16089,6 +16160,7 @@ async def alerting_settings( field_description="Enable slack alerting for monitoring proxy in production: llm outages, budgets, spend tracking failures.", field_value=is_slack_enabled, stored_in_db=True if alerting_values is not None else False, + source=source_for(settings, "alerting"), field_default_value=None, premium_field=False, ) @@ -16096,6 +16168,7 @@ async def alerting_settings( for field_name, field_info in SlackAlertingArgs.model_fields.items(): if field_name in allowed_args: + field_default: JsonValue = _get_field_default(field_info) _stored_in_db: bool | None = None if field_name in alerting_args_dict: _stored_in_db = True @@ -16106,9 +16179,16 @@ async def alerting_settings( field_name=field_name, field_type=allowed_args[field_name], field_description=field_info.description or "", - field_value=_slack_alerting_args_dict.get(field_name, None), + field_value=_slack_alerting_args_dict.get(field_name, field_default), stored_in_db=_stored_in_db, - field_default_value=field_info.default, + source=_nested_setting_source( + settings, + alerting_args_dict, + "alerting_args", + field_name, + field_default, + ), + field_default_value=field_default, premium_field=(True if field_name == "region_outage_alert_ttl" else False), ) return_val.append(_response_obj) @@ -16647,6 +16727,7 @@ async def onboarding(invite_link: str, request: Request): auth_header_name=general_settings.get("litellm_key_header_name", "Authorization"), disabled_non_admin_personal_key_creation=disabled_non_admin_personal_key_creation, server_root_path=get_server_root_path(), + password_reset_required=False, ) jwt_token: Final = jwt.encode( cast(dict, returned_ui_token_object), @@ -16757,6 +16838,7 @@ async def _generate_onboarding_ui_session_token(user_obj: _UserTableRow) -> str: auth_header_name=general_settings.get("litellm_key_header_name", "Authorization"), disabled_non_admin_personal_key_creation=disabled_non_admin_personal_key_creation, server_root_path=get_server_root_path(), + password_reset_required=False, ) assert master_key is not None return jwt.encode( @@ -16827,6 +16909,7 @@ async def claim_onboarding_link(data: InvitationClaim, request: Request): ) validate_password_policy(data.password, general_settings) + await validate_password_not_breached(data.password, general_settings) hashed_pw: Final = hash_password(data.password) current_time = litellm.utils.get_utc_datetime() async with prisma_client.db.tx() as tx: @@ -16846,7 +16929,12 @@ async def claim_onboarding_link(data: InvitationClaim, request: Request): ### UPDATE USER OBJECT ### user_obj: Final[_UserTableRow | None] = await tx.litellm_usertable.update( - where={"user_id": invite_obj.user_id}, data={"password": hashed_pw} + where={"user_id": invite_obj.user_id}, + data={ + "password": hashed_pw, + "password_reset_required": False, + "last_breach_check_at": None, + }, ) if user_obj is None: @@ -19284,6 +19372,7 @@ app.include_router(pass_through_router) app.include_router(health_router) app.include_router(key_management_router) app.include_router(internal_user_router) +app.include_router(password_management_router) app.include_router(team_router) app.include_router(ui_sso_router) app.include_router(organization_router) diff --git a/litellm/proxy/schema.prisma b/litellm/proxy/schema.prisma index 2d7e557a9d1..85996430bc5 100644 --- a/litellm/proxy/schema.prisma +++ b/litellm/proxy/schema.prisma @@ -73,6 +73,7 @@ model LiteLLM_AgentsTable { static_headers Json? @default("{}") extra_headers String[] @default([]) agent_access_groups String[] @default([]) + access_group_ids String[] @default([]) object_permission_id String? object_permission LiteLLM_ObjectPermissionTable? @relation(fields: [object_permission_id], references: [object_permission_id]) spend Float @default(0.0) @@ -246,6 +247,8 @@ model LiteLLM_UserTable { organization_id String? object_permission_id String? password String? + password_reset_required Boolean? + last_breach_check_at DateTime? teams String[] @default([]) user_role String? max_budget Float? @@ -1621,6 +1624,47 @@ model LiteLLM_AutoRouterSession { @@index([last_turn_at], map: "idx_autorouter_session_last_turn") } +model LiteLLM_AutoRouterUserSession { + user_id String + api_key String + session_id String + router_name String + router_type String + first_turn_at DateTime + last_turn_at DateTime + last_model String + models Json @default("{}") + turns Int @default(0) + unordered_turns Int @default(0) + covered_turns Int @default(0) + cache_hits Int @default(0) + same_model_turns Int @default(0) + same_model_hits Int @default(0) + first_visit_turns Int @default(0) + first_visit_hits Int @default(0) + return_turns Int @default(0) + return_hits Int @default(0) + return_expired_misses Int @default(0) + return_within_ttl_misses Int @default(0) + ttl_5m_turns Int @default(0) + ttl_1h_turns Int @default(0) + total_tokens BigInt @default(0) + spend Float @default(0) + saved_spend Float @default(0) + savings_estimated_turns Int @default(0) + savings_estimated_actual_spend Float @default(0) + savings_estimated_saved_spend Float @default(0) + savings_estimated_baseline_models Json @default("{}") + classifier_cost Float @default(0) + classifier_cost_recorded_turns Int @default(0) + tier_turns Json @default("{}") + baseline_models Json @default("{}") + + @@id([user_id, api_key, session_id, router_name]) + @@index([last_turn_at], map: "idx_autorouter_user_session_last_turn") + @@index([user_id, last_turn_at], map: "idx_autorouter_user_session_user_last_turn") +} + // Shadow eval: evaluation of an auto-router against one or more keys' live traffic, in // either direction. forward duplicates the requests the keys did not route through the // router through it, answering whether they should adopt it; reverse duplicates the diff --git a/litellm/proxy/shutdown/scheduled_jobs.py b/litellm/proxy/shutdown/scheduled_jobs.py new file mode 100644 index 00000000000..7889c35cf4e --- /dev/null +++ b/litellm/proxy/shutdown/scheduled_jobs.py @@ -0,0 +1,79 @@ +# pyright: reportMissingTypeStubs=false # apscheduler ships no type information + +import asyncio +from collections.abc import Collection +from typing import Final, Protocol + +from apscheduler.executors.asyncio import AsyncIOExecutor + +from litellm._logging import verbose_proxy_logger +from litellm.constants import ( + SCHEDULED_JOB_SHUTDOWN_CANCEL_TIMEOUT_SECONDS, + SCHEDULED_JOB_SHUTDOWN_FINISH_TIMEOUT_SECONDS, +) + + +class StoppableScheduler(Protocol): + """The slice of ``AsyncIOScheduler`` shutdown uses, which ships no type information""" + + @property + def running(self) -> bool: ... + + def pause(self) -> None: ... + + def shutdown(self, wait: bool = ...) -> None: ... + + +class AwaitableAsyncIOExecutor(AsyncIOExecutor): # pyright: ignore[reportUntypedBaseClass] # apscheduler ships no type information and is absent from the type-check env + """``AsyncIOExecutor`` whose in-flight job tasks can be awaited after ``shutdown`` cancels them""" + + _pending_futures: Collection["asyncio.Future[object]"] + + def in_flight_jobs(self) -> tuple["asyncio.Future[object]", ...]: + """The job tasks that are running right now, as a snapshot""" + return tuple(future for future in self._pending_futures if not future.done()) + + +def pause_scheduled_jobs(scheduler: StoppableScheduler) -> None: + """Stop the scheduler from starting jobs that shutdown would only cancel; running jobs continue""" + if scheduler.running: + scheduler.pause() + + +async def stop_in_flight_scheduler_jobs( + scheduler: StoppableScheduler, + executor: AwaitableAsyncIOExecutor, + *, + finish_timeout_seconds: float = SCHEDULED_JOB_SHUTDOWN_FINISH_TIMEOUT_SECONDS, + cancel_timeout_seconds: float = SCHEDULED_JOB_SHUTDOWN_CANCEL_TIMEOUT_SECONDS, +) -> None: + """ + Let in-flight jobs finish for up to finish_timeout_seconds, then stop the scheduler and wait, bounded by + cancel_timeout_seconds, for the jobs it cancels. + + Must run before the database is disconnected: a write job that finishes needs its connection, + and a job's cancellation handler is what records the run's outcome. + """ + if not scheduler.running: + return + in_flight: Final = executor.in_flight_jobs() + if in_flight: + verbose_proxy_logger.info( + "Waiting up to %ss for %d in-flight scheduled job(s) to finish", + finish_timeout_seconds, + len(in_flight), + ) + still_running: Final = ( + (await asyncio.wait(in_flight, timeout=finish_timeout_seconds))[1] if in_flight else frozenset() + ) + scheduler.shutdown(wait=False) + if not still_running: + return + verbose_proxy_logger.info("Cancelling %d in-flight scheduled job(s) for shutdown", len(still_running)) + _done, pending = await asyncio.wait(still_running, timeout=cancel_timeout_seconds) + if pending: + verbose_proxy_logger.warning( + "%d scheduled job(s) did not finish within %ss of cancellation; giving up on them", + len(pending), + cancel_timeout_seconds, + ) diff --git a/litellm/proxy/spend_tracking/budget_reservation.py b/litellm/proxy/spend_tracking/budget_reservation.py index f9e5c4ff1e4..ac9b07a55f7 100644 --- a/litellm/proxy/spend_tracking/budget_reservation.py +++ b/litellm/proxy/spend_tracking/budget_reservation.py @@ -366,6 +366,7 @@ async def reserve_budget_for_request( "reserved_cost": reservation_cost, "entries": applied_entries, "finalized": False, + "callback_bound": False, "input_cost": min(float(input_cost or 0.0), reservation_cost), "input_tokens": max(input_token_counts.values(), default=None), } @@ -474,6 +475,19 @@ async def release_or_invalidate_budget_reservation( budget_reservation["finalized"] = True +async def release_unbound_budget_reservation(budget_reservation: Mapping[str, object]) -> None: + """Release a reservation no logging callback took ownership of, once the request ended. + + A handler whose litellm call never builds a logging object (batch cancel, file + content, anything without the client decorator) runs no cost callback, so nothing + else would ever reconcile its reservation. A bound reservation is left alone: its + success or failure handler settles it, possibly after the response has been sent. + """ + if not isinstance(budget_reservation, dict) or budget_reservation.get("callback_bound") is True: + return + await release_or_invalidate_budget_reservation(budget_reservation=budget_reservation) + + async def _get_budget_counters( request_body: dict, valid_token: UserAPIKeyAuth, diff --git a/litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py b/litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py index 7bdadeadf86..f4d4ccf5851 100644 --- a/litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py +++ b/litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py @@ -15,8 +15,8 @@ from typing import ( from urllib.parse import urlparse from fastapi import APIRouter, Body, Depends, File, HTTPException, UploadFile -from pydantic import ConfigDict, JsonValue, TypeAdapter, ValidationError, create_model -from pydantic.fields import FieldInfo +from pydantic import BaseModel, ConfigDict, JsonValue, TypeAdapter, ValidationError, create_model +from pydantic.fields import FieldInfo, PydanticUndefined from typing_extensions import NotRequired, ReadOnly, TypedDict import litellm @@ -25,6 +25,7 @@ from litellm.litellm_core_utils.sensitive_data_masker import mask_sensitive_keys from litellm.proxy._experimental.mcp_server.tool_search import MCP_TOOL_SEARCH_SETTINGS_KEY from litellm.proxy._types import * from litellm.proxy.auth.user_api_key_auth import user_api_key_auth +from litellm.proxy.config_resolvers import FieldSource, SettingsStore, source_for from litellm.proxy.config_resolvers.settings_store import ConfigOwnedKeyError from litellm.proxy.config_resolvers.sso import ( SSO_FIELD_ENV_VARS, @@ -35,7 +36,10 @@ from litellm.proxy.management_endpoints.team_admin_field_permissions import ( SUPPORTED_TEAM_ADMIN_PERMISSIONS, TEAM_ADMIN_EDITABLE_TEAM_FIELDS_SETTING, ) -from litellm.proxy.spend_tracking.ptu_feature_flag import is_ptu_cost_attribution_enabled +from litellm.proxy.spend_tracking.ptu_feature_flag import ( + PTU_COST_ATTRIBUTION_ENV_VAR, + is_ptu_cost_attribution_enabled, +) from litellm.proxy.utils import invalidate_config_param from litellm.repositories.config_repository import ConfigRepository from litellm.repositories.organization_repository import OrganizationRepository @@ -45,6 +49,7 @@ from litellm.repositories.table_repositories import ( UISettingsRepository, ) from litellm.repositories.team_repository import TeamRepository +from litellm.secret_managers.main import get_secret from litellm.types.mcp import MCPToolSearchSettings from litellm.types.proxy.management_endpoints.ui_sso import ( DefaultTeamSSOParams, @@ -199,6 +204,11 @@ class SettingsResponse(BaseModel): """Schema information including descriptions and property types for UI display""" +class _SettingsWithSchema(BaseModel): + values: dict[str, object] + field_schema: dict[str, object] + + class SSOSettingsResponse(SettingsResponse): """Response model for SSO settings""" @@ -330,6 +340,8 @@ class UISettings(BaseModel): class UISettingsResponse(SettingsResponse): """Response model for UI settings""" + source: dict[str, FieldSource] + # Allowlist of UI settings that can be stored ALLOWED_UI_SETTINGS_FIELDS: Final = { @@ -748,6 +760,25 @@ def _root_schema(settings_class: type[BaseModel]) -> _RootSchema: ) +def _model_field_default(settings_class: type[BaseModel], field_name: str) -> object: + field_info: Final = settings_class.model_fields.get(field_name) + if field_info is None or field_info.default is PydanticUndefined: + return None + return cast(object, field_info.default) # cast-ok: Pydantic field defaults are untyped + + +def _ui_setting_source( + key: str, + value: object, + settings: SettingsStore, + settings_class: type[BaseModel], +) -> FieldSource: + if key == ENABLE_PTU_COST_ATTRIBUTION_UI_SETTING: + configured_value: Final = get_secret(PTU_COST_ATTRIBUTION_ENV_VAR, None) + return "config" if configured_value is not None or value is True else "default" + return source_for(settings, key, _model_field_default(settings_class, key)) + + async def _get_settings_with_schema( settings_key: str, settings_class: type[BaseModel], @@ -1705,7 +1736,7 @@ async def get_ui_settings(): Get UI-specific configuration flags. All authenticated users can fetch these settings for client-side behavior. """ - from litellm.proxy.proxy_server import prisma_client + from litellm.proxy.proxy_server import prisma_client, proxy_config if prisma_client is None: raise HTTPException( @@ -1730,20 +1761,43 @@ async def get_ui_settings(): await user_api_key_cache.async_set_cache(key=UI_SETTINGS_CACHE_KEY, value=ui_settings, ttl=UI_SETTINGS_CACHE_TTL) - # Build config-like object for schema helper - config: Final[dict[str, object]] = {"litellm_settings": {"ui_settings": ui_settings}} - - settings: Final = await _get_settings_with_schema( - settings_key="ui_settings", - settings_class=_get_effective_ui_settings_class(), - config=config, + effective_ui_settings: Final[Mapping[str, object]] = MappingProxyType( + { + **ui_settings, + **{key: proxy_config.settings[key] for key in ALLOWED_UI_SETTINGS_FIELDS if key in proxy_config.settings}, + } + ) + config: Final[Mapping[str, object]] = MappingProxyType( + {"litellm_settings": MappingProxyType({"ui_settings": effective_ui_settings})} + ) + settings_class: Final = _get_effective_ui_settings_class() + resolved_settings: Final = _SettingsWithSchema.model_validate( + await _get_settings_with_schema( + settings_key="ui_settings", + settings_class=settings_class, + config=config, + ) + ) + values: Final[Mapping[str, object]] = MappingProxyType( + { + **resolved_settings.values, + ENABLE_PTU_COST_ATTRIBUTION_UI_SETTING: is_ptu_cost_attribution_enabled(), + } + ) + source: Final[Mapping[str, FieldSource]] = MappingProxyType( + { + key: ( + _ui_setting_source(key, values[key], proxy_config.settings, settings_class) + if key in proxy_config.settings or key not in ui_settings + else "db" + ) + for key in values + } ) return UISettingsResponse( - values={ - **settings["values"], - ENABLE_PTU_COST_ATTRIBUTION_UI_SETTING: is_ptu_cost_attribution_enabled(), - }, - field_schema=settings["field_schema"], + values=values, + field_schema=resolved_settings.field_schema, + source=source, ) diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index de5f545f109..bfd412db1a7 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -148,6 +148,7 @@ from litellm.proxy._types import ( Member, UserAPIKeyAuth, ) +from litellm.proxy.agent_endpoints.auth.agent_access_groups import CeilingResolver, resolve_agent_access_group_ceiling from litellm.proxy.auth.route_checks import RouteChecks from litellm.proxy.common_utils.callback_utils import add_guardrail_to_applied_guardrails_header from litellm.proxy.common_utils.config_sync_pubsub import publish_config_param_change @@ -8261,6 +8262,51 @@ async def _get_access_group_models( return tuple(dict.fromkeys((*team_group_models, *key_group_models))) +async def _agent_access_group_visible_models( + user_api_key_dict: "UserAPIKeyAuth", + llm_router: "Router | None", + include_model_access_groups: bool, + return_wildcard_routes: bool, + team_id: str | None, + resolve_agent_ceiling: CeilingResolver, +) -> frozenset[str] | None: + """Models an agent key may still list once its attached access groups cap it, ``None`` when + nothing caps it, so ``/v1/models`` never advertises a model the same key would be denied on.""" + from litellm.proxy.auth.model_checks import get_complete_model_list, get_team_models + + if not user_api_key_dict.agent_id: + return None + ceiling: Final = await resolve_agent_ceiling(user_api_key_dict.agent_id) + if ceiling is None: + return None + if llm_router is None: + return ceiling.models + proxy_model_list: Final = llm_router.get_model_names() + model_access_groups: Final = llm_router.get_model_access_groups() + granted: Final = get_team_models( + team_models=sorted(ceiling.models), + proxy_model_list=proxy_model_list, + model_access_groups=model_access_groups, + include_model_access_groups=include_model_access_groups, + ) + if not granted: + return frozenset() + return frozenset( + get_complete_model_list( + key_models=granted, + team_models=(), + proxy_model_list=proxy_model_list, + user_model=None, + infer_model_from_keys=False, + return_wildcard_routes=return_wildcard_routes, + llm_router=llm_router, + model_access_groups=model_access_groups, + include_model_access_groups=include_model_access_groups, + team_id=team_id, + ) + ) + + async def get_available_models_for_user( user_api_key_dict: "UserAPIKeyAuth", llm_router: Optional["Router"], @@ -8273,6 +8319,7 @@ async def get_available_models_for_user( only_model_access_groups: bool = False, return_wildcard_routes: bool = False, user_api_key_cache: Optional["UserApiKeyCache"] = None, + resolve_agent_ceiling: CeilingResolver = resolve_agent_access_group_ceiling, ) -> list[str]: """ Get the list of models available to a user based on their API key and team permissions. @@ -8376,7 +8423,18 @@ async def get_available_models_for_user( team_id=effective_team_id, ) - return all_models + agent_visible: Final = await _agent_access_group_visible_models( + user_api_key_dict=user_api_key_dict, + llm_router=llm_router, + include_model_access_groups=include_model_access_groups, + return_wildcard_routes=return_wildcard_routes, + team_id=effective_team_id, + resolve_agent_ceiling=resolve_agent_ceiling, + ) + if agent_visible is None: + return all_models + capped: Final = [m for m in all_models if m in agent_visible] # mutable-ok: callers expect the list all_models is + return capped def _safe_get_model_info(model: str, get_model_info: Callable[[str], ModelInfo]) -> ModelInfo | None: diff --git a/litellm/responses/main.py b/litellm/responses/main.py index 5a4a08b760c..c5032536df4 100644 --- a/litellm/responses/main.py +++ b/litellm/responses/main.py @@ -433,13 +433,18 @@ def _bridges_to_chat_completions( return responses_api_provider_config is None or use_chat_completions_api is True +_RESPONSES_ONLY_REQUEST_FIELDS_NEVER_BRIDGED: Final = frozenset({"client_metadata"}) + + def _bridge_kwargs( kwargs: Mapping[str, object], responses_api_provider_config: BaseResponsesAPIConfig | None, allowed_openai_params: Sequence[str] | None, ) -> Mapping[str, object]: if responses_api_provider_config is None: - return kwargs + return MappingProxyType( + {key: value for key, value in kwargs.items() if key not in _RESPONSES_ONLY_REQUEST_FIELDS_NEVER_BRIDGED} + ) forwarded_keys: Final = frozenset( ( *litellm.OPENAI_CHAT_COMPLETION_PARAMS, @@ -448,7 +453,7 @@ def _bridge_kwargs( *GenericLiteLLMParams.model_fields, *(allowed_openai_params or ()), ) - ) + ).difference(_RESPONSES_ONLY_REQUEST_FIELDS_NEVER_BRIDGED) return MappingProxyType({key: value for key, value in kwargs.items() if key in forwarded_keys}) diff --git a/litellm/responses/mcp/mcp_streaming_iterator.py b/litellm/responses/mcp/mcp_streaming_iterator.py index c60020ab979..3b5cb85862d 100644 --- a/litellm/responses/mcp/mcp_streaming_iterator.py +++ b/litellm/responses/mcp/mcp_streaming_iterator.py @@ -91,16 +91,6 @@ async def create_mcp_list_tools_events( # Use the pre-processed MCP tools that were already fetched, filtered, and deduplicated by the parent filtered_mcp_tools: Final = pre_processed_mcp_tools - # Convert tools to dict format for the event - _mcp_tools_dict: Final = [ - tool.model_dump() - if hasattr(tool, "model_dump") and callable(getattr(tool, "model_dump", None)) - else tool.__dict__ - if hasattr(tool, "__dict__") - else {"name": getattr(tool, "name", str(tool))} - for tool in filtered_mcp_tools - ] - # Emit list tools completed event completed_event: Final = MCPListToolsCompletedEvent( type=ResponsesAPIStreamEvents.MCP_LIST_TOOLS_COMPLETED, diff --git a/litellm/router_strategy/complexity_router/complexity_router.py b/litellm/router_strategy/complexity_router/complexity_router.py index 27affc09337..f0fef3974f4 100644 --- a/litellm/router_strategy/complexity_router/complexity_router.py +++ b/litellm/router_strategy/complexity_router/complexity_router.py @@ -1779,7 +1779,7 @@ class ComplexityRouter(CustomLogger): conversation_continuing: bool = True, tier_litellm_params: Mapping[str, object] | None = None, context_escalation_original_tier: ComplexityTier | str | None = None, - heuristic_v2_forecast: StandardLoggingHeuristicV2Forecast | None = None, + previous_decision: StandardLoggingRoutingDecision | None = None, ) -> StandardLoggingRoutingDecision: """Assemble the per-request provenance record for this router's decision. @@ -1839,8 +1839,15 @@ class ComplexityRouter(CustomLogger): masked_tier_litellm_params: Final = mask_credentials_in_payload(tier_litellm_params) if isinstance(masked_tier_litellm_params, Mapping): decision["tier_litellm_params"] = masked_tier_litellm_params - return ( - decision if heuristic_v2_forecast is None else {**decision, "heuristic_v2_forecast": heuristic_v2_forecast} + forecast_fields: Final = MappingProxyType( + { + field: value + for field, value in (previous_decision.items() if previous_decision is not None else ()) + if field.startswith("classifier_") or field == "heuristic_v2_forecast" + } + ) + return cast( # cast-ok: retaining optional keys from a typed decision preserves their declared values + StandardLoggingRoutingDecision, {**forecast_fields, **decision} ) async def aclassify( @@ -3595,7 +3602,7 @@ class ComplexityRouter(CustomLogger): context_escalation_original_tier=( decision.get("context_escalation_original_tier") if decision is not None else None ), - heuristic_v2_forecast=decision.get("heuristic_v2_forecast") if decision is not None else None, + previous_decision=decision, ) from litellm.types.router import PreRoutingHookResponse as HookResponse @@ -3775,7 +3782,7 @@ class ComplexityRouter(CustomLogger): conversation_continuing=bool(decision.get("conversation_continuing", True)), tier_litellm_params=self._litellm_params_for_model(candidate_tier, new_model), context_escalation_original_tier=decision.get("context_escalation_original_tier"), - heuristic_v2_forecast=decision.get("heuristic_v2_forecast"), + previous_decision=decision, ) return response.model_copy( update={ # mutable-ok: model_copy types update as a plain dict @@ -3820,7 +3827,7 @@ class ComplexityRouter(CustomLogger): conversation_continuing=bool(decision.get("conversation_continuing", True)), tier_litellm_params=self._litellm_params_for_model(None, default_model), context_escalation_original_tier=decision.get("context_escalation_original_tier"), - heuristic_v2_forecast=decision.get("heuristic_v2_forecast"), + previous_decision=decision, ) return response.model_copy( update={ # mutable-ok: model_copy types update as a plain dict diff --git a/litellm/router_strategy/complexity_router/config.py b/litellm/router_strategy/complexity_router/config.py index 1537e3a540c..c4dbe68ebe2 100644 --- a/litellm/router_strategy/complexity_router/config.py +++ b/litellm/router_strategy/complexity_router/config.py @@ -1321,7 +1321,7 @@ class ComplexityRouterConfig(BaseModel): ) enable_context_window_escalation: bool = Field( - default=True, + default=False, description=( "Escalate a request off a tier whose models provably cannot hold its prompt, before " "dispatch. The classifier scores complexity and never prompt size, so a long agentic " @@ -1331,7 +1331,8 @@ class ComplexityRouterConfig(BaseModel): "moves to the lowest configured tier with a model whose declared window fits; when " "only some of the tier's models fit, the pick is restricted to those and the tier " "keeps the request. Models with no resolvable window are never escalated away from " - "and never escalated onto. Set false to dispatch on complexity alone, as before." + "and never escalated onto. Disabled by default: omit or set false to dispatch on " + "complexity alone; set true to enable context-window escalation." ), ) context_window_escalation_buffer: float = Field( diff --git a/litellm/router_strategy/complexity_router/jev_classifier.py b/litellm/router_strategy/complexity_router/jev_classifier.py index a41df18b55f..c0d0d1de8e3 100644 --- a/litellm/router_strategy/complexity_router/jev_classifier.py +++ b/litellm/router_strategy/complexity_router/jev_classifier.py @@ -165,6 +165,7 @@ class HttpJevClassifierClient: end_time=end_time, cache_hit=False, request_body=MappingProxyType({"model": request.model}), + custom_llm_provider="typesafe", litellm_params=params, ) success_handlers: Final = logging_obj.dispatch_success_handlers( diff --git a/litellm/rust_bridge/_native.pyi b/litellm/rust_bridge/_native.pyi index 05a6df6d5af..31623a53dd0 100644 --- a/litellm/rust_bridge/_native.pyi +++ b/litellm/rust_bridge/_native.pyi @@ -93,6 +93,133 @@ class ResponsesWebSocketConnection: def recv_text(self) -> Future[str | None]: ... def close(self) -> Future[None]: ... +@final +class _CacheTestBinding: + @property + def kind(self) -> str: ... + def lookup( + self, + request: object, + *, + callback_kwargs: Mapping[str, object] | Sequence[object] | None = None, + ) -> object: ... + def store( + self, + request: object, + response: object, + *, + callback_kwargs: Mapping[str, object] | None = None, + ) -> None: ... + def lookup_batch( + self, + requests: Sequence[object], + *, + callback_kwargs: Sequence[object] | None = None, + ) -> object: ... + def async_lookup( + self, + request: object, + *, + callback_kwargs: Mapping[str, object] | None = None, + ) -> Future[object]: ... + def async_store( + self, + request: object, + response: object, + *, + callback_kwargs: Mapping[str, object] | None = None, + ) -> Future[None]: ... + def async_lookup_batch( + self, + requests: Sequence[object], + *, + callback_kwargs: Sequence[object] | None = None, + ) -> Future[object]: ... + def async_store_batch( + self, + requests: Sequence[object], + responses: Sequence[object], + *, + callback_result: object = None, + callback_kwargs: Mapping[str, object] | None = None, + ) -> Future[object]: ... + def async_flush(self) -> Future[None]: ... + def ping(self) -> Future[object]: ... + +@final +class _CacheTestHandle: + def __new__(cls, _uninstantiable: Never, /) -> Never: ... + @staticmethod + def memory( + *, + capacity: int = 200, + ttl_seconds: float = 600.0, + max_entry_bytes: int = 1048576, + ) -> _CacheTestHandle: ... + @staticmethod + def redis( + url: str, + *, + ttl_seconds: float = 60.0, + namespace: str | None = None, + startup_nodes: Sequence[tuple[str, int]] | None = None, + ) -> _CacheTestHandle: ... + @staticmethod + def disk(directory: str) -> _CacheTestHandle: ... + @staticmethod + def qdrant_semantic( + url: str, + *, + collection_name: str, + similarity_threshold: float, + vector_size: int, + embedding_model: str = "text-embedding-3-small", + api_key: str | None = None, + embedding_api_key: str | None = None, + embedding_api_base: str | None = None, + embedding_timeout_seconds: float | None = None, + quantization: str = "binary", + ) -> _CacheTestHandle: ... + @staticmethod + def azure_blob(account_url: str, container: str) -> _CacheTestHandle: ... + @staticmethod + def redis_semantic(backend: object) -> _CacheTestHandle: ... + @staticmethod + def valkey_semantic( + url: str, + similarity_threshold: float, + index_name: str, + embedder: object, + ) -> _CacheTestHandle: ... + @staticmethod + def gcs( + bucket_name: str, + *, + gcs_path: str | None = None, + path_service_account: str | None = None, + endpoint: str | None = None, + token: str | None = None, + ) -> _CacheTestHandle: ... + @staticmethod + def s3( + bucket: str, + *, + region: str, + endpoint_url: str | None = None, + key_prefix: str = "", + access_key_id: str | None = None, + secret_access_key: str | None = None, + session_token: str | None = None, + ) -> _CacheTestHandle: ... + @property + def backend(self) -> str: ... + def _bind_facade(self, facade: object) -> None: ... + +@final +class _CacheTestResolver: + def __new__(cls, namespace: object) -> _CacheTestResolver: ... + def resolve(self) -> _CacheTestBinding: ... + @final class TokenCounter: def __new__(cls, tokenizer_json: str) -> TokenCounter: ... diff --git a/litellm/rust_bridge/callbacks_legacy_python.py b/litellm/rust_bridge/callbacks_legacy_python.py index 30aa1d97bfc..6bbf2ffed6b 100644 --- a/litellm/rust_bridge/callbacks_legacy_python.py +++ b/litellm/rust_bridge/callbacks_legacy_python.py @@ -59,9 +59,17 @@ def setup( } supplied: Final = arguments.get("litellm_logging_obj") if isinstance(supplied, Logging): - return CallSetup(supplied, arguments) + return _claim_budget_reservation(CallSetup(supplied, arguments), asynchronous) logger, prepared = function_setup(call_type, Rules(), start_time, *args, is_async_call=asynchronous, **arguments) - return CallSetup(logger, prepared) + return _claim_budget_reservation(CallSetup(logger, prepared), asynchronous) + + +def _claim_budget_reservation(call_setup: CallSetup, asynchronous: bool) -> CallSetup: + from litellm.litellm_core_utils.core_helpers import bind_budget_reservation_to_callbacks + + if asynchronous and not is_internal_call(): + bind_budget_reservation_to_callbacks(call_setup.logger.litellm_params) + return call_setup def check_limits(kwargs: Mapping[str, object]) -> None: @@ -96,6 +104,9 @@ def finalize( class LoggingSurface(Protocol): + @property + def litellm_params(self) -> Mapping[str, object]: ... + def update_from_kwargs( self, kwargs: dict[str, object], @@ -236,8 +247,12 @@ def sync_success_for_async_call( def failure_handler( logger: LoggingSurface, error: Exception, start: datetime.datetime, end: datetime.datetime, asynchronous: bool ) -> Coroutine[object, object, None] | None: + from litellm.litellm_core_utils.core_helpers import unbind_budget_reservation_from_callbacks + trace: Final = "".join(traceback.format_exception(error)) if asynchronous: + if not is_internal_call(): + unbind_budget_reservation_from_callbacks(logger.litellm_params) return logger.async_failure_handler(error, trace, start, end) logger.failure_handler(error, trace, start, end) return None diff --git a/litellm/types/agents.py b/litellm/types/agents.py index dbaaab62d86..7f8d8c6af66 100644 --- a/litellm/types/agents.py +++ b/litellm/types/agents.py @@ -2,7 +2,7 @@ from collections.abc import Mapping, Sequence from datetime import datetime from typing import TYPE_CHECKING, Any, Final, Literal -from pydantic import BaseModel, PrivateAttr, StrictInt +from pydantic import BaseModel, ConfigDict, PrivateAttr, StrictInt from typing_extensions import ReadOnly, Required, TypedDict from litellm.types.llms.base import LiteLLMPydanticObjectBase @@ -189,6 +189,7 @@ class AgentConfig(TypedDict, total=False): session_rpm_limit: int | None static_headers: dict[str, str] | None extra_headers: list[str] | None + access_group_ids: ReadOnly[Sequence[str] | None] class PatchAgentRequest(TypedDict, total=False): @@ -202,6 +203,21 @@ class PatchAgentRequest(TypedDict, total=False): session_rpm_limit: int | None static_headers: dict[str, str] | None extra_headers: list[str] | None + access_group_ids: ReadOnly[Sequence[str] | None] + + +AGENT_CALLER_USER_ID_HEADER: Final = "x-litellm-user-id" +AGENT_CALLER_TEAM_ID_HEADER: Final = "x-litellm-team-id" + + +class AgentCaller(BaseModel): + """The user and team that invoked an agent, echoed back by the agent on its own proxy calls. + Only ever narrows what the agent's key may do.""" + + model_config = ConfigDict(frozen=True) + + user_id: str | None = None + team_id: str | None = None # Request/Response models for CRUD endpoints @@ -226,6 +242,7 @@ class AgentResponse(BaseModel): session_rpm_limit: int | None = None static_headers: dict[str, str] | None = None extra_headers: list[str] | None = None + access_group_ids: Sequence[str] | None = None keys: list[AgentKeySummary] | None = None search_score: float | None = None created_at: datetime | None = None diff --git a/litellm/types/integrations/prometheus.py b/litellm/types/integrations/prometheus.py index f4893e857d1..c929ee2ee79 100644 --- a/litellm/types/integrations/prometheus.py +++ b/litellm/types/integrations/prometheus.py @@ -131,6 +131,7 @@ EXCEPTION_STATUS: Final = "exception_status" EXCEPTION_CLASS: Final = "exception_class" RATE_LIMIT_CATEGORY: Final = "rate_limit_category" RATE_LIMIT_TYPE: Final = "rate_limit_type" +ZERO_COST_REASON_LABEL: Final = "reason" STATUS_CODE: Final = "status_code" EXCEPTION_LABELS: Final = [EXCEPTION_STATUS, EXCEPTION_CLASS] LATENCY_BUCKETS: Final = ( @@ -279,6 +280,7 @@ DEFINED_PROMETHEUS_METRICS = Literal[ "litellm_guardrail_latency_seconds", "litellm_guardrail_errors_total", "litellm_guardrail_requests_total", + "litellm_zero_cost_requests_total", # Cache metrics "litellm_cache_hits_metric", "litellm_cache_misses_metric", @@ -590,6 +592,14 @@ class PrometheusMetricLabels: UserAPIKeyLabelNames.SERVICE_TIER.value, ] + litellm_zero_cost_requests_total = ( + UserAPIKeyLabelNames.REQUESTED_MODEL.value, + UserAPIKeyLabelNames.v1_LITELLM_MODEL_NAME.value, + UserAPIKeyLabelNames.MODEL_ID.value, + UserAPIKeyLabelNames.API_PROVIDER.value, + ZERO_COST_REASON_LABEL, + ) + litellm_input_tokens_metric = [ UserAPIKeyLabelNames.END_USER.value, UserAPIKeyLabelNames.API_KEY_HASH.value, diff --git a/litellm/types/llms/custom_http.py b/litellm/types/llms/custom_http.py index d80d7410aae..793893451df 100644 --- a/litellm/types/llms/custom_http.py +++ b/litellm/types/llms/custom_http.py @@ -31,6 +31,7 @@ class httpxSpecialProvider(str, Enum): UI = "ui" Sandbox = "sandbox" ModelCostMap = "model_cost_map" + PasswordBreachCheck = "password_breach_check" VerifyTypes = str | bool | ssl.SSLContext diff --git a/litellm/types/proxy/ui_sso.py b/litellm/types/proxy/ui_sso.py index 0d7e0b99cf0..03b0b92a4d1 100644 --- a/litellm/types/proxy/ui_sso.py +++ b/litellm/types/proxy/ui_sso.py @@ -1,6 +1,6 @@ from typing import Literal -from typing_extensions import TypedDict +from typing_extensions import ReadOnly, TypedDict class ReturnedUITokenObject(TypedDict): @@ -17,6 +17,7 @@ class ReturnedUITokenObject(TypedDict): auth_header_name: str disabled_non_admin_personal_key_creation: bool server_root_path: str # e.g. `/litellm` + password_reset_required: ReadOnly[bool] class ParsedOpenIDResult(TypedDict, total=False): diff --git a/litellm/types/utils.py b/litellm/types/utils.py index e1d43b7fccb..2e8c869b89d 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -1,7 +1,7 @@ import json import re import time -from collections.abc import Mapping, Sequence +from collections.abc import Collection, Mapping, Sequence from enum import Enum from types import MappingProxyType from typing import ( @@ -36,12 +36,14 @@ from pydantic import ( BaseModel, ConfigDict, Field, + FieldSerializationInfo, JsonValue, PrivateAttr, SkipValidation, field_serializer, field_validator, ) +from pydantic.main import IncEx from typing_extensions import NotRequired, ReadOnly, Required, TypedDict from litellm._logging import verbose_logger @@ -80,6 +82,27 @@ from .llms.openai import ( ) from .rerank import RerankResponse as RerankResponse + +def _nested_selector( + selector: IncEx | None, + index: int, + count: int, + is_include: bool, +) -> tuple[bool, IncEx | None]: + if selector is None: + return True, None + if isinstance(selector, Mapping): + value: Final = selector.get(index, selector.get(index - count, selector.get("__all__"))) + keep: Final = value is not None if is_include else value is not True + per_item_selector: Final = None if value is True or value is None else value + return keep, per_item_selector + if isinstance(selector, Collection) and not isinstance(selector, (str, bytes)): + if all(isinstance(item, int) for item in selector): + addressed: Final = index in selector or index - count in selector + return (addressed if is_include else not addressed), None + return True, selector + + if TYPE_CHECKING: from .vector_stores import VectorStoreSearchResponse else: @@ -333,6 +356,9 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False): output_cost_per_second_768p: ReadOnly[float | None] output_cost_per_second_2k: ReadOnly[float | None] output_cost_per_second_4k: ReadOnly[float | None] + output_cost_per_image_512: ReadOnly[float | None] + output_cost_per_image_1024: ReadOnly[float | None] + output_cost_per_image_1536: ReadOnly[float | None] ocr_cost_per_page: float | None # for OCR models ocr_cost_per_page_batches: ReadOnly[float | None] ocr_cost_per_credit: float | None # for OCR models priced by credit @@ -459,6 +485,8 @@ class CallTypes(str, Enum): ######################################################### create_video = "create_video" acreate_video = "acreate_video" + video_generation = "video_generation" + avideo_generation = "avideo_generation" avideo_retrieve = "avideo_retrieve" video_retrieve = "video_retrieve" avideo_content = "avideo_content" @@ -2555,8 +2583,35 @@ class ImageResponse(OpenAIImageResponse, BaseLiteLLMOpenAIResponseObject): model_config = ConfigDict(extra="allow", protected_namespaces=()) @field_serializer("data") - def _serialize_image_data(self, data: Sequence[OpenAIImage] | None) -> Sequence[Mapping[str, object]] | None: - return None if data is None else [image.model_dump() for image in data] + def _serialize_image_data( + self, + data: Sequence[OpenAIImage] | None, + info: FieldSerializationInfo, + ) -> Sequence[Mapping[str, object]] | None: + if data is None: + return None + include: Final = info.include + exclude: Final = info.exclude + + def _serialize_image(index: int, image: OpenAIImage) -> Mapping[str, object] | None: + include_keep, include_selector = _nested_selector(include, index, len(data), is_include=True) + exclude_keep, exclude_selector = _nested_selector(exclude, index, len(data), is_include=False) + if not include_keep or not exclude_keep: + return None + return image.model_dump( + mode=info.mode, + include=include_selector, + exclude=exclude_selector, + context=info.context, + exclude_none=info.exclude_none, + exclude_unset=info.exclude_unset, + exclude_defaults=info.exclude_defaults, + round_trip=info.round_trip, + by_alias=info.by_alias, + ) + + serialized_images: Final = tuple(_serialize_image(index, image) for index, image in enumerate(data)) + return [image for image in serialized_images if image is not None] def __init__( self, @@ -3154,6 +3209,15 @@ class StandardLoggingModelCostFailureDebugInformation(TypedDict, total=False): custom_pricing: bool | None +ZeroCostReason = Literal["missing_pricing_key", "pricing_not_applied", "cost_calculation_error"] + + +class StandardLoggingZeroCostDiagnostic(TypedDict): + reason: ReadOnly[ZeroCostReason] + pricing_model: ReadOnly[str] + missing_pricing_keys: ReadOnly[tuple[str, ...]] + + class StandardLoggingPayloadErrorInformation(TypedDict, total=False): error_code: str | None error_class: str | None @@ -3472,6 +3536,7 @@ class StandardLoggingPayload(ClassifierAudit): autorouter_savings_estimate: ReadOnly[Mapping[str, JsonValue] | None] autorouter_baseline_observation: ReadOnly[str | None] response_cost_failure_debug_info: StandardLoggingModelCostFailureDebugInformation | None + zero_cost_diagnostic: NotRequired[ReadOnly[StandardLoggingZeroCostDiagnostic | None]] status: StandardLoggingPayloadStatus status_fields: StandardLoggingPayloadStatusFields custom_llm_provider: str | None @@ -3620,6 +3685,9 @@ class CustomPricingLiteLLMParams(MirroredPricingParams): output_cost_per_second_768p: float | None = None output_cost_per_second_2k: float | None = None output_cost_per_second_4k: float | None = None + output_cost_per_image_512: float | None = None + output_cost_per_image_1024: float | None = None + output_cost_per_image_1536: float | None = None input_cost_per_pixel: float | None = None output_cost_per_pixel: float | None = None @@ -3790,6 +3858,24 @@ def echoed_cost_map_pricing_fields(model_info: Mapping[str, Any]) -> tuple[str, return tuple(sorted(k for k in model_info if is_server_derived_pricing_key(k))) +def echoed_cost_map_fields( + model_info: Mapping[str, object], *cost_map_entries: Mapping[str, object] +) -> tuple[str, ...]: + """Fields a ``/model/info`` echo copied from the cost map unchanged. + + Only ``litellm.get_model_info`` emits ``key``, so a blob carrying it is an echo of that + response. Anything in it that still equals a resolved cost-map entry is a display value + nobody typed; a value the operator edited differs from every entry and stays a real override. + Callers pass both the live entry, which the router rewrites with each deployment's own + overrides, and the catalog entry as loaded, so a reset to the catalog value reads as an echo either way. + """ + if COST_MAP_LOOKUP_KEY not in model_info: + return () + return tuple( + sorted(k for k, v in model_info.items() if any(k in entry and entry[k] == v for entry in cost_map_entries)) + ) + + def pricing_override_fields(*sources: Mapping[str, Any]) -> tuple[str, ...]: return tuple( sorted( diff --git a/litellm/utils.py b/litellm/utils.py index 709f3f6d1dd..812299560c7 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -81,7 +81,11 @@ from litellm.constants import ( PROVIDERS_THAT_AUTHENTICATE_ON_PROVIDER_INFO, TOOL_CHOICE_OBJECT_TOKEN_COUNT, ) -from litellm.litellm_core_utils.core_helpers import normalize_drop_params +from litellm.litellm_core_utils.core_helpers import ( + bind_budget_reservation_to_callbacks, + normalize_drop_params, + unbind_budget_reservation_from_callbacks, +) from litellm.litellm_core_utils.fallback_generalizations import ( match_capability_generalizations, match_fill_missing_generalizations, @@ -1880,6 +1884,8 @@ def client(original_function): # Type assertion: logging_obj is guaranteed to be non-None after function_setup assert logging_obj is not None, "logging_obj should not be None after function_setup" + if not _is_litellm_internal_call: + bind_budget_reservation_to_callbacks(logging_obj.litellm_params) kwargs["litellm_logging_obj"] = logging_obj modified_kwargs: Final = await async_pre_call_deployment_hook(kwargs, call_type) @@ -2081,6 +2087,7 @@ def client(original_function): # the failure hook ran, so a slow callback doesn't inflate the reported duration. end_time = _deployment_call_end_time if _deployment_call_end_time is not None else datetime.datetime.now() # noqa: DTZ005 # matches the naive datetimes this whole function already times start_time/end_time with if logging_obj and not _is_litellm_internal_call: + unbind_budget_reservation_from_callbacks(logging_obj.litellm_params) try: logging_obj.failure_handler( e, traceback_exception, start_time, end_time @@ -8340,6 +8347,7 @@ class ProviderConfigManager: False, ), LlmProviders.EDENAI: (litellm.EdenAIChatConfig, False), + LlmProviders.FAL_AI: (litellm.FalAIChatConfig, False), LlmProviders.COMETAPI: (lambda: litellm.CometAPIConfig(), False), LlmProviders.DATAROBOT: (lambda: litellm.DataRobotConfig(), False), LlmProviders.GEMINI: (lambda: litellm.GoogleAIStudioGeminiConfig(), False), @@ -9569,9 +9577,9 @@ class ProviderConfigManager: return BlackForestLabsImageEditConfig() elif LlmProviders.FAL_AI == provider: - from litellm.llms.fal_ai.image_edit import FalAIImageEditConfig + from litellm.llms.fal_ai.image_edit import get_fal_ai_image_edit_config - return FalAIImageEditConfig() + return get_fal_ai_image_edit_config(model) elif LlmProviders.AZURE_AI == provider: from litellm.llms.azure_ai.image_edit import get_azure_ai_image_edit_config diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 6b49b1d47a5..70c54b91cbd 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -24966,6 +24966,52 @@ "/v1/images/generations" ] }, + "fal_ai/fal-ai/trellis": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.02, + "source": "https://fal.ai/models/fal-ai/trellis", + "metadata": { + "comment": "image-to-3D, returns a GLB mesh; served through the /fal_ai pass-through route" + } + }, + "fal_ai/fal-ai/trellis-2": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.3, + "output_cost_per_image_512": 0.25, + "output_cost_per_image_1024": 0.3, + "output_cost_per_image_1536": 0.35, + "source": "https://fal.ai/models/fal-ai/trellis-2", + "metadata": { + "comment": "image-to-3D, returns a GLB mesh; priced by the request's resolution field (default 1024); served through the /fal_ai pass-through route" + } + }, + "fal_ai/fal-ai/flux-lora-depth": { + "litellm_provider": "fal_ai", + "metadata": { + "notes": "fal bills fal-ai/flux-lora-depth at $0.035 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries" + }, + "mode": "image_generation", + "output_cost_per_image": 0.035, + "output_cost_per_pixel": 3.337860107421875e-08, + "source": "https://fal.ai/models/fal-ai/flux-lora-depth", + "supported_endpoints": [ + "/v1/images/edits" + ] + }, + "fal_ai/fal-ai/moondream3-preview/query": { + "input_cost_per_token": 4e-07, + "litellm_provider": "fal_ai", + "mode": "chat", + "output_cost_per_token": 3.5e-06, + "source": "https://fal.ai/models/fal-ai/moondream3-preview/query", + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_reasoning": true, + "supports_vision": true + }, "featherless_ai/featherless-ai/Qwerky-72B": { "litellm_provider": "featherless_ai", "max_input_tokens": 32768, @@ -43037,21 +43083,21 @@ "supports_web_search": false }, "openrouter/deepseek/deepseek-v4-pro": { - "input_cost_per_token": 8.95578e-07, + "input_cost_per_token": 9.5526e-07, "input_cost_per_token_cache_hit": 4.4e-08, "litellm_provider": "openrouter", "max_input_tokens": 1048576, "max_output_tokens": 384000, "max_tokens": 384000, "mode": "chat", - "output_cost_per_token": 1.791156e-06, + "output_cost_per_token": 1.91052e-06, "source": "https://openrouter.ai/api/v1/models", "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "cache_read_input_token_cost": 7.46315e-08, + "cache_read_input_token_cost": 7.9605e-08, "supports_audio_input": false, "supports_pdf_input": false, "supports_vision": false, @@ -44812,26 +44858,6 @@ "max_tokens": 128000, "mode": "chat" }, - "openrouter/stealth/union-alpha": { - "deprecation_date": "2098-12-31", - "input_cost_per_token": 0.0, - "output_cost_per_token": 0.0, - "litellm_provider": "openrouter", - "max_input_tokens": 262144, - "max_output_tokens": 131072, - "max_tokens": 131072, - "mode": "chat", - "source": "https://openrouter.ai/api/v1/models", - "supports_audio_input": false, - "supports_function_calling": true, - "supports_pdf_input": false, - "supports_prompt_caching": false, - "supports_reasoning": false, - "supports_tool_choice": true, - "supports_response_schema": true, - "supports_vision": true, - "supports_web_search": false - }, "ovhcloud/DeepSeek-R1-Distill-Llama-70B": { "input_cost_per_token": 6.7e-07, "litellm_provider": "ovhcloud", @@ -54217,7 +54243,7 @@ "max_tokens": 1000000, "mode": "responses", "output_cost_per_token": 2.5e-06, - "source": "https://docs.x.ai/docs/models", + "source": "https://api.x.ai/v1/language-models", "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, @@ -54227,6 +54253,7 @@ "input_cost_per_token_above_200k_tokens": 2.5e-06, "output_cost_per_token_above_200k_tokens": 5e-06, "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_image_token": 1.25e-06, "supports_response_schema": true, "supported_endpoints": [ "/v1/responses" @@ -54241,7 +54268,7 @@ "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, - "source": "https://docs.x.ai/docs/models", + "source": "https://api.x.ai/v1/language-models", "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, @@ -54251,6 +54278,7 @@ "input_cost_per_token_above_200k_tokens": 2.5e-06, "output_cost_per_token_above_200k_tokens": 5e-06, "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_image_token": 1.25e-06, "supports_response_schema": true }, "xai/grok-4.20-0309-reasoning": { @@ -54262,7 +54290,7 @@ "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, - "source": "https://docs.x.ai/docs/models", + "source": "https://api.x.ai/v1/language-models", "supports_function_calling": true, "supports_reasoning": true, "supports_tool_choice": true, @@ -54271,6 +54299,7 @@ "input_cost_per_token_above_200k_tokens": 2.5e-06, "output_cost_per_token_above_200k_tokens": 5e-06, "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_image_token": 1.25e-06, "supports_prompt_caching": true, "supports_response_schema": true }, @@ -54283,7 +54312,7 @@ "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, - "source": "https://docs.x.ai/docs/models", + "source": "https://api.x.ai/v1/language-models", "supports_function_calling": true, "supports_prompt_caching": true, "supports_tool_choice": true, @@ -54292,11 +54321,13 @@ "input_cost_per_token_above_200k_tokens": 2.5e-06, "output_cost_per_token_above_200k_tokens": 5e-06, "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_image_token": 1.25e-06, "supports_response_schema": true }, "xai/grok-4.3": { "cache_read_input_token_cost": 2e-07, "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_image_token": 1.25e-06, "input_cost_per_token": 1.25e-06, "input_cost_per_token_above_200k_tokens": 2.5e-06, "litellm_provider": "xai", @@ -54306,7 +54337,7 @@ "mode": "chat", "output_cost_per_token": 2.5e-06, "output_cost_per_token_above_200k_tokens": 5e-06, - "source": "https://docs.x.ai/docs/models", + "source": "https://api.x.ai/v1/language-models", "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, @@ -54318,6 +54349,7 @@ "xai/grok-4.3-latest": { "cache_read_input_token_cost": 2e-07, "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_image_token": 1.25e-06, "input_cost_per_token": 1.25e-06, "input_cost_per_token_above_200k_tokens": 2.5e-06, "litellm_provider": "xai", @@ -54327,7 +54359,7 @@ "mode": "chat", "output_cost_per_token": 2.5e-06, "output_cost_per_token_above_200k_tokens": 5e-06, - "source": "https://docs.x.ai/docs/models", + "source": "https://api.x.ai/v1/language-models", "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, @@ -54339,6 +54371,7 @@ "xai/grok-4.5": { "cache_read_input_token_cost": 3e-07, "cache_read_input_token_cost_above_200k_tokens": 6e-07, + "input_cost_per_image_token": 2e-06, "input_cost_per_token": 2e-06, "input_cost_per_token_above_200k_tokens": 4e-06, "litellm_provider": "xai", @@ -54348,7 +54381,7 @@ "mode": "chat", "output_cost_per_token": 6e-06, "output_cost_per_token_above_200k_tokens": 1.2e-05, - "source": "https://docs.x.ai/docs/models", + "source": "https://api.x.ai/v1/language-models", "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, @@ -54360,6 +54393,7 @@ "xai/grok-4.5-latest": { "cache_read_input_token_cost": 3e-07, "cache_read_input_token_cost_above_200k_tokens": 6e-07, + "input_cost_per_image_token": 2e-06, "input_cost_per_token": 2e-06, "input_cost_per_token_above_200k_tokens": 4e-06, "litellm_provider": "xai", @@ -54369,7 +54403,7 @@ "mode": "chat", "output_cost_per_token": 6e-06, "output_cost_per_token_above_200k_tokens": 1.2e-05, - "source": "https://docs.x.ai/docs/models", + "source": "https://api.x.ai/v1/language-models", "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, @@ -54381,6 +54415,7 @@ "xai/grok-build-latest": { "cache_read_input_token_cost": 3e-07, "cache_read_input_token_cost_above_200k_tokens": 6e-07, + "input_cost_per_image_token": 2e-06, "input_cost_per_token": 2e-06, "input_cost_per_token_above_200k_tokens": 4e-06, "litellm_provider": "xai", @@ -54390,7 +54425,7 @@ "mode": "chat", "output_cost_per_token": 6e-06, "output_cost_per_token_above_200k_tokens": 1.2e-05, - "source": "https://docs.x.ai/developers/models", + "source": "https://api.x.ai/v1/language-models", "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, @@ -54402,6 +54437,7 @@ "xai/grok-4.6": { "cache_read_input_token_cost": 5e-07, "cache_read_input_token_cost_above_200k_tokens": 1e-06, + "input_cost_per_image_token": 2e-06, "input_cost_per_token": 2e-06, "input_cost_per_token_above_200k_tokens": 4e-06, "litellm_provider": "xai", @@ -54411,7 +54447,7 @@ "mode": "chat", "output_cost_per_token": 6e-06, "output_cost_per_token_above_200k_tokens": 1.2e-05, - "source": "https://docs.x.ai/developers/models", + "source": "https://api.x.ai/v1/language-models", "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, @@ -54423,6 +54459,7 @@ "xai/grok-4.7": { "cache_read_input_token_cost": 5e-07, "cache_read_input_token_cost_above_200k_tokens": 1e-06, + "input_cost_per_image_token": 2e-06, "input_cost_per_token": 2e-06, "input_cost_per_token_above_200k_tokens": 4e-06, "litellm_provider": "xai", @@ -54432,7 +54469,7 @@ "mode": "chat", "output_cost_per_token": 6e-06, "output_cost_per_token_above_200k_tokens": 1.2e-05, - "source": "https://docs.x.ai/developers/models", + "source": "https://api.x.ai/v1/language-models", "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, @@ -54450,7 +54487,7 @@ "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 2e-06, - "source": "https://docs.x.ai/docs/models", + "source": "https://api.x.ai/v1/language-models", "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, @@ -54460,7 +54497,8 @@ "cache_read_input_token_cost_above_200k_tokens": 4e-07, "supports_response_schema": true, "supports_vision": true, - "deprecation_date": "2026-05-15" + "deprecation_date": "2026-05-15", + "input_cost_per_image_token": 1e-06 }, "xai/grok-code-fast-1": { "cache_read_input_token_cost": 2e-07, @@ -54471,7 +54509,7 @@ "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 2e-06, - "source": "https://docs.x.ai/docs/models", + "source": "https://api.x.ai/v1/language-models", "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, @@ -54481,7 +54519,8 @@ "cache_read_input_token_cost_above_200k_tokens": 4e-07, "supports_response_schema": true, "supports_vision": true, - "deprecation_date": "2026-05-15" + "deprecation_date": "2026-05-15", + "input_cost_per_image_token": 1e-06 }, "xai/grok-code-fast-1-0825": { "cache_read_input_token_cost": 2e-07, @@ -54492,7 +54531,7 @@ "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 2e-06, - "source": "https://docs.x.ai/docs/models", + "source": "https://api.x.ai/v1/language-models", "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, @@ -54502,7 +54541,8 @@ "cache_read_input_token_cost_above_200k_tokens": 4e-07, "supports_response_schema": true, "supports_vision": true, - "deprecation_date": "2026-05-15" + "deprecation_date": "2026-05-15", + "input_cost_per_image_token": 1e-06 }, "zai.glm-4.7": { "input_cost_per_token": 6e-07, @@ -61978,7 +62018,7 @@ "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, - "source": "https://docs.x.ai/docs/models", + "source": "https://api.x.ai/v1/language-models", "supports_function_calling": true, "supports_prompt_caching": true, "supports_tool_choice": true, @@ -61987,6 +62027,7 @@ "input_cost_per_token_above_200k_tokens": 2.5e-06, "output_cost_per_token_above_200k_tokens": 5e-06, "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_image_token": 1.25e-06, "supports_response_schema": true }, "xai/grok-4.20-multi-agent-0309": { @@ -61998,7 +62039,7 @@ "max_tokens": 1000000, "mode": "responses", "output_cost_per_token": 2.5e-06, - "source": "https://docs.x.ai/docs/models", + "source": "https://api.x.ai/v1/language-models", "supports_function_calling": false, "supports_prompt_caching": true, "supports_reasoning": true, @@ -62008,6 +62049,7 @@ "input_cost_per_token_above_200k_tokens": 2.5e-06, "output_cost_per_token_above_200k_tokens": 5e-06, "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_image_token": 1.25e-06, "supports_response_schema": true, "supported_endpoints": [ "/v1/responses" @@ -62022,7 +62064,7 @@ "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 2e-06, - "source": "https://docs.x.ai/docs/models", + "source": "https://api.x.ai/v1/language-models", "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, @@ -62030,6 +62072,7 @@ "input_cost_per_token_above_200k_tokens": 2e-06, "output_cost_per_token_above_200k_tokens": 4e-06, "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_image_token": 1e-06, "supports_response_schema": true, "supports_vision": true }, @@ -65225,7 +65268,7 @@ "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, - "source": "https://docs.x.ai/docs/models", + "source": "https://api.x.ai/v1/language-models", "supports_function_calling": true, "supports_reasoning": true, "supports_tool_choice": true, @@ -65234,6 +65277,7 @@ "input_cost_per_token_above_200k_tokens": 2.5e-06, "output_cost_per_token_above_200k_tokens": 5e-06, "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_image_token": 1.25e-06, "supports_prompt_caching": true, "supports_response_schema": true }, @@ -65246,7 +65290,7 @@ "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, - "source": "https://docs.x.ai/docs/models", + "source": "https://api.x.ai/v1/language-models", "supports_function_calling": true, "supports_reasoning": true, "supports_tool_choice": true, @@ -65255,6 +65299,7 @@ "input_cost_per_token_above_200k_tokens": 2.5e-06, "output_cost_per_token_above_200k_tokens": 5e-06, "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_image_token": 1.25e-06, "supports_prompt_caching": true, "supports_response_schema": true }, @@ -65267,7 +65312,7 @@ "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, - "source": "https://docs.x.ai/docs/models", + "source": "https://api.x.ai/v1/language-models", "supports_function_calling": true, "supports_reasoning": true, "supports_tool_choice": true, @@ -65276,6 +65321,7 @@ "input_cost_per_token_above_200k_tokens": 2.5e-06, "output_cost_per_token_above_200k_tokens": 5e-06, "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_image_token": 1.25e-06, "supports_prompt_caching": true, "supports_response_schema": true }, @@ -65519,7 +65565,7 @@ "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, - "source": "https://docs.x.ai/docs/models", + "source": "https://api.x.ai/v1/language-models", "supports_function_calling": true, "supports_prompt_caching": true, "supports_tool_choice": true, @@ -65528,6 +65574,7 @@ "input_cost_per_token_above_200k_tokens": 2.5e-06, "output_cost_per_token_above_200k_tokens": 5e-06, "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_image_token": 1.25e-06, "supports_response_schema": true }, "xai/grok-4.20-non-reasoning-latest": { @@ -65539,7 +65586,7 @@ "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, - "source": "https://docs.x.ai/docs/models", + "source": "https://api.x.ai/v1/language-models", "supports_function_calling": true, "supports_prompt_caching": true, "supports_tool_choice": true, @@ -65548,6 +65595,7 @@ "input_cost_per_token_above_200k_tokens": 2.5e-06, "output_cost_per_token_above_200k_tokens": 5e-06, "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_image_token": 1.25e-06, "supports_response_schema": true }, "xai/grok-4.20-multi-agent": { @@ -65559,7 +65607,7 @@ "max_tokens": 1000000, "mode": "responses", "output_cost_per_token": 2.5e-06, - "source": "https://docs.x.ai/docs/models", + "source": "https://api.x.ai/v1/language-models", "supported_endpoints": [ "/v1/responses" ], @@ -65572,6 +65620,7 @@ "input_cost_per_token_above_200k_tokens": 2.5e-06, "output_cost_per_token_above_200k_tokens": 5e-06, "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_image_token": 1.25e-06, "supports_response_schema": true }, "xai/grok-4.20-multi-agent-latest": { @@ -65583,7 +65632,7 @@ "max_tokens": 1000000, "mode": "responses", "output_cost_per_token": 2.5e-06, - "source": "https://docs.x.ai/docs/models", + "source": "https://api.x.ai/v1/language-models", "supported_endpoints": [ "/v1/responses" ], @@ -65596,6 +65645,7 @@ "input_cost_per_token_above_200k_tokens": 2.5e-06, "output_cost_per_token_above_200k_tokens": 5e-06, "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_image_token": 1.25e-06, "supports_response_schema": true }, "groq/qwen/qwen3.8-27b": { @@ -68212,13 +68262,13 @@ "supports_web_search": false }, "openrouter/z-ai/glm-5.3-flash": { - "input_cost_per_token": 7.5e-08, - "output_cost_per_token": 2.5e-07, - "cache_read_input_token_cost": 2e-08, + "input_cost_per_token": 1.5e-07, + "output_cost_per_token": 5e-07, + "cache_read_input_token_cost": 5e-08, "litellm_provider": "openrouter", "max_input_tokens": 1310720, - "max_output_tokens": 102400, - "max_tokens": 102400, + "max_output_tokens": 943718, + "max_tokens": 943718, "mode": "chat", "source": "https://openrouter.ai/api/v1/models", "supports_audio_input": false, @@ -68252,9 +68302,9 @@ "supports_web_search": false }, "openrouter/z-ai/glm-5.3": { - "input_cost_per_token": 9.1e-07, - "output_cost_per_token": 2.86e-06, - "cache_read_input_token_cost": 1.69e-07, + "input_cost_per_token": 8.4e-07, + "output_cost_per_token": 2.64e-06, + "cache_read_input_token_cost": 1.56e-07, "litellm_provider": "openrouter", "max_input_tokens": 1310720, "max_output_tokens": 131072, @@ -68941,9 +68991,9 @@ "supports_web_search": true }, "openrouter/deepseek/deepseek-v4-flash": { - "input_cost_per_token": 5.544e-08, - "output_cost_per_token": 1.1088e-07, - "cache_read_input_token_cost": 1.1088e-08, + "input_cost_per_token": 8.8606e-08, + "output_cost_per_token": 1.77212e-07, + "cache_read_input_token_cost": 1.77212e-08, "litellm_provider": "openrouter", "max_input_tokens": 1048576, "max_output_tokens": 384000, @@ -70299,8 +70349,8 @@ "supports_web_search": true }, "openrouter/meta-llama/llama-4-maverick": { - "input_cost_per_token": 2e-07, - "output_cost_per_token": 8e-07, + "input_cost_per_token": 1.875e-07, + "output_cost_per_token": 6.525e-07, "litellm_provider": "openrouter", "max_input_tokens": 1048576, "max_output_tokens": 16384, @@ -72854,6 +72904,16 @@ "supports_reasoning": true, "supports_vision": true }, + "openrouter/typesafe/jev-1.13": { + "input_cost_per_token": 4.2e-08, + "litellm_provider": "openrouter", + "max_input_tokens": 32000, + "max_output_tokens": 28800, + "max_tokens": 28800, + "mode": "evaluation", + "output_cost_per_token": 0.0, + "source": "https://openrouter.ai/typesafe/jev-1.13" + }, "typesafe/jev-1.13.0": { "input_cost_per_token": 4.2e-08, "litellm_provider": "typesafe", @@ -73252,14 +73312,14 @@ "supports_web_search": true }, "openrouter/~z-ai/glm-flash-latest": { - "cache_read_input_token_cost": 2e-08, - "input_cost_per_token": 7.5e-08, + "cache_read_input_token_cost": 5e-08, + "input_cost_per_token": 1.5e-07, "litellm_provider": "openrouter", "max_input_tokens": 1310720, - "max_output_tokens": 102400, - "max_tokens": 102400, + "max_output_tokens": 943718, + "max_tokens": 943718, "mode": "chat", - "output_cost_per_token": 2.5e-07, + "output_cost_per_token": 5e-07, "source": "https://openrouter.ai/api/v1/models", "supports_audio_input": false, "supports_function_calling": true, @@ -73272,14 +73332,14 @@ "supports_web_search": false }, "openrouter/~z-ai/glm-latest": { - "cache_read_input_token_cost": 1.69e-07, - "input_cost_per_token": 9.1e-07, + "cache_read_input_token_cost": 1.56e-07, + "input_cost_per_token": 8.4e-07, "litellm_provider": "openrouter", "max_input_tokens": 1310720, "max_output_tokens": 131072, "max_tokens": 131072, "mode": "chat", - "output_cost_per_token": 2.86e-06, + "output_cost_per_token": 2.64e-06, "source": "https://openrouter.ai/api/v1/models", "supports_audio_input": false, "supports_function_calling": true, @@ -73794,6 +73854,7 @@ "supports_web_search": false }, "openrouter/bytedance-seed/seed-1.6": { + "deprecation_date": "2026-11-11", "input_cost_per_token": 2.5e-07, "input_cost_per_token_above_128k_tokens": 5e-07, "litellm_provider": "openrouter", @@ -73815,6 +73876,7 @@ "supports_web_search": false }, "openrouter/bytedance-seed/seed-1.6-flash": { + "deprecation_date": "2026-11-11", "input_cost_per_token": 7.5e-08, "input_cost_per_token_above_128k_tokens": 1e-07, "litellm_provider": "openrouter", @@ -73855,6 +73917,7 @@ "supports_web_search": false }, "openrouter/bytedance-seed/seed-2.0-code": { + "deprecation_date": "2026-11-11", "input_cost_per_token": 5e-07, "input_cost_per_token_above_128k_tokens": 1e-06, "litellm_provider": "openrouter", @@ -76876,6 +76939,26 @@ "supports_vision": true, "supports_web_search": true }, + "moonshotai.kimi-k3": { + "cache_creation_input_token_cost": 3.75e-06, + "cache_read_input_token_cost": 3e-07, + "input_cost_per_token": 3e-06, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 1000000, + "max_output_tokens": 131072, + "max_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 1.5e-05, + "source": "https://aws.amazon.com/bedrock/pricing/", + "supports_audio_input": false, + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true + }, "global.moonshotai.kimi-k3": { "cache_creation_input_token_cost": 3.75e-06, "cache_read_input_token_cost": 3e-07, @@ -76975,5 +77058,473 @@ "supports_tool_choice": true, "supports_vision": true, "supports_web_search": false + }, + "xiaomi_mimo/mimo-v2.6-pro": { + "cache_read_input_token_cost": 3.6e-09, + "input_cost_per_token": 4.35e-07, + "litellm_provider": "xiaomi_mimo", + "max_input_tokens": 1048576, + "max_output_tokens": 131072, + "max_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 8.7e-07, + "source": "https://platform.xiaomimimo.com/static/docs/price/pay-as-you-go.md", + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_audio_input": true, + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_video_input": true, + "supports_vision": true + }, + "xiaomi_mimo/mimo-v2.6-flash": { + "cache_read_input_token_cost": 2.8e-09, + "input_cost_per_token": 1.4e-07, + "litellm_provider": "xiaomi_mimo", + "max_input_tokens": 1048576, + "max_output_tokens": 131072, + "max_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 2.8e-07, + "source": "https://platform.xiaomimimo.com/static/docs/price/pay-as-you-go.md", + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_audio_input": true, + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_video_input": true, + "supports_vision": true + }, + "xai/grok-4.20-0309": { + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 1.25e-06, + "litellm_provider": "xai", + "max_input_tokens": 1000000, + "max_output_tokens": 1000000, + "max_tokens": 1000000, + "mode": "chat", + "output_cost_per_token": 2.5e-06, + "source": "https://api.x.ai/v1/language-models", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true, + "input_cost_per_token_above_200k_tokens": 2.5e-06, + "output_cost_per_token_above_200k_tokens": 5e-06, + "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_image_token": 1.25e-06, + "supports_prompt_caching": true, + "supports_response_schema": true + }, + "xai/grok-4.20-beta": { + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 1.25e-06, + "litellm_provider": "xai", + "max_input_tokens": 1000000, + "max_output_tokens": 1000000, + "max_tokens": 1000000, + "mode": "chat", + "output_cost_per_token": 2.5e-06, + "source": "https://api.x.ai/v1/language-models", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true, + "input_cost_per_token_above_200k_tokens": 2.5e-06, + "output_cost_per_token_above_200k_tokens": 5e-06, + "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_image_token": 1.25e-06, + "supports_prompt_caching": true, + "supports_response_schema": true + }, + "xai/grok-4.20-beta-0309": { + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 1.25e-06, + "litellm_provider": "xai", + "max_input_tokens": 1000000, + "max_output_tokens": 1000000, + "max_tokens": 1000000, + "mode": "chat", + "output_cost_per_token": 2.5e-06, + "source": "https://api.x.ai/v1/language-models", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true, + "input_cost_per_token_above_200k_tokens": 2.5e-06, + "output_cost_per_token_above_200k_tokens": 5e-06, + "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_image_token": 1.25e-06, + "supports_prompt_caching": true, + "supports_response_schema": true + }, + "xai/grok-4.20-beta-latest": { + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 1.25e-06, + "litellm_provider": "xai", + "max_input_tokens": 1000000, + "max_output_tokens": 1000000, + "max_tokens": 1000000, + "mode": "chat", + "output_cost_per_token": 2.5e-06, + "source": "https://api.x.ai/v1/language-models", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true, + "input_cost_per_token_above_200k_tokens": 2.5e-06, + "output_cost_per_token_above_200k_tokens": 5e-06, + "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_image_token": 1.25e-06, + "supports_prompt_caching": true, + "supports_response_schema": true + }, + "xai/grok-4.20-beta-latest-non-reasoning": { + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 1.25e-06, + "litellm_provider": "xai", + "max_input_tokens": 1000000, + "max_output_tokens": 1000000, + "max_tokens": 1000000, + "mode": "chat", + "output_cost_per_token": 2.5e-06, + "source": "https://api.x.ai/v1/language-models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true, + "input_cost_per_token_above_200k_tokens": 2.5e-06, + "output_cost_per_token_above_200k_tokens": 5e-06, + "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_image_token": 1.25e-06, + "supports_response_schema": true + }, + "xai/grok-4.20-beta-latest-reasoning": { + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 1.25e-06, + "litellm_provider": "xai", + "max_input_tokens": 1000000, + "max_output_tokens": 1000000, + "max_tokens": 1000000, + "mode": "chat", + "output_cost_per_token": 2.5e-06, + "source": "https://api.x.ai/v1/language-models", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true, + "input_cost_per_token_above_200k_tokens": 2.5e-06, + "output_cost_per_token_above_200k_tokens": 5e-06, + "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_image_token": 1.25e-06, + "supports_prompt_caching": true, + "supports_response_schema": true + }, + "xai/grok-4.20-beta-non-reasoning": { + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 1.25e-06, + "litellm_provider": "xai", + "max_input_tokens": 1000000, + "max_output_tokens": 1000000, + "max_tokens": 1000000, + "mode": "chat", + "output_cost_per_token": 2.5e-06, + "source": "https://api.x.ai/v1/language-models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true, + "input_cost_per_token_above_200k_tokens": 2.5e-06, + "output_cost_per_token_above_200k_tokens": 5e-06, + "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_image_token": 1.25e-06, + "supports_response_schema": true + }, + "xai/grok-4.20-beta-reasoning": { + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 1.25e-06, + "litellm_provider": "xai", + "max_input_tokens": 1000000, + "max_output_tokens": 1000000, + "max_tokens": 1000000, + "mode": "chat", + "output_cost_per_token": 2.5e-06, + "source": "https://api.x.ai/v1/language-models", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true, + "input_cost_per_token_above_200k_tokens": 2.5e-06, + "output_cost_per_token_above_200k_tokens": 5e-06, + "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_image_token": 1.25e-06, + "supports_prompt_caching": true, + "supports_response_schema": true + }, + "xai/grok-4.20-experimental-beta-0304": { + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 1.25e-06, + "litellm_provider": "xai", + "max_input_tokens": 1000000, + "max_output_tokens": 1000000, + "max_tokens": 1000000, + "mode": "chat", + "output_cost_per_token": 2.5e-06, + "source": "https://api.x.ai/v1/language-models", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true, + "input_cost_per_token_above_200k_tokens": 2.5e-06, + "output_cost_per_token_above_200k_tokens": 5e-06, + "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_image_token": 1.25e-06, + "supports_prompt_caching": true, + "supports_response_schema": true + }, + "xai/grok-4.20-experimental-beta-0304-non-reasoning": { + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 1.25e-06, + "litellm_provider": "xai", + "max_input_tokens": 1000000, + "max_output_tokens": 1000000, + "max_tokens": 1000000, + "mode": "chat", + "output_cost_per_token": 2.5e-06, + "source": "https://api.x.ai/v1/language-models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true, + "input_cost_per_token_above_200k_tokens": 2.5e-06, + "output_cost_per_token_above_200k_tokens": 5e-06, + "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_image_token": 1.25e-06, + "supports_response_schema": true + }, + "xai/grok-4.20-experimental-beta-0304-reasoning": { + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 1.25e-06, + "litellm_provider": "xai", + "max_input_tokens": 1000000, + "max_output_tokens": 1000000, + "max_tokens": 1000000, + "mode": "chat", + "output_cost_per_token": 2.5e-06, + "source": "https://api.x.ai/v1/language-models", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true, + "input_cost_per_token_above_200k_tokens": 2.5e-06, + "output_cost_per_token_above_200k_tokens": 5e-06, + "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_image_token": 1.25e-06, + "supports_prompt_caching": true, + "supports_response_schema": true + }, + "xai/grok-4.20-experimental-beta-latest": { + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 1.25e-06, + "litellm_provider": "xai", + "max_input_tokens": 1000000, + "max_output_tokens": 1000000, + "max_tokens": 1000000, + "mode": "chat", + "output_cost_per_token": 2.5e-06, + "source": "https://api.x.ai/v1/language-models", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true, + "input_cost_per_token_above_200k_tokens": 2.5e-06, + "output_cost_per_token_above_200k_tokens": 5e-06, + "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_image_token": 1.25e-06, + "supports_prompt_caching": true, + "supports_response_schema": true + }, + "xai/grok-4.20-experimental-beta-non-reasoning-latest": { + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 1.25e-06, + "litellm_provider": "xai", + "max_input_tokens": 1000000, + "max_output_tokens": 1000000, + "max_tokens": 1000000, + "mode": "chat", + "output_cost_per_token": 2.5e-06, + "source": "https://api.x.ai/v1/language-models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true, + "input_cost_per_token_above_200k_tokens": 2.5e-06, + "output_cost_per_token_above_200k_tokens": 5e-06, + "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_image_token": 1.25e-06, + "supports_response_schema": true + }, + "xai/grok-4.20-experimental-beta-reasoning-latest": { + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 1.25e-06, + "litellm_provider": "xai", + "max_input_tokens": 1000000, + "max_output_tokens": 1000000, + "max_tokens": 1000000, + "mode": "chat", + "output_cost_per_token": 2.5e-06, + "source": "https://api.x.ai/v1/language-models", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true, + "input_cost_per_token_above_200k_tokens": 2.5e-06, + "output_cost_per_token_above_200k_tokens": 5e-06, + "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_image_token": 1.25e-06, + "supports_prompt_caching": true, + "supports_response_schema": true + }, + "xai/grok-4.20-multi-agent-beta-latest": { + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 1.25e-06, + "litellm_provider": "xai", + "max_input_tokens": 1000000, + "max_output_tokens": 1000000, + "max_tokens": 1000000, + "mode": "responses", + "output_cost_per_token": 2.5e-06, + "source": "https://api.x.ai/v1/language-models", + "supports_function_calling": false, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": false, + "supports_vision": true, + "supports_web_search": true, + "input_cost_per_token_above_200k_tokens": 2.5e-06, + "output_cost_per_token_above_200k_tokens": 5e-06, + "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_image_token": 1.25e-06, + "supports_response_schema": true, + "supported_endpoints": [ + "/v1/responses" + ] + }, + "xai/grok-4.20-multi-agent-experimental-beta-0304": { + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 1.25e-06, + "litellm_provider": "xai", + "max_input_tokens": 1000000, + "max_output_tokens": 1000000, + "max_tokens": 1000000, + "mode": "responses", + "output_cost_per_token": 2.5e-06, + "source": "https://api.x.ai/v1/language-models", + "supports_function_calling": false, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": false, + "supports_vision": true, + "supports_web_search": true, + "input_cost_per_token_above_200k_tokens": 2.5e-06, + "output_cost_per_token_above_200k_tokens": 5e-06, + "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_image_token": 1.25e-06, + "supports_response_schema": true, + "supported_endpoints": [ + "/v1/responses" + ] + }, + "xai/grok-4.20-multi-agent-experimental-beta-latest": { + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 1.25e-06, + "litellm_provider": "xai", + "max_input_tokens": 1000000, + "max_output_tokens": 1000000, + "max_tokens": 1000000, + "mode": "responses", + "output_cost_per_token": 2.5e-06, + "source": "https://api.x.ai/v1/language-models", + "supports_function_calling": false, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": false, + "supports_vision": true, + "supports_web_search": true, + "input_cost_per_token_above_200k_tokens": 2.5e-06, + "output_cost_per_token_above_200k_tokens": 5e-06, + "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_image_token": 1.25e-06, + "supports_response_schema": true, + "supported_endpoints": [ + "/v1/responses" + ] + }, + "xai/grok-4.20-non-reasoning-gv2": { + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 1.25e-06, + "litellm_provider": "xai", + "max_input_tokens": 1000000, + "max_output_tokens": 1000000, + "max_tokens": 1000000, + "mode": "chat", + "output_cost_per_token": 2.5e-06, + "source": "https://api.x.ai/v1/language-models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true, + "input_cost_per_token_above_200k_tokens": 2.5e-06, + "output_cost_per_token_above_200k_tokens": 5e-06, + "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_image_token": 1.25e-06, + "supports_response_schema": true + }, + "xai/grok-4.20-reasoning-gv2": { + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 1.25e-06, + "litellm_provider": "xai", + "max_input_tokens": 1000000, + "max_output_tokens": 1000000, + "max_tokens": 1000000, + "mode": "chat", + "output_cost_per_token": 2.5e-06, + "source": "https://api.x.ai/v1/language-models", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true, + "input_cost_per_token_above_200k_tokens": 2.5e-06, + "output_cost_per_token_above_200k_tokens": 5e-06, + "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_image_token": 1.25e-06, + "supports_prompt_caching": true, + "supports_response_schema": true } } diff --git a/model_prices_and_context_window.schema.json b/model_prices_and_context_window.schema.json index 0509516ac32..f3a4e614f59 100644 --- a/model_prices_and_context_window.schema.json +++ b/model_prices_and_context_window.schema.json @@ -586,6 +586,18 @@ "type": "number", "minimum": 0 }, + "output_cost_per_image_1024": { + "type": "number", + "minimum": 0 + }, + "output_cost_per_image_1536": { + "type": "number", + "minimum": 0 + }, + "output_cost_per_image_512": { + "type": "number", + "minimum": 0 + }, "output_cost_per_image_token": { "type": "number", "minimum": 0 diff --git a/pyproject.toml b/pyproject.toml index 95da93df41e..a1b276e4e8c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,12 +15,14 @@ dependencies = [ # When changing a floor, verify it installs + imports on every supported # Python with: `uv pip install --resolution=lowest-direct .` "fastuuid>=0.14.0,<1.0", + "filelock>=3.16.1,<4.0", "httpx[http2]>=0.28.0,<1.0", "openai>=2.20.0,<3.0.0", "python-dotenv>=1.0.0,<2.0", "tiktoken>=0.8.0,<1.0; python_version < '3.14'", "tiktoken>=0.12.0,<1.0; python_version >= '3.14'", "importlib-metadata>=8.0.0,<9.0", + "packaging>=24.0", "tokenizers>=0.21.0,<1.0", "click>=8.0.0,<9.0", "jinja2>=3.1.6,<4.0", diff --git a/schema.prisma b/schema.prisma index 2d7e557a9d1..85996430bc5 100644 --- a/schema.prisma +++ b/schema.prisma @@ -73,6 +73,7 @@ model LiteLLM_AgentsTable { static_headers Json? @default("{}") extra_headers String[] @default([]) agent_access_groups String[] @default([]) + access_group_ids String[] @default([]) object_permission_id String? object_permission LiteLLM_ObjectPermissionTable? @relation(fields: [object_permission_id], references: [object_permission_id]) spend Float @default(0.0) @@ -246,6 +247,8 @@ model LiteLLM_UserTable { organization_id String? object_permission_id String? password String? + password_reset_required Boolean? + last_breach_check_at DateTime? teams String[] @default([]) user_role String? max_budget Float? @@ -1621,6 +1624,47 @@ model LiteLLM_AutoRouterSession { @@index([last_turn_at], map: "idx_autorouter_session_last_turn") } +model LiteLLM_AutoRouterUserSession { + user_id String + api_key String + session_id String + router_name String + router_type String + first_turn_at DateTime + last_turn_at DateTime + last_model String + models Json @default("{}") + turns Int @default(0) + unordered_turns Int @default(0) + covered_turns Int @default(0) + cache_hits Int @default(0) + same_model_turns Int @default(0) + same_model_hits Int @default(0) + first_visit_turns Int @default(0) + first_visit_hits Int @default(0) + return_turns Int @default(0) + return_hits Int @default(0) + return_expired_misses Int @default(0) + return_within_ttl_misses Int @default(0) + ttl_5m_turns Int @default(0) + ttl_1h_turns Int @default(0) + total_tokens BigInt @default(0) + spend Float @default(0) + saved_spend Float @default(0) + savings_estimated_turns Int @default(0) + savings_estimated_actual_spend Float @default(0) + savings_estimated_saved_spend Float @default(0) + savings_estimated_baseline_models Json @default("{}") + classifier_cost Float @default(0) + classifier_cost_recorded_turns Int @default(0) + tier_turns Json @default("{}") + baseline_models Json @default("{}") + + @@id([user_id, api_key, session_id, router_name]) + @@index([last_turn_at], map: "idx_autorouter_user_session_last_turn") + @@index([user_id, last_turn_at], map: "idx_autorouter_user_session_user_last_turn") +} + // Shadow eval: evaluation of an auto-router against one or more keys' live traffic, in // either direction. forward duplicates the requests the keys did not route through the // router through it, answering whether they should adopt it; reverse duplicates the diff --git a/terraform/provider/CHANGELOG.md b/terraform/provider/CHANGELOG.md index ee5b42fe0b7..8f40ed6dfb7 100644 --- a/terraform/provider/CHANGELOG.md +++ b/terraform/provider/CHANGELOG.md @@ -18,6 +18,15 @@ longer signal it. - **team_member_add**: `tpm_limit`, `rpm_limit`, `budget_duration`, and `allowed_models` attributes on `litellm_team_member_add`, applied to every member of the resource; `budget_duration` and `allowed_models` ride on `/team/member_add`, while the limits are sent through `/team/member_update`, which is where the proxy accepts them - **team**: Optional `team_id` argument on `litellm_team`, so teams can be created with a stable, human-readable ID instead of a provider-generated UUID; changing it forces replacement +- `litellm_jwt_key_mapping` accepts `token_id` as an alternative to `key`, so a + mapping can name its virtual key by the SHA-256 hash the proxy stores instead + of by the plaintext. Exactly one of the two is required. This is what lets a + mapping reference a key managed in the same configuration + (`token_id = litellm_key.foo.token_id`), which `key` cannot do, because + `litellm_key` marks its generated key write-only and referencing it fails at + plan time. `POST /jwt/key/mapping/new` and `/jwt/key/mapping/update` gained a + matching `token` field, validated as 64 lowercase hex characters so a + plaintext key sent by mistake is rejected instead of hashed twice - **jwt_key_mapping**: New `litellm_jwt_key_mapping` resource for the proxy's JWT to virtual key mappings, so JWT clients identified by a claim (`client_id`, `azp`, `sub`) map to virtual keys and inherit their models, budgets and rate limits. Supports `description` and `is_active`, rotating the mapped key in place, and forces replacement when the claim name or value changes - **team**: `soft_budget`, `tags`, and `soft_budget_alerting_emails` attributes on `litellm_team`, matching what `/team/new` and `/team/update` already accept; `soft_budget_alerting_emails` is sent under `metadata`, where the proxy reads it - **user**: New `litellm_user` resource and `litellm_user` / `litellm_users` data sources for managing internal users diff --git a/terraform/provider/docs/resources/jwt_key_mapping.md b/terraform/provider/docs/resources/jwt_key_mapping.md index fbc30947113..726c4b16021 100644 --- a/terraform/provider/docs/resources/jwt_key_mapping.md +++ b/terraform/provider/docs/resources/jwt_key_mapping.md @@ -65,7 +65,8 @@ resource "litellm_jwt_key_mapping" "developer" { - `jwt_claim_name` - (Required, ForceNew) Name of the JWT claim to match on, for example `client_id`, `azp` or `sub`. Must match `virtual_key_claim_field` in the proxy JWT config - `jwt_claim_value` - (Required, ForceNew) Value of the claim identifying the JWT client. Unique together with `jwt_claim_name`, so a second mapping for the same pair fails with a 409 -- `key` - (Required, Sensitive) The virtual key this claim value maps to. It has to exist already, otherwise the proxy rejects the mapping with `The provided key does not match an existing virtual key` +- `key` - (Optional, Sensitive) The virtual key this claim value maps to, as plaintext. It has to exist already, otherwise the proxy rejects the mapping with `The provided key does not match an existing virtual key`. Exactly one of `key` or `token_id` is required. `litellm_key` marks its generated `key` write-only, so this cannot reference a `litellm_key` resource -- use `token_id` for that, or supply the plaintext from a variable or a secret manager +- `token_id` - (Optional) The SHA-256 hash of the virtual key this claim value maps to, which is what the proxy stores. `litellm_key` exposes it as `token_id`, so unlike `key` it can be referenced directly from a `litellm_key` resource. Not a secret, so it is not marked sensitive. Exactly one of `key` or `token_id` is required - `description` - (Optional) Description of the mapping - `is_active` - (Optional) Whether the mapping is active. Inactive mappings are ignored during JWT auth. Defaults to `true` diff --git a/terraform/provider/litellm/resource_jwt_key_mapping.go b/terraform/provider/litellm/resource_jwt_key_mapping.go index e606e865737..ea968821527 100644 --- a/terraform/provider/litellm/resource_jwt_key_mapping.go +++ b/terraform/provider/litellm/resource_jwt_key_mapping.go @@ -29,10 +29,17 @@ func resourceLiteLLMJWTKeyMapping() *schema.Resource { Description: "Value of the claim identifying the JWT client. Unique together with jwt_claim_name", }, "key": { - Type: schema.TypeString, - Required: true, - Sensitive: true, - Description: "The virtual key this claim value maps to. The proxy stores only a hash of it and never returns it, so drift on this attribute cannot be detected and Terraform tracks the configured value", + Type: schema.TypeString, + Optional: true, + Sensitive: true, + ExactlyOneOf: []string{"key", "token_id"}, + Description: "The virtual key this claim value maps to, as plaintext. The proxy stores only a hash of it and never returns it, so drift on this attribute cannot be detected and Terraform tracks the configured value. litellm_key marks its generated key write-only, so this cannot reference a litellm_key resource; use token_id for that, or supply the plaintext from a variable or a secret manager", + }, + "token_id": { + Type: schema.TypeString, + Optional: true, + ExactlyOneOf: []string{"key", "token_id"}, + Description: "The SHA-256 hash of the virtual key this claim value maps to, which is what the proxy stores. litellm_key exposes it as token_id, so unlike key it can be referenced directly from a litellm_key resource. Not a secret, so it is not marked sensitive", }, "description": { Type: schema.TypeString, diff --git a/terraform/provider/litellm/resource_jwt_key_mapping_crud.go b/terraform/provider/litellm/resource_jwt_key_mapping_crud.go index 725235305f6..6c3883de2e1 100644 --- a/terraform/provider/litellm/resource_jwt_key_mapping_crud.go +++ b/terraform/provider/litellm/resource_jwt_key_mapping_crud.go @@ -19,6 +19,7 @@ func resourceLiteLLMJWTKeyMappingCreate(d *schema.ResourceData, m interface{}) e JWTClaimName: d.Get("jwt_claim_name").(string), JWTClaimValue: d.Get("jwt_claim_value").(string), Key: d.Get("key").(string), + Token: d.Get("token_id").(string), Description: d.Get("description").(string), } @@ -95,6 +96,7 @@ func resourceLiteLLMJWTKeyMappingUpdate(d *schema.ResourceData, m interface{}) e client := m.(*Client) oldKey, _ := d.GetChange("key") + oldTokenID, _ := d.GetChange("token_id") oldDescription, _ := d.GetChange("description") oldIsActive, _ := d.GetChange("is_active") @@ -104,6 +106,7 @@ func resourceLiteLLMJWTKeyMappingUpdate(d *schema.ResourceData, m interface{}) e // attempting to resync, so a failed refresh can't leave the rejected // values persisted into state. d.Set("key", oldKey) + d.Set("token_id", oldTokenID) d.Set("description", oldDescription) d.Set("is_active", oldIsActive) if readErr := resourceLiteLLMJWTKeyMappingRead(d, m); readErr != nil { @@ -146,6 +149,7 @@ func updateJWTKeyMapping(d *schema.ResourceData, client *Client) error { updateRequest := JWTKeyMappingUpdateRequest{ ID: d.Id(), Key: d.Get("key").(string), + Token: d.Get("token_id").(string), Description: d.Get("description").(string), IsActive: d.Get("is_active").(bool), } diff --git a/terraform/provider/litellm/resource_jwt_key_mapping_crud_test.go b/terraform/provider/litellm/resource_jwt_key_mapping_crud_test.go index 8007d1d4e08..27b75849fe6 100644 --- a/terraform/provider/litellm/resource_jwt_key_mapping_crud_test.go +++ b/terraform/provider/litellm/resource_jwt_key_mapping_crud_test.go @@ -628,3 +628,99 @@ func TestJWTKeyMappingCreateDoesNotLeakKeyInErrors(t *testing.T) { t.Fatalf("the virtual key must be redacted in errors, got %v", err) } } + +func TestJWTKeyMappingCreateSendsTokenIDAndOmitsKey(t *testing.T) { + srv, calls := jwtKeyMappingTestServer(t, jwtKeyMappingFixture()) + defer srv.Close() + + const tokenHash = "1923314ae0efc8b2523c7d421bac5a7cf88df291273b139948b526d396974a41" + + client := NewClient(srv.URL, "test-key", true) + d := schema.TestResourceDataRaw(t, resourceLiteLLMJWTKeyMapping().Schema, map[string]interface{}{ + "jwt_claim_name": "client_id", + "jwt_claim_value": "dev-alice", + "token_id": tokenHash, + "is_active": true, + }) + + if err := resourceLiteLLMJWTKeyMappingCreate(d, client); err != nil { + t.Fatalf("create failed: %v", err) + } + + create := (*calls)[0] + if create.Body["token"] != tokenHash { + t.Fatalf("token hash not sent: %v", create.Body["token"]) + } + if _, sent := create.Body["key"]; sent { + t.Fatalf("key must be omitted when token_id is used, got: %v", create.Body) + } +} + +func TestJWTKeyMappingCreateOmitsTokenWhenKeyIsUsed(t *testing.T) { + srv, calls := jwtKeyMappingTestServer(t, jwtKeyMappingFixture()) + defer srv.Close() + + client := NewClient(srv.URL, "test-key", true) + d := schema.TestResourceDataRaw(t, resourceLiteLLMJWTKeyMapping().Schema, map[string]interface{}{ + "jwt_claim_name": "client_id", + "jwt_claim_value": "dev-alice", + "key": "sk-abc123", + "is_active": true, + }) + + if err := resourceLiteLLMJWTKeyMappingCreate(d, client); err != nil { + t.Fatalf("create failed: %v", err) + } + + create := (*calls)[0] + if create.Body["key"] != "sk-abc123" { + t.Fatalf("virtual key not sent: %v", create.Body["key"]) + } + if _, sent := create.Body["token"]; sent { + t.Fatalf("token must be omitted when key is used, got: %v", create.Body) + } +} + +func TestJWTKeyMappingUpdateSendsTokenIDAndOmitsKey(t *testing.T) { + srv, calls := jwtKeyMappingTestServer(t, jwtKeyMappingFixture()) + defer srv.Close() + + const oldHash = "1111111111111111111111111111111111111111111111111111111111111111" + const newHash = "2222222222222222222222222222222222222222222222222222222222222222" + + client := NewClient(srv.URL, "test-key", true) + d := resourceDataWithChange(t, + map[string]string{ + "id": "map-abc-123", + "jwt_claim_name": "client_id", + "jwt_claim_value": "dev-alice", + "token_id": oldHash, + "is_active": "true", + }, + map[string]interface{}{ + "jwt_claim_name": "client_id", + "jwt_claim_value": "dev-alice", + "token_id": newHash, + "is_active": true, + }) + + if err := resourceLiteLLMJWTKeyMappingUpdate(d, client); err != nil { + t.Fatalf("update failed: %v", err) + } + + var update *jwtKeyMappingCall + for i := range *calls { + if (*calls)[i].Path == "/jwt/key/mapping/update" { + update = &(*calls)[i] + } + } + if update == nil { + t.Fatalf("expected an update call, got %v", *calls) + } + if update.Body["token"] != newHash { + t.Fatalf("new token hash not sent: %v", update.Body["token"]) + } + if _, sent := update.Body["key"]; sent { + t.Fatalf("key must be omitted when token_id is used, got: %v", update.Body) + } +} diff --git a/terraform/provider/litellm/types.go b/terraform/provider/litellm/types.go index 7bef44409fd..8bcf7dc4fe3 100644 --- a/terraform/provider/litellm/types.go +++ b/terraform/provider/litellm/types.go @@ -276,13 +276,15 @@ type VectorStoreInfoRequest struct { type JWTKeyMappingRequest struct { JWTClaimName string `json:"jwt_claim_name"` JWTClaimValue string `json:"jwt_claim_value"` - Key string `json:"key"` + Key string `json:"key,omitempty"` + Token string `json:"token,omitempty"` Description string `json:"description,omitempty"` } type JWTKeyMappingUpdateRequest struct { ID string `json:"id"` Key string `json:"key,omitempty"` + Token string `json:"token,omitempty"` Description string `json:"description"` IsActive bool `json:"is_active"` } diff --git a/terraform/provider/tools/endpointaudit/coverage_allowlist.txt b/terraform/provider/tools/endpointaudit/coverage_allowlist.txt index 4ea64b152f1..f8277c83a64 100644 --- a/terraform/provider/tools/endpointaudit/coverage_allowlist.txt +++ b/terraform/provider/tools/endpointaudit/coverage_allowlist.txt @@ -86,6 +86,7 @@ POST /team/key/bulk_update POST /team/permissions_bulk_update POST /team/{team_id}/disable_logging POST /user/bulk_update +POST /user/password/change # Alternate method or path for functionality the provider already manages elsewhere GET /credentials/by_model/{model_id} diff --git a/tests/e2e/batches/COVERAGE.md b/tests/e2e/batches/COVERAGE.md index d18bed6c088..1731b4c620d 100644 --- a/tests/e2e/batches/COVERAGE.md +++ b/tests/e2e/batches/COVERAGE.md @@ -21,6 +21,7 @@ failures are hard test failures (see `tests/e2e/AGENTS.md`). | Vertex AI | yes | yes | yes | yes | yes (provider-transformed) | GCS (`gcs_bucket_name` / `GCS_BUCKET_NAME` on model) | | Bedrock | yes (unified only) | yes | yes | yes (unfiltered managed list) | yes (provider-transformed) | S3 (`s3_bucket_name` + `aws_*` + `AWS_BATCH_ROLE_ARN` on model) | | Bedrock GovCloud (`us-gov-west-1`) | yes (unified only) | yes | no | no | yes (provider-transformed) | S3 (`s3_bucket_name` + `aws_*` on model, resolved from `AWS_GOVCLOUD_ACCESS_KEY_ID` / `AWS_GOVCLOUD_SECRET_ACCESS_KEY` / `AWS_GOVCLOUD_BATCH_S3_BUCKET` / `AWS_GOVCLOUD_BATCH_ROLE_ARN`) | +| Bedrock split S3 identity | no | no | no | no | yes (file upload, content, delete) | S3 signed with `s3_access_key_id` / `s3_secret_access_key` (`AWS_S3_ONLY_ACCESS_KEY_ID` / `AWS_S3_ONLY_SECRET_ACCESS_KEY`, object rights on `AWS_BATCH_S3_BUCKET` only) while `aws_*` is `AWS_BEDROCK_ONLY_ACCESS_KEY_ID` / `AWS_BEDROCK_ONLY_SECRET_ACCESS_KEY`, an identity with no S3 rights on that bucket | Bedrock cancel maps to `StopModelInvocationJob` and comes back `cancelling`; the lifecycle asserts it the same way it does for OpenAI (`_CANCEL_ASSERTED_PROVIDERS`). diff --git a/tests/e2e/batches/test_batches_e2e.py b/tests/e2e/batches/test_batches_e2e.py index 9b3c06d9a1b..9bb6d05bec8 100644 --- a/tests/e2e/batches/test_batches_e2e.py +++ b/tests/e2e/batches/test_batches_e2e.py @@ -1024,6 +1024,72 @@ class TestBedrockBatchAssumeRole: assert fetched.id == batch.id +def _split_s3_identity_params() -> LiteLLMParamsBody: + return LiteLLMParamsBody( + model=ASSUME_ROLE_RAW_MODEL, + aws_access_key_id="os.environ/AWS_BEDROCK_ONLY_ACCESS_KEY_ID", + aws_secret_access_key="os.environ/AWS_BEDROCK_ONLY_SECRET_ACCESS_KEY", + aws_region_name="os.environ/AWS_REGION", + s3_region_name="os.environ/AWS_REGION", + s3_bucket_name="os.environ/AWS_BATCH_S3_BUCKET", + s3_access_key_id="os.environ/AWS_S3_ONLY_ACCESS_KEY_ID", + s3_secret_access_key="os.environ/AWS_S3_ONLY_SECRET_ACCESS_KEY", + aws_batch_role_arn="os.environ/AWS_BATCH_ROLE_ARN", + ) + + +class TestBedrockBatchSplitS3Credentials: + """Bedrock batch deployment whose aws_* identity cannot touch the bucket. + + AWS_BEDROCK_ONLY_* is an IAM user with no S3 rights on AWS_BATCH_S3_BUCKET; + AWS_S3_ONLY_* is an IAM user with object rights on that bucket only. Every + S3 call the proxy signs (PutObject on upload, GetObject on content, + DeleteObject on delete) must use the s3_* pair, otherwise S3 answers 403. + """ + + @pytest.mark.covers( + "llm.files.bedrock.split_s3_credentials.nonstream.works", + exercised_on=["files"], + ) + def test_file_lifecycle_signs_s3_with_s3_credentials( + self, client: BatchClient, resources: ResourceManager + ) -> None: + model_name = batch_model_name("bedrock-split-s3-batch") + model_id = client.create_model(model_name, _split_s3_identity_params()) + resources.defer(lambda: client.delete_model(model_id)) + key = resources.key() + + uploaded = client.upload_file( + content=render_jsonl(ASSUME_ROLE_RAW_MODEL), + form=FileUploadForm(purpose="batch", target_model_names=model_name), + key=key, + ) + assert isinstance(uploaded, Success), ( + f"upload must sign the S3 PutObject with s3_access_key_id, got {uploaded!r}" + ) + file = uploaded.data + resources.defer(lambda: cleanup_file(client, file.id, key=key)) + assert_file_object(file, provider="bedrock") + + downloaded = client.proxy.transport.download( + f"/v1/files/{file.id}/content", + headers=client.proxy.transport.bearer(key), + ) + assert downloaded.status_code == 200, ( + f"content must sign the S3 GetObject with s3_access_key_id, " + f"got {downloaded.status_code}: {downloaded.body[:300]}" + ) + assert all(json.loads(line) for line in downloaded.body.strip().splitlines()), ( + f"content download returned non-JSONL body: {downloaded.body[:200]}" + ) + + deleted = client.delete_file(file.id, key=key) + assert isinstance(deleted, Success), ( + f"delete must sign the S3 DeleteObject with s3_access_key_id, got {deleted!r}" + ) + assert deleted.data.id == file.id, f"delete confirmed a different file: {deleted.data!r}" + + GOVCLOUD_REGION: Final = "us-gov-west-1" GOVCLOUD_RAW_MODEL: Final = "bedrock/amazon.nova-lite-v1:0" diff --git a/tests/e2e/coverage_registry/guardrail.yaml b/tests/e2e/coverage_registry/guardrail.yaml index 81832bebf49..86eb44f6cb1 100644 --- a/tests/e2e/coverage_registry/guardrail.yaml +++ b/tests/e2e/coverage_registry/guardrail.yaml @@ -6,6 +6,7 @@ - {id: guardrail.presidio.pre_call.logs_masked_entities, module: guardrail, tier: P0, hook_point: pre_call, assertions: [logs_masked_entities], exercised_on: [chat_completions], source: "guardrail_hooks/presidio.py", rationale: "A masking run must record itself on the spend log: the dashboard's guardrail panel renders the masked-entity counts and per-entity scores straight off metadata.guardrail_information, so a run that masks but records nothing leaves an operator unable to audit it"} - {id: guardrail.bedrock.pre_call.blocks, module: guardrail, tier: P0, hook_point: pre_call, assertions: [blocks], exercised_on: [chat_completions], source: "guardrail_hooks/bedrock_guardrails.py", rationale: "AWS content guardrail blocks harmful input"} - {id: guardrail.litellm_content_filter.pre_call.blocks, module: guardrail, tier: P0, hook_point: pre_call, assertions: [blocks], exercised_on: [chat_completions], source: "test_team_disable_global_guardrail_e2e.py", rationale: "Local content-filter default-on blocks banned keyword pre-call"} +- {id: guardrail.litellm_content_filter.pre_call.blocks_video, module: guardrail, tier: P0, hook_point: pre_call, assertions: [blocks], exercised_on: [videos], source: "test_key_guardrail_video_e2e.py", fail_before_fix: proven, rationale: "A content-filter guardrail attached to a key (metadata.guardrails) blocks a banned prompt on POST /v1/videos before the provider is called; before the fix the route's call type was unknown to the unified guardrail hook and the prompt went to the provider unscanned (LIT-6685)"} - {id: guardrail.litellm_content_filter.pre_call.allows, module: guardrail, tier: P0, hook_point: pre_call, assertions: [allows], exercised_on: [chat_completions], source: "test_team_disable_global_guardrail_e2e.py", rationale: "Team disable_global_guardrails bypasses default-on content filter"} - {id: guardrail.litellm_content_filter.apply_endpoint.blocks, module: guardrail, tier: P0, hook_point: apply_endpoint, assertions: [blocks], exercised_on: [chat_completions], source: "guardrail_endpoints.py:apply_guardrail", rationale: "POST /guardrails/apply_guardrail blocks banned content for customers that call the apply surface directly"} - {id: guardrail.litellm_content_filter.apply_endpoint.allows, module: guardrail, tier: P0, hook_point: apply_endpoint, assertions: [allows], exercised_on: [chat_completions], source: "guardrail_endpoints.py:apply_guardrail", rationale: "POST /guardrails/apply_guardrail returns clean text for allowed input"} diff --git a/tests/e2e/coverage_registry/llm_nonconversational.yaml b/tests/e2e/coverage_registry/llm_nonconversational.yaml index 50f9b9808b2..c58c8af44ff 100644 --- a/tests/e2e/coverage_registry/llm_nonconversational.yaml +++ b/tests/e2e/coverage_registry/llm_nonconversational.yaml @@ -47,6 +47,7 @@ - {id: llm.files.vertex.upload.nonstream.works, module: llm, tier: P0, subject_endpoint: files, route: vertex, capability: basic, streaming: nonstream, assertions: [works], source: "batches/capabilities.py:52", rationale: "Vertex file upload to GCS"} - {id: llm.files.bedrock.upload.nonstream.works, module: llm, tier: P0, subject_endpoint: files, route: bedrock_converse, capability: basic, streaming: nonstream, assertions: [works], source: "batches/capabilities.py:59", rationale: "Bedrock file upload to S3"} - {id: llm.files.bedrock.govcloud_partition.nonstream.works, module: llm, tier: P0, subject_endpoint: files, route: bedrock_converse, capability: govcloud_partition, streaming: nonstream, assertions: [works], source: "test_batches_e2e.py", rationale: "Bedrock file upload to an S3 bucket in the us-gov-west-1 partition"} +- {id: llm.files.bedrock.split_s3_credentials.nonstream.works, module: llm, tier: P0, subject_endpoint: files, route: bedrock_converse, capability: split_s3_credentials, streaming: nonstream, assertions: [works], source: "test_batches_e2e.py / LIT-8297", rationale: "Bedrock file upload, content and delete sign S3 with s3_access_key_id / s3_secret_access_key when they differ from the aws_* identity"} - {id: llm.files.gemini.upload.nonstream.works, module: llm, tier: P1, subject_endpoint: files, route: gemini, capability: basic, streaming: nonstream, assertions: [works], source: "test_batches_e2e.py", rationale: "Gemini Files API upload via proxy"} - {id: llm.files.hosted_vllm.upload.nonstream.works, module: llm, tier: P1, subject_endpoint: files, route: hosted_vllm, capability: basic, streaming: nonstream, assertions: [works], source: "test_batches_e2e.py", rationale: "hosted_vllm OpenAI-compatible file upload"} - {id: llm.files.openai.require_managed_files_upload.nonstream.works, module: llm, tier: P1, subject_endpoint: files, route: openai, capability: input_validation, streaming: nonstream, assertions: [works], source: "test_managed_files_enforcement_e2e.py / LIT-5902", rationale: "With require_managed_files enabled, an upload without target_model_names and an upload carrying a model param are both rejected 400; runs only in the sequential managed-files stack phase (E2E_MANAGED_FILES_STACK)"} diff --git a/tests/e2e/coverage_registry/reliability.yaml b/tests/e2e/coverage_registry/reliability.yaml index 65354100f58..334780eda53 100644 --- a/tests/e2e/coverage_registry/reliability.yaml +++ b/tests/e2e/coverage_registry/reliability.yaml @@ -12,6 +12,7 @@ - {id: reliability.cooldown.429.trips_then_recovers, module: reliability, tier: P0, behavior: cooldown, variant: "429", assertions: [trips_then_recovers], exercised_on: [chat_completions], source: "cooldown_handlers.py:69", rationale: "Cools on 429, avoids hammering exhausted provider"} - {id: reliability.cooldown.auth.trips_then_recovers, module: reliability, tier: P1, behavior: cooldown, variant: auth, assertions: [trips_then_recovers], exercised_on: [chat_completions], source: "cooldown_handlers.py:74", rationale: "Cools on 401 auth error"} - {id: reliability.cooldown.timeout.trips_then_recovers, module: reliability, tier: P1, behavior: cooldown, variant: timeout, assertions: [trips_then_recovers], exercised_on: [chat_completions], source: "cooldown_handlers.py:77", rationale: "Cools on 408 timeout"} +- {id: reliability.cooldown.client_disconnect.stays_healthy, module: reliability, tier: P0, behavior: cooldown, variant: client_disconnect, assertions: [stays_healthy], exercised_on: [chat_completions], source: "llms/azure/azure.py:484", fail_before_fix: proven, rationale: "A client hanging up mid-request under cancel_on_disconnect never benches the Azure deployment it was talking to: the cancellation used to surface as a fake 500 that tripped the cooldown and sent every caller behind it to billed fallbacks (GitHub issues #35329 and #42222)"} - {id: reliability.routing.simple_shuffle.picks_healthy_deployment, module: reliability, tier: P1, behavior: routing, variant: simple_shuffle, assertions: [picks_healthy_deployment], exercised_on: [chat_completions], source: "router_strategy/simple_shuffle.py", rationale: "Baseline weighted/uniform pick"} - {id: reliability.routing.latency_based.picks_lowest_latency, module: reliability, tier: P1, behavior: routing, variant: latency_based, assertions: [picks_lowest_latency], exercised_on: [chat_completions], source: "router_strategy/lowest_latency.py", rationale: "Routes to lowest-latency deployment"} - {id: reliability.routing.cost_based.picks_lowest_cost, module: reliability, tier: P1, behavior: routing, variant: cost_based, assertions: [picks_lowest_cost], exercised_on: [chat_completions], source: "router_strategy/lowest_cost.py", rationale: "Spend-aware routing"} diff --git a/tests/e2e/coverage_registry/schema.py b/tests/e2e/coverage_registry/schema.py index 8b0d38a083c..f3ac1ef8a83 100644 --- a/tests/e2e/coverage_registry/schema.py +++ b/tests/e2e/coverage_registry/schema.py @@ -58,6 +58,7 @@ LlmRoute = Literal[ "openai", "together_ai", "vertex", + "xiaomi_mimo", ] LlmCapability = Literal[ @@ -66,6 +67,7 @@ LlmCapability = Literal[ "batch_deployment", "count_tokens", "govcloud_partition", + "split_s3_credentials", "input_validation", "long_context_1m", "mid_conversation_system", diff --git a/tests/e2e/e2e_http.py b/tests/e2e/e2e_http.py index d4978601b20..97f1e1671f8 100644 --- a/tests/e2e/e2e_http.py +++ b/tests/e2e/e2e_http.py @@ -676,6 +676,35 @@ def send( return streaming_outcome(resp, stream, sent_at=sent_at) +class AbandonedRequest(BaseModel): + """A non-streaming request whose socket the client closed ``after`` seconds in, + before the proxy had answered.""" + + kind: Literal["abandoned"] = "abandoned" + after: float + + +def abandon( + url: URL, *, headers: BaseModel, json: BaseModel, after: float, connect_timeout: float = 10.0 +) -> AbandonedRequest | StreamingResponse: + """POST and close the connection ``after`` seconds if no response head has arrived + by then; returns the response instead when the proxy answered first.""" + sent_at: Final = time.monotonic() + session: Final = requests.Session() + try: + resp = session.post( + str(url), + headers=_headers(headers), + json=wire_body(json), + timeout=(connect_timeout, after), + ) + except requests.exceptions.ReadTimeout: + return AbandonedRequest(after=after) + finally: + session.close() + return streaming_outcome(resp, False, sent_at=sent_at) + + def stream(url: URL, *, headers: BaseModel, json: BaseModel, timeout: float = 60.0) -> StreamingResponse: """Streaming (SSE) call: consumes the stream counting events, and captures the x-litellm-call-id + content-type headers. Body is elided.""" diff --git a/tests/e2e/gateway/stage_mirror_ci_config.yml b/tests/e2e/gateway/stage_mirror_ci_config.yml index 352caddf588..2d02fedddae 100644 --- a/tests/e2e/gateway/stage_mirror_ci_config.yml +++ b/tests/e2e/gateway/stage_mirror_ci_config.yml @@ -10,6 +10,7 @@ general_settings: store_prompts_in_spend_logs: true database_connection_pool_limit: 10 forward_client_headers_to_llm_api: false + cancel_on_disconnect: true maximum_spend_logs_retention_period: "60d" maximum_spend_logs_cleanup_cron: "0 1 * * *" proxy_budget_rescheduler_min_time: 15 diff --git a/tests/e2e/guardrails/guardrails_client.py b/tests/e2e/guardrails/guardrails_client.py index ed112a79b9b..3ceac737399 100644 --- a/tests/e2e/guardrails/guardrails_client.py +++ b/tests/e2e/guardrails/guardrails_client.py @@ -20,6 +20,7 @@ from models import ( ChatResponse, ChatTool, KeyGenerateBody, + KeyMetadata, LiteLLMParamsBody, TeamDeleteBody, TeamInfoParams, @@ -27,6 +28,8 @@ from models import ( TeamMetadata, TeamNewBody, TeamNewResponse, + VideoCreateBody, + VideoCreateResponse, ) from proxy_client import ProxyClient from pydantic import BaseModel @@ -151,12 +154,12 @@ class _ResponsesGuardrailBody(BaseModel): class GuardrailsClient: proxy: ProxyClient - def create_content_filter_guardrail(self, name: str, blocked_keyword: str) -> str: + def create_content_filter_guardrail(self, name: str, blocked_keyword: str, *, default_on: bool = True) -> str: return self.register( name, ContentFilterParamsBody( mode="pre_call", - default_on=True, + default_on=default_on, blocked_words=[BlockedWordBody(keyword=blocked_keyword, action="BLOCK")], ), ) @@ -266,6 +269,21 @@ class GuardrailsClient: def create_key_in_team(self, team_id: str) -> str: return self.proxy.generate_key(KeyGenerateBody(team_id=team_id, user_id="e2e-guardrails-user")) + def create_key_with_guardrails(self, resources: ResourceManager, guardrails: list[str]) -> str: + key = self.proxy.generate_key( + KeyGenerateBody(user_id="e2e-guardrails-user", metadata=KeyMetadata(guardrails=guardrails)) + ) + resources.defer(lambda: self.proxy.delete_key(key)) + return key + + def create_video(self, key: str, model: str, prompt: str) -> Result[VideoCreateResponse]: + return self.proxy.transport.post( + "/v1/videos", + headers=self.proxy.transport.bearer(key), + json=VideoCreateBody(model=model, prompt=prompt, seconds="4"), + response_type=VideoCreateResponse, + ) + def chat( self, key: str, diff --git a/tests/e2e/guardrails/test_key_guardrail_video_e2e.py b/tests/e2e/guardrails/test_key_guardrail_video_e2e.py new file mode 100644 index 00000000000..5f318e141a5 --- /dev/null +++ b/tests/e2e/guardrails/test_key_guardrail_video_e2e.py @@ -0,0 +1,69 @@ +from __future__ import annotations + +import pytest +from e2e_config import unique_marker +from e2e_http import Success, UnknownApiError +from guardrails_client import GuardrailsClient, poll_until_blocked +from lifecycle import ResourceManager +from models import LiteLLMParamsBody + +pytestmark = pytest.mark.e2e + +CHAT_MODEL = "gemini-2.5-flash" +VIDEO_BACKEND = "vertex_ai/veo-3.1-fast-generate-001" + + +def _video_prompt_with(banned_keyword: str) -> str: + return f"A short clip of a paper boat floating down a stream. {banned_keyword}" + + +def _create_video_model(client: GuardrailsClient, resources: ResourceManager) -> str: + model_name = f"e2e-guard-video-{unique_marker()}" + model_id = client.proxy.create_model( + model_name, + LiteLLMParamsBody( + model=VIDEO_BACKEND, + vertex_project="os.environ/VERTEXAI_PROJECT", + vertex_location="os.environ/VERTEXAI_LOCATION", + vertex_credentials="os.environ/VERTEXAI_CREDENTIALS", + ), + provider_live=True, + ) + resources.defer(lambda: client.proxy.delete_model(model_id)) + return model_name + + +class TestKeyAttachedGuardrailOnVideos: + @pytest.mark.covers( + "guardrail.litellm_content_filter.pre_call.blocks_video", + exercised_on=["videos"], + ) + def test_key_attached_content_filter_blocks_banned_video_prompt( + self, client: GuardrailsClient, resources: ResourceManager + ) -> None: + banned = unique_marker() + guardrail_name = f"e2e-video-filter-{banned}" + guardrail_id = client.create_content_filter_guardrail(guardrail_name, banned, default_on=False) + resources.defer(lambda: client.delete_guardrail(guardrail_id)) + key = client.create_key_with_guardrails(resources, [guardrail_name]) + model = _create_video_model(client, resources) + + synced = poll_until_blocked(lambda: client.chat(key, CHAT_MODEL, _video_prompt_with(banned))) + assert isinstance(synced, UnknownApiError) and synced.status_code == 400, ( + f"key guardrail {guardrail_name!r} never synced to the proxy on /chat/completions: {synced}" + ) + + result = client.create_video(key, model, _video_prompt_with(banned)) + match result: + case UnknownApiError(status_code=status, body=body): + assert status == 400, f"expected a 400 guardrail block, got {status}: {body[:300]}" + assert "content blocked" in body.lower() or banned in body, ( + f"block response missing content-filter reason: {body[:300]}" + ) + case Success(data=video): + pytest.fail( + f"key-attached guardrail {guardrail_name!r} was skipped on /v1/videos: " + f"the banned prompt reached the provider and started video job {video.id}" + ) + case _: + pytest.fail(f"unexpected /v1/videos outcome for a banned prompt: {result}") diff --git a/tests/e2e/llm_translation/test_chat_completions_regression_e2e.py b/tests/e2e/llm_translation/test_chat_completions_regression_e2e.py index 87bd32d8dab..363b2a7e02e 100644 --- a/tests/e2e/llm_translation/test_chat_completions_regression_e2e.py +++ b/tests/e2e/llm_translation/test_chat_completions_regression_e2e.py @@ -45,6 +45,10 @@ pytestmark = pytest.mark.e2e COHERE_BACKEND = "cohere/command-r-08-2024" GEMINI_BACKEND = "gemini/gemini-2.5-flash" +VERTEX_BACKEND: Final = "vertex_ai/gemini-2.5-flash" +AZURE_OPENAI_BACKEND: Final = "azure/gpt-5.4-nano" +AZURE_OPENAI_API_VERSION: Final = "v1" +AZURE_FOUNDRY_BACKEND: Final = "azure_ai/claude-haiku-4-5" OPENAI_BACKEND = "openai/gpt-5.6" ANTHROPIC_BACKEND = "anthropic/claude-haiku-4-5-20251001" BEDROCK_CONVERSE_BACKEND = "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0" @@ -108,7 +112,7 @@ def _assert_describes_cat(response: ChatResponse) -> None: assert response.choices, f"vision returned no choices: {response}" message = response.choices[0].message content = (message.content if message else None) or "" - assert "cat" in content.lower() or "feline" in content.lower(), ( + assert any(term in content.lower() for term in ("cat", "feline", "kitten", "kitty")), ( f"vision response did not describe the image: {content[:200]}" ) @@ -208,7 +212,6 @@ class TestChatCompletionsRegression: @pytest.mark.covers( "llm.chat_completions.openai.basic.nonstream.works", "llm.chat_completions.anthropic.basic.nonstream.works", - "llm.chat_completions.vertex.basic.nonstream.works", exercised_on=[], ) def test_chat_returns_real_completion( @@ -336,6 +339,232 @@ class TestGeminiChatCompletions: assert row.status == "success", f"gemini chat spend status={row.status!r}" +class TestVertexChatCompletions: + def _register(self, client: PassthroughClient, resources: ResourceManager, prefix: str) -> str: + model = f"{prefix}-{unique_marker()}" + model_id = client.proxy.create_model( + model, + LiteLLMParamsBody( + model=VERTEX_BACKEND, + vertex_project="os.environ/VERTEXAI_PROJECT", + vertex_location="us-central1", + ), + ) + resources.defer(lambda: client.proxy.delete_model(model_id)) + return model + + @pytest.mark.covers( + "llm.chat_completions.vertex.basic.nonstream.works", + exercised_on=["chat_completions"], + ) + def test_vertex_chat_returns_content( + self, client: PassthroughClient, resources: ResourceManager + ) -> None: + model = self._register(client, resources, "e2e-vertex-chat") + key = resources.key() + + response = unwrap( + client.proxy.chat( + key, + ChatBody( + model=model, + messages=[ + ChatMessage( + role="user", + content=f"Reply with the single word pong. {unique_marker()}", + ) + ], + max_tokens=32, + ), + ) + ) + assert response.choices, f"vertex chat returned no choices: {response}" + content = response.choices[0].message.content if response.choices[0].message else None + assert content and content.strip(), f"vertex chat returned empty content: {response}" + + @pytest.mark.covers( + "llm.chat_completions.vertex.tool_use.nonstream.works", + exercised_on=["chat_completions"], + ) + def test_vertex_chat_returns_tool_call( + self, client: PassthroughClient, resources: ResourceManager + ) -> None: + model = self._register(client, resources, "e2e-vertex-tool") + key = resources.key() + + response = unwrap( + client.proxy.chat( + key, + ChatBody( + model=model, + messages=[ + ChatMessage( + role="user", + content="What is the weather in San Francisco? Use the get_weather tool.", + ) + ], + tools=[_WEATHER_TOOL], + tool_choice="required", + max_tokens=128, + ), + ) + ) + _assert_weather_tool_call(response) + + @pytest.mark.covers( + "llm.chat_completions.vertex.vision.nonstream.works", + exercised_on=["chat_completions"], + ) + def test_vertex_chat_vision_describes_image( + self, client: PassthroughClient, resources: ResourceManager + ) -> None: + model = self._register(client, resources, "e2e-vertex-vision") + key = resources.key() + + response = unwrap(client.proxy.chat(key, ChatBody(model=model, messages=_vision_messages(), max_tokens=32))) + _assert_describes_cat(response) + + @pytest.mark.covers( + "llm.chat_completions.vertex.basic.stream.works", + exercised_on=["chat_completions"], + ) + def test_vertex_chat_streams_real_content( + self, client: PassthroughClient, resources: ResourceManager + ) -> None: + model = self._register(client, resources, "e2e-vertex-stream") + key = resources.key() + + result = client.proxy.chat_stream( + key, + ChatBody( + model=model, + messages=[ + ChatMessage( + role="user", + content=f"Count from 1 to 5, one number per line. {unique_marker()}", + ) + ], + max_tokens=64, + stream=True, + ), + ) + _assert_streamed_completion(result) + + +class TestAzureOpenAIChatCompletions: + def _register(self, client: PassthroughClient, resources: ResourceManager, prefix: str) -> str: + model = f"{prefix}-{unique_marker()}" + model_id = client.proxy.create_model( + model, + LiteLLMParamsBody( + model=AZURE_OPENAI_BACKEND, + api_base="os.environ/AZURE_API_BASE", + api_key="os.environ/AZURE_API_KEY", + api_version=AZURE_OPENAI_API_VERSION, + ), + ) + resources.defer(lambda: client.proxy.delete_model(model_id)) + return model + + @pytest.mark.covers( + "llm.chat_completions.azure_openai.basic.nonstream.works", + exercised_on=["chat_completions"], + ) + def test_azure_openai_chat_returns_content( + self, client: PassthroughClient, resources: ResourceManager + ) -> None: + model = self._register(client, resources, "e2e-azure-openai-chat") + key = resources.key() + + response = unwrap( + client.proxy.chat( + key, + ChatBody( + model=model, + messages=[ + ChatMessage( + role="user", + content=f"Reply with the single word pong. {unique_marker()}", + ) + ], + max_tokens=32, + ), + ) + ) + assert response.choices, f"azure openai chat returned no choices: {response}" + content = response.choices[0].message.content if response.choices[0].message else None + assert content and content.strip(), f"azure openai chat returned empty content: {response}" + + @pytest.mark.covers( + "llm.chat_completions.azure_openai.tool_use.nonstream.works", + exercised_on=["chat_completions"], + ) + def test_azure_openai_chat_returns_tool_call( + self, client: PassthroughClient, resources: ResourceManager + ) -> None: + model = self._register(client, resources, "e2e-azure-openai-tool") + key = resources.key() + + response = unwrap( + client.proxy.chat( + key, + ChatBody( + model=model, + messages=[ + ChatMessage( + role="user", + content="What is the weather in San Francisco? Use the get_weather tool.", + ) + ], + tools=[_WEATHER_TOOL], + tool_choice="required", + max_tokens=128, + ), + ) + ) + _assert_weather_tool_call(response) + + +class TestAzureFoundryChatCompletions: + @pytest.mark.covers( + "llm.chat_completions.azure_foundry.basic.nonstream.works", + exercised_on=["chat_completions"], + ) + def test_azure_foundry_chat_returns_content( + self, client: PassthroughClient, resources: ResourceManager + ) -> None: + model = f"e2e-azure-foundry-chat-{unique_marker()}" + model_id = client.proxy.create_model( + model, + LiteLLMParamsBody( + model=AZURE_FOUNDRY_BACKEND, + api_base="os.environ/AZURE_AI_API_BASE", + api_key="os.environ/AZURE_AI_API_KEY", + ), + ) + resources.defer(lambda: client.proxy.delete_model(model_id)) + key = resources.key() + + response = unwrap( + client.proxy.chat( + key, + ChatBody( + model=model, + messages=[ + ChatMessage( + role="user", + content=f"Reply with the single word pong. {unique_marker()}", + ) + ], + max_tokens=32, + ), + ) + ) + assert response.choices, f"azure foundry chat returned no choices: {response}" + content = response.choices[0].message.content if response.choices[0].message else None + assert content and content.strip(), f"azure foundry chat returned empty content: {response}" + + class TestHostedVllmChat: """hosted_vllm (self-hosted OpenAI-compatible server) via /chat/completions.""" @@ -764,6 +993,90 @@ class TestAnthropicChatCompletions: resources.defer(lambda: client.proxy.delete_model(model_id)) return model + @pytest.mark.covers( + "llm.chat_completions.anthropic.structured_output.nonstream.works", + exercised_on=["chat_completions"], + ) + def test_anthropic_chat_structured_output_conforms_to_schema( + self, client: PassthroughClient, resources: ResourceManager + ) -> None: + model = self._register(client, resources, "e2e-anthropic-schema") + key = resources.key() + + response = unwrap( + client.proxy.chat( + key, + ChatBody( + model=model, + messages=[ + ChatMessage( + role="user", + content="Extract the person. John Doe is 42 years old.", + ) + ], + response_format=_PERSON_SCHEMA, + max_tokens=128, + ), + ) + ) + assert response.choices, f"anthropic structured output returned no choices: {response}" + message = response.choices[0].message + content = message.content if message else None + assert content, f"anthropic structured output returned empty content: {response}" + person = _Person.model_validate_json(content) + assert person.name.strip() and person.age == 42, f"anthropic schema output was wrong: {person}" + + @pytest.mark.covers( + "llm.chat_completions.anthropic.thinking.nonstream.works", + exercised_on=["chat_completions"], + ) + def test_anthropic_chat_returns_thinking_content( + self, client: PassthroughClient, resources: ResourceManager + ) -> None: + model = self._register(client, resources, "e2e-anthropic-thinking") + key = resources.key() + + response = unwrap( + client.proxy.chat( + key, + ChatBody( + model=model, + messages=[ + ChatMessage( + role="user", + content=( + "Prove that the sum of two odd integers is even, then find the smallest prime " + "greater than 100 such that p+2 is also prime." + ), + ) + ], + thinking=ThinkingParam(type="enabled", budget_tokens=1024), + max_tokens=2048, + ), + ) + ) + assert response.choices, f"anthropic thinking returned no choices: {response}" + message = response.choices[0].message + assert message and message.content and message.content.strip(), ( + f"anthropic thinking returned no answer content: {response}" + ) + assert message.reasoning_content and message.reasoning_content.strip(), ( + f"anthropic thinking returned no reasoning content: {response}" + ) + + @pytest.mark.covers( + "llm.chat_completions.anthropic.vision.nonstream.works", + exercised_on=["chat_completions"], + ) + def test_anthropic_chat_vision_describes_image( + self, client: PassthroughClient, resources: ResourceManager + ) -> None: + model = self._register(client, resources, "e2e-anthropic-vision") + key = resources.key() + + response = unwrap(client.proxy.chat(key, ChatBody(model=model, messages=_vision_messages(), max_tokens=32))) + _assert_describes_cat(response) + @pytest.mark.covers( "llm.chat_completions.anthropic.basic.stream.works", exercised_on=["chat_completions"], diff --git a/tests/e2e/llm_translation/test_responses_e2e.py b/tests/e2e/llm_translation/test_responses_e2e.py index 9cc70da63b0..6fa77694eb8 100644 --- a/tests/e2e/llm_translation/test_responses_e2e.py +++ b/tests/e2e/llm_translation/test_responses_e2e.py @@ -45,6 +45,9 @@ class _OptionalResponsesBody(BaseModel): BEDROCK_CONVERSE_BACKEND = "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0" +VERTEX_BACKEND: Final = "vertex_ai/gemini-2.5-flash" +AZURE_OPENAI_BACKEND: Final = "azure/gpt-5.4-nano" +AZURE_OPENAI_API_VERSION: Final = "v1" INSTRUCTIONS = "You are a helpful assistant" CAT_IMAGE_URL = "https://upload.wikimedia.org/wikipedia/commons/3/3a/Cat03.jpg" BEDROCK_EDGE_REGION: Final = "us-east-1" @@ -105,6 +108,23 @@ def _bedrock_params() -> LiteLLMParamsBody: ) +def _vertex_params() -> LiteLLMParamsBody: + return LiteLLMParamsBody( + model=VERTEX_BACKEND, + vertex_project="os.environ/VERTEXAI_PROJECT", + vertex_location="us-central1", + ) + + +def _azure_openai_params() -> LiteLLMParamsBody: + return LiteLLMParamsBody( + model=AZURE_OPENAI_BACKEND, + api_base="os.environ/AZURE_API_BASE", + api_key="os.environ/AZURE_API_KEY", + api_version=AZURE_OPENAI_API_VERSION, + ) + + def _register( proxy: ProxyClient, resources: ResourceManager, params: LiteLLMParamsBody, prefix: str = "e2e-responses" ) -> str: @@ -291,6 +311,66 @@ class TestResponses: ) _assert_weather_call(response) + @pytest.mark.covers("llm.responses.vertex.basic.nonstream.works") + def test_responses_vertex_returns_completion( + self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients + ) -> None: + model = _register(proxy, resources, _vertex_params(), prefix="e2e-responses-vertex") + client = sdk.openai(resources.key()) + + response = client.responses.create( + model=model, input="reply with one word", instructions=INSTRUCTIONS, extra_body=NO_PROXY_CACHE + ) + assert response.output_text.strip(), f"/responses over vertex returned no output text: {response.output!r}" + + @pytest.mark.covers("llm.responses.vertex.tool_use.nonstream.works") + def test_responses_vertex_returns_function_call( + self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients + ) -> None: + model = _register(proxy, resources, _vertex_params(), prefix="e2e-responses-vertex-tool") + client = sdk.openai(resources.key()) + + response = client.responses.create( + model=model, + input="What is the weather in San Francisco? Use the get_weather tool.", + instructions=INSTRUCTIONS, + tools=[WEATHER_TOOL], + tool_choice="required", + extra_body=NO_PROXY_CACHE, + ) + _assert_weather_call(response) + + @pytest.mark.covers("llm.responses.azure_openai.basic.nonstream.works") + def test_responses_azure_openai_returns_completion( + self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients + ) -> None: + model = _register(proxy, resources, _azure_openai_params(), prefix="e2e-responses-azure-openai") + client = sdk.openai(resources.key()) + + response = client.responses.create( + model=model, input="reply with one word", instructions=INSTRUCTIONS, extra_body=NO_PROXY_CACHE + ) + assert response.output_text.strip(), ( + f"/responses over azure openai returned no output text: {response.output!r}" + ) + + @pytest.mark.covers("llm.responses.azure_openai.tool_use.nonstream.works") + def test_responses_azure_openai_returns_function_call( + self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients + ) -> None: + model = _register(proxy, resources, _azure_openai_params(), prefix="e2e-responses-azure-openai-tool") + client = sdk.openai(resources.key()) + + response = client.responses.create( + model=model, + input="What is the weather in San Francisco? Use the get_weather tool.", + instructions=INSTRUCTIONS, + tools=[WEATHER_TOOL], + tool_choice="required", + extra_body=NO_PROXY_CACHE, + ) + _assert_weather_call(response) + @pytest.mark.provider_edge_host @pytest.mark.parametrize("endpoint", ["/v1/responses", "/v1/chat/completions"]) def test_bedrock_forwards_allowed_safety_identifier_as_additional_model_request_field( diff --git a/tests/e2e/models.py b/tests/e2e/models.py index 355329585fb..2608da2e4fc 100644 --- a/tests/e2e/models.py +++ b/tests/e2e/models.py @@ -60,6 +60,7 @@ class KeyMetadata(BaseModel): priority: str | None = None batch_enqueued_token_limit: int | None = None tag: str | None = None + guardrails: list[str] | None = None class ObjectPermission(BaseModel): @@ -697,6 +698,20 @@ class EmbedResponse(BaseModel): model: str | None = None +# ---------- videos ---------- + + +class VideoCreateBody(BaseModel): + model: str + prompt: str + seconds: str | None = None + + +class VideoCreateResponse(BaseModel): + id: str + status: str | None = None + + # ---------- rerank ---------- @@ -936,6 +951,23 @@ class RouterSettingsResponse(BaseModel): current_values: RouterCurrentValues +class ConfigListParams(BaseModel): + config_type: Literal["general_settings"] + + +class ConfigField(BaseModel): + """One row of GET /config/list: a general_settings field and the value the + proxy is running with, the two fields a test preconditions on.""" + + model_config = ConfigDict(extra="ignore") + field_name: str + field_value: JsonValue = None + + +class ConfigFieldList(RootModel[tuple[ConfigField, ...]]): + """GET /config/list answers with a bare array of general_settings fields.""" + + class CostMapEntry(BaseModel): model_config = ConfigDict(extra="ignore") litellm_provider: str | None = None @@ -1051,6 +1083,7 @@ class ModelInfoBody(BaseModel): mode: ModelMode | None = None access_groups: list[str] | None = None team_id: str | None = None + allowed_fails: int | None = None allowed_fails_policy: dict[str, int] | None = None diff --git a/tests/e2e/proxy_client.py b/tests/e2e/proxy_client.py index 2f32361e083..76c57452c69 100644 --- a/tests/e2e/proxy_client.py +++ b/tests/e2e/proxy_client.py @@ -47,6 +47,8 @@ from models import ( AnthropicMessagesResponse, ChatBody, ChatResponse, + ConfigFieldList, + ConfigListParams, CostMap, CostMapEntry, CountTokensBody, @@ -630,6 +632,19 @@ class ProxyClient: provider_live=provider_live, ) + def general_setting_enabled(self, field_name: str) -> bool: + """Whether the proxy is running with the named general_settings flag on, for + a test whose behavior only exists under a config flag the stack has to carry.""" + fields = unwrap( + self.transport.get( + "/config/list", + headers=self.transport.master, + params=ConfigListParams(config_type="general_settings"), + response_type=ConfigFieldList, + ) + ).root + return any(entry.field_name == field_name and entry.field_value is True for entry in fields) + def register_model( self, body: ModelNewBody, listed_for: str | None = None, *, provider_live: bool = False ) -> str: diff --git a/tests/e2e/router/reliability_support.py b/tests/e2e/router/reliability_support.py index 976c05ffceb..3d5b76f6408 100644 --- a/tests/e2e/router/reliability_support.py +++ b/tests/e2e/router/reliability_support.py @@ -42,7 +42,7 @@ REAL_KEY = "os.environ/OPENAI_API_KEY" CACHING_MODEL = "anthropic/claude-haiku-4-5" CACHING_KEY = "os.environ/ANTHROPIC_API_KEY" -CONTENT_FILTERED_MODEL = "azure/gpt-5.4-nano" +AZURE_MODEL = "azure/gpt-5.4-nano" AZURE_KEY = "os.environ/AZURE_API_KEY" AZURE_BASE = "os.environ/AZURE_API_BASE" AZURE_API_VERSION = "2024-10-21" @@ -53,6 +53,7 @@ CONTENT_POLICY_PROMPT = ( ) COOLDOWN_SECONDS = 30.0 +REPLICA_PROPAGATION_SECONDS = 15.0 # The smallest-context chat model OpenAI still serves (16385 tokens). A prompt # past that limit comes back as a real `context_length_exceeded` 400, which is @@ -111,7 +112,7 @@ def create_content_filtered_deployment(proxy: ProxyClient, name: str) -> str: return proxy.create_model( name, LiteLLMParamsBody( - model=CONTENT_FILTERED_MODEL, + model=AZURE_MODEL, api_key=AZURE_KEY, api_base=AZURE_BASE, api_version=AZURE_API_VERSION, @@ -120,6 +121,26 @@ def create_content_filtered_deployment(proxy: ProxyClient, name: str) -> str: ) +def create_azure_benched_on_first_failure_deployment(proxy: ProxyClient, name: str, cooldown_time: float) -> str: + """The live Azure OpenAI deployment holding all of the group's shuffle weight, + benched on its first failure of any class, with the client's own retries off.""" + return proxy.register_model( + ModelNewBody( + model_name=name, + litellm_params=LiteLLMParamsBody( + model=AZURE_MODEL, + api_key=AZURE_KEY, + api_base=AZURE_BASE, + api_version=AZURE_API_VERSION, + max_retries=0, + weight=1, + cooldown_time=cooldown_time, + ), + model_info=ModelInfoBody(allowed_fails=0), + ) + ) + + def create_caching_deployment(proxy: ProxyClient, name: str) -> str: """Register the Anthropic deployment whose prompt cache the affinity check pins to.""" return proxy.create_model(name, LiteLLMParamsBody(model=CACHING_MODEL, api_key=CACHING_KEY, weight=1)) diff --git a/tests/e2e/router/test_reliability_cancel_on_disconnect_e2e.py b/tests/e2e/router/test_reliability_cancel_on_disconnect_e2e.py new file mode 100644 index 00000000000..06174e97d20 --- /dev/null +++ b/tests/e2e/router/test_reliability_cancel_on_disconnect_e2e.py @@ -0,0 +1,139 @@ +"""Live e2e: a client hanging up mid-request under cancel_on_disconnect never +benches the deployment it was talking to. + +The group is the cooldown suite's pair: the live Azure deployment holding all of +the shuffle weight, benched on its first failure of any class with a cooldown that +outlasts the test, plus a healthy backup at weight 0 the shuffle only reaches once +the Azure deployment is benched. A cheap call first proves the Azure deployment +answers the key and warms its auth path. The test then asks for an answer far +longer than CLIENT_HANGS_UP_AFTER_SECONDS of generation, retries off, and hangs up +that many seconds in: late enough that the proxy has handed the call to Azure (a +hang-up before the provider call is in flight cancels nothing the router could +bench, so the cell would pass vacuously). An answer that comes back inside the +window proves nothing and benches nothing either, since a success never counts +against the deployment, so the cell asks again up to HANG_UP_ATTEMPTS times and +fails out loud naming the window only when every ask came back early. After the +cooldown suite's replica propagation window, every one of the next calls has to +come back 200 from the Azure deployment itself, named in x-litellm-model-id; a +single answer from the backup means the hang-up was booked as a failure. + +The test reads `cancel_on_disconnect` back from the proxy first: without the flag +the hang-up cancels nothing and the cell would pass vacuously. +""" + +from __future__ import annotations + +import time + +import pytest +from complexity_router_client import ComplexityRouterClient +from e2e_config import unique_marker +from e2e_http import AbandonedRequest, StreamingResponse +from lifecycle import ResourceManager +from models import ChatMessage, ReliabilityChatBody, RouterSettingsOverride +from reliability_support import ( + REPLICA_PROPAGATION_SECONDS, + chat_override, + create_azure_benched_on_first_failure_deployment, + create_zero_weight_backup_deployment, + model_id_of, +) + +pytestmark = pytest.mark.e2e + +CLIENT_HANGS_UP_AFTER_SECONDS = 5.0 +HANG_UP_ATTEMPTS = 3 +LONG_ANSWER_MAX_TOKENS = 16384 +BENCH_OUTLASTS_TEST_SECONDS = 300.0 +CALLS_AFTER_HANGUP = 6 + + +def _say_hi(client: ComplexityRouterClient, key: str, group: str) -> StreamingResponse: + return chat_override( + client.proxy, + key, + group, + f"say hi {unique_marker()}", + override=RouterSettingsOverride(num_retries=0), + ) + + +def _ask_for_a_long_answer_then_hang_up( + client: ComplexityRouterClient, key: str, group: str +) -> AbandonedRequest | StreamingResponse: + return client.proxy.transport.abandon( + "/chat/completions", + headers=client.proxy.transport.bearer(key), + json=ReliabilityChatBody( + model=group, + messages=[ + ChatMessage( + role="user", + content=( + "Write an essay on the history of the telegraph with one section per decade from the 1830s " + f"to the 2020s, each section at least 300 words. {unique_marker()}" + ), + ) + ], + max_tokens=LONG_ANSWER_MAX_TOKENS, + router_settings_override=RouterSettingsOverride(num_retries=0), + ), + after=CLIENT_HANGS_UP_AFTER_SECONDS, + ) + + +def _hang_up_mid_answer(client: ComplexityRouterClient, key: str, group: str) -> None: + for attempt in range(1, HANG_UP_ATTEMPTS + 1): + match _ask_for_a_long_answer_then_hang_up(client, key, group): + case AbandonedRequest(): + return + case StreamingResponse(status_code=200): + continue + case StreamingResponse(status_code=status_code, body=body): + pytest.fail( + f"hang-up attempt {attempt} should have found the long answer still in flight after " + f"{CLIENT_HANGS_UP_AFTER_SECONDS:.0f}s, but the proxy answered {status_code}: {body[:300]}" + ) + pytest.fail( + f"the proxy answered all {HANG_UP_ATTEMPTS} long asks within {CLIENT_HANGS_UP_AFTER_SECONDS:.0f}s, so the " + "client never hung up with a call still in flight and the bench this cell guards against could not happen" + ) + + +class TestReliabilityCancelOnDisconnect: + @pytest.mark.covers("reliability.cooldown.client_disconnect.stays_healthy") + def test_client_hanging_up_never_benches_the_deployment( + self, client: ComplexityRouterClient, resources: ResourceManager, scoped_key: str + ) -> None: + assert client.proxy.general_setting_enabled("cancel_on_disconnect"), ( + "this cell needs general_settings.cancel_on_disconnect: true in the proxy config; without it the " + "hang-up cancels nothing and the bench it guards against can never happen" + ) + + group = f"reliability-cooldown-disconnect-{unique_marker()}" + azure = create_azure_benched_on_first_failure_deployment( + client.proxy, group, cooldown_time=BENCH_OUTLASTS_TEST_SECONDS + ) + resources.defer(lambda: client.proxy.delete_model(azure)) + backup = create_zero_weight_backup_deployment(client.proxy, group) + resources.defer(lambda: client.proxy.delete_model(backup)) + + warm_up = _say_hi(client, scoped_key, group) + assert warm_up.status_code == 200 and model_id_of(warm_up) == azure, ( + f"before any hang-up the Azure deployment {azure} should answer the group, got {warm_up.status_code} " + f"from {model_id_of(warm_up)!r}: {warm_up.body[:300]}" + ) + + _hang_up_mid_answer(client, scoped_key, group) + time.sleep(REPLICA_PROPAGATION_SECONDS) + + for call in range(1, CALLS_AFTER_HANGUP + 1): + resp = _say_hi(client, scoped_key, group) + assert resp.status_code == 200, ( + f"call {call} after the hang-up should have been a plain 200 from the group, got " + f"{resp.status_code}: {resp.body[:300]}" + ) + assert model_id_of(resp) == azure, ( + f"call {call} after the hang-up should have been served by the Azure deployment {azure}, the proxy " + f"named {model_id_of(resp)!r}: the cancelled call was booked as a failure and benched it" + ) diff --git a/tests/e2e/router/test_reliability_cooldowns_e2e.py b/tests/e2e/router/test_reliability_cooldowns_e2e.py index 5b5cec09f06..769971e1533 100644 --- a/tests/e2e/router/test_reliability_cooldowns_e2e.py +++ b/tests/e2e/router/test_reliability_cooldowns_e2e.py @@ -43,6 +43,7 @@ from lifecycle import ResourceManager from models import KeyGenerateBody, RouterSettingsOverride from reliability_support import ( COOLDOWN_SECONDS, + REPLICA_PROPAGATION_SECONDS, chat_override, create_always_5xx_deployment, create_always_rate_limited_deployment, @@ -57,7 +58,6 @@ from reliability_support import ( pytestmark = pytest.mark.e2e RECOVERY_GRACE_SECONDS = 10 -REPLICA_PROPAGATION_SECONDS = 15.0 PROPAGATION_POLL_SECONDS = 0.25 BENCH_MARGIN_SECONDS = 4.0 diff --git a/tests/e2e/transport.py b/tests/e2e/transport.py index 0022c0c4355..a3eec815441 100644 --- a/tests/e2e/transport.py +++ b/tests/e2e/transport.py @@ -13,6 +13,7 @@ from typing import Protocol import e2e_http from e2e_http import ( URL, + AbandonedRequest, AuthHeaders, BinaryStream, NetworkError, @@ -58,6 +59,10 @@ class Transport(Protocol): stream: bool = False, ) -> StreamingResponse: ... + def abandon( + self, path: str, *, headers: BaseModel, json: BaseModel, after: float + ) -> AbandonedRequest | StreamingResponse: ... + def get[R: BaseModel]( self, path: str, @@ -243,6 +248,11 @@ class HttpTransport: timeout=self.request_timeout, ) + def abandon( + self, path: str, *, headers: BaseModel, json: BaseModel, after: float + ) -> AbandonedRequest | StreamingResponse: + return e2e_http.abandon(self._url(path), headers=headers, json=json, after=after) + def probe(self, path: str, *, params: BaseModel, headers: BaseModel | None = None) -> ProbeResult: return e2e_http.probe( self._url(path), @@ -420,6 +430,11 @@ class SplitTransport: ) -> StreamingResponse: return self._route(path).send(path, headers=headers, json=json, params=params, stream=stream) + def abandon( + self, path: str, *, headers: BaseModel, json: BaseModel, after: float + ) -> AbandonedRequest | StreamingResponse: + return self._route(path).abandon(path, headers=headers, json=json, after=after) + def probe(self, path: str, *, params: BaseModel, headers: BaseModel | None = None) -> ProbeResult: return self._route(path).probe(path, params=params, headers=headers) diff --git a/tests/e2e/ui/helpers/mcp.ts b/tests/e2e/ui/helpers/mcp.ts index 178644df95f..19b2b9599b4 100644 --- a/tests/e2e/ui/helpers/mcp.ts +++ b/tests/e2e/ui/helpers/mcp.ts @@ -1,8 +1,21 @@ import { expect, Page as PwPage } from "@playwright/test"; +import { Client } from "@modelcontextprotocol/sdk/client/index.js"; +import { StreamableHTTPClientTransport } from "@modelcontextprotocol/sdk/client/streamableHttp.js"; import { navigateToPage } from "./navigation"; import { Page } from "../fixtures/pages"; import { masterKey } from "./traffic"; +export async function listUpstreamToolNames(url: string): Promise { + const client = new Client({ name: "litellm-ui-e2e", version: "0.0.0" }); + await client.connect(new StreamableHTTPClientTransport(new URL(url))); + try { + const { tools } = await client.listTools(); + return tools.map((tool) => tool.name); + } finally { + await client.close(); + } +} + /** Creates an MCP server through the UI's discovery to custom-form flow and returns its name. */ export async function createMcpServer(page: PwPage, url: string): Promise { await navigateToPage(page, Page.McpServers); diff --git a/tests/e2e/ui/helpers/roundTrip.ts b/tests/e2e/ui/helpers/roundTrip.ts index 8d6e264e622..91e48b7087c 100644 --- a/tests/e2e/ui/helpers/roundTrip.ts +++ b/tests/e2e/ui/helpers/roundTrip.ts @@ -12,17 +12,111 @@ export async function captureRequestBody( match: { method: string; urlIncludes: string }, action: () => Promise, ): Promise> { - const pending = page.waitForRequest((req) => req.method() === match.method && req.url().includes(match.urlIncludes)); + const pending = page.waitForRequest( + (req) => + req.method() === match.method && req.url().includes(match.urlIncludes), + ); await action(); const request = await pending; return JSON.parse(request.postData() ?? "{}") as Record; } /** Reads an endpoint as the master key, so a failure is bad data and not an expired UI token. */ -export async function readBack(page: Page, endpoint: string): Promise { +export async function readBack( + page: Page, + endpoint: string, +): Promise { const res = await page.request.get(endpoint, { headers: { Authorization: `Bearer ${masterKey()}` }, }); expect(res.ok(), `GET ${endpoint}`).toBe(true); return (await res.json()) as T; } + +type OperationOutcome = + | { readonly status: "success" } + | { readonly status: "failure"; readonly error: unknown }; + +type RunFailure = + | { readonly status: "action_failure"; readonly error: unknown } + | { readonly status: "cleanup_failure"; readonly error: unknown } + | { + readonly status: "action_and_cleanup_failure"; + readonly actionError: unknown; + readonly cleanupError: unknown; + }; + +function toRunFailure( + actionOutcome: OperationOutcome, + cleanupOutcome: OperationOutcome, +): RunFailure | null { + if ( + actionOutcome.status === "failure" && + cleanupOutcome.status === "failure" + ) { + return { + status: "action_and_cleanup_failure", + actionError: actionOutcome.error, + cleanupError: cleanupOutcome.error, + }; + } + if (actionOutcome.status === "failure") { + return { status: "action_failure", error: actionOutcome.error }; + } + if (cleanupOutcome.status === "failure") { + return { status: "cleanup_failure", error: cleanupOutcome.error }; + } + return null; +} + +function raiseRunFailure(failure: RunFailure): never { + switch (failure.status) { + case "action_failure": + throw failure.error; + case "cleanup_failure": + throw failure.error; + case "action_and_cleanup_failure": + throw new AggregateError( + [failure.actionError, failure.cleanupError], + "Action and cleanup failed", + ); + } +} + +async function runAction( + action: () => void | Promise, +): Promise { + return Promise.resolve() + .then(action) + .then( + () => ({ status: "success" as const }), + (error: unknown) => ({ status: "failure" as const, error }), + ); +} + +async function runCleanup( + cleanup: () => boolean | Promise, +): Promise { + return Promise.resolve() + .then(cleanup) + .then( + (succeeded) => + succeeded + ? { status: "success" as const } + : { + status: "failure" as const, + error: new Error("Failed to clean up UI E2E resource"), + }, + (error: unknown) => ({ status: "failure" as const, error }), + ); +} + +export async function runWithCleanup( + action: () => void | Promise, + cleanup: () => boolean | Promise, +): Promise { + const actionOutcome = await runAction(action); + const cleanupOutcome = await runCleanup(cleanup); + const failure = toRunFailure(actionOutcome, cleanupOutcome); + if (failure !== null) raiseRunFailure(failure); +} diff --git a/tests/e2e/ui/package-lock.json b/tests/e2e/ui/package-lock.json index b22673a3535..f56e00506e9 100644 --- a/tests/e2e/ui/package-lock.json +++ b/tests/e2e/ui/package-lock.json @@ -8,11 +8,66 @@ "name": "litellm-ui-e2e", "version": "0.0.0", "devDependencies": { + "@modelcontextprotocol/sdk": "1.30.0", "@playwright/test": "1.58.1", "@types/node": "20.19.37", "typescript": "5.9.3" } }, + "node_modules/@hono/node-server": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/@hono/node-server/-/node-server-2.1.1.tgz", + "integrity": "sha512-ELuehkj5VCBdgEw9zs+ivkKwyzzUCSQuE96YmiPvn1ECBoZCczbFXJLeEGMTYjphP6gydh4pHMqEYPVMYUVgQg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=20" + }, + "peerDependencies": { + "hono": "^4" + } + }, + "node_modules/@modelcontextprotocol/sdk": { + "version": "1.30.0", + "resolved": "https://registry.npmjs.org/@modelcontextprotocol/sdk/-/sdk-1.30.0.tgz", + "integrity": "sha512-xKd8OIzlqNzcqcNumGAa6g+PW2kjD5vrpcKOnfldAUPP3j7lnqMPwlTXQm8gF+UwH72z0lqaRbjr9hqGz0eITA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@hono/node-server": "^1.19.9 || ^2.0.5", + "ajv": "^8.17.1", + "ajv-formats": "^3.0.1", + "content-type": "^1.0.5", + "cors": "^2.8.5", + "cross-spawn": "^7.0.5", + "eventsource": "^3.0.2", + "eventsource-parser": "^3.0.0", + "express": "^5.2.1", + "express-rate-limit": "^8.2.1", + "hono": "^4.11.4", + "jose": "^6.1.3", + "json-schema-typed": "^8.0.2", + "pkce-challenge": "^5.0.0", + "raw-body": "^3.0.0", + "zod": "^3.25 || ^4.0", + "zod-to-json-schema": "^3.25.1" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@cfworker/json-schema": "^4.1.1", + "zod": "^3.25 || ^4.0" + }, + "peerDependenciesMeta": { + "@cfworker/json-schema": { + "optional": true + }, + "zod": { + "optional": false + } + } + }, "node_modules/@playwright/test": { "version": "1.58.1", "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.58.1.tgz", @@ -39,6 +94,475 @@ "undici-types": "~6.21.0" } }, + "node_modules/accepts": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/accepts/-/accepts-2.0.0.tgz", + "integrity": "sha512-5cvg6CtKwfgdmVqY1WIiXKc3Q1bkRqGLi+2W/6ao+6Y7gu/RCwRuAhGEzh5B4KlszSuTLgZYuqFqo5bImjNKng==", + "dev": true, + "license": "MIT", + "dependencies": { + "mime-types": "^3.0.0", + "negotiator": "^1.0.0" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/ajv": { + "version": "8.20.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.20.0.tgz", + "integrity": "sha512-Thbli+OlOj+iMPYFBVBfJ3OmCAnaSyNn4M1vz9T6Gka5Jt9ba/HIR56joy65tY6kx/FCF5VXNB819Y7/GUrBGA==", + "dev": true, + "license": "MIT", + "dependencies": { + "fast-deep-equal": "^3.1.3", + "fast-uri": "^3.0.1", + "json-schema-traverse": "^1.0.0", + "require-from-string": "^2.0.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, + "node_modules/ajv-formats": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/ajv-formats/-/ajv-formats-3.0.1.tgz", + "integrity": "sha512-8iUql50EUR+uUcdRQ3HDqa6EVyo3docL8g5WJ3FNcWmu62IbkGUue/pEyLBW8VGKKucTPgqeks4fIU1DA4yowQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "ajv": "^8.0.0" + }, + "peerDependencies": { + "ajv": "^8.0.0" + }, + "peerDependenciesMeta": { + "ajv": { + "optional": true + } + } + }, + "node_modules/body-parser": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-2.3.0.tgz", + "integrity": "sha512-2cGmJupaNgg+QUwVLAucDuWuoMZ6EX9iHDRswZ5lsNYEmwPaRknMPCLZz07yTzVq/83p4o/wzbDZbBrTvGGTIw==", + "dev": true, + "license": "MIT", + "dependencies": { + "bytes": "^3.1.2", + "content-type": "^2.0.0", + "debug": "^4.4.3", + "http-errors": "^2.0.1", + "iconv-lite": "^0.7.2", + "on-finished": "^2.4.1", + "qs": "^6.15.2", + "raw-body": "^3.0.2", + "type-is": "^2.1.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/body-parser/node_modules/content-type": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-2.1.0.tgz", + "integrity": "sha512-mj7UPXE0jaqaOsukNZRUEfEi2AcL7C/vwmwcHV0O97eO1E1pxBZuyjlZrx5seTaNBg1U6+o35wpa35Qfcc+7ag==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/bytes": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", + "integrity": "sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/call-bind-apply-helpers": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz", + "integrity": "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/call-bound": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/call-bound/-/call-bound-1.0.4.tgz", + "integrity": "sha512-+ys997U96po4Kx/ABpBCqhA9EuxJaQWDQg7295H4hBphv3IZg0boBKuwYpt4YXp6MZ5AmZQnU/tyMTlRpaSejg==", + "dev": true, + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.2", + "get-intrinsic": "^1.3.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/content-disposition": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-1.1.0.tgz", + "integrity": "sha512-5jRCH9Z/+DRP7rkvY83B+yGIGX96OYdJmzngqnw2SBSxqCFPd0w2km3s5iawpGX8krnwSGmF0FW5Nhr0Hfai3g==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/content-type": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.5.tgz", + "integrity": "sha512-nTjqfcBFEipKdXCv4YDQWCfmcLZKm81ldF0pAopTvyrFGVbcR6P/VAAd5G7N+0tTr8QqiU0tFadD6FK4NtJwOA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/cookie": { + "version": "0.7.2", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.7.2.tgz", + "integrity": "sha512-yki5XnKuf750l50uGTllt6kKILY4nQ1eNIQatoXEByZ5dWgnKqbnqmTrBE5B4N7lrMJKQ2ytWMiTO2o0v6Ew/w==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/cookie-signature": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.2.2.tgz", + "integrity": "sha512-D76uU73ulSXrD1UXF4KE2TMxVVwhsnCgfAyTg9k8P6KGZjlXKrOLe4dJQKI3Bxi5wjesZoFXJWElNWBjPZMbhg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.6.0" + } + }, + "node_modules/cors": { + "version": "2.8.6", + "resolved": "https://registry.npmjs.org/cors/-/cors-2.8.6.tgz", + "integrity": "sha512-tJtZBBHA6vjIAaF6EnIaq6laBBP9aq/Y3ouVJjEfoHbRBcHBAHYcMh/w8LDrk2PvIMMq8gmopa5D4V8RmbrxGw==", + "dev": true, + "license": "MIT", + "dependencies": { + "object-assign": "^4", + "vary": "^1" + }, + "engines": { + "node": ">= 0.10" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/cross-spawn": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", + "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==", + "dev": true, + "license": "MIT", + "dependencies": { + "path-key": "^3.1.0", + "shebang-command": "^2.0.0", + "which": "^2.0.1" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "dev": true, + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/depd": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", + "integrity": "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/dunder-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz", + "integrity": "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==", + "dev": true, + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.1", + "es-errors": "^1.3.0", + "gopd": "^1.2.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/ee-first": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", + "integrity": "sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==", + "dev": true, + "license": "MIT" + }, + "node_modules/encodeurl": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-2.0.0.tgz", + "integrity": "sha512-Q0n9HRi4m6JuGIV1eFlmvJB7ZEVxu93IrMyiMsGC0lrMJMWzRgx6WGquyfQgZVb31vhGgXnfmPNNXmxnOkRBrg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/es-define-property": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz", + "integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-errors": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz", + "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-object-atoms": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.1.2.tgz", + "integrity": "sha512-HWcBoN6NileqtSydK2FqHbS/LoDd2pqrnQHLyJzBj4kOp/ky2MWMN694xOfkK8/SnUsW2DH7EfyVlydKCsm1Zw==", + "dev": true, + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/escape-html": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz", + "integrity": "sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow==", + "dev": true, + "license": "MIT" + }, + "node_modules/etag": { + "version": "1.8.1", + "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz", + "integrity": "sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/eventsource": { + "version": "3.0.7", + "resolved": "https://registry.npmjs.org/eventsource/-/eventsource-3.0.7.tgz", + "integrity": "sha512-CRT1WTyuQoD771GW56XEZFQ/ZoSfWid1alKGDYMmkt2yl8UXrVR4pspqWNEcqKvVIzg6PAltWjxcSSPrboA4iA==", + "dev": true, + "license": "MIT", + "dependencies": { + "eventsource-parser": "^3.0.1" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/eventsource-parser": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/eventsource-parser/-/eventsource-parser-3.1.1.tgz", + "integrity": "sha512-EKN1vKAMcZ8MlYMpaNuxN6R9yakzH6uajHcHVTqWJzvu5pWw9DyhbP35HH8MVBQ+dZjAfDxk+A8NiR9KWaXiyQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/express": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/express/-/express-5.2.1.tgz", + "integrity": "sha512-hIS4idWWai69NezIdRt2xFVofaF4j+6INOpJlVOLDO8zXGpUVEVzIYk12UUi2JzjEzWL3IOAxcTubgz9Po0yXw==", + "dev": true, + "license": "MIT", + "dependencies": { + "accepts": "^2.0.0", + "body-parser": "^2.2.1", + "content-disposition": "^1.0.0", + "content-type": "^1.0.5", + "cookie": "^0.7.1", + "cookie-signature": "^1.2.1", + "debug": "^4.4.0", + "depd": "^2.0.0", + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "etag": "^1.8.1", + "finalhandler": "^2.1.0", + "fresh": "^2.0.0", + "http-errors": "^2.0.0", + "merge-descriptors": "^2.0.0", + "mime-types": "^3.0.0", + "on-finished": "^2.4.1", + "once": "^1.4.0", + "parseurl": "^1.3.3", + "proxy-addr": "^2.0.7", + "qs": "^6.14.0", + "range-parser": "^1.2.1", + "router": "^2.2.0", + "send": "^1.1.0", + "serve-static": "^2.2.0", + "statuses": "^2.0.1", + "type-is": "^2.0.1", + "vary": "^1.1.2" + }, + "engines": { + "node": ">= 18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/express-rate-limit": { + "version": "8.7.0", + "resolved": "https://registry.npmjs.org/express-rate-limit/-/express-rate-limit-8.7.0.tgz", + "integrity": "sha512-hOwV7WOxXfjRpAM1DSJWZDXx3GhplwD8IfwuwvogD8i1Qnkgosw/H45s4ZnFAUHDAhPjlY9hLBvJhKmGMyY26g==", + "dev": true, + "license": "MIT", + "dependencies": { + "debug": "^4.4.3", + "ip-address": "^10.2.0" + }, + "engines": { + "node": ">= 16" + }, + "funding": { + "url": "https://github.com/sponsors/express-rate-limit" + }, + "peerDependencies": { + "express": ">= 4.11" + } + }, + "node_modules/fast-deep-equal": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", + "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", + "dev": true, + "license": "MIT" + }, + "node_modules/fast-uri": { + "version": "3.1.8", + "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.8.tgz", + "integrity": "sha512-GZMtZUTNRpOVIECoXwLNZS5xUGE+mVNbTB8h/7Rwh2TFWcBQiPzTgyZi05BF9UMZKkLJv8XBRJTlU7zg8+ZfMg==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fastify" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fastify" + } + ], + "license": "BSD-3-Clause" + }, + "node_modules/finalhandler": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-2.1.1.tgz", + "integrity": "sha512-S8KoZgRZN+a5rNwqTxlZZePjT/4cnm0ROV70LedRHZ0p8u9fRID0hJUZQpkKLzro8LfmC8sx23bY6tVNxv8pQA==", + "dev": true, + "license": "MIT", + "dependencies": { + "debug": "^4.4.0", + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "on-finished": "^2.4.1", + "parseurl": "^1.3.3", + "statuses": "^2.0.1" + }, + "engines": { + "node": ">= 18.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/forwarded": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz", + "integrity": "sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/fresh": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/fresh/-/fresh-2.0.0.tgz", + "integrity": "sha512-Rx/WycZ60HOaqLKAi6cHRKKI7zxWbJ31MhntmtwMoaTeF7XFH9hhBp8vITaMidfljRQ6eYWCKkaTK+ykVJHP2A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, "node_modules/fsevents": { "version": "2.3.2", "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", @@ -54,6 +578,396 @@ "node": "^8.16.0 || ^10.6.0 || >=11.0.0" } }, + "node_modules/function-bind": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", + "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", + "dev": true, + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/get-intrinsic": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.3.0.tgz", + "integrity": "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.2", + "es-define-property": "^1.0.1", + "es-errors": "^1.3.0", + "es-object-atoms": "^1.1.1", + "function-bind": "^1.1.2", + "get-proto": "^1.0.1", + "gopd": "^1.2.0", + "has-symbols": "^1.1.0", + "hasown": "^2.0.2", + "math-intrinsics": "^1.1.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/get-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/get-proto/-/get-proto-1.0.1.tgz", + "integrity": "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==", + "dev": true, + "license": "MIT", + "dependencies": { + "dunder-proto": "^1.0.1", + "es-object-atoms": "^1.0.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/gopd": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz", + "integrity": "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/has-symbols": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz", + "integrity": "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/hasown": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.4.tgz", + "integrity": "sha512-T2UbfbBEF32wiepXIsMlTW9+dDYC6wMh/t/vYA4tuOMKqWz/n3vr1NFSxQiyP+zk2mXsoMA/i/7qV6LKut1t1A==", + "dev": true, + "license": "MIT", + "dependencies": { + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/hono": { + "version": "4.13.8", + "resolved": "https://registry.npmjs.org/hono/-/hono-4.13.8.tgz", + "integrity": "sha512-/Gng7NfoykZl2pjukW5Z6+8Yxm3BPRf86GTbQnt0SbySkvax4fyL4H3HhY1cCpBGmiW9XDRFzRV+CXK2W8QudQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=16.9.0" + } + }, + "node_modules/http-errors": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.1.tgz", + "integrity": "sha512-4FbRdAX+bSdmo4AUFuS0WNiPz8NgFt+r8ThgNWmlrjQjt1Q7ZR9+zTlce2859x4KSXrwIsaeTqDoKQmtP8pLmQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "depd": "~2.0.0", + "inherits": "~2.0.4", + "setprototypeof": "~1.2.0", + "statuses": "~2.0.2", + "toidentifier": "~1.0.1" + }, + "engines": { + "node": ">= 0.8" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/iconv-lite": { + "version": "0.7.3", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.7.3.tgz", + "integrity": "sha512-IKXpvIzjnC9XTAUbVBcMfGS0EPaIXtW6v+zr+RRp+hqULEpo0owZax6wyRwPOJbWbzjYspQwusTsfVr0ifh4uQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3.0.0" + }, + "engines": { + "node": ">=0.10.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", + "dev": true, + "license": "ISC" + }, + "node_modules/ip-address": { + "version": "10.7.2", + "resolved": "https://registry.npmjs.org/ip-address/-/ip-address-10.7.2.tgz", + "integrity": "sha512-7H/2gFSIitxc0hG3nOI1glS8QLo/EHBFFLk8vEUjXY/xu0AdL8jZ9U1IzO2PUm0d2D/ofQcAifb0g6OBkt8U7w==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 12" + } + }, + "node_modules/ipaddr.js": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", + "integrity": "sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/is-promise": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/is-promise/-/is-promise-4.0.0.tgz", + "integrity": "sha512-hvpoI6korhJMnej285dSg6nu1+e6uxs7zG3BYAm5byqDsgJNWwxzM6z6iZiAgQR4TJ30JmBTOwqZUw3WlyH3AQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/isexe": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", + "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", + "dev": true, + "license": "ISC" + }, + "node_modules/jose": { + "version": "6.2.12", + "resolved": "https://registry.npmjs.org/jose/-/jose-6.2.12.tgz", + "integrity": "sha512-9NiFmJEex0sy2Dk58j2UGBSHgUs2ypF9eZSu4L6vjOX3Dp96Sw1F3uL+H+D1sx02jZZdzUT0HgvCy59CuvXcWw==", + "dev": true, + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/panva" + } + }, + "node_modules/json-schema-traverse": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", + "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==", + "dev": true, + "license": "MIT" + }, + "node_modules/json-schema-typed": { + "version": "8.0.2", + "resolved": "https://registry.npmjs.org/json-schema-typed/-/json-schema-typed-8.0.2.tgz", + "integrity": "sha512-fQhoXdcvc3V28x7C7BMs4P5+kNlgUURe2jmUT1T//oBRMDrqy1QPelJimwZGo7Hg9VPV3EQV5Bnq4hbFy2vetA==", + "dev": true, + "license": "BSD-2-Clause" + }, + "node_modules/math-intrinsics": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", + "integrity": "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/media-typer": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-1.1.1.tgz", + "integrity": "sha512-yz3xRaG20c6/BOzvYoDaGtPmGscs7YivItZEEqe6GbwNfHuxu9YNmvnEkMzKldAGY4/80pRcQRZSEnhquk9XuQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.8" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/merge-descriptors": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-2.0.0.tgz", + "integrity": "sha512-Snk314V5ayFLhp3fkUREub6WtjBfPdCPY1Ln8/8munuLuiYhsABgBVWsozAG+MWMbVEvcdcpbi9R7ww22l9Q3g==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/mime-db": { + "version": "1.54.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.54.0.tgz", + "integrity": "sha512-aU5EJuIN2WDemCcAp2vFBfp/m4EAhWJnUNSSw0ixs7/kXbd6Pg64EmwJkNdFhB8aWt1sH2CTXrLxo/iAGV3oPQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mime-types": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-3.0.2.tgz", + "integrity": "sha512-Lbgzdk0h4juoQ9fCKXW4by0UJqj+nOOrI9MJ1sSj4nI8aI2eo1qmvQEie4VD1glsS250n15LsWsYtCugiStS5A==", + "dev": true, + "license": "MIT", + "dependencies": { + "mime-db": "^1.54.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "dev": true, + "license": "MIT" + }, + "node_modules/negotiator": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-1.1.0.tgz", + "integrity": "sha512-NMPBRMJgiQHjbd8phG3Vebdx4kZ1H121rbl5IkMqeOsahptB9BKo/d7oJ3zTXqTgagn2bWlNSXkh0QUGM31RYg==", + "dev": true, + "license": "MIT", + "dependencies": { + "content-type": "^2.1.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/negotiator/node_modules/content-type": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-2.1.0.tgz", + "integrity": "sha512-mj7UPXE0jaqaOsukNZRUEfEi2AcL7C/vwmwcHV0O97eO1E1pxBZuyjlZrx5seTaNBg1U6+o35wpa35Qfcc+7ag==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/object-assign": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", + "integrity": "sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/object-inspect": { + "version": "1.13.4", + "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.4.tgz", + "integrity": "sha512-W67iLl4J2EXEGTbfeHCffrjDfitvLANg0UlX3wFUUSTx92KXRFegMHUVgSqE+wvhAbi4WqjGg9czysTV2Epbew==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/on-finished": { + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.4.1.tgz", + "integrity": "sha512-oVlzkg3ENAhCk2zdv7IJwd/QUD4z2RxRwpkcGY8psCVcCYZNq4wYnVWALHM+brtuJjePWiYF/ClmuDr8Ch5+kg==", + "dev": true, + "license": "MIT", + "dependencies": { + "ee-first": "1.1.1" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", + "dev": true, + "license": "ISC", + "dependencies": { + "wrappy": "1" + } + }, + "node_modules/parseurl": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz", + "integrity": "sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/path-key": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", + "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/path-to-regexp": { + "version": "8.4.2", + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-8.4.2.tgz", + "integrity": "sha512-qRcuIdP69NPm4qbACK+aDogI5CBDMi1jKe0ry5rSQJz8JVLsC7jV8XpiJjGRLLol3N+R5ihGYcrPLTno6pAdBA==", + "dev": true, + "license": "MIT", + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/pkce-challenge": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/pkce-challenge/-/pkce-challenge-5.0.1.tgz", + "integrity": "sha512-wQ0b/W4Fr01qtpHlqSqspcj3EhBvimsdh0KlHhH8HRZnMsEa0ea2fTULOXOS9ccQr3om+GcGRk4e+isrZWV8qQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=16.20.0" + } + }, "node_modules/playwright": { "version": "1.58.1", "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.58.1.tgz", @@ -86,6 +1000,311 @@ "node": ">=18" } }, + "node_modules/proxy-addr": { + "version": "2.0.8", + "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.8.tgz", + "integrity": "sha512-5nnx0yGyVUcY6t9RnWcARWtwT9F1D8O9rt08htPvnd49W1IgZtmLkhu9WfMzQj1cFxjHIO6connUNVW5k7AVyQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "forwarded": "0.2.0", + "ipaddr.js": "1.9.1" + }, + "engines": { + "node": ">= 0.10" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/qs": { + "version": "6.16.0", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.16.0.tgz", + "integrity": "sha512-h6fhOIaRrID2CbEY2fqs+7t+UXZo+MLAnU5gRIq85uFtdiUPCdsApMlHhXogKVM4HM2DVbIjGNTTYH2OcmP1vA==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "es-define-property": "^1.0.1", + "side-channel": "^1.1.1" + }, + "engines": { + "node": ">=0.6" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/range-parser": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.3.0.tgz", + "integrity": "sha512-hek2mFQpPuI4E1BBKrSto+BU3e3x4xuarsbiwr3+lf7p44juvFMV0XFWQAP3xUyqXA4RrXLIoaSUGbSt056ZMw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.6" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/raw-body": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-3.0.2.tgz", + "integrity": "sha512-K5zQjDllxWkf7Z5xJdV0/B0WTNqx6vxG70zJE4N0kBs4LovmEYWJzQGxC9bS9RAKu3bgM40lrd5zoLJ12MQ5BA==", + "dev": true, + "license": "MIT", + "dependencies": { + "bytes": "~3.1.2", + "http-errors": "~2.0.1", + "iconv-lite": "~0.7.0", + "unpipe": "~1.0.0" + }, + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/require-from-string": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz", + "integrity": "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/router": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/router/-/router-2.2.0.tgz", + "integrity": "sha512-nLTrUKm2UyiL7rlhapu/Zl45FwNgkZGaCpZbIHajDYgwlJCOzLSk+cIPAnsEqV955GjILJnKbdQC1nVPz+gAYQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "debug": "^4.4.0", + "depd": "^2.0.0", + "is-promise": "^4.0.0", + "parseurl": "^1.3.3", + "path-to-regexp": "^8.0.0" + }, + "engines": { + "node": ">= 18" + } + }, + "node_modules/safer-buffer": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", + "dev": true, + "license": "MIT" + }, + "node_modules/send": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/send/-/send-1.2.1.tgz", + "integrity": "sha512-1gnZf7DFcoIcajTjTwjwuDjzuz4PPcY2StKPlsGAQ1+YH20IRVrBaXSWmdjowTJ6u8Rc01PoYOGHXfP1mYcZNQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "debug": "^4.4.3", + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "etag": "^1.8.1", + "fresh": "^2.0.0", + "http-errors": "^2.0.1", + "mime-types": "^3.0.2", + "ms": "^2.1.3", + "on-finished": "^2.4.1", + "range-parser": "^1.2.1", + "statuses": "^2.0.2" + }, + "engines": { + "node": ">= 18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/serve-static": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-2.2.1.tgz", + "integrity": "sha512-xRXBn0pPqQTVQiC8wyQrKs2MOlX24zQ0POGaj0kultvoOCstBQM5yvOhAVSUwOMjQtTvsPWoNCHfPGwaaQJhTw==", + "dev": true, + "license": "MIT", + "dependencies": { + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "parseurl": "^1.3.3", + "send": "^1.2.0" + }, + "engines": { + "node": ">= 18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/setprototypeof": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz", + "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==", + "dev": true, + "license": "ISC" + }, + "node_modules/shebang-command": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", + "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", + "dev": true, + "license": "MIT", + "dependencies": { + "shebang-regex": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/shebang-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", + "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/side-channel": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.1.1.tgz", + "integrity": "sha512-6x6dK6zJdpTzF4sQeNYxwtvBzf6Eg4GtlesS94HOvTudUeyK2WXAaIfmDgsyslYrRBeFIlsi54AYsFGUuhmvrQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "object-inspect": "^1.13.4", + "side-channel-list": "^1.0.1", + "side-channel-map": "^1.0.1", + "side-channel-weakmap": "^1.0.2" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-list": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/side-channel-list/-/side-channel-list-1.0.1.tgz", + "integrity": "sha512-mjn/0bi/oUURjc5Xl7IaWi/OJJJumuoJFQJfDDyO46+hBWsfaVM65TBHq2eoZBhzl9EchxOijpkbRC8SVBQU0w==", + "dev": true, + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "object-inspect": "^1.13.4" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-map": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/side-channel-map/-/side-channel-map-1.0.1.tgz", + "integrity": "sha512-VCjCNfgMsby3tTdo02nbjtM/ewra6jPHmpThenkTYh8pG9ucZ/1P8So4u4FGBek/BjpOVsDCMoLA/iuBKIFXRA==", + "dev": true, + "license": "MIT", + "dependencies": { + "call-bound": "^1.0.2", + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.5", + "object-inspect": "^1.13.3" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-weakmap": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/side-channel-weakmap/-/side-channel-weakmap-1.0.2.tgz", + "integrity": "sha512-WPS/HvHQTYnHisLo9McqBHOJk2FkHO/tlpvldyrnem4aeQp4hai3gythswg6p01oSoTl58rcpiFAjF2br2Ak2A==", + "dev": true, + "license": "MIT", + "dependencies": { + "call-bound": "^1.0.2", + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.5", + "object-inspect": "^1.13.3", + "side-channel-map": "^1.0.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/statuses": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.2.tgz", + "integrity": "sha512-DvEy55V3DB7uknRo+4iOGT5fP1slR8wQohVdknigZPMpMstaKJQWhwiYBACJE3Ul2pTnATihhBYnRhZQHGBiRw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/toidentifier": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.1.tgz", + "integrity": "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.6" + } + }, + "node_modules/type-is": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/type-is/-/type-is-2.1.0.tgz", + "integrity": "sha512-faYHw0anBbc/kWF3zFTEnxSFOAGUX9GFbOBthvDdLsIlEoWOFOtS0zgCiQYwIskL9iGXZL3kAXD8OoZ4GmMATA==", + "dev": true, + "license": "MIT", + "dependencies": { + "content-type": "^2.0.0", + "media-typer": "^1.1.0", + "mime-types": "^3.0.0" + }, + "engines": { + "node": ">= 18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/type-is/node_modules/content-type": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-2.1.0.tgz", + "integrity": "sha512-mj7UPXE0jaqaOsukNZRUEfEi2AcL7C/vwmwcHV0O97eO1E1pxBZuyjlZrx5seTaNBg1U6+o35wpa35Qfcc+7ag==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, "node_modules/typescript": { "version": "5.9.3", "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", @@ -106,6 +1325,69 @@ "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==", "dev": true, "license": "MIT" + }, + "node_modules/unpipe": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", + "integrity": "sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/vary": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", + "integrity": "sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/which": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", + "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", + "dev": true, + "license": "ISC", + "dependencies": { + "isexe": "^2.0.0" + }, + "bin": { + "node-which": "bin/node-which" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", + "dev": true, + "license": "ISC" + }, + "node_modules/zod": { + "version": "4.6.5", + "resolved": "https://registry.npmjs.org/zod/-/zod-4.6.5.tgz", + "integrity": "sha512-v5l/aFXZQeai4awLbOpSoHecE9UiMrnfx75tEXLjNonXVARxQ5mOeipTjROUchszUNCqnE+hqAMujRsRHsut2Q==", + "dev": true, + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/colinhacks" + } + }, + "node_modules/zod-to-json-schema": { + "version": "3.25.2", + "resolved": "https://registry.npmjs.org/zod-to-json-schema/-/zod-to-json-schema-3.25.2.tgz", + "integrity": "sha512-O/PgfnpT1xKSDeQYSCfRI5Gy3hPf91mKVDuYLUHZJMiDFptvP41MSnWofm8dnCm0256ZNfZIM7DSzuSMAFnjHA==", + "dev": true, + "license": "ISC", + "peerDependencies": { + "zod": "^3.25.28 || ^4" + } } } } diff --git a/tests/e2e/ui/package.json b/tests/e2e/ui/package.json index ede759d97cb..78130412be9 100644 --- a/tests/e2e/ui/package.json +++ b/tests/e2e/ui/package.json @@ -9,6 +9,7 @@ "e2e:migration:root": "playwright test --config migration.serverRootPath.config.ts" }, "devDependencies": { + "@modelcontextprotocol/sdk": "1.30.0", "@playwright/test": "1.58.1", "@types/node": "20.19.37", "typescript": "5.9.3" diff --git a/tests/e2e/ui/tests/mcp/mcpTools.spec.ts b/tests/e2e/ui/tests/mcp/mcpTools.spec.ts index 3e6093c5c3f..921f73a928c 100644 --- a/tests/e2e/ui/tests/mcp/mcpTools.spec.ts +++ b/tests/e2e/ui/tests/mcp/mcpTools.spec.ts @@ -1,17 +1,20 @@ import { test, expect, Locator } from "@playwright/test"; import { ADMIN_STORAGE_PATH } from "../../constants"; -import { createMcpServer, deleteMcpServerByName, openMcpToolsTab } from "../../helpers/mcp"; +import { createMcpServer, deleteMcpServerByName, listUpstreamToolNames, openMcpToolsTab } from "../../helpers/mcp"; // Listing and calling MCP tools, which needs a server that really answers; the create-only spec // points at an unreachable URL on purpose. // -// This spec makes a read-only network call to DeepWiki's public MCP server, from the proxy rather -// than the browser. It needs no credentials, so there is no secret to leak from a public repo. +// This spec makes read-only network calls to DeepWiki's public MCP server: from the proxy, and from +// the test runner to learn which tools the upstream advertises today, so the tool list is never +// pinned here. It needs no credentials, so there is no secret to leak from a public repo. // // A DeepWiki outage turns this red for something that is not a litellm regression. That is left // visible rather than auto-skipped: skipping on connection trouble also skips when the proxy's own // MCP client breaks, which is the regression this exists to catch. E2E_SKIP_EXTERNAL_MCP=1 opts out. const MCP_SERVER_URL = "https://mcp.deepwiki.com/mcp"; +// Read from DeepWiki's tools/list on 2026-09-22. One name has to be pinned so the call-tool test can +// fill a known input (repoName); the listing test checks it is still advertised before the UI checks. const TOOL_NAME = "read_wiki_structure"; const TOOL_ARG_REPO = "BerriAI/litellm"; @@ -36,14 +39,17 @@ test.describe("MCP Tools", () => { }); test("MCP Catalog tab lists the tools the upstream server advertises", async ({ page }) => { + const upstreamTools = await listUpstreamToolNames(MCP_SERVER_URL); + expect(upstreamTools).toContain(TOOL_NAME); + // Fetched through the proxy on mount, so allow for a cold upstream connection. const toolList = page.locator(".mcp-tools-scrollable"); await expect(toolList).toBeVisible({ timeout: 30_000 }); - // Non-empty would still pass if the proxy returned some other server's tools. - await expect(toolCard(toolList, TOOL_NAME)).toBeVisible(); - await expect(toolCard(toolList, "ask_question")).toBeVisible(); - await expect(toolCard(toolList, "read_wiki_contents")).toBeVisible(); + for (const name of upstreamTools) { + await expect(toolCard(toolList, name)).toBeVisible(); + } + await expect(toolList.locator("h4.font-mono")).toHaveCount(upstreamTools.length); // No other tool's name or description contains this string, so exactly one card survives. await page.getByPlaceholder("Search tools...").fill(TOOL_NAME); diff --git a/tests/e2e/ui/tests/prompts/addPrompt.spec.ts b/tests/e2e/ui/tests/prompts/addPrompt.spec.ts new file mode 100644 index 00000000000..9d85236c4a6 --- /dev/null +++ b/tests/e2e/ui/tests/prompts/addPrompt.spec.ts @@ -0,0 +1,75 @@ +import { test, expect } from "@playwright/test"; + +import { ADMIN_STORAGE_PATH } from "../../constants"; +import { Page as DashboardPage } from "../../fixtures/pages"; +import { navigateToPage } from "../../helpers/navigation"; +import { runWithCleanup } from "../../helpers/roundTrip"; +import { masterKey, uniqueSuffix } from "../../helpers/traffic"; + +test.use({ storageState: ADMIN_STORAGE_PATH }); + +test.describe("Prompt upload form", () => { + test("uploads a prompt file and reads the created prompt back", async ({ + page, + }) => { + const promptId = `e2e-prompt-${uniqueSuffix()}`; + const promptContent = "Hello {{name}}"; + + await runWithCleanup( + async () => { + await navigateToPage(page, DashboardPage.Prompts); + await page.getByRole("button", { name: "Upload .prompt File" }).click(); + await expect( + page.getByRole("dialog", { name: "Add New Prompt" }), + ).toBeVisible(); + await page.getByLabel("Prompt ID").fill(promptId); + await page.locator('input[type="file"]').setInputFiles({ + name: "e2e.prompt", + mimeType: "text/plain", + buffer: Buffer.from( + `---\nmodel: fake-openai-gpt-4\n---\n${promptContent}\n`, + ), + }); + await expect(page.getByText("Selected: e2e.prompt")).toBeVisible(); + await page.getByRole("button", { name: "Create Prompt" }).click(); + + await expect + .poll(async () => { + const response = await page.request.get( + `/prompts/${encodeURIComponent(promptId)}/info?environment=development`, + { + headers: { Authorization: `Bearer ${masterKey()}` }, + }, + ); + return response.ok(); + }) + .toBe(true); + await expect + .poll(async () => { + const response = await page.request.get( + `/prompts/${encodeURIComponent(promptId)}/info?environment=development`, + { + headers: { Authorization: `Bearer ${masterKey()}` }, + }, + ); + if (!response.ok()) return undefined; + const promptInfo = (await response.json()) as { + raw_prompt_template?: { content?: string }; + }; + return promptInfo.raw_prompt_template?.content; + }) + .toBe(promptContent); + await expect(page.getByText(promptId, { exact: true })).toBeVisible(); + }, + async () => { + const response = await page.request.delete( + `/prompts/${encodeURIComponent(promptId)}?environment=development`, + { + headers: { Authorization: `Bearer ${masterKey()}` }, + }, + ); + return response.ok(); + }, + ); + }); +}); diff --git a/tests/e2e/ui/tests/tagManagement/tagManagement.spec.ts b/tests/e2e/ui/tests/tagManagement/tagManagement.spec.ts new file mode 100644 index 00000000000..fe659080eab --- /dev/null +++ b/tests/e2e/ui/tests/tagManagement/tagManagement.spec.ts @@ -0,0 +1,84 @@ +import { test, expect } from "@playwright/test"; + +import { ADMIN_STORAGE_PATH } from "../../constants"; +import { Page as DashboardPage } from "../../fixtures/pages"; +import { navigateToPage } from "../../helpers/navigation"; +import { + captureRequestBody, + readBack, + runWithCleanup, +} from "../../helpers/roundTrip"; +import { masterKey, uniqueSuffix } from "../../helpers/traffic"; + +test.use({ storageState: ADMIN_STORAGE_PATH }); + +test.describe("Tag management", () => { + test("creates, edits, reopens, and reads back a tag", async ({ page }) => { + const tagName = `e2e-tag-${uniqueSuffix()}`; + const description = "synthetic tag description"; + const updatedDescription = `${description} updated`; + + await runWithCleanup( + async () => { + await navigateToPage(page, DashboardPage.TagManagement); + await page.getByRole("button", { name: "+ Create New Tag" }).click(); + await expect( + page.getByRole("dialog", { name: "Create New Tag" }), + ).toBeVisible(); + await page.getByLabel("Tag Name").fill(tagName); + await page.getByLabel("Description").fill(description); + await page.getByRole("button", { name: "Create Tag" }).click(); + + await expect + .poll(async () => { + const response = await readBack>( + page, + "/tag/list", + ); + return response.some((tag) => tag.name === tagName); + }) + .toBe(true); + await expect(page.getByText(tagName, { exact: true })).toBeVisible(); + + await page.getByText(tagName, { exact: true }).click(); + await expect(page.getByText("Tag Name:")).toBeVisible(); + await page.getByRole("button", { name: "Edit Tag" }).click(); + await page.getByLabel("Description").fill(updatedDescription); + const updateBody = await captureRequestBody( + page, + { method: "POST", urlIncludes: "/tag/update" }, + () => page.getByRole("button", { name: "Save Changes" }).click(), + ); + expect(updateBody).toMatchObject({ + name: tagName, + description: updatedDescription, + }); + + await expect + .poll(async () => { + const infoResponse = await page.request.post("/tag/info", { + headers: { Authorization: `Bearer ${masterKey()}` }, + data: { names: [tagName] }, + }); + expect(infoResponse.ok()).toBe(true); + const info = (await infoResponse.json()) as Record< + string, + { description?: string } + >; + return info[tagName]?.description; + }) + .toBe(updatedDescription); + }, + async () => { + const response = await page.request.post("/tag/delete", { + headers: { + Authorization: `Bearer ${masterKey()}`, + "Content-Type": "application/json", + }, + data: { name: tagName }, + }); + return response.ok(); + }, + ); + }); +}); diff --git a/tests/integration/contracts.json b/tests/integration/contracts.json index 5d9a17acc49..e1b5940935f 100644 --- a/tests/integration/contracts.json +++ b/tests/integration/contracts.json @@ -172,12 +172,39 @@ "tests/integration/providers/test_fal_ai_image_wire.py::test_fal_gpt_image_25_generation_sends_quality_and_size_and_charges_keyed_row": [ "other.provider_wire.fal_ai.gpt_image_generation_quality_size_wire_and_keyed_pricing" ], + "tests/integration/providers/test_fal_ai_image_wire.py::test_fal_gpt_image_25_generation_prices_non_canonical_size_from_nearest_row": [ + "other.provider_wire.fal_ai.gpt_image_generation_noncanonical_size_uses_nearest_keyed_row" + ], + "tests/integration/providers/test_fal_ai_image_wire.py::test_fal_gpt_image_sdk_response_honors_dump_options": [ + "other.provider_wire.fal_ai.sdk_image_response_dump_options" + ], "tests/integration/providers/test_fal_ai_image_wire.py::test_fal_flux_dev_generation_targets_dev_endpoint_and_charges_per_image": [ "other.provider_wire.fal_ai.flux_dev_endpoint_and_per_image_pricing" ], + "tests/integration/providers/test_fal_ai_passthrough_wire.py::test_fal_queue_submit_charges_and_polls_pass_through_free": [ + "other.provider_wire.fal_ai.passthrough_queue_submit_charges_and_polls_do_not" + ], "tests/integration/providers/test_fal_ai_image_wire.py::test_fal_gpt_image_25_edit_inlines_upload_as_data_url_and_charges_keyed_row": [ "other.provider_wire.fal_ai.image_edit_json_data_urls_and_keyed_pricing" ], + "tests/integration/providers/test_fal_ai_image_wire.py::test_fal_flux_lora_depth_edit_sends_single_image_url_and_charges_flat_row": [ + "other.provider_wire.fal_ai.flux_lora_depth_edit_single_image_url_and_flat_pricing" + ], + "tests/integration/providers/test_fal_ai_chat_wire.py::test_fal_moondream3_chat_sends_prompt_image_and_reasoning": [ + "other.provider_wire.fal_ai.moondream3_chat_query_wire_and_token_pricing" + ], + "tests/integration/providers/test_xiaomi_mimo_wire.py::test_xiaomi_mimo_nonstream_surfaces_reasoning_and_charges_registry_price[mimo-v2.6-pro]": [ + "other.provider_wire.xiaomi_mimo.reasoning_content_and_registry_pricing" + ], + "tests/integration/providers/test_xiaomi_mimo_wire.py::test_xiaomi_mimo_nonstream_surfaces_reasoning_and_charges_registry_price[mimo-v2.6-flash]": [ + "other.provider_wire.xiaomi_mimo.reasoning_content_and_registry_pricing" + ], + "tests/integration/providers/test_xiaomi_mimo_wire.py::test_xiaomi_mimo_stream_delivers_reasoning_then_answer_deltas": [ + "other.provider_wire.xiaomi_mimo.reasoning_and_answer_stream_as_deltas" + ], + "tests/integration/providers/test_xiaomi_mimo_wire.py::test_xiaomi_mimo_tool_call_is_forwarded_and_returned": [ + "other.provider_wire.xiaomi_mimo.tool_call_survives_translation" + ], "tests/integration/providers/test_fal_ai_video_wire.py::test_fal_h3_video_create_uses_canonical_body_and_status_path": [ "other.provider_wire.fal_ai.video_queue_create_status_and_content_download" ], diff --git a/tests/integration/cost_calculation/cost_tracking_case.py b/tests/integration/cost_calculation/cost_tracking_case.py index ea8bf230d05..ac3fcd33d2e 100644 --- a/tests/integration/cost_calculation/cost_tracking_case.py +++ b/tests/integration/cost_calculation/cost_tracking_case.py @@ -1,5 +1,6 @@ from __future__ import annotations +import json from collections.abc import Mapping from pathlib import Path from types import MappingProxyType @@ -8,6 +9,7 @@ from typing import Annotated, Final, Literal, TypeAlias from pydantic import BaseModel, ConfigDict, Field, JsonValue, field_validator, model_validator CASES_PATH: Final = Path(__file__).resolve().parent / "cost_tracking_cases.json" +PRIOR_RESPONSE_ID_MARKER: Final = "$PRIOR_RESPONSE_ID" class SearchContextCostPerQuery(BaseModel): @@ -312,6 +314,21 @@ class CostTrackingTestCase(BaseModel): usage: Final = self.response.body.get("usage") return isinstance(usage, dict) and isinstance(usage.get("cost"), (int, float)) + @property + def chains_prior_response(self) -> bool: + return self.request.get("previous_response_id") == PRIOR_RESPONSE_ID_MARKER + + @property + def can_chain_prior_response(self) -> bool: + return ( + self.chains_prior_response + and self.endpoint == "/v1/responses" + and isinstance(self.response, JsonResponse) + and isinstance(self.response.body.get("id"), str) + and not isinstance(self.expected, FailureExpected) + and not (isinstance(self.expected, ExactExpected) and self.expected.rollups) + ) + class BatchOutputLine(BaseModel): model_config = ConfigDict(frozen=True, extra="forbid") @@ -681,6 +698,11 @@ def data_errors() -> tuple[str, ...]: for marker in ('"id": "call_$REQUEST_ID"', '"id": "toolu_$REQUEST_ID"') ) ) + invalid_prior_response_chains: Final = sorted( + case.name + for case in CASES + if PRIOR_RESPONSE_ID_MARKER in json.dumps(case.request) and not case.can_chain_prior_response + ) return tuple( message for message in ( @@ -700,6 +722,10 @@ def data_errors() -> tuple[str, ...]: f"pinned tool IDs contain $REQUEST_ID: {invalid_pinned_tool_ids}" if invalid_pinned_tool_ids else None, + f"{PRIOR_RESPONSE_ID_MARKER} needs a non-rollup, non-failure /v1/responses JSON response with a string id" + f" as previous_response_id: {invalid_prior_response_chains}" + if invalid_prior_response_chains + else None, ) if message is not None ) diff --git a/tests/integration/cost_calculation/cost_tracking_cases.json b/tests/integration/cost_calculation/cost_tracking_cases.json index 17ebc793fae..09dfa66012f 100644 --- a/tests/integration/cost_calculation/cost_tracking_cases.json +++ b/tests/integration/cost_calculation/cost_tracking_cases.json @@ -27733,7 +27733,7 @@ "request": { "model": "$MODEL", "input": "continue this text", - "previous_response_id": "resp_scripted_prior" + "previous_response_id": "$PRIOR_RESPONSE_ID" }, "response": { "content_type": "application/json", diff --git a/tests/integration/cost_calculation/test_cost_tracking.py b/tests/integration/cost_calculation/test_cost_tracking.py index efab17acba4..82878634677 100644 --- a/tests/integration/cost_calculation/test_cost_tracking.py +++ b/tests/integration/cost_calculation/test_cost_tracking.py @@ -9,13 +9,14 @@ import time import uuid import wave import zlib +from collections.abc import Mapping from hashlib import sha256 from itertools import islice from typing import Final, cast import httpx import pytest -from integration._support.client import JSON_OBJECT, Gateway +from integration._support.client import JSON_OBJECT, Gateway, string_value from integration._support.upstream import delete_scenario, register_scenario from integration.cost_calculation.assertions import assert_exact, assert_recount from integration.cost_calculation.conftest import ( @@ -112,6 +113,19 @@ def _replace_model(value: JsonValue, model_name: str) -> JsonValue: return value +def _prime_prior_response( + gateway: Gateway, request_path: str, request_values: Mapping[str, JsonValue], key: str +) -> str: + primed: Final = gateway.request( + "POST", + request_path, + {field: value for field, value in request_values.items() if field != "previous_response_id"}, + key=key, + ) + assert primed.is_success, f"priming response failed: {primed.status_code}: {primed.text[:400]}" + return string_value(JSON_OBJECT.validate_json(primed.content)["id"]) + + @pytest.mark.parametrize("case", _CASES) def test_case_bills_expected_cost(gateway: Gateway, case: CostTrackingTestCase) -> None: marker: Final = sha256(case.name.encode()).hexdigest()[:12] @@ -175,21 +189,6 @@ def test_case_bills_expected_cost(gateway: Gateway, case: CostTrackingTestCase) if isinstance(expected, ExactExpected) and expected.rollups else None ) - request_body: Final = JSON_OBJECT.validate_python( - { - **base_request_values, - **( - {"model": fallback_deployment.model_name, "fallbacks": [model_name]} - if fallback_deployment is not None - else {} - ), - **( - {"user": end_user_id, "cache": {"no-cache": True}} - if end_user_id is not None - else {} - ), - } - ) request_headers: Final = ( { "x-pass-x-scripted-scenario": scenario_id, @@ -207,6 +206,27 @@ def test_case_bills_expected_cost(gateway: Gateway, case: CostTrackingTestCase) if passthrough_provider is not None else case.endpoint ) + prior_response_id: Final = ( + _prime_prior_response(gateway, request_path, base_request_values, key) + if case.chains_prior_response + else None + ) + request_body: Final = JSON_OBJECT.validate_python( + { + **base_request_values, + **( + {"model": fallback_deployment.model_name, "fallbacks": [model_name]} + if fallback_deployment is not None + else {} + ), + **( + {"user": end_user_id, "cache": {"no-cache": True}} + if end_user_id is not None + else {} + ), + **({"previous_response_id": prior_response_id} if prior_response_id is not None else {}), + } + ) if case.disconnect_after_frames is not None: with gateway.client.stream( "POST", @@ -250,7 +270,7 @@ def test_case_bills_expected_cost(gateway: Gateway, case: CostTrackingTestCase) assert response.is_success, f"{case.name}: proxy returned {response.status_code}: {response.text[:400]}" if case.response.content_type == "text/event-stream": _assert_stream_has_no_error(response.text) - rows: Final = poll_rows(key, len(responses)) + rows: Final = poll_rows(key, len(responses) + (prior_response_id is not None)) if isinstance(expected, RecountExpected): row: Final = rows[0] assert_recount(case.name, expected, row) diff --git a/tests/integration/providers/test_fal_ai_chat_wire.py b/tests/integration/providers/test_fal_ai_chat_wire.py new file mode 100644 index 00000000000..2bb1ac3f168 --- /dev/null +++ b/tests/integration/providers/test_fal_ai_chat_wire.py @@ -0,0 +1,99 @@ +import json +from pathlib import Path +from typing import Final + +import httpx +import pytest +from integration._support.client import Gateway +from integration._support.wire import Reply, Request, wire_server +from pydantic import JsonValue, TypeAdapter + +_MODEL: Final = "fal-ai/moondream3-preview/query" +_PROMPT: Final = "what is in this image?" +_COST_MAP_PATH: Final = Path(__file__).resolve().parents[3] / "model_prices_and_context_window.json" +_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) +_COST_MAP: Final = TypeAdapter(dict[str, dict[str, object]]) + + +def _catalog_cost(key: str, field: str) -> float: + cost_map: Final = _COST_MAP.validate_json(_COST_MAP_PATH.read_bytes()) + cost_value: Final = cost_map[key][field] + assert isinstance(cost_value, (int, float)) + return float(cost_value) + + +def _approx(value: float) -> object: + return pytest.approx(value, rel=1e-6) # pyright: ignore[reportUnknownMemberType] # pytest lacks typed approx stubs + + +@pytest.mark.covers("other.provider_wire.fal_ai.moondream3_chat_query_wire_and_token_pricing") +def test_fal_moondream3_chat_sends_prompt_image_and_reasoning(gateway: Gateway) -> None: + def respond(request: Request) -> Reply: + assert request.method == "POST" + assert request.headers["authorization"] == "Key synthetic-fal-key" + assert request.target == f"/{_MODEL}" + assert request.headers["content-type"] == "application/json" + assert _JSON_OBJECT.validate_json(request.body) == { + "prompt": _PROMPT, + "image_url": "https://example.com/pic.png", + "reasoning": False, + "temperature": 0.2, + } + return Reply( + body=json.dumps( + { + "output": "a red circle on a blue background", + "reasoning": "inspected the shapes", + "finish_reason": "stop", + "usage_info": { + "input_tokens": 11, + "output_tokens": 7, + "prefill_time_ms": 1.0, + "decode_time_ms": 2.0, + "ttft_ms": 1.5, + }, + } + ).encode() + ) + + with wire_server(respond) as wire, gateway.scenario() as scenario: + wire_url: Final = wire.url + model: Final = scenario.model(model=f"fal_ai/{_MODEL}", api_base=wire.url, api_key="synthetic-fal-key") + response: Final = gateway.request( + "POST", + "/v1/chat/completions", + { + "model": model, + "messages": [ + { + "role": "user", + "content": [ + {"type": "text", "text": _PROMPT}, + {"type": "image_url", "image_url": {"url": "https://example.com/pic.png"}}, + ], + } + ], + "reasoning_effort": "none", + "temperature": 0.2, + }, + ) + assert response.status_code == 200, response.text + payload: Final = _JSON_OBJECT.validate_json(response.content) + assert payload["choices"] == [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "role": "assistant", + "content": "a red circle on a blue background", + "reasoning_content": "inspected the shapes", + }, + } + ] + assert payload["usage"] == {"prompt_tokens": 11, "completion_tokens": 7, "total_tokens": 18} + cost: Final = float(response.headers["x-litellm-response-cost"]) + assert cost == _approx( + 11 * _catalog_cost(f"fal_ai/{_MODEL}", "input_cost_per_token") + + 7 * _catalog_cost(f"fal_ai/{_MODEL}", "output_cost_per_token") + ) + assert [(request.method, request.target) for request in wire.drain()] == [("POST", f"/{_MODEL}")] diff --git a/tests/integration/providers/test_fal_ai_image_wire.py b/tests/integration/providers/test_fal_ai_image_wire.py index f9ceac0b037..02f24f9e369 100644 --- a/tests/integration/providers/test_fal_ai_image_wire.py +++ b/tests/integration/providers/test_fal_ai_image_wire.py @@ -4,6 +4,7 @@ from pathlib import Path from typing import Final import httpx +import litellm import pytest from integration._support.client import Gateway from integration._support.wire import Reply, Request, wire_server @@ -120,6 +121,63 @@ def test_fal_gpt_image_25_generation_sends_quality_and_size_and_charges_keyed_ro ] +@pytest.mark.covers("other.provider_wire.fal_ai.gpt_image_generation_noncanonical_size_uses_nearest_keyed_row") +def test_fal_gpt_image_25_generation_prices_non_canonical_size_from_nearest_row(gateway: Gateway) -> None: + def respond(request: Request) -> Reply: + assert request.method == "POST" + assert request.headers["authorization"] == "Key synthetic-fal-key" + assert request.target == "/openai/gpt-image-2.5/flare/text-to-image" + body: Final = _JSON_OBJECT.validate_json(request.body) + assert body == {"prompt": _PROMPT, "quality": "low", "image_size": {"width": 1536, "height": 1024}} + return Reply(body=_image_response(((f"{wire_url}/files/noncanonical.png", 1536, 1024),), _PROMPT)) + + with wire_server(respond) as wire, gateway.scenario() as scenario: + wire_url: Final = wire.url + model: Final = scenario.model( + model=f"fal_ai/{_GPT_IMAGE_MODEL}", api_base=wire.url, api_key="synthetic-fal-key" + ) + response: Final = gateway.request( + "POST", + "/v1/images/generations", + {"model": model, "prompt": _PROMPT, "quality": "low", "size": "1536x1024"}, + ) + assert response.status_code == 200, response.text + cost: Final = _response_cost(response) + assert cost == _approx(_catalog_cost("fal_ai/low/1024-x-1536/openai/gpt-image-2.5/flare/text-to-image")) + assert [(request.method, request.target) for request in wire.drain()] == [ + ("POST", "/openai/gpt-image-2.5/flare/text-to-image") + ] + + +@pytest.mark.covers("other.provider_wire.fal_ai.sdk_image_response_dump_options") +def test_fal_gpt_image_sdk_response_honors_dump_options() -> None: + def respond(request: Request) -> Reply: + assert request.method == "POST" + assert request.headers["authorization"] == "Key synthetic-fal-key" + assert request.target == "/openai/gpt-image-2.5/flare/text-to-image" + assert _JSON_OBJECT.validate_json(request.body) == {"prompt": _PROMPT, "quality": "low"} + return Reply(body=_image_response((("https://example.com/fal.png", 1024, 1536),), _PROMPT)) + + with wire_server(respond) as wire: + response: Final = litellm.image_generation( + model=_GPT_IMAGE_MODEL, + prompt=_PROMPT, + quality="low", + api_base=wire.url, + api_key="synthetic-fal-key", + custom_llm_provider="fal_ai", + ) + assert response.model_dump(exclude_none=True)["data"] == [ + { + "url": "https://example.com/fal.png", + "provider_specific_fields": {"width": 1024, "height": 1536, "content_type": "image/png"}, + } + ] + assert [(request.method, request.target) for request in wire.drain()] == [ + ("POST", "/openai/gpt-image-2.5/flare/text-to-image") + ] + + @pytest.mark.covers("other.provider_wire.fal_ai.flux_dev_endpoint_and_per_image_pricing") def test_fal_flux_dev_generation_targets_dev_endpoint_and_charges_per_image(gateway: Gateway) -> None: def respond(request: Request) -> Reply: @@ -205,3 +263,44 @@ def test_fal_gpt_image_25_edit_inlines_upload_as_data_url_and_charges_keyed_row( assert [(request.method, request.target) for request in wire.drain()] == [ ("POST", "/openai/gpt-image-2.5/flare/edit") ] + + +@pytest.mark.covers("other.provider_wire.fal_ai.flux_lora_depth_edit_single_image_url_and_flat_pricing") +def test_fal_flux_lora_depth_edit_sends_single_image_url_and_charges_flat_row(gateway: Gateway) -> None: + def respond(request: Request) -> Reply: + assert request.method == "POST" + assert request.headers["authorization"] == "Key synthetic-fal-key" + assert request.target == "/fal-ai/flux-lora-depth" + assert request.headers["content-type"] == "application/json" + assert _JSON_OBJECT.validate_json(request.body) == { + "prompt": _PROMPT, + "image_url": "data:image/png;base64," + base64.b64encode(_PNG_BYTES).decode(), + } + return Reply(body=_image_response(((f"{wire_url}/files/depth.png", 1024, 1024),), _PROMPT)) + + with wire_server(respond) as wire, gateway.scenario() as scenario: + wire_url: Final = wire.url + model: Final = scenario.model( + model="fal_ai/fal-ai/flux-lora-depth", api_base=wire.url, api_key="synthetic-fal-key" + ) + response: Final = gateway.client.post( + "/v1/images/edits", + data={"model": model, "prompt": _PROMPT}, + files={"image": ("red_circle.png", _PNG_BYTES, "image/png")}, + headers={"Authorization": f"Bearer {gateway.key}"}, + ) + assert response.status_code == 200, response.text + payload: Final = _JSON_OBJECT.validate_json(response.content) + assert payload["data"] == [ + { + "url": f"{wire.url}/files/depth.png", + "b64_json": None, + "revised_prompt": None, + "provider_specific_fields": {"width": 1024, "height": 1024, "content_type": "image/png"}, + } + ] + cost: Final = _response_cost(response) + assert cost == _approx(_catalog_cost("fal_ai/fal-ai/flux-lora-depth")) + assert [(request.method, request.target) for request in wire.drain()] == [ + ("POST", "/fal-ai/flux-lora-depth") + ] diff --git a/tests/integration/providers/test_fal_ai_passthrough_wire.py b/tests/integration/providers/test_fal_ai_passthrough_wire.py new file mode 100644 index 00000000000..f103135124e --- /dev/null +++ b/tests/integration/providers/test_fal_ai_passthrough_wire.py @@ -0,0 +1,86 @@ +import json +from typing import Final + +import pytest +from integration._support.client import Gateway, eventually +from integration._support.database import read_rows +from integration._support.process import owned_proxy +from integration._support.wire import Reply, Request, wire_server + +_MODEL: Final = "fal-ai/trellis-2" +_REQUEST_BODY: Final = {"image_url": "https://example.com/in.png", "resolution": 1536} +_UPSTREAM_BODY: Final = { + "model_glb": { + "url": "https://fal.media/model.glb", + "content_type": "model/gltf-binary", + "file_name": "model.glb", + "file_size": 123, + } +} +_EXPECTED_SPEND: Final = 0.35 + + +@pytest.mark.covers("other.provider_wire.fal_ai.passthrough_queue_submit_charges_and_polls_do_not") +def test_fal_queue_submit_charges_and_polls_pass_through_free(gateway: Gateway, tmp_path) -> None: + def respond(request: Request) -> Reply: + assert request.headers["authorization"] == "Key synthetic-fal-key" + if request.method == "POST": + assert request.target == f"/{_MODEL}" + assert json.loads(request.body) == _REQUEST_BODY + return Reply(body=json.dumps({"request_id": "req-1", "status": "IN_QUEUE"}).encode()) + if request.target == f"/{_MODEL}/requests/req-1/status": + return Reply(body=json.dumps({"status": "COMPLETED"}).encode()) + assert request.target == f"/{_MODEL}/requests/req-1" + return Reply(body=json.dumps(_UPSTREAM_BODY).encode()) + + config: Final = tmp_path / "proxy_config.yaml" + config.write_text( + "model_list: []\n" + "general_settings:\n" + " master_key: os.environ/LITELLM_MASTER_KEY\n" + " database_url: os.environ/DATABASE_URL\n" + " store_model_in_db: true\n" + " disable_spend_logs: false\n" + " proxy_batch_write_at: 1\n" + "router_settings:\n" + " disable_cooldowns: true\n" + ) + with wire_server(respond) as wire: + with owned_proxy( + gateway, + tmp_path, + {"FAL_AI_QUEUE_API_BASE": wire.url, "FAL_AI_API_KEY": "synthetic-fal-key"}, + config=config, + ) as candidate: + submit: Final = candidate.request("POST", f"/fal_ai/{_MODEL}", _REQUEST_BODY) + assert submit.status_code == 200, submit.text + assert json.loads(submit.content) == {"request_id": "req-1", "status": "IN_QUEUE"} + status_response: Final = candidate.request("GET", f"/fal_ai/{_MODEL}/requests/req-1/status") + assert status_response.status_code == 200, status_response.text + assert json.loads(status_response.content) == {"status": "COMPLETED"} + result_response: Final = candidate.request("GET", f"/fal_ai/{_MODEL}/requests/req-1") + assert result_response.status_code == 200, result_response.text + assert json.loads(result_response.content) == _UPSTREAM_BODY + submit_spend: Final = eventually( + lambda: read_rows( + 'SELECT spend FROM "LiteLLM_SpendLogs" WHERE request_id=%s', + (submit.headers["x-litellm-call-id"],), + ), + lambda values: len(values) == 1, + seconds=70, + ) + assert float(submit_spend[0]["spend"]) == pytest.approx(_EXPECTED_SPEND) + poll_rows: Final = eventually( + lambda: read_rows( + 'SELECT spend FROM "LiteLLM_SpendLogs" WHERE request_id=ANY(%s)', + ([status_response.headers["x-litellm-call-id"], result_response.headers["x-litellm-call-id"]],), + ), + lambda values: len(values) == 2, + seconds=70, + ) + assert sorted(float(row["spend"]) for row in poll_rows) == [0.0, 0.0] + assert [(request.method, request.target) for request in wire.drain()] == [ + ("POST", f"/{_MODEL}"), + ("GET", f"/{_MODEL}/requests/req-1/status"), + ("GET", f"/{_MODEL}/requests/req-1"), + ] diff --git a/tests/integration/providers/test_fal_ai_video_wire.py b/tests/integration/providers/test_fal_ai_video_wire.py index 827818c6780..276ea2868a7 100644 --- a/tests/integration/providers/test_fal_ai_video_wire.py +++ b/tests/integration/providers/test_fal_ai_video_wire.py @@ -76,6 +76,9 @@ def test_fal_h3_video_create_uses_canonical_body_and_status_path(gateway: Gatewa request_id: Final = "fal-h3-req-" + uuid.uuid4().hex def respond(request: Request) -> Reply: + if request.target == f"/files/{request_id}.mp4": + assert request.method == "GET" + return Reply(body=_MP4, content_type="video/mp4") assert request.headers["authorization"] == "Key synthetic-fal-key" if request.method == "POST": assert request.target == f"/{_H3_MODEL}" diff --git a/tests/integration/providers/test_xiaomi_mimo_wire.py b/tests/integration/providers/test_xiaomi_mimo_wire.py new file mode 100644 index 00000000000..96b9dc17bdf --- /dev/null +++ b/tests/integration/providers/test_xiaomi_mimo_wire.py @@ -0,0 +1,258 @@ +import json +import uuid +from collections.abc import Mapping +from pathlib import Path +from typing import Final + +import pytest +from integration._support.client import Gateway, eventually +from integration._support.database import read_rows +from integration._support.wire import Reply, Request, wire_server +from pydantic import BaseModel, ConfigDict, JsonValue, TypeAdapter + +_BACKENDS: Final = ("mimo-v2.6-pro", "mimo-v2.6-flash") +_API_KEY: Final = "synthetic-xiaomi-key" +_ARITHMETIC_PROMPT: Final = "What is 17 + 26? Answer with just the number." +_WEATHER_PROMPT: Final = "What is the weather in Paris? Use the tool." +_COUNTING_PROMPT: Final = "Count from 1 to 5, one number per line." +_WEATHER_TOOL: Final[JsonValue] = { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the current weather for a city", + "parameters": { + "type": "object", + "properties": {"city": {"type": "string"}}, + "required": ["city"], + }, + }, +} +_COST_MAP_PATH: Final = Path(__file__).resolve().parents[3] / "model_prices_and_context_window.json" +_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) +_COST_MAP: Final = TypeAdapter(dict[str, dict[str, object]]) + + +class _Delta(BaseModel): + model_config = ConfigDict(extra="ignore") + content: str | None = None + reasoning_content: str | None = None + + +class _Choice(BaseModel): + model_config = ConfigDict(extra="ignore") + delta: _Delta + finish_reason: str | None = None + + +class _Chunk(BaseModel): + model_config = ConfigDict(extra="ignore") + id: str + choices: tuple[_Choice, ...] + + +def _catalog_cost(backend: str, field: str) -> float: + cost_map: Final = _COST_MAP.validate_json(_COST_MAP_PATH.read_bytes()) + cost_value: Final = cost_map[f"xiaomi_mimo/{backend}"][field] + assert isinstance(cost_value, (int, float)) + return float(cost_value) + + +def _approx(value: float) -> object: + return pytest.approx(value, rel=1e-6) # pyright: ignore[reportUnknownMemberType] # pytest lacks typed approx stubs + + +def _completion(identity: str, backend: str, message: Mapping[str, object], finish: str) -> bytes: + return json.dumps( + { + "id": identity, + "object": "chat.completion", + "created": 1, + "model": backend, + "choices": [{"index": 0, "message": message, "finish_reason": finish}], + "usage": {"prompt_tokens": 23, "completion_tokens": 41, "total_tokens": 64}, + } + ).encode() + + +def _frame(identity: str, backend: str, delta: Mapping[str, object], finish: str | None = None) -> bytes: + value: Final = { + "id": identity, + "object": "chat.completion.chunk", + "created": 1, + "model": backend, + "choices": [{"index": 0, "delta": delta, "finish_reason": finish}], + } + return b"data: " + json.dumps(value).encode() + b"\n\n" + + +def _assert_provider_request(request: Request, backend: str, prompt: str) -> dict[str, JsonValue]: + assert request.method == "POST" + assert request.target == "/chat/completions" + assert request.headers["authorization"] == f"Bearer {_API_KEY}" + assert request.headers["content-type"] == "application/json" + body: Final = _JSON_OBJECT.validate_json(request.body) + assert body["model"] == backend + assert body["messages"] == [{"role": "user", "content": prompt}] + return body + + +@pytest.mark.covers("other.provider_wire.xiaomi_mimo.reasoning_content_and_registry_pricing") +@pytest.mark.parametrize("backend", _BACKENDS) +def test_xiaomi_mimo_nonstream_surfaces_reasoning_and_charges_registry_price(gateway: Gateway, backend: str) -> None: + identity: Final = f"xiaomi-cost-{uuid.uuid4().hex}" + + def respond(request: Request) -> Reply: + body: Final = _assert_provider_request(request, backend, _ARITHMETIC_PROMPT) + assert body["max_tokens"] == 256 + assert "max_completion_tokens" not in body + return Reply( + body=_completion( + identity, + backend, + {"role": "assistant", "content": "43", "reasoning_content": "17 plus 26 is 43."}, + "stop", + ) + ) + + with wire_server(respond) as wire, gateway.scenario() as scenario: + model: Final = scenario.model(model=f"xiaomi_mimo/{backend}", api_base=wire.url, api_key=_API_KEY) + response: Final = gateway.request( + "POST", + "/v1/chat/completions", + { + "model": model, + "messages": [{"role": "user", "content": _ARITHMETIC_PROMPT}], + "max_completion_tokens": 256, + }, + ) + assert response.status_code == 200, response.text + payload: Final = _JSON_OBJECT.validate_json(response.content) + assert payload["id"] == identity + assert payload["choices"] == [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "role": "assistant", + "content": "43", + "reasoning_content": "17 plus 26 is 43.", + "provider_specific_fields": {"refusal": None}, + }, + "provider_specific_fields": {}, + } + ] + assert payload["usage"] == {"prompt_tokens": 23, "completion_tokens": 41, "total_tokens": 64} + expected_cost: Final = 23 * _catalog_cost(backend, "input_cost_per_token") + 41 * _catalog_cost( + backend, "output_cost_per_token" + ) + assert float(response.headers["x-litellm-response-cost"]) == _approx(expected_cost) + assert [(request.method, request.target) for request in wire.drain()] == [("POST", "/chat/completions")] + rows: Final = eventually( + lambda: read_rows( + 'SELECT spend, prompt_tokens, completion_tokens FROM "LiteLLM_SpendLogs" WHERE request_id=%s', + (identity,), + ), + lambda values: len(values) == 1, + seconds=70, + ) + assert (rows[0]["prompt_tokens"], rows[0]["completion_tokens"]) == (23, 41) + spend: Final = rows[0]["spend"] + assert isinstance(spend, (int, float, str)) + assert float(spend) == _approx(expected_cost) + + +@pytest.mark.covers("other.provider_wire.xiaomi_mimo.reasoning_and_answer_stream_as_deltas") +def test_xiaomi_mimo_stream_delivers_reasoning_then_answer_deltas(gateway: Gateway) -> None: + backend: Final = _BACKENDS[0] + identity: Final = f"xiaomi-stream-{uuid.uuid4().hex}" + frames: Final = ( + _frame(identity, backend, {"role": "assistant", "reasoning_content": "Count "}), + _frame(identity, backend, {"reasoning_content": "up by one."}), + _frame(identity, backend, {"content": "1\n2\n"}), + _frame(identity, backend, {"content": "3\n4\n5"}), + _frame(identity, backend, {}, finish="stop"), + b"data: [DONE]\n\n", + ) + + def respond(request: Request) -> Reply: + body: Final = _assert_provider_request(request, backend, _COUNTING_PROMPT) + assert body["stream"] is True + return Reply(content_type="text/event-stream", chunks=frames) + + with wire_server(respond) as wire, gateway.scenario() as scenario: + model: Final = scenario.model(model=f"xiaomi_mimo/{backend}", api_base=wire.url, api_key=_API_KEY) + with gateway.client.stream( + "POST", + "/v1/chat/completions", + json={"model": model, "messages": [{"role": "user", "content": _COUNTING_PROMPT}], "stream": True}, + headers={"Authorization": f"Bearer {gateway.key}"}, + ) as response: + assert response.status_code == 200, response.read() + lines: Final = tuple(line for line in response.iter_lines() if line.startswith("data: ")) + assert lines[-1] == "data: [DONE]" + chunks: Final = tuple(_Chunk.model_validate_json(line.removeprefix("data: ")) for line in lines[:-1]) + assert {chunk.id for chunk in chunks} == {identity} + choices: Final = tuple(choice for chunk in chunks for choice in chunk.choices) + assert "".join(choice.delta.reasoning_content or "" for choice in choices) == "Count up by one." + assert "".join(choice.delta.content or "" for choice in choices) == "1\n2\n3\n4\n5" + assert tuple(choice.finish_reason for choice in choices if choice.finish_reason) == ("stop",) + assert [(request.method, request.target) for request in wire.drain()] == [("POST", "/chat/completions")] + + +@pytest.mark.covers("other.provider_wire.xiaomi_mimo.tool_call_survives_translation") +def test_xiaomi_mimo_tool_call_is_forwarded_and_returned(gateway: Gateway) -> None: + backend: Final = _BACKENDS[1] + identity: Final = f"xiaomi-tool-{uuid.uuid4().hex}" + tool_call: Final = { + "id": "call_paris", + "type": "function", + "function": {"name": "get_weather", "arguments": json.dumps({"city": "Paris"})}, + } + + def respond(request: Request) -> Reply: + body: Final = _assert_provider_request(request, backend, _WEATHER_PROMPT) + assert body["tools"] == [_WEATHER_TOOL] + assert body["tool_choice"] == "auto" + return Reply( + body=_completion( + identity, + backend, + { + "role": "assistant", + "content": None, + "reasoning_content": "Need the tool.", + "tool_calls": [tool_call], + }, + "tool_calls", + ) + ) + + with wire_server(respond) as wire, gateway.scenario() as scenario: + model: Final = scenario.model(model=f"xiaomi_mimo/{backend}", api_base=wire.url, api_key=_API_KEY) + response: Final = gateway.request( + "POST", + "/v1/chat/completions", + { + "model": model, + "messages": [{"role": "user", "content": _WEATHER_PROMPT}], + "tools": [_WEATHER_TOOL], + "tool_choice": "auto", + }, + ) + assert response.status_code == 200, response.text + payload: Final = _JSON_OBJECT.validate_json(response.content) + assert payload["choices"] == [ + { + "finish_reason": "tool_calls", + "index": 0, + "message": { + "role": "assistant", + "content": None, + "reasoning_content": "Need the tool.", + "tool_calls": [tool_call], + "provider_specific_fields": {"refusal": None}, + }, + "provider_specific_fields": {}, + } + ] + assert [(request.method, request.target) for request in wire.drain()] == [("POST", "/chat/completions")] diff --git a/tests/pass_through_unit_tests/test_vertex_ai_anthropic_streaming_cost_injection.py b/tests/pass_through_unit_tests/test_vertex_ai_anthropic_streaming_cost_injection.py index 498f0a734a3..ab5fd7f80ee 100644 --- a/tests/pass_through_unit_tests/test_vertex_ai_anthropic_streaming_cost_injection.py +++ b/tests/pass_through_unit_tests/test_vertex_ai_anthropic_streaming_cost_injection.py @@ -53,6 +53,7 @@ async def test_vertex_ai_anthropic_streaming_cost_injection_enabled(): # Setup logging object with model info litellm_logging_obj = MagicMock(spec=LiteLLMLoggingObj) + litellm_logging_obj.litellm_params = {} litellm_logging_obj.model_call_details = {"model": "claude-sonnet-4@20250514"} litellm_logging_obj.completion_start_time = None litellm_logging_obj.async_success_handler = AsyncMock() @@ -132,6 +133,7 @@ async def test_vertex_ai_anthropic_streaming_cost_injection_disabled(): response.aiter_bytes = mock_aiter_bytes litellm_logging_obj = MagicMock(spec=LiteLLMLoggingObj) + litellm_logging_obj.litellm_params = {} litellm_logging_obj.model_call_details = {"model": "claude-sonnet-4@20250514"} litellm_logging_obj.completion_start_time = None litellm_logging_obj.async_success_handler = AsyncMock() @@ -194,6 +196,7 @@ async def test_vertex_ai_anthropic_streaming_cost_injection_no_usage_chunk(): response.aiter_bytes = mock_aiter_bytes litellm_logging_obj = MagicMock(spec=LiteLLMLoggingObj) + litellm_logging_obj.litellm_params = {} litellm_logging_obj.model_call_details = {"model": "claude-sonnet-4@20250514"} litellm_logging_obj.completion_start_time = None litellm_logging_obj.async_success_handler = AsyncMock() @@ -249,6 +252,7 @@ async def test_vertex_ai_anthropic_streaming_model_extraction(): response.aiter_bytes = mock_aiter_bytes litellm_logging_obj = MagicMock(spec=LiteLLMLoggingObj) + litellm_logging_obj.litellm_params = {} litellm_logging_obj.model_call_details = {} litellm_logging_obj.completion_start_time = None litellm_logging_obj.async_success_handler = AsyncMock() diff --git a/tests/proxy_behavior/spend/test_autorouter_session_rollup.py b/tests/proxy_behavior/spend/test_autorouter_session_rollup.py index f3c68b489a5..77549b527d8 100644 --- a/tests/proxy_behavior/spend/test_autorouter_session_rollup.py +++ b/tests/proxy_behavior/spend/test_autorouter_session_rollup.py @@ -6,17 +6,24 @@ tests/test_litellm/proxy/db/test_autorouter_session_rollup.py. """ import asyncio +import time import uuid from datetime import datetime, timedelta, timezone -from typing import Final +from types import SimpleNamespace +from typing import Final, TypedDict, cast import pytest from prisma import Prisma +from prisma.errors import RawQueryError +from typing_extensions import ReadOnly from litellm.proxy.db.autorouter_session_rollup import ( AUTOROUTER_BENCHMARKS_SQL, UPSERT_AUTOROUTER_SESSION_SQL, + AutoRouterTurnTransaction, + flush_autorouter_turn_transactions, ) +from litellm.proxy.db.db_transaction_queue.spend_log_cleanup import SpendLogCleanup pytestmark = pytest.mark.asyncio(loop_scope="session") @@ -45,6 +52,7 @@ async def _turn( tier: "str | None" = None, baseline: "str | None" = None, estimated: bool = True, + user_id: str = "", ) -> None: touched: Final = 1 if (hit or ttl is not None or not covered) else 0 await db.execute_raw( @@ -68,6 +76,7 @@ async def _turn( int(estimated), spend if estimated else 0.0, saved if estimated else 0.0, + user_id, ) @@ -217,7 +226,7 @@ async def test_subtotal_coverage_survives_legacy_and_rolling_writers(db, writers assert row["savings_estimated_actual_spend"] == pytest.approx(0.01 * sum(writers)) assert row["savings_estimated_saved_spend"] == pytest.approx(0.02 * sum(writers)) groups: Final = await db.query_raw( - AUTOROUTER_BENCHMARKS_SQL, T0.isoformat(), (T0 + timedelta(days=1)).isoformat(), key + AUTOROUTER_BENCHMARKS_SQL, T0.isoformat(), (T0 + timedelta(days=1)).isoformat(), key, None ) assert len(groups) == 1 assert groups[0]["classifier_cost"] == row["classifier_cost"] @@ -242,7 +251,7 @@ async def test_unknown_and_legacy_turns_preserve_actual_spend_without_entering_t assert row["saved_spend"] == pytest.approx(-0.03) assert row["savings_estimated_baseline_models"] == {"opus": 1} groups: Final = await db.query_raw( - AUTOROUTER_BENCHMARKS_SQL, T0.isoformat(), (T0 + timedelta(days=1)).isoformat(), key + AUTOROUTER_BENCHMARKS_SQL, T0.isoformat(), (T0 + timedelta(days=1)).isoformat(), key, None ) assert len(groups) == 1 for actual in (row, groups[0]): @@ -277,6 +286,7 @@ async def test_the_benchmarks_aggregate_reads_only_overlapping_sessions(db): (T0 - timedelta(days=1)).isoformat(), (T0 + timedelta(days=1)).isoformat(), None, + None, ) matching = [row for row in rows if row["router_name"] == router] assert len(matching) == 1 @@ -304,6 +314,7 @@ async def test_the_benchmarks_aggregate_can_filter_to_one_key(db): (T0 - timedelta(days=1)).isoformat(), (T0 + timedelta(days=1)).isoformat(), first_key, + None, ) matching = [row for row in rows if row["router_name"] == router] assert len(matching) == 1 @@ -317,10 +328,160 @@ async def test_the_benchmarks_aggregate_can_filter_to_one_key(db): (T0 - timedelta(days=1)).isoformat(), (T0 + timedelta(days=1)).isoformat(), f"k-{uuid.uuid4()}", + None, ) assert [row for row in unknown_key_rows if row["router_name"] == router] == [] +class _BenchmarkRow(TypedDict): + sessions: ReadOnly[int] + turns: ReadOnly[int] + same_model_turns: ReadOnly[int] + first_visit_turns: ReadOnly[int] + spend: ReadOnly[float] + saved_spend: ReadOnly[float] + tier_turns: ReadOnly[dict[str, int]] + cache_hits: ReadOnly[int] + savings_estimated_turns: ReadOnly[int] + savings_estimated_actual_spend: ReadOnly[float] + savings_estimated_saved_spend: ReadOnly[float] + + +async def _scoped_benchmarks( + db: Prisma, router: str, user_id: str | None = None, key: str | None = None +) -> tuple[_BenchmarkRow, ...]: + rows: Final = await db.query_raw( + AUTOROUTER_BENCHMARKS_SQL, + (T0 - timedelta(days=1)).isoformat(), + (T0 + timedelta(days=1)).isoformat(), + key, + user_id, + ) + return tuple(cast(_BenchmarkRow, row) for row in rows if row["router_name"] == router) + + +async def test_users_keep_written_identity_across_shared_keys_and_keyless_sessions(db: Prisma) -> None: + router: Final = f"r-{uuid.uuid4()}" + alice: Final = f"u-{uuid.uuid4()}" + bob: Final = f"u-{uuid.uuid4()}" + first_key: Final = f"k-{uuid.uuid4()}" + second_key: Final = f"k-{uuid.uuid4()}" + await _legacy_turn(db, first_key, T0, router=router) + await _turn(db, first_key, "A", T0 + timedelta(seconds=10), router=router, user_id=alice, tier="simple") + await _turn( + db, first_key, "B", T0 + timedelta(seconds=20), router=router, user_id=bob, spend=0.03, saved=0.06, tier="complex" + ) + await _turn(db, second_key, "C", T0, router=router, user_id=alice, spend=0.02, saved=0.04) + await _turn(db, "", "A", T0, router=router, user_id=alice, ttl=300) + await _turn(db, "", "A", T0 + timedelta(seconds=1), router=router, user_id=alice, hit=1) + await _turn(db, "", "B", T0, router=router, user_id=bob, spend=0.04, saved=0.08) + await _turn(db, second_key, "C", T0 - timedelta(days=40), router=router, user_id=alice, session_id="expired") + + alice_rows: Final = await _scoped_benchmarks(db, router, user_id=alice) + bob_rows: Final = await _scoped_benchmarks(db, router, user_id=bob) + global_rows: Final = await _scoped_benchmarks(db, router) + key_rows: Final = await _scoped_benchmarks(db, router, key=first_key) + intersection: Final = await _scoped_benchmarks(db, router, user_id=alice, key=first_key) + assert len(alice_rows) == len(bob_rows) == len(global_rows) == len(key_rows) == len(intersection) == 1 + assert (alice_rows[0]["sessions"], alice_rows[0]["turns"], alice_rows[0]["same_model_turns"]) == (3, 4, 1) + assert (bob_rows[0]["sessions"], bob_rows[0]["turns"], bob_rows[0]["first_visit_turns"]) == (2, 2, 2) + assert alice_rows[0]["spend"] == pytest.approx(0.05) + assert bob_rows[0]["spend"] == pytest.approx(0.07) + assert alice_rows[0]["tier_turns"] == {"simple": 1} + assert bob_rows[0]["tier_turns"] == {"complex": 1} + assert (alice_rows[0]["cache_hits"], bob_rows[0]["cache_hits"]) == (1, 0) + assert (global_rows[0]["sessions"], global_rows[0]["turns"]) == (4, 7) + assert (alice_rows[0]["savings_estimated_turns"], bob_rows[0]["savings_estimated_turns"]) == (4, 2) + assert global_rows[0]["savings_estimated_turns"] == 6 + for scoped in (alice_rows[0], bob_rows[0]): + assert scoped["savings_estimated_actual_spend"] == pytest.approx(scoped["spend"]) + assert scoped["savings_estimated_saved_spend"] == pytest.approx(scoped["saved_spend"]) + assert global_rows[0]["spend"] == pytest.approx(alice_rows[0]["spend"] + bob_rows[0]["spend"] + 0.01) + assert global_rows[0]["saved_spend"] == pytest.approx(alice_rows[0]["saved_spend"] + bob_rows[0]["saved_spend"] + 0.02) + assert global_rows[0]["tier_turns"] == {"simple": 1, "complex": 1} + assert (key_rows[0]["sessions"], key_rows[0]["turns"]) == (1, 3) + assert key_rows[0]["spend"] == pytest.approx(0.05) + assert (intersection[0]["sessions"], intersection[0]["turns"]) == (1, 1) + assert intersection[0]["spend"] == pytest.approx(0.01) + assert await _scoped_benchmarks(db, router, user_id=bob, key=second_key) == () + assert await _scoped_benchmarks(db, router, user_id=f"u-{uuid.uuid4()}") == () + assert await _scoped_benchmarks(db, router, user_id="") == () + + +async def test_a_failed_user_projection_rolls_back_the_keys_increment(db: Prisma) -> None: + key: Final = f"k-{uuid.uuid4()}" + user_id: Final = "".join(str(uuid.uuid4()) for _ in range(200)) + await _turn(db, key, "A", T0) + before: Final = await _row(db, key) + + with pytest.raises(RawQueryError, match=r"index row (requires|size)"): + await _turn(db, key, "B", T0 + timedelta(seconds=1), user_id=user_id) + + assert await _row(db, key) == before + assert await db.query_raw('SELECT user_id FROM "LiteLLM_AutoRouterUserSession" WHERE user_id = $1', user_id) == [] + + first_user: Final = f"u-{uuid.uuid4()}" + second_user: Final = f"u-{uuid.uuid4()}" + turns: Final = tuple( + AutoRouterTurnTransaction( + api_key=key, + user_id=user, + session_id="s1", + router_name="auto-1", + router_type="complexity", + model=model, + turn_at=T0 + timedelta(seconds=second), + total_tokens=100, + spend=0.01, + saved_spend=0.02, + classifier_cost=0.0, + covered=True, + cache_hit=False, + cache_ttl_seconds=None, + cache_touched=False, + ) + for user, model, second in ( + (first_user, "A", 1), + (user_id, "B", 2), + (first_user, "B", 3), + (second_user, "C", 4), + (first_user, "B", 5), + (second_user, "C", 6), + (user_id, "A", 7), + ) + ) + await flush_autorouter_turn_transactions(SimpleNamespace(db=db), tuple(reversed(turns)), n_retry_times=0) + + key_row: Final = await _row(db, key) + assert (key_row["turns"], key_row["last_model"], key_row["unordered_turns"]) == (2, "A", 0) + assert key_row["spend"] == pytest.approx(0.02) + user_rows: Final = await db.query_raw('SELECT * FROM "LiteLLM_AutoRouterUserSession" WHERE api_key = $1', key) + by_user: Final = {row["user_id"]: row for row in user_rows} + assert set(by_user) == {first_user, second_user} + for user, count, model in ((first_user, 3, "B"), (second_user, 2, "C")): + row: Final = by_user[user] + assert (row["turns"], row["same_model_turns"], row["unordered_turns"], row["last_model"]) == (count, 1, 0, model) + assert row["spend"] == pytest.approx(count * 0.01) + assert row["saved_spend"] == pytest.approx(count * 0.02) + + +async def test_user_session_cleanup_keeps_another_users_recent_keyless_session(db: Prisma) -> None: + router: Final = f"r-{uuid.uuid4()}" + expired_user: Final = f"u-{uuid.uuid4()}" + recent_user: Final = f"u-{uuid.uuid4()}" + await _turn(db, "", "A", T0 - timedelta(days=1), router=router, user_id=expired_user) + await _turn(db, "", "A", T0 + timedelta(days=1), router=router, user_id=recent_user) + cleaner: Final = SpendLogCleanup(general_settings={}) + + await cleaner._delete_old_autorouter_user_session_rows( + SimpleNamespace(db=db), T0.replace(tzinfo=timezone.utc), time.monotonic() + 60 + ) + + assert await db.query_raw( + 'SELECT user_id, turns FROM "LiteLLM_AutoRouterUserSession" WHERE router_name = $1', router + ) == [{"user_id": recent_user, "turns": 1}] + + async def test_a_reconfigured_alias_reports_each_router_type_as_its_own_group(db): key = f"k-{uuid.uuid4()}" router = f"r-{uuid.uuid4()}" @@ -334,6 +495,7 @@ async def test_a_reconfigured_alias_reports_each_router_type_as_its_own_group(db (T0 - timedelta(days=1)).isoformat(), (T0 + timedelta(days=1)).isoformat(), None, + None, ) matching = sorted( (row for row in rows if row["router_name"] == router), @@ -418,6 +580,7 @@ async def test_the_benchmarks_aggregate_sums_tier_turns_across_sessions(db): (T0 - timedelta(days=1)).isoformat(), (T0 + timedelta(days=1)).isoformat(), None, + None, ) grouped = next(row for row in rows if row["router_name"] == router) assert grouped["tier_turns"] == {"simple": 2, "complex": 1} @@ -446,6 +609,7 @@ async def test_tier_maps_stay_separate_per_router_type_on_a_reconfigured_alias(d (T0 - timedelta(days=1)).isoformat(), (T0 + timedelta(days=1)).isoformat(), None, + None, ) by_type = {row["router_type"]: row["tier_turns"] for row in rows if row["router_name"] == router} assert by_type == {"complexity": {"medium": 1}, "quality": {"2": 1}} @@ -461,6 +625,7 @@ async def test_a_window_with_no_tiered_turns_aggregates_to_an_empty_map(db): (T0 - timedelta(days=1)).isoformat(), (T0 + timedelta(days=1)).isoformat(), None, + None, ) grouped = next(row for row in rows if row["router_name"] == router) assert grouped["tier_turns"] == {} diff --git a/tests/proxy_behavior/spend/test_baseline_accounting.py b/tests/proxy_behavior/spend/test_baseline_accounting.py index e187a44c29d..3504751d132 100644 --- a/tests/proxy_behavior/spend/test_baseline_accounting.py +++ b/tests/proxy_behavior/spend/test_baseline_accounting.py @@ -56,7 +56,9 @@ def record() -> Callable[..., BaselineAccountingRecord]: }, ) - def create(label: str = "first", started: float = 10000.0, identical: bool = True) -> BaselineAccountingRecord: + def create( + label: str = "first", started: float = 10000.0, identical: bool = True, user_id: str = "" + ) -> BaselineAccountingRecord: return BaselineAccountingRecord( scope="autorouter-baseline:v3:" + run * 2, api_key=run, session_id=run, router_name="test-router", baseline_model="anthropic/claude-opus-5", @@ -76,6 +78,7 @@ def record() -> Callable[..., BaselineAccountingRecord]: total_tokens=6230, spend=0.17, saved_spend=0.0, classifier_cost=0.0, covered=True, cache_hit=False, cache_ttl_seconds=3600, cache_touched=True, baseline_model="anthropic/claude-opus-5", + user_id=user_id, ), daily=DailyBaselineAttribution( date="2026-09-15", api_key=run, model="claude-opus-5", custom_llm_provider="anthropic", @@ -99,21 +102,36 @@ async def _session(db: Prisma, record: BaselineAccountingRecord): return rows[0] +async def _user_sessions(db: Prisma, record: BaselineAccountingRecord) -> dict[str, dict[str, object]]: + rows: Final = await db.query_raw('SELECT * FROM "LiteLLM_AutoRouterUserSession" WHERE api_key=$1', record.api_key) + return {str(row["user_id"]): row for row in rows} + + async def test_late_replay_updates_all_projections_without_rebilling(db: Prisma, record: Callable[..., BaselineAccountingRecord]) -> None: store: Final = _store(db) - late: Final = record("late", 10001.0) - early: Final = record("early", identical=False) + late: Final = record("late", 10001.0, user_id="late-user") + early: Final = record("early", identical=False, user_id="early-user") await _log(db, late) assert await store.append(late) == "recorded" assert await store.project(late.scope) == "published" before: Final = await _session(db, late) assert before["savings_estimated_actual_spend"] == before["spend"] == 0.17 assert before["saved_spend"] == 0.0 + before_users: Final = await _user_sessions(db, late) + assert set(before_users) == {"late-user"} + assert before_users["late-user"]["savings_estimated_turns"] == 1 + assert before_users["late-user"]["savings_estimated_baseline_models"] == {late.baseline_model: 1} await _log(db, early) assert await store.append(early) == "recorded" pending: Final = await _session(db, late) assert pending["spend"] == 0.34 and pending["savings_estimated_turns"] == 0 assert pending["saved_spend"] == pending["savings_estimated_actual_spend"] == 0.0 + pending_users: Final = await _user_sessions(db, late) + assert set(pending_users) == {"late-user", "early-user"} + for user in pending_users.values(): + assert user["turns"] == 1 and user["spend"] == 0.17 + assert user["savings_estimated_turns"] == user["savings_estimated_actual_spend"] == user["saved_spend"] == 0 + assert user["savings_estimated_baseline_models"] == {} waiting: Final = await db.query_raw('SELECT metadata FROM "LiteLLM_SpendLogs" WHERE request_id=$1', late.observation.request_id) assert waiting[0]["metadata"]["autorouter_savings"] is None assert waiting[0]["metadata"]["autorouter_savings_estimate"]["reason"] == "pending_projection" @@ -125,37 +143,69 @@ async def test_late_replay_updates_all_projections_without_rebilling(db: Prisma, assert logs[0]["spend"] == 0.17 assert logs[0]["metadata"]["autorouter_savings_estimate"]["provenance"] == "modeled" assert after["saved_spend"] == pytest.approx(logs[0]["metadata"]["autorouter_savings"]) + after_users: Final = await _user_sessions(db, late) + assert after_users["early-user"] == pending_users["early-user"] + for field in ( + "saved_spend", "savings_estimated_turns", "savings_estimated_actual_spend", + "savings_estimated_saved_spend", "savings_estimated_baseline_models", + ): + assert after_users["late-user"][field] == after[field] + assert after_users["late-user"]["turns"] == 1 and after_users["late-user"]["spend"] == 0.17 for table in ("DailyUserSpend", "DailyTeamSpend", "DailyOrganizationSpend", "DailyEndUserSpend", "DailyAgentSpend", "DailyTagSpend"): rows: Final = await db.query_raw(f'SELECT spend,api_requests,autorouter_savings_spend FROM "LiteLLM_{table}" WHERE api_key=$1', late.api_key) assert rows[0]["spend"] == rows[0]["api_requests"] == 0 assert rows[0]["autorouter_savings_spend"] == pytest.approx(after["saved_spend"]) -async def test_commit_ack_loss_and_concurrent_duplicate_delivery_are_idempotent(db: Prisma, record: Callable[..., BaselineAccountingRecord]) -> None: - event: Final = record() +@pytest.mark.parametrize("attributed", [True, False]) +async def test_commit_ack_loss_and_concurrent_duplicate_delivery_are_idempotent( + db: Prisma, record: Callable[..., BaselineAccountingRecord], attributed: bool +) -> None: + event: Final = record(user_id="first-user" if attributed else "") + other: Final = record("other", 10001.0, user_id="second-user" if attributed else "") await _log(db, event) assert await _store(db, after_commit=True).append(event) == "unavailable" store: Final = _store(db) assert set(await asyncio.gather(*(store.append(event) for _ in range(4)))) == {"recorded"} + await _log(db, other) + assert await store.append(other) == "recorded" + if not attributed: + await db.execute_raw( + 'UPDATE "LiteLLM_AutoRouterBaselineObservation" SET data=(data::jsonb #- \'{turn,user_id}\')::text WHERE scope=$1', + event.scope, + ) assert await store.project(event.scope) == "published" assert await store.project(event.scope) == "unchanged" session: Final = await _session(db, event) - assert session["turns"] == session["savings_estimated_turns"] == 1 - assert session["spend"] == session["savings_estimated_actual_spend"] == 0.17 + assert session["turns"] == session["savings_estimated_turns"] == 2 + assert session["spend"] == session["savings_estimated_actual_spend"] == 0.34 + users: Final = await _user_sessions(db, event) + assert set(users) == ({"first-user", "second-user"} if attributed else set()) + for user in users.values(): + assert user["turns"] == user["savings_estimated_turns"] == 1 + assert user["spend"] == user["savings_estimated_actual_spend"] == 0.17 + assert user["savings_estimated_baseline_models"] == {event.baseline_model: 1} async def test_publication_rollback_keeps_dirty_revision_for_retry(db: Prisma, record: Callable[..., BaselineAccountingRecord]) -> None: - event: Final = record() + event: Final = record(user_id="rollback-user") await _log(db, event) store: Final = _store(db) assert await store.append(event) == "recorded" assert await _store(db, before_commit=True).project(event.scope) == "unavailable" session: Final = await _session(db, event) assert session["spend"] == 0.17 and session["savings_estimated_turns"] == 0 + before_users: Final = await _user_sessions(db, event) + assert before_users["rollback-user"]["spend"] == 0.17 + assert before_users["rollback-user"]["savings_estimated_turns"] == 0 + assert before_users["rollback-user"]["savings_estimated_baseline_models"] == {} revisions: Final = await db.query_raw('SELECT revision,published_revision FROM "LiteLLM_AutoRouterBaselineComparison" WHERE scope=$1', event.scope) assert revisions[0]["revision"] > revisions[0]["published_revision"] assert await store.project(event.scope) == "published" assert (await _session(db, event))["savings_estimated_turns"] == 1 + after_users: Final = await _user_sessions(db, event) + assert after_users["rollback-user"]["turns"] == after_users["rollback-user"]["savings_estimated_turns"] == 1 + assert after_users["rollback-user"]["spend"] == after_users["rollback-user"]["savings_estimated_actual_spend"] == 0.17 async def test_conflicting_duplicate_cannot_restore_an_observed_estimate(db: Prisma, record: Callable[..., BaselineAccountingRecord]) -> None: @@ -196,6 +246,7 @@ async def test_native_observation_enters_spend_pipeline_once_with_shared_daily_a db: Prisma, record: Callable[..., BaselineAccountingRecord], monkeypatch: pytest.MonkeyPatch, ) -> None: import os + from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache from litellm.proxy.db.db_spend_update_writer import DBSpendUpdateWriter from litellm.proxy.hooks.autorouter_baseline_cache import CapturedBaselineObservation diff --git a/tests/proxy_unit_tests/test_jwt_key_mapping.py b/tests/proxy_unit_tests/test_jwt_key_mapping.py index 3f2c04336a7..e95ed42013b 100644 --- a/tests/proxy_unit_tests/test_jwt_key_mapping.py +++ b/tests/proxy_unit_tests/test_jwt_key_mapping.py @@ -19,6 +19,8 @@ from litellm.proxy._types import ( ) from litellm.proxy.management_endpoints.jwt_key_mapping_endpoints import ( _to_response, + _token_hash_for_create, + _token_hash_for_update, create_jwt_key_mapping, delete_jwt_key_mapping, info_jwt_key_mapping, @@ -1515,6 +1517,132 @@ def test_jwt_client_id_field_does_not_raise_on_duplicate(): assert auth.virtual_key_claim_field == "new_field" +# ────────────────────────────────────────────── +# Tests: identifying the mapped key by hash instead of plaintext +# ────────────────────────────────────────────── + +_TOKEN_HASH = "1923314ae0efc8b2523c7d421bac5a7cf88df291273b139948b526d396974a41" + + +def test_create_stores_a_supplied_token_hash_verbatim(): + """A caller that holds only the hash gets it stored as given, not hashed again.""" + from litellm.proxy._types import CreateJWTKeyMappingRequest + + data = CreateJWTKeyMappingRequest( + jwt_claim_name="email", jwt_claim_value="user@example.com", token=_TOKEN_HASH + ) + + assert _token_hash_for_create(data) == _TOKEN_HASH + + +def test_create_hashes_a_supplied_plaintext_key(): + """Supplying `key` keeps the original behaviour, so existing configs are unaffected.""" + from litellm.proxy._types import CreateJWTKeyMappingRequest, hash_token + + data = CreateJWTKeyMappingRequest( + jwt_claim_name="email", jwt_claim_value="user@example.com", key="sk-test-key" + ) + + assert _token_hash_for_create(data) == hash_token("sk-test-key") + + +@pytest.mark.parametrize( + "kwargs", + [ + pytest.param({}, id="neither"), + pytest.param({"key": "sk-test-key", "token": _TOKEN_HASH}, id="both"), + ], +) +def test_create_requires_exactly_one_identifier(kwargs): + """Neither or both is a 400, so a mapping can never be created ambiguously.""" + from litellm.proxy._types import CreateJWTKeyMappingRequest + + data = CreateJWTKeyMappingRequest( + jwt_claim_name="email", jwt_claim_value="user@example.com", **kwargs + ) + + with pytest.raises(HTTPException) as exc_info: + _token_hash_for_create(data) + + assert exc_info.value.status_code == 400 + assert "exactly one" in exc_info.value.detail.lower() + + +@pytest.mark.parametrize( + "token", + [ + pytest.param("sk-not-a-hash", id="plaintext-key"), + pytest.param("abc123", id="too-short"), + pytest.param(_TOKEN_HASH.upper(), id="uppercase"), + pytest.param(_TOKEN_HASH + "0", id="too-long"), + pytest.param(_TOKEN_HASH[:-1] + "g", id="non-hex-character"), + ], +) +def test_create_rejects_a_token_that_is_not_a_sha256_hash(token): + """hash_token hashes unconditionally, so a bad `token` would be stored as a hash + of a hash and then silently match nothing at auth time.""" + from litellm.proxy._types import CreateJWTKeyMappingRequest + + data = CreateJWTKeyMappingRequest( + jwt_claim_name="email", jwt_claim_value="user@example.com", token=token + ) + + with pytest.raises(HTTPException) as exc_info: + _token_hash_for_create(data) + + assert exc_info.value.status_code == 400 + assert "SHA-256" in exc_info.value.detail + + +def test_update_leaves_the_mapped_key_alone_when_neither_is_given(): + """Updating only the description must not blank out the mapped key.""" + from litellm.proxy._types import UpdateJWTKeyMappingRequest + + data = UpdateJWTKeyMappingRequest(id="mapping-1", description="new text") + + assert _token_hash_for_update(data) is None + + +def test_update_stores_a_supplied_token_hash_verbatim(): + from litellm.proxy._types import UpdateJWTKeyMappingRequest + + data = UpdateJWTKeyMappingRequest(id="mapping-1", token=_TOKEN_HASH) + + assert _token_hash_for_update(data) == _TOKEN_HASH + + +def test_update_hashes_a_supplied_plaintext_key(): + from litellm.proxy._types import UpdateJWTKeyMappingRequest, hash_token + + data = UpdateJWTKeyMappingRequest(id="mapping-1", key="sk-rotated") + + assert _token_hash_for_update(data) == hash_token("sk-rotated") + + +def test_update_rejects_both_identifiers(): + from litellm.proxy._types import UpdateJWTKeyMappingRequest + + data = UpdateJWTKeyMappingRequest(id="mapping-1", key="sk-abc", token=_TOKEN_HASH) + + with pytest.raises(HTTPException) as exc_info: + _token_hash_for_update(data) + + assert exc_info.value.status_code == 400 + assert "at most one" in exc_info.value.detail.lower() + + +def test_update_rejects_a_token_that_is_not_a_sha256_hash(): + from litellm.proxy._types import UpdateJWTKeyMappingRequest + + data = UpdateJWTKeyMappingRequest(id="mapping-1", token="sk-not-a-hash") + + with pytest.raises(HTTPException) as exc_info: + _token_hash_for_update(data) + + assert exc_info.value.status_code == 400 + assert "SHA-256" in exc_info.value.detail + + # ────────────────────────────────────────────── # Tests: cache eviction must happen AFTER the DB write commits # ────────────────────────────────────────────── diff --git a/tests/test_litellm/caching/test_qdrant_semantic_cache.py b/tests/test_litellm/caching/test_qdrant_semantic_cache.py index a0a9b71787c..ca7303e4c6d 100644 --- a/tests/test_litellm/caching/test_qdrant_semantic_cache.py +++ b/tests/test_litellm/caching/test_qdrant_semantic_cache.py @@ -578,6 +578,7 @@ def test_qdrant_semantic_cache_set_cache(): assert ( upsert_payload[QdrantSemanticCache.CACHE_KEY_FIELD_NAME] == "test_key" ) + assert qdrant_cache.sync_client.put.call_args.kwargs["params"] == {"wait": "true"} @pytest.mark.asyncio @@ -650,6 +651,7 @@ async def test_qdrant_semantic_cache_async_set_cache(): assert ( upsert_payload[QdrantSemanticCache.CACHE_KEY_FIELD_NAME] == "test_key" ) + assert qdrant_cache.async_client.put.call_args.kwargs["params"] == {"wait": "true"} def test_qdrant_semantic_cache_custom_vector_size(): diff --git a/tests/test_litellm/integrations/test_prometheus_zero_cost_metric.py b/tests/test_litellm/integrations/test_prometheus_zero_cost_metric.py new file mode 100644 index 00000000000..989009b309a --- /dev/null +++ b/tests/test_litellm/integrations/test_prometheus_zero_cost_metric.py @@ -0,0 +1,179 @@ +import datetime +from typing import Final + +import pytest +from prometheus_client import REGISTRY +from prometheus_client.samples import Sample + +import litellm +from litellm.integrations.prometheus import PrometheusLogger +from litellm.types.utils import StandardLoggingZeroCostDiagnostic + +METRIC: Final = "litellm_zero_cost_requests_total" +MISSING_KEY_DIAGNOSTIC: Final[StandardLoggingZeroCostDiagnostic] = { + "reason": "missing_pricing_key", + "pricing_model": "dep-1", + "missing_pricing_keys": ("input_cost_per_token", "output_cost_per_token"), +} + + +def _clear_prometheus_registry() -> None: + for collector in list(REGISTRY._collector_to_names.keys()): + try: + REGISTRY.unregister(collector) + except Exception: + pass + + +def _samples(metric_name: str) -> list[Sample]: + return [sample for metric in REGISTRY.collect() for sample in metric.samples if sample.name == metric_name] + + +def _payload(zero_cost_diagnostic: StandardLoggingZeroCostDiagnostic | None) -> dict[str, object]: + return { + "id": "t", + "call_type": "completion", + "response_cost": 0.0, + "status": "success", + "total_tokens": 30, + "prompt_tokens": 20, + "completion_tokens": 10, + "startTime": 1.0, + "endTime": 2.0, + "completionStartTime": 1.5, + "model": "openai/gpt-5.4-nano", + "model_id": "dep-1", + "model_group": "per-second-priced-chat", + "api_base": "https://api.openai.com", + "custom_llm_provider": "openai", + "request_tags": [], + "end_user": None, + "cache_hit": False, + "stream": False, + "response": {"id": "chatcmpl-1"}, + "model_parameters": {}, + "zero_cost_diagnostic": zero_cost_diagnostic, + "metadata": { + "user_api_key_hash": "h", + "user_api_key_alias": "a", + "user_api_key_team_id": "t", + "user_api_key_team_alias": "ta", + "user_api_key_user_id": "u", + "user_api_key_user_email": "e@x.com", + "user_api_key_org_id": None, + "user_api_key_org_alias": None, + "requester_metadata": None, + "user_api_key_end_user_id": None, + "usage_object": None, + }, + "hidden_params": {"litellm_overhead_time_ms": None, "additional_headers": None}, + } + + +async def _log_success( + logger: PrometheusLogger, zero_cost_diagnostic: StandardLoggingZeroCostDiagnostic | None +) -> None: + now: Final = datetime.datetime.now() + kwargs: Final = { + "model": "openai/gpt-5.4-nano", + "litellm_params": {"metadata": {}}, + "standard_logging_object": _payload(zero_cost_diagnostic), + "stream": False, + "start_time": now - datetime.timedelta(seconds=3), + "api_call_start_time": now - datetime.timedelta(seconds=2), + "completion_start_time": now - datetime.timedelta(seconds=1), + "end_time": now, + } + await logger.async_log_success_event(kwargs, None, now, now) + + +async def _log_failure( + logger: PrometheusLogger, zero_cost_diagnostic: StandardLoggingZeroCostDiagnostic | None +) -> None: + now: Final = datetime.datetime.now() + kwargs: Final = { + "model": "openai/gpt-5.4-nano", + "litellm_params": {"metadata": {}}, + "standard_logging_object": {**_payload(zero_cost_diagnostic), "status": "failure"}, + "exception": Exception("stream cut off after the usage chunk"), + "stream": True, + "start_time": now - datetime.timedelta(seconds=3), + "end_time": now, + } + await logger.async_log_failure_event(kwargs, None, now, now) + + +@pytest.mark.asyncio +async def test_failure_event_counts_a_zero_cost_request_by_model_and_reason() -> None: + _clear_prometheus_registry() + try: + logger: Final = PrometheusLogger() + await _log_failure(logger, None) + assert _samples(METRIC) == [] + + await _log_failure(logger, MISSING_KEY_DIAGNOSTIC) + + samples: Final = _samples(METRIC) + assert len(samples) == 1 + assert samples[0].labels == { + "requested_model": "per-second-priced-chat", + "model": "openai/gpt-5.4-nano", + "model_id": "dep-1", + "api_provider": "openai", + "reason": "missing_pricing_key", + } + assert samples[0].value == 1.0 + finally: + _clear_prometheus_registry() + + +@pytest.mark.asyncio +async def test_success_event_counts_a_zero_cost_request_by_model_and_reason() -> None: + _clear_prometheus_registry() + try: + logger: Final = PrometheusLogger() + await _log_success(logger, MISSING_KEY_DIAGNOSTIC) + await _log_success(logger, MISSING_KEY_DIAGNOSTIC) + + samples: Final = _samples(METRIC) + assert len(samples) == 1 + assert samples[0].labels == { + "requested_model": "per-second-priced-chat", + "model": "openai/gpt-5.4-nano", + "model_id": "dep-1", + "api_provider": "openai", + "reason": "missing_pricing_key", + } + assert samples[0].value == 2.0 + finally: + _clear_prometheus_registry() + + +@pytest.mark.asyncio +async def test_request_without_a_diagnostic_leaves_the_counter_untouched() -> None: + _clear_prometheus_registry() + try: + await _log_success(PrometheusLogger(), None) + + assert _samples(METRIC) == [] + finally: + _clear_prometheus_registry() + + +@pytest.mark.asyncio +async def test_label_filter_that_drops_reason_still_counts_the_request() -> None: + _clear_prometheus_registry() + previous_config: Final = litellm.prometheus_metrics_config + litellm.prometheus_metrics_config = [ + {"group": "zero_cost", "metrics": [METRIC], "include_labels": ["requested_model"]} + ] + try: + await _log_success(PrometheusLogger(), MISSING_KEY_DIAGNOSTIC) + + samples: Final = _samples(METRIC) + assert len(samples) == 1 + assert samples[0].labels == {"requested_model": "per-second-priced-chat"} + assert samples[0].value == 1.0 + finally: + litellm.prometheus_metrics_config = previous_config + _clear_prometheus_registry() diff --git a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_zero_cost_diagnostic.py b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_zero_cost_diagnostic.py new file mode 100644 index 00000000000..0e453e3f5eb --- /dev/null +++ b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_zero_cost_diagnostic.py @@ -0,0 +1,157 @@ +from collections.abc import Mapping +from typing import Final + +import pytest + +from litellm.litellm_core_utils.llm_cost_calc.zero_cost_diagnostic import ( + ZERO_COST_COUNTER_NAME, + diagnose_zero_cost, + used_pricing_keys, + zero_cost_warning, +) +from litellm.types.utils import CompletionTokensDetailsWrapper, PromptTokensDetailsWrapper, Usage + +PER_SECOND_ENTRY: Final = {"input_cost_per_second": 0.00042, "output_cost_per_second": 0.00042} +FREE_ENTRY: Final = {"input_cost_per_token": 0, "output_cost_per_token": 0, "cache_read_input_token_cost": 2e-08} +PRICED_ENTRY: Final = {"input_cost_per_token": 1e-06, "output_cost_per_token": 2e-06} +TEXT_USAGE: Final = Usage(prompt_tokens=10, completion_tokens=20, total_tokens=30) + + +def test_missing_pricing_key_names_every_rate_the_usage_needs() -> None: + diagnostic = diagnose_zero_cost( + usage=TEXT_USAGE, pricing_model="dep-1", pricing_entry=PER_SECOND_ENTRY, calculation_failed=False + ) + + assert diagnostic == { + "reason": "missing_pricing_key", + "pricing_model": "dep-1", + "missing_pricing_keys": ("input_cost_per_token", "output_cost_per_token"), + } + + +def test_only_the_absent_rate_is_reported() -> None: + diagnostic = diagnose_zero_cost( + usage=TEXT_USAGE, pricing_model="dep-1", pricing_entry={"input_cost_per_token": 1e-06}, calculation_failed=False + ) + + assert diagnostic is not None + assert diagnostic["missing_pricing_keys"] == ("output_cost_per_token",) + + +def test_free_model_stays_silent() -> None: + assert ( + diagnose_zero_cost(usage=TEXT_USAGE, pricing_model="dep-1", pricing_entry=FREE_ENTRY, calculation_failed=False) + is None + ) + + +@pytest.mark.parametrize("calculation_failed", [False, True]) +def test_request_without_usage_stays_silent(calculation_failed: bool) -> None: + usage = Usage(prompt_tokens=0, completion_tokens=0, total_tokens=0) + + assert ( + diagnose_zero_cost( + usage=usage, pricing_model="dep-1", pricing_entry=PER_SECOND_ENTRY, calculation_failed=calculation_failed + ) + is None + ) + + +@pytest.mark.parametrize( + "entry", + [ + {"litellm_provider": "openai", "mode": "chat", "supports_prompt_caching": True}, + {"tiered_pricing": [{"range": [0, 128000], "input_cost_per_token": 0, "output_cost_per_token": 0}]}, + {"tiered_pricing": "not a tier table", "litellm_provider": "openai"}, + ], +) +def test_entry_that_declares_no_rate_stays_silent(entry: Mapping[str, object]) -> None: + assert ( + diagnose_zero_cost(usage=TEXT_USAGE, pricing_model="dep-1", pricing_entry=entry, calculation_failed=False) + is None + ) + + +def test_tiered_rate_counts_as_a_declared_rate() -> None: + entry = {"tiered_pricing": [{"range": [0, 128000], "input_cost_per_token": 1e-06, "output_cost_per_token": 2e-06}]} + + diagnostic = diagnose_zero_cost( + usage=TEXT_USAGE, pricing_model="dep-1", pricing_entry=entry, calculation_failed=False + ) + + assert diagnostic is not None + assert diagnostic["reason"] == "missing_pricing_key" + + +def test_priced_entry_that_still_prices_to_zero_is_pricing_not_applied() -> None: + diagnostic = diagnose_zero_cost( + usage=TEXT_USAGE, pricing_model="dep-1", pricing_entry=PRICED_ENTRY, calculation_failed=False + ) + + assert diagnostic == {"reason": "pricing_not_applied", "pricing_model": "dep-1", "missing_pricing_keys": ()} + + +def test_calculator_failure_on_a_priced_entry_is_cost_calculation_error() -> None: + diagnostic = diagnose_zero_cost( + usage=TEXT_USAGE, pricing_model="dep-1", pricing_entry=PRICED_ENTRY, calculation_failed=True + ) + + assert diagnostic == {"reason": "cost_calculation_error", "pricing_model": "dep-1", "missing_pricing_keys": ()} + + +def test_calculator_failure_on_a_free_entry_stays_silent() -> None: + assert ( + diagnose_zero_cost(usage=TEXT_USAGE, pricing_model="dep-1", pricing_entry=FREE_ENTRY, calculation_failed=True) + is None + ) + + +def test_calculator_failure_on_an_entry_that_declares_no_rate_stays_silent() -> None: + entry: Final = {"litellm_provider": "openai", "mode": "chat", "supports_prompt_caching": True} + assert ( + diagnose_zero_cost(usage=TEXT_USAGE, pricing_model="dep-1", pricing_entry=entry, calculation_failed=True) + is None + ) + + +def test_audio_tokens_need_the_audio_rates() -> None: + usage = Usage( + prompt_tokens=10, + completion_tokens=20, + total_tokens=30, + prompt_tokens_details=PromptTokensDetailsWrapper(audio_tokens=10, text_tokens=0), + completion_tokens_details=CompletionTokensDetailsWrapper(audio_tokens=5, text_tokens=15), + ) + + assert used_pricing_keys(usage) == ( + "input_cost_per_audio_token", + "output_cost_per_token", + "output_cost_per_audio_token", + ) + diagnostic = diagnose_zero_cost( + usage=usage, pricing_model="gemini-audio", pricing_entry=PRICED_ENTRY, calculation_failed=False + ) + assert diagnostic is not None + assert diagnostic["missing_pricing_keys"] == ("input_cost_per_audio_token", "output_cost_per_audio_token") + + +def test_warning_names_the_request_the_entry_the_missing_keys_and_the_counter() -> None: + diagnostic = diagnose_zero_cost( + usage=TEXT_USAGE, pricing_model="dep-1", pricing_entry=PER_SECOND_ENTRY, calculation_failed=False + ) + assert diagnostic is not None + + message = zero_cost_warning( + diagnostic, + model_group="per-second-priced-chat", + model="openai/gpt-5.4-nano", + custom_llm_provider="openai", + usage=TEXT_USAGE, + ) + + assert "model_group=per-second-priced-chat" in message + assert "model=openai/gpt-5.4-nano" in message + assert "provider=openai" in message + assert "prompt_tokens=10 completion_tokens=20" in message + assert "pricing entry 'dep-1' has no input_cost_per_token, output_cost_per_token" in message + assert f'{ZERO_COST_COUNTER_NAME}{{reason="missing_pricing_key"}}' in message diff --git a/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py b/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py index c67f72680a8..62cf5680266 100644 --- a/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py +++ b/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py @@ -20,6 +20,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( handle_any_messages_to_chat_completion_str_messages_conversion, hoist_images_from_tool_messages, is_encrypted_reasoning_block, + merge_consecutive_system_messages, responses_reasoning_items_from_thinking_blocks, split_concatenated_json_objects, strip_encrypted_reasoning_from_messages, @@ -1846,3 +1847,95 @@ class TestEncryptedReasoningReplay: strip_encrypted_reasoning_from_messages(messages) assert messages == before + + +class TestMergeConsecutiveSystemMessages: + def test_merges_each_run_of_string_system_messages_with_a_blank_line(self): + messages = [ + {"role": "system", "content": "You are terse.", "cache_control": {"type": "ephemeral"}}, + {"role": "system", "content": "Skills: none."}, + {"role": "user", "content": "Hello"}, + {"role": "assistant", "content": "Hi"}, + {"role": "system", "content": "Reminder A"}, + {"role": "system", "content": "Reminder B"}, + {"role": "user", "content": "Bye"}, + ] + + merged = merge_consecutive_system_messages(messages) + + assert merged == [ + {"role": "system", "content": "You are terse.\n\nSkills: none.", "cache_control": {"type": "ephemeral"}}, + {"role": "user", "content": "Hello"}, + {"role": "assistant", "content": "Hi"}, + {"role": "system", "content": "Reminder A\n\nReminder B"}, + {"role": "user", "content": "Bye"}, + ] + + def test_merges_into_text_parts_when_any_system_content_is_a_list(self): + cached_part = {"type": "text", "text": "Skills: none.", "cache_control": {"type": "ephemeral"}} + messages = [ + {"role": "system", "content": "You are terse."}, + {"role": "system", "content": [cached_part, {"type": "text", "text": "Be brief."}]}, + {"role": "system", "content": "Answer in English."}, + {"role": "user", "content": "Hello"}, + ] + + merged = merge_consecutive_system_messages(messages) + + assert merged == [ + { + "role": "system", + "content": [ + {"type": "text", "text": "You are terse."}, + cached_part, + {"type": "text", "text": "Be brief."}, + {"type": "text", "text": "Answer in English."}, + ], + }, + {"role": "user", "content": "Hello"}, + ] + assert merged[0]["content"][1] is cached_part + + @pytest.mark.parametrize( + "messages", + [ + [{"role": "system", "content": "You are terse."}, {"role": "user", "content": "Hello"}], + [{"role": "user", "content": "Hello"}, {"role": "assistant", "content": "Hi"}], + [ + {"role": "system", "content": "You are terse."}, + {"role": "user", "content": "Hello"}, + {"role": "system", "content": "Reminder"}, + ], + [], + ], + ids=["single-system", "no-system", "separated-systems", "empty"], + ) + def test_leaves_messages_without_consecutive_system_messages_untouched(self, messages): + before = copy.deepcopy(messages) + + merged = merge_consecutive_system_messages(messages) + + assert merged == before + assert [message is original for message, original in zip(merged, messages)] == [True] * len(messages) + + @pytest.mark.parametrize( + ("messages", "expected_content"), + [ + ([{"role": "system"}, {"role": "system", "content": "Skills: none."}], "Skills: none."), + ([{"role": "system", "content": "You are terse."}, {"role": "system"}], "You are terse."), + ( + [{"role": "system"}, {"role": "system", "content": [{"type": "text", "text": "Be brief."}]}], + [{"type": "text", "text": "Be brief."}], + ), + ], + ids=["missing-then-str", "str-then-missing", "missing-then-list"], + ) + def test_skips_system_messages_without_content_when_merging(self, messages, expected_content): + merged = merge_consecutive_system_messages([*messages, {"role": "user", "content": "Hello"}]) + + assert merged == [{"role": "system", "content": expected_content}, {"role": "user", "content": "Hello"}] + + def test_keeps_the_first_message_when_no_system_message_in_the_run_has_content(self): + merged = merge_consecutive_system_messages([{"role": "system"}, {"role": "system"}, {"role": "user", "content": "Hi"}]) + + assert merged == [{"role": "system"}, {"role": "user", "content": "Hi"}] diff --git a/tests/test_litellm/litellm_core_utils/test_agentic_followup_kwargs.py b/tests/test_litellm/litellm_core_utils/test_agentic_followup_kwargs.py new file mode 100644 index 00000000000..af0fbdcc35b --- /dev/null +++ b/tests/test_litellm/litellm_core_utils/test_agentic_followup_kwargs.py @@ -0,0 +1,66 @@ +from collections.abc import Mapping +from typing import Final + +from litellm.litellm_core_utils.agentic_followup_kwargs import build_agentic_followup_kwargs + + +def _build( + *, + request_kwargs: dict[str, object], + patch_kwargs: dict[str, object], + request_params: set[str], + fingerprints: list[str] | None = None, +) -> Mapping[str, object]: + return build_agentic_followup_kwargs( + request_kwargs=request_kwargs, + patch_kwargs=patch_kwargs, + request_params=request_params, + depth=0, + max_loops=3, + fingerprints=fingerprints if fingerprints is not None else [], + fingerprint="fp", + ) + + +def test_followup_kwargs_never_repeat_a_request_param(): + """Neither source may re-add a key the caller already sends as a request param, or the follow-up call raises a duplicate keyword""" + followup: Final = _build( + request_kwargs={"prompt_cache_key": "thread-1", "api_base": "https://a"}, + patch_kwargs={"prompt_cache_key": "thread-1", "metadata": {"user": "u1"}}, + request_params={"prompt_cache_key", "model", "input"}, + ) + + assert followup.keys().isdisjoint({"prompt_cache_key", "model", "input"}) + assert followup["api_base"] == "https://a" + assert followup["metadata"] == {"user": "u1"} + + +def test_followup_kwargs_let_the_plan_override_the_request(): + followup: Final = _build( + request_kwargs={"api_base": "https://request", "timeout": 5}, + patch_kwargs={"api_base": "https://plan"}, + request_params=set(), + ) + + assert followup["api_base"] == "https://plan" + assert followup["timeout"] == 5 + + +def test_followup_kwargs_carry_the_loop_bookkeeping_without_touching_the_inputs(): + fingerprints: Final = ["earlier"] + request_kwargs: Final = {"_agentic_loop_depth": 0, "max_agentic_loops": 9} + patch_kwargs: Final = {"_agentic_loop_fingerprints": ["stale"]} + + followup: Final = _build( + request_kwargs=request_kwargs, + patch_kwargs=patch_kwargs, + request_params=set(), + fingerprints=fingerprints, + ) + + assert followup["_agentic_loop_depth"] == 1 + assert followup["max_agentic_loops"] == 3 + assert followup["_agentic_loop_fingerprints"] == ["earlier", "fp"] + assert fingerprints == ["earlier"] + assert request_kwargs == {"_agentic_loop_depth": 0, "max_agentic_loops": 9} + assert patch_kwargs == {"_agentic_loop_fingerprints": ["stale"]} diff --git a/tests/test_litellm/litellm_core_utils/test_chat_completion_agentic_loop.py b/tests/test_litellm/litellm_core_utils/test_chat_completion_agentic_loop.py index 434daab6ab5..0d7f735e6e5 100644 --- a/tests/test_litellm/litellm_core_utils/test_chat_completion_agentic_loop.py +++ b/tests/test_litellm/litellm_core_utils/test_chat_completion_agentic_loop.py @@ -343,6 +343,40 @@ async def test_dispatcher_runs_followup_with_incremented_depth_and_patched_messa assert logger.cleanup_calls == 1 +@pytest.mark.asyncio +async def test_dispatcher_followup_does_not_repeat_a_request_param_found_in_request_kwargs( + restore_callbacks, +): + """Request kwargs that repeat a request param must not crash the follow-up + with a duplicate keyword, whether or not the plan copies them too.""" + followup = _plain_model_response("done") + request_kwargs = {"temperature": 0.2, "api_base": "https://a"} + plan = AgenticLoopPlan( + run_agentic_loop=True, + request_patch=AgenticLoopRequestPatch(messages=_patched_messages(), kwargs=dict(request_kwargs)), + ) + litellm.callbacks = [_GateOnlyLogger(plan=plan, tool_calls={"tool_calls": [{"id": "call_abc"}]})] + + acompletion_mock = AsyncMock(return_value=followup) + with patch.object(litellm, "acompletion", acompletion_mock): + result = await maybe_run_chat_completion_agentic_loop( + response=_tool_call_model_response(), + model="gpt-4o-mini", + messages=[{"role": "user", "content": "what is 6*7?"}], + optional_params={"temperature": 0.2}, + kwargs=dict(request_kwargs), + logging_obj=_LoggingStub(), + custom_llm_provider="openai", + stream=False, + ) + + assert result is followup + acompletion_mock.assert_awaited_once() + call_kwargs = acompletion_mock.await_args.kwargs + assert call_kwargs["temperature"] == 0.2 + assert call_kwargs["api_base"] == "https://a" + + @pytest.mark.asyncio async def test_dispatcher_raises_when_depth_reaches_max_agentic_loops( restore_callbacks, diff --git a/tests/test_litellm/litellm_core_utils/test_core_helpers.py b/tests/test_litellm/litellm_core_utils/test_core_helpers.py index b2ad13c205e..bca61a0e76f 100644 --- a/tests/test_litellm/litellm_core_utils/test_core_helpers.py +++ b/tests/test_litellm/litellm_core_utils/test_core_helpers.py @@ -6,6 +6,8 @@ import pytest from litellm.litellm_core_utils.core_helpers import ( _FINISH_REASON_MAP, + bind_budget_reservation_to_callbacks, + budget_reservation_from_metadata, drop_params_env_flag, drop_params_flag, get_or_create_metadata_bucket, @@ -13,7 +15,60 @@ from litellm.litellm_core_utils.core_helpers import ( normalize_drop_params, reconstruct_model_name, redact_nested_match_and_regex_keys, + unbind_budget_reservation_from_callbacks, ) +from litellm.proxy._types import UserAPIKeyAuth + + +class TestBudgetReservationBinding: + """The request-end release skips a reservation a cost callback has claimed, so the claim + must land on the one dict auth stamped, through whichever metadata field or auth object + carries it, and a failed call must be able to hand it back.""" + + @staticmethod + def _reservation() -> dict: + return {"reserved_cost": 0.5, "entries": [], "finalized": False, "callback_bound": False} + + @pytest.mark.parametrize("metadata_variable_name", ["metadata", "litellm_metadata"]) + def test_reservation_stamped_on_the_metadata_is_bound(self, metadata_variable_name: str): + reservation = self._reservation() + + bind_budget_reservation_to_callbacks({metadata_variable_name: {"user_api_key_budget_reservation": reservation}}) + + assert reservation["callback_bound"] is True + + def test_reservation_reachable_only_through_the_auth_object_is_bound(self): + reservation = self._reservation() + user_api_key_auth = UserAPIKeyAuth(token="hashed") + user_api_key_auth.budget_reservation = reservation + + bind_budget_reservation_to_callbacks({"metadata": {"user_api_key_auth": user_api_key_auth}}) + + assert reservation["callback_bound"] is True + + def test_reservation_reachable_only_through_a_dumped_auth_object_is_bound(self): + reservation = self._reservation() + + bind_budget_reservation_to_callbacks({"metadata": {"user_api_key_auth": {"budget_reservation": reservation}}}) + + assert reservation["callback_bound"] is True + + def test_unbind_hands_a_claimed_reservation_back(self): + reservation = self._reservation() + litellm_params = {"litellm_metadata": {"user_api_key_budget_reservation": reservation}} + bind_budget_reservation_to_callbacks(litellm_params) + + unbind_budget_reservation_from_callbacks(litellm_params) + + assert reservation["callback_bound"] is False + + def test_request_without_a_reservation_binds_nothing(self): + metadata = {"user_api_key_auth": UserAPIKeyAuth(token="hashed")} + + bind_budget_reservation_to_callbacks({"metadata": metadata, "litellm_metadata": None}) + + assert budget_reservation_from_metadata(metadata) is None + assert "user_api_key_budget_reservation" not in metadata class TestGetOrCreateMetadataBucket: diff --git a/tests/test_litellm/litellm_core_utils/test_get_litellm_params.py b/tests/test_litellm/litellm_core_utils/test_get_litellm_params.py index a34bc2af59d..4c963d14ada 100644 --- a/tests/test_litellm/litellm_core_utils/test_get_litellm_params.py +++ b/tests/test_litellm/litellm_core_utils/test_get_litellm_params.py @@ -67,6 +67,15 @@ class TestGetLitellmParamsKwargsExtraction: assert "s3_endpoint_url" not in result_without_s3_kwargs assert "s3_region_name" not in result_without_s3_kwargs + def test_s3_credential_kwargs_are_forwarded_for_s3_signing(self): + result = get_litellm_params(s3_access_key_id="s3-key", s3_secret_access_key="s3-secret") + assert result["s3_access_key_id"] == "s3-key" + assert result["s3_secret_access_key"] == "s3-secret" + + result_without_s3_kwargs = get_litellm_params() + assert "s3_access_key_id" not in result_without_s3_kwargs + assert "s3_secret_access_key" not in result_without_s3_kwargs + def test_subset_of_kwargs_only_includes_provided(self): """Only provided kwargs appear, others remain absent.""" result = get_litellm_params(azure_ad_token="token123") diff --git a/tests/test_litellm/litellm_core_utils/test_get_model_cost_map.py b/tests/test_litellm/litellm_core_utils/test_get_model_cost_map.py index 53fee36b3a8..262dabb7c1b 100644 --- a/tests/test_litellm/litellm_core_utils/test_get_model_cost_map.py +++ b/tests/test_litellm/litellm_core_utils/test_get_model_cost_map.py @@ -448,6 +448,23 @@ async def test_refetch_records_the_blob_id_of_the_bytes_served_and_the_fetch_eta assert get_model_cost_map_provenance() == {"source_revision": git_blob_id(body), "etag": 'W/"abc123"'} +@pytest.mark.asyncio +async def test_loaded_catalog_snapshot_follows_the_fetched_map_and_ignores_later_registrations(monkeypatch): + import litellm + + edited = json.loads(_real_map_bytes()) + edited["gpt-5.4-mini"]["max_input_tokens"] = 777 + client, _ = _mock_client([httpx.Response(200, content=json.dumps(edited).encode())]) + + result = await refetch_model_cost_map(url=_URL, sleep=_SleepRecorder(), rng=random.Random(0), client=client) + + assert isinstance(result, ModelCostMapReloaded) + monkeypatch.setattr(litellm, "model_cost", result.model_cost_map) + litellm.register_model({"gpt-5.4-mini": {"max_input_tokens": 2048}}, persist_across_reloads=False) + assert litellm.model_cost["gpt-5.4-mini"]["max_input_tokens"] == 2048 + assert GetModelCostMap.loaded_model_cost_map()["gpt-5.4-mini"]["max_input_tokens"] == 777 + + @pytest.mark.asyncio async def test_refetch_revision_follows_the_bytes_not_the_url(): edited = json.loads(_real_map_bytes()) diff --git a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py index 277ae33a076..1bfc55d1486 100644 --- a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py +++ b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py @@ -1,9 +1,10 @@ import asyncio import contextlib import datetime +import logging import os import sys -from collections.abc import Callable +from collections.abc import Callable, Iterator, Mapping from typing import Final, Literal from unittest.mock import AsyncMock, MagicMock, patch @@ -25,6 +26,7 @@ from litellm.litellm_core_utils.litellm_logging import ( set_callbacks, ) from litellm.llms.base_llm.ocr.transformation import OCRUsageInfo +from litellm.proxy._types import UserAPIKeyAuth from litellm.types.llms.openai import ResponseAPIUsage, ResponseCompletedEvent, ResponsesAPIResponse from litellm.types.utils import ( CallTypes, @@ -302,6 +304,398 @@ def test_response_cost_calculator_uses_router_model_id_from_litellm_metadata(): litellm.model_cost.pop(custom_model_id, None) +class TestZeroCostDiagnostic: + DEPLOYMENT_ID: Final = "lit7898-per-second-priced-deployment" + MODEL_GROUP: Final = "per-second-priced-chat" + PER_SECOND_PRICING: Final = {"input_cost_per_second": 0.00042, "output_cost_per_second": 0.00042} + FREE_PRICING: Final = {"input_cost_per_token": 0, "output_cost_per_token": 0} + + @pytest.fixture(params=["per_second", "free"]) + def deployment_pricing(self, request: pytest.FixtureRequest) -> Iterator[Mapping[str, float]]: + pricing: Final = self.PER_SECOND_PRICING if request.param == "per_second" else self.FREE_PRICING + litellm.register_model(model_cost={self.DEPLOYMENT_ID: pricing}, persist_across_reloads=False) + try: + yield pricing + finally: + litellm.model_cost.pop(self.DEPLOYMENT_ID, None) + + def _logging_obj( + self, + pricing: Mapping[str, object], + stream: bool = False, + model: str = "openai/gpt-5.4-nano", + call_type: str = "completion", + deployment_id: str | None = DEPLOYMENT_ID, + custom_llm_provider: str = "openai", + ) -> LitellmLogging: + logging_obj: Final = LitellmLogging( + model=model, + messages=[{"role": "user", "content": "Hi"}], + stream=stream, + call_type=call_type, + start_time=time.time(), + litellm_call_id="lit7898", + function_id="fn", + ) + self._route_to_deployment( + logging_obj, pricing, model=model, deployment_id=deployment_id, custom_llm_provider=custom_llm_provider + ) + return logging_obj + + def _route_to_deployment( + self, + logging_obj: LitellmLogging, + pricing: Mapping[str, object], + model: str = "openai/gpt-5.4-nano", + deployment_id: str | None = DEPLOYMENT_ID, + custom_llm_provider: str = "openai", + ) -> None: + model_info: Final = pricing if deployment_id is None else {"id": deployment_id, **pricing} + logging_obj.update_environment_variables( + model=model, + user="", + optional_params={}, + litellm_params={"metadata": {"model_group": self.MODEL_GROUP, "model_info": model_info}}, + custom_llm_provider=custom_llm_provider, + ) + + @staticmethod + def _response( + usage: litellm.Usage | None = None, model: str = "gpt-5.4-nano", **hidden_params: object + ) -> ModelResponse: + response: Final = ModelResponse( + model=model, + choices=[litellm.Choices(message=litellm.Message(role="assistant", content="hello"))], + usage=usage, + ) + response._hidden_params = {"custom_llm_provider": "openai", **hidden_params} + return response + + @staticmethod + def _zero_cost_warnings(caplog: pytest.LogCaptureFixture) -> list[str]: + return [ + record.getMessage() + for record in caplog.records + if record.name == "LiteLLM" and record.levelno == logging.WARNING and "priced at $0" in record.getMessage() + ] + + def _assert_flagged(self, logging_obj: LitellmLogging, caplog: pytest.LogCaptureFixture) -> None: + assert logging_obj.model_call_details["zero_cost_diagnostic"] == { + "reason": "missing_pricing_key", + "pricing_model": self.DEPLOYMENT_ID, + "missing_pricing_keys": ("input_cost_per_token", "output_cost_per_token"), + } + warnings: Final = self._zero_cost_warnings(caplog) + assert len(warnings) == 1 + assert f"model_group={self.MODEL_GROUP}" in warnings[0] + assert f"pricing entry '{self.DEPLOYMENT_ID}' has no input_cost_per_token, output_cost_per_token" in warnings[0] + + def test_zero_cost_with_a_missing_rate_warns_once_and_is_recorded( + self, deployment_pricing: Mapping[str, float], caplog: pytest.LogCaptureFixture + ) -> None: + usage: Final = litellm.Usage(prompt_tokens=10, completion_tokens=20, total_tokens=30) + logging_obj: Final = self._logging_obj(deployment_pricing) + + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + first_cost: Final = logging_obj._response_cost_calculator(result=self._response(usage)) + second_cost: Final = logging_obj._response_cost_calculator(result=self._response(usage)) + + assert first_cost == 0.0 + assert second_cost == 0.0 + if deployment_pricing is self.FREE_PRICING: + assert logging_obj.model_call_details["zero_cost_diagnostic"] is None + assert self._zero_cost_warnings(caplog) == [] + return + self._assert_flagged(logging_obj, caplog) + + def test_usage_less_stream_chunk_does_not_hide_the_final_response_diagnostic( + self, deployment_pricing: Mapping[str, float], caplog: pytest.LogCaptureFixture + ) -> None: + usage: Final = litellm.Usage(prompt_tokens=8, completion_tokens=2, total_tokens=10) + logging_obj: Final = self._logging_obj(deployment_pricing, stream=True) + + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + logging_obj._response_cost_calculator(result=self._response(usage=None)) + logging_obj._response_cost_calculator(result=self._response(usage)) + + if deployment_pricing is self.FREE_PRICING: + assert logging_obj.model_call_details["zero_cost_diagnostic"] is None + assert self._zero_cost_warnings(caplog) == [] + return + self._assert_flagged(logging_obj, caplog) + + def test_terminal_responses_stream_event_is_judged_by_its_inner_response( + self, deployment_pricing: Mapping[str, float], caplog: pytest.LogCaptureFixture + ) -> None: + logging_obj: Final = self._logging_obj(deployment_pricing, stream=True, call_type="aresponses") + event: Final = ResponseCompletedEvent( + type="response.completed", + response=ResponsesAPIResponse( + id="resp-lit7898", + created_at=1, + object="response", + status="completed", + model="gpt-5.4-nano", + output=[], + usage=ResponseAPIUsage(input_tokens=10, output_tokens=20, total_tokens=30), + ), + ) + + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + cost: Final = logging_obj._response_cost_calculator(result=event) + + assert cost == 0.0 + if deployment_pricing is self.FREE_PRICING: + assert logging_obj.model_call_details["zero_cost_diagnostic"] is None + assert self._zero_cost_warnings(caplog) == [] + return + self._assert_flagged(logging_obj, caplog) + + def test_precomputed_zero_hidden_cost_is_flagged_and_lands_in_the_payload( + self, deployment_pricing: Mapping[str, float], caplog: pytest.LogCaptureFixture + ) -> None: + usage: Final = litellm.Usage(prompt_tokens=10, completion_tokens=20, total_tokens=30) + logging_obj: Final = self._logging_obj(deployment_pricing) + response: Final = self._response(usage, response_cost=0.0, model_id=self.DEPLOYMENT_ID) + + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + logging_obj._process_hidden_params_and_response_cost( + response, start_time=datetime.datetime.now(), end_time=datetime.datetime.now() + ) + + payload: Final = logging_obj.model_call_details["standard_logging_object"] + assert payload["response_cost"] == 0.0 + if deployment_pricing is self.FREE_PRICING: + assert payload["zero_cost_diagnostic"] is None + assert self._zero_cost_warnings(caplog) == [] + return + self._assert_flagged(logging_obj, caplog) + assert payload["zero_cost_diagnostic"] == logging_obj.model_call_details["zero_cost_diagnostic"] + + def test_uncomputed_hidden_cost_is_not_a_zero_cost( + self, deployment_pricing: Mapping[str, float], caplog: pytest.LogCaptureFixture + ) -> None: + usage: Final = litellm.Usage(prompt_tokens=10, completion_tokens=20, total_tokens=30) + logging_obj: Final = self._logging_obj(deployment_pricing) + response: Final = self._response(usage, response_cost=None, model_id=self.DEPLOYMENT_ID) + + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + logging_obj._process_hidden_params_and_response_cost( + response, start_time=datetime.datetime.now(), end_time=datetime.datetime.now() + ) + + assert logging_obj.model_call_details["standard_logging_object"]["zero_cost_diagnostic"] is None + assert self._zero_cost_warnings(caplog) == [] + + def test_unbilled_read_route_with_usage_stays_silent( + self, deployment_pricing: Mapping[str, float], caplog: pytest.LogCaptureFixture + ) -> None: + usage: Final = litellm.Usage(prompt_tokens=10, completion_tokens=20, total_tokens=30) + logging_obj: Final = self._logging_obj(deployment_pricing, call_type="aget_responses") + response: Final = self._response(usage, response_cost=0.0, model_id=self.DEPLOYMENT_ID) + + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + logging_obj._process_hidden_params_and_response_cost( + response, start_time=datetime.datetime.now(), end_time=datetime.datetime.now() + ) + + assert logging_obj.model_call_details["standard_logging_object"]["zero_cost_diagnostic"] is None + assert self._zero_cost_warnings(caplog) == [] + + def test_unmapped_model_that_fails_cost_calculation_stays_silent(self, caplog: pytest.LogCaptureFixture) -> None: + usage: Final = litellm.Usage(prompt_tokens=10, completion_tokens=20, total_tokens=30) + logging_obj: Final = self._logging_obj( + {}, model="openai/lit7898-unmapped-model", deployment_id="lit7898-unmapped-deployment" + ) + + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + cost: Final = logging_obj._response_cost_calculator( + result=self._response(usage, model="lit7898-unmapped-model") + ) + + assert cost is None + assert logging_obj.model_call_details["response_cost_failure_debug_information"] is not None + assert logging_obj.model_call_details.get("zero_cost_diagnostic") is None + assert self._zero_cost_warnings(caplog) == [] + + def test_malformed_usage_never_raises_out_of_the_cost_calculator( + self, deployment_pricing: Mapping[str, float], caplog: pytest.LogCaptureFixture + ) -> None: + logging_obj: Final = self._logging_obj(deployment_pricing) + + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + cost: Final = logging_obj._response_cost_calculator( + result={"model": "gpt-5.4-nano", "usage": {"prompt_tokens": "n/a", "completion_tokens": 3}} + ) + + assert cost is None + assert logging_obj.model_call_details.get("zero_cost_diagnostic") is None + assert self._zero_cost_warnings(caplog) == [] + + def test_usage_less_evaluation_between_two_zero_cost_findings_does_not_warn_twice( + self, deployment_pricing: Mapping[str, float], caplog: pytest.LogCaptureFixture + ) -> None: + usage: Final = litellm.Usage(prompt_tokens=8, completion_tokens=2, total_tokens=10) + logging_obj: Final = self._logging_obj(deployment_pricing, stream=True, call_type="anthropic_messages") + + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + logging_obj._response_cost_calculator(result=self._response(usage=None)) + logging_obj._response_cost_calculator(result=self._response(usage)) + logging_obj._response_cost_calculator(result=self._response(usage=None)) + logging_obj._response_cost_calculator(result=self._response(usage)) + + if deployment_pricing is self.FREE_PRICING: + assert logging_obj.model_call_details["zero_cost_diagnostic"] is None + assert self._zero_cost_warnings(caplog) == [] + return + self._assert_flagged(logging_obj, caplog) + + def test_retry_that_prices_clears_the_diagnostic_and_a_later_zero_cost_is_recorded_silently( + self, caplog: pytest.LogCaptureFixture + ) -> None: + priced_id: Final = "lit7898-priced-deployment" + priced_pricing: Final = {"input_cost_per_token": 1e-06, "output_cost_per_token": 2e-06} + usage: Final = litellm.Usage(prompt_tokens=10, completion_tokens=20, total_tokens=30) + litellm.register_model( + model_cost={self.DEPLOYMENT_ID: self.PER_SECOND_PRICING, priced_id: priced_pricing}, + persist_across_reloads=False, + ) + try: + logging_obj: Final = self._logging_obj(self.PER_SECOND_PRICING) + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + assert logging_obj._response_cost_calculator(result=self._response(usage)) == 0.0 + self._assert_flagged(logging_obj, caplog) + + self._route_to_deployment(logging_obj, priced_pricing, deployment_id=priced_id) + assert logging_obj._response_cost_calculator(result=self._response(usage)) == pytest.approx(5e-05) + assert logging_obj.model_call_details["zero_cost_diagnostic"] is None + + self._route_to_deployment(logging_obj, self.PER_SECOND_PRICING) + assert logging_obj._response_cost_calculator(result=self._response(usage)) == 0.0 + + assert logging_obj.model_call_details["zero_cost_diagnostic"]["reason"] == "missing_pricing_key" + assert len(self._zero_cost_warnings(caplog)) == 1 + finally: + litellm.model_cost.pop(self.DEPLOYMENT_ID, None) + litellm.model_cost.pop(priced_id, None) + + def test_one_request_evaluated_against_two_cost_map_entries_warns_once( + self, caplog: pytest.LogCaptureFixture + ) -> None: + dated_model: Final = "lit7898-nano-2026-03-17" + requested_model: Final = "lit7898-nano" + usage: Final = litellm.Usage(prompt_tokens=10, completion_tokens=20, total_tokens=30) + cost_map_entry: Final = {"litellm_provider": "openai", "mode": "chat", **self.PER_SECOND_PRICING} + litellm.register_model( + model_cost={dated_model: cost_map_entry, requested_model: cost_map_entry}, persist_across_reloads=False + ) + try: + logging_obj: Final = self._logging_obj( + {}, model=f"openai/{requested_model}", deployment_id="lit7898-cost-map-deployment" + ) + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + assert logging_obj._response_cost_calculator(result=self._response(usage, model=dated_model)) == 0.0 + assert logging_obj._response_cost_calculator(result=self._response(usage, model=requested_model)) == 0.0 + + assert logging_obj.model_call_details["zero_cost_diagnostic"]["pricing_model"] == requested_model + warnings: Final = self._zero_cost_warnings(caplog) + assert len(warnings) == 1 + assert f"pricing entry '{dated_model}' has no input_cost_per_token, output_cost_per_token" in warnings[0] + finally: + litellm.model_cost.pop(dated_model, None) + litellm.model_cost.pop(requested_model, None) + + def test_free_deployment_without_a_router_id_is_judged_by_its_own_pricing( + self, caplog: pytest.LogCaptureFixture + ) -> None: + global_model: Final = "lit7898-priced-global" + usage: Final = litellm.Usage(prompt_tokens=10, completion_tokens=20, total_tokens=30) + litellm.register_model( + model_cost={ + global_model: { + "litellm_provider": "openai", + "mode": "chat", + "input_cost_per_token": 1e-06, + "output_cost_per_token": 2e-06, + } + }, + persist_across_reloads=False, + ) + try: + logging_obj: Final = self._logging_obj(self.FREE_PRICING, model=global_model, deployment_id=None) + response: Final = self._response(usage, model=global_model, response_cost=0.0) + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + logging_obj._process_hidden_params_and_response_cost( + response, start_time=datetime.datetime.now(), end_time=datetime.datetime.now() + ) + assert logging_obj.model_call_details["zero_cost_diagnostic"] is None + assert self._zero_cost_warnings(caplog) == [] + finally: + litellm.model_cost.pop(global_model, None) + + def test_cache_hit_priced_for_saved_cost_stays_silent( + self, deployment_pricing: Mapping[str, float], caplog: pytest.LogCaptureFixture + ) -> None: + usage: Final = litellm.Usage(prompt_tokens=10, completion_tokens=20, total_tokens=30) + logging_obj: Final = self._logging_obj(deployment_pricing) + logging_obj.model_call_details["cache_hit"] = True + + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + assert logging_obj._response_cost_calculator(result=self._response(usage), cache_hit=False) == 0.0 + + assert logging_obj.model_call_details["zero_cost_diagnostic"] is None + assert self._zero_cost_warnings(caplog) == [] + + @pytest.mark.parametrize("spilled_over", [True, False]) + def test_ptu_deployment_is_judged_by_the_entry_the_calculator_priced_with( + self, spilled_over: bool, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture + ) -> None: + router_model_id: Final = "lit7898-ptu-router-model-id" + served_model: Final = "azure/lit7898-ptu-served-model" + ptu_model_info: Final = { + "team_id": "team-1", + "ptu_count": 100, + "cost_per_ptu_per_hour": 1.0, + "ptu_effective_from": "2026-01-01", + **self.FREE_PRICING, + } + usage: Final = litellm.Usage(prompt_tokens=10, completion_tokens=20, total_tokens=30) + litellm.register_model( + model_cost={ + router_model_id: {**self.FREE_PRICING, "litellm_provider": "azure", "mode": "chat"}, + served_model: {**self.PER_SECOND_PRICING, "litellm_provider": "azure", "mode": "chat"}, + }, + persist_across_reloads=False, + ) + monkeypatch.setenv("LITELLM_ENABLE_PTU_COST_ATTRIBUTION", "True") + try: + logging_obj: Final = self._logging_obj( + ptu_model_info, model=served_model, deployment_id=router_model_id, custom_llm_provider="azure" + ) + spillover_headers: Final = {"llm_provider-x-ms-is-spilled-over": "true"} if spilled_over else {} + response: Final = self._response( + usage, model=served_model, custom_llm_provider="azure", additional_headers=spillover_headers + ) + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + assert logging_obj._response_cost_calculator(result=response) == 0.0 + + warnings: Final = self._zero_cost_warnings(caplog) + if not spilled_over: + assert logging_obj.model_call_details["zero_cost_diagnostic"] is None + assert warnings == [] + return + assert logging_obj.model_call_details["zero_cost_diagnostic"] == { + "reason": "missing_pricing_key", + "pricing_model": served_model, + "missing_pricing_keys": ("input_cost_per_token", "output_cost_per_token"), + } + assert len(warnings) == 1 + assert f"pricing entry '{served_model}' has no input_cost_per_token, output_cost_per_token" in warnings[0] + finally: + litellm.model_cost.pop(router_model_id, None) + litellm.model_cost.pop(served_model, None) + + class TestGetRouterModelId: """Tests for the get_router_model_id helper method.""" @@ -407,7 +801,6 @@ class TestGetRouterDeploymentModelInfo: logging_obj.litellm_params = {"api_base": ""} assert logging_obj.get_router_deployment_model_info() is None - def test_a_published_batch_rate_never_displaces_a_declared_standard_rate(self) -> None: """Ownership is per token direction, not per field. @@ -1111,7 +1504,9 @@ async def test_arealtime_marks_litellm_params_async(monkeypatch): @pytest.mark.asyncio -async def test_aresponses_websocket_hands_back_the_provider_failure_without_a_success_log(monkeypatch: pytest.MonkeyPatch): +async def test_aresponses_websocket_hands_back_the_provider_failure_without_a_success_log( + monkeypatch: pytest.MonkeyPatch, +): from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER from litellm.responses.main import base_llm_http_handler @@ -7066,22 +7461,41 @@ async def test_classifier_audit_matches_provider_transport(provider: str) -> Non return httpx.Response(200, json=mock_responses_api_response(content).model_dump()) if provider == "anthropic": - return httpx.Response(200, json={ - "id": "msg-audit", "type": "message", "role": "assistant", "model": "claude-haiku-4-5", - "content": [{"type": "text", "text": content}], "stop_reason": "end_turn", - "usage": {"input_tokens": 10, "output_tokens": 5}, - }) + return httpx.Response( + 200, + json={ + "id": "msg-audit", + "type": "message", + "role": "assistant", + "model": "claude-haiku-4-5", + "content": [{"type": "text", "text": content}], + "stop_reason": "end_turn", + "usage": {"input_tokens": 10, "output_tokens": 5}, + }, + ) if provider == "bedrock": - return httpx.Response(200, json={ - "output": {"message": {"role": "assistant", "content": [{"text": content}]}}, - "stopReason": "end_turn", "usage": {"inputTokens": 10, "outputTokens": 5, "totalTokens": 15}, - "metrics": {"latencyMs": 1}, - }) - return httpx.Response(200, json={ - "id": "chatcmpl-audit", "object": "chat.completion", "created": 0, "model": "gpt-5.6", - "choices": [{"index": 0, "message": {"role": "assistant", "content": content}, "finish_reason": "stop"}], - "usage": {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15}, - }) + return httpx.Response( + 200, + json={ + "output": {"message": {"role": "assistant", "content": [{"text": content}]}}, + "stopReason": "end_turn", + "usage": {"inputTokens": 10, "outputTokens": 5, "totalTokens": 15}, + "metrics": {"latencyMs": 1}, + }, + ) + return httpx.Response( + 200, + json={ + "id": "chatcmpl-audit", + "object": "chat.completion", + "created": 0, + "model": "gpt-5.6", + "choices": [ + {"index": 0, "message": {"role": "assistant", "content": content}, "finish_reason": "stop"} + ], + "usage": {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15}, + }, + ) async def capture(kwargs, response_obj, start_time, end_time): logs.put_nowait(kwargs["standard_logging_object"]) @@ -7092,11 +7506,15 @@ async def test_classifier_audit_matches_provider_transport(provider: str) -> Non handler.client = http_client client: Final = ( AsyncAzureOpenAI( - api_key="transport-only", azure_endpoint="https://azure.invalid", - api_version="2025-04-01-preview", http_client=http_client, + api_key="transport-only", + azure_endpoint="https://azure.invalid", + api_version="2025-04-01-preview", + http_client=http_client, ) - if provider == "azure" else AsyncOpenAI(api_key="transport-only", http_client=http_client) - if provider == "openai" else handler + if provider == "azure" + else AsyncOpenAI(api_key="transport-only", http_client=http_client) + if provider == "openai" + else handler ) model: Final = { "openai": "openai/gpt-5.6", @@ -7109,23 +7527,44 @@ async def test_classifier_audit_matches_provider_transport(provider: str) -> Non async def run(marker: str) -> None: if provider == "responses": await litellm.aresponses( - model=model, api_key="transport-only", client=client, max_output_tokens=128, - instructions="classifier-rubric", input=marker, + model=model, + api_key="transport-only", + client=client, + max_output_tokens=128, + instructions="classifier-rubric", + input=marker, metadata={"internal_call_origin": "autorouter_classifier"}, proxy_server_request={"body": {}, "originating_request_masked": {"input": f"source-only-{marker}"}}, - success_callback=[capture], num_retries=0, + success_callback=[capture], + num_retries=0, ) return await litellm.acompletion( - model=model, api_key="transport-only", client=client, max_tokens=128, - aws_access_key_id="transport-only", aws_secret_access_key="transport-only", aws_region_name="us-east-1", + model=model, + api_key="transport-only", + client=client, + max_tokens=128, + aws_access_key_id="transport-only", + aws_secret_access_key="transport-only", + aws_region_name="us-east-1", messages=[{"role": "system", "content": "classifier-rubric"}, {"role": "user", "content": marker}], metadata={"internal_call_origin": "autorouter_classifier"}, proxy_server_request={"body": {}, "originating_request_masked": {"input": f"source-only-{marker}"}}, - success_callback=[capture], num_retries=0, - **({"api_base": "https://azure.invalid", "api_version": "2025-04-01-preview"} if provider == "azure" else {}), - **({"extra_body": {"audit_context": "provider-extra"}, "extra_headers": {"X-Audit": "header-only-secret"}} - if provider in ("openai", "azure") else {}), + success_callback=[capture], + num_retries=0, + **( + {"api_base": "https://azure.invalid", "api_version": "2025-04-01-preview"} + if provider == "azure" + else {} + ), + **( + { + "extra_body": {"audit_context": "provider-extra"}, + "extra_headers": {"X-Audit": "header-only-secret"}, + } + if provider in ("openai", "azure") + else {} + ), ) await asyncio.gather(run("request-one"), run("request-two")) @@ -7149,14 +7588,17 @@ async def test_classifier_audit_matches_provider_transport(provider: str) -> Non @pytest.mark.parametrize("redaction", ["none", "global", "request", "header"]) @pytest.mark.parametrize("status", ["success", "failure"]) @pytest.mark.parametrize("call_type", ["completion", "acompletion", "responses", "aresponses"]) -def test_classifier_audit_obeys_message_logging_before_payload_emission(logging_obj, monkeypatch, redaction, status, call_type): +def test_classifier_audit_obeys_message_logging_before_payload_emission( + logging_obj, monkeypatch, redaction, status, call_type +): from litellm.litellm_core_utils.litellm_logging import get_standard_logging_object_payload monkeypatch.setattr(litellm, "turn_off_message_logging", redaction == "global") params: Final = { - "metadata": {"internal_call_origin": "autorouter_classifier", **( - {"headers": {"x-litellm-enable-message-redaction": "true"}} if redaction == "header" else {} - )}, + "metadata": { + "internal_call_origin": "autorouter_classifier", + **({"headers": {"x-litellm-enable-message-redaction": "true"}} if redaction == "header" else {}), + }, "proxy_server_request": {"body": {}, "originating_request_masked": {"input": "source-only"}}, } logging_obj.call_type = call_type @@ -7169,8 +7611,12 @@ def test_classifier_audit_obeys_message_logging_before_payload_emission(logging_ ) now: Final = datetime.datetime.now() payload: Final = get_standard_logging_object_payload( - kwargs={**logging_obj.model_call_details, "call_type": call_type}, init_response_obj={}, - start_time=now, end_time=now, logging_obj=logging_obj, status=status, + kwargs={**logging_obj.model_call_details, "call_type": call_type}, + init_response_obj={}, + start_time=now, + end_time=now, + logging_obj=logging_obj, + status=status, ) assert payload is not None if redaction == "none": @@ -7421,7 +7867,13 @@ def _completed_responses_event(usage: ResponseAPIUsage) -> ResponseCompletedEven return ResponseCompletedEvent( type="response.completed", response=ResponsesAPIResponse( - id="resp-1", created_at=1, object="response", status="completed", model="codex-mini-latest", output=[], usage=usage + id="resp-1", + created_at=1, + object="response", + status="completed", + model="codex-mini-latest", + output=[], + usage=usage, ), ) @@ -7441,7 +7893,9 @@ def test_get_assembled_streaming_response_bills_a_provider_reported_usage_cost() now = datetime.datetime.now() assembled = logging_obj._get_assembled_streaming_response( - result=_completed_responses_event(ResponseAPIUsage(input_tokens=12, output_tokens=2, total_tokens=14, cost=0.0042)), + result=_completed_responses_event( + ResponseAPIUsage(input_tokens=12, output_tokens=2, total_tokens=14, cost=0.0042) + ), start_time=now, end_time=now, is_async=True, @@ -7467,3 +7921,40 @@ def test_get_assembled_streaming_response_without_usage_cost_leaves_pricing_to_t assert "additional_headers" not in assembled._hidden_params price_map_cost = logging_obj._response_cost_calculator(result=assembled) assert price_map_cost is not None and 0 < price_map_cost != 0.0042 + + +def test_response_cost_calculator_prices_terminal_responses_event_from_its_response(): + logging_obj: Final = _responses_stream_logging_obj() + inner_response: Final = ResponsesAPIResponse( + id="resp-priced", + created_at=1, + object="response", + status="completed", + model="gpt-4o-mini", + output=[], + usage=ResponseAPIUsage(input_tokens=1840, output_tokens=412, total_tokens=2252), + ) + event: Final = ResponseCompletedEvent(type="response.completed", response=inner_response) + + event_cost: Final = logging_obj._response_cost_calculator(result=event) + inner_cost: Final = logging_obj._response_cost_calculator(result=inner_response) + + assert event_cost is not None and event_cost > 0 + assert event_cost == inner_cost + assert logging_obj.cost_breakdown["input_cost"] is not None and logging_obj.cost_breakdown["input_cost"] > 0 + + +class TestBudgetReservationBinding: + """The proxy builds a logging object for every route before calling anything, so a + logging object seeing the reservation is no promise that a cost callback will settle + it: the claim belongs to the call wrapper, and this object must leave it unbound.""" + + def test_update_environment_variables_leaves_the_reservation_unbound(self, logging_obj): + reservation: Final = {"reserved_cost": 0.5, "entries": [], "finalized": False, "callback_bound": False} + + logging_obj.update_environment_variables( + litellm_params={"metadata": {"user_api_key_budget_reservation": reservation}}, optional_params={} + ) + + assert logging_obj.litellm_params["metadata"]["user_api_key_budget_reservation"] is reservation + assert reservation["callback_bound"] is False diff --git a/tests/test_litellm/litellm_core_utils/test_logging_worker.py b/tests/test_litellm/litellm_core_utils/test_logging_worker.py index 1553e788472..2bb93a58531 100644 --- a/tests/test_litellm/litellm_core_utils/test_logging_worker.py +++ b/tests/test_litellm/litellm_core_utils/test_logging_worker.py @@ -180,6 +180,53 @@ class TestLoggingWorker: assert sorted(fired) == ["first", "second"] + @pytest.mark.parametrize("stranded", ["still_queued", "dequeued_never_started"]) + def test_flush_on_new_loop_drains_tasks_stranded_on_previous_loop(self, stranded): + """ + Regression: ``flush()`` from a new event loop used to ``join()`` the queue bound to the + previous loop, whose unfinished counter nothing on the new loop ever decrements. The first + such flush hung until pytest-timeout killed it and every later one raised + ``RuntimeError: ... is bound to a different event loop`` from the queue's Event. + """ + worker = LoggingWorker(timeout=1.0, max_queue_size=10) + callback = AsyncMock() + + async def enqueue_on_first_loop(): + if stranded == "still_queued": + worker._ensure_queue() + worker.enqueue(callback()) + return + worker.ensure_initialized_and_enqueue(callback()) + + asyncio.run(enqueue_on_first_loop()) + assert worker._queue is not None + expected_shape = (1, 0) if stranded == "still_queued" else (0, 1) + assert (worker._queue.qsize(), len(worker._unstarted_dequeued_tasks())) == expected_shape + assert callback.await_count == 0, "precondition: the callback never ran before the first loop closed" + + async def flush_twice_on_second_loop(): + await asyncio.wait_for(worker.flush(), timeout=5) + await asyncio.wait_for(worker.flush(), timeout=5) + + asyncio.run(flush_twice_on_second_loop()) + + assert callback.await_count == 1 + + def test_flush_starts_a_worker_when_the_queue_has_none(self): + """``flush()`` must drain a queue that exists on the current loop without a running worker.""" + worker = LoggingWorker(timeout=1.0, max_queue_size=10) + callback = AsyncMock() + + async def enqueue_then_flush(): + worker._ensure_queue() + worker.enqueue(callback()) + assert worker._worker_task is None, "precondition: nothing is draining the queue yet" + await asyncio.wait_for(worker.flush(), timeout=3) + + asyncio.run(enqueue_then_flush()) + + assert callback.await_count == 1 + def test_flush_on_exit_swallows_cancellation_and_drains_remaining(self): """A callback raising CancelledError must not abort the atexit flush of later events.""" worker = LoggingWorker(timeout=1.0, max_queue_size=10) diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/context_management/test_compact.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/context_management/test_compact.py index fc5d807bc23..d835db63d83 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/context_management/test_compact.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/context_management/test_compact.py @@ -16,6 +16,7 @@ import json from typing import Any, Dict, List from unittest.mock import AsyncMock, MagicMock, patch +import httpx import pytest import litellm @@ -1507,6 +1508,55 @@ async def test_summary_model_denied_when_team_member_scope_excludes_it(): assert result.applied_edits[0].get("error") == "summary_model_access_denied" +async def test_summary_model_denied_when_team_membership_read_hits_a_db_outage(): + """A member-level scope that cannot be read fails closed: the summary + model is not invoked while the membership row is unreachable.""" + messages = _simple_messages() + mock_call = AsyncMock(return_value=_make_mock_response("x")) + + auth = _fake_user_api_key_auth(key_models=["all-proxy-models"], team_id="team-outage") + auth.user_id = "user-outage" + + class _UnreachableMembershipPrisma: + class db: + class litellm_teammembership: + @staticmethod + async def find_unique(where: dict[str, dict[str, str]], include: dict[str, bool]) -> None: + raise httpx.ConnectError("All connection attempts failed") + + with ( + patch( + "litellm.llms.anthropic.experimental_pass_through.context_management.editors.compact._read_summary_model_setting", + return_value="claude-haiku-4-5", + ), + patch("litellm.token_counter", return_value=200_000), + patch( + "litellm.llms.anthropic.experimental_pass_through.context_management.editors.compact._call_summary_model", + mock_call, + ), + patch( + "litellm.proxy.auth.auth_checks.get_user_object", + AsyncMock(return_value=None), + ), + patch( + "litellm.proxy.auth.auth_checks.get_project_object", + AsyncMock(return_value=None), + ), + patch("litellm.proxy.proxy_server.prisma_client", _UnreachableMembershipPrisma()), + ): + result = await apply_compact_20260112( + model=MODEL, + messages=messages, + tools=None, + system=None, + edit_spec=_EDIT_SPEC_DEFAULT, + user_api_key_auth=auth, + ) + + mock_call.assert_not_awaited() + assert result.applied_edits[0].get("error") == "summary_model_access_denied" + + async def test_summary_model_denied_when_key_over_model_budget(): """A caller whose per-model budget for the summary model is exhausted cannot trigger the summary call via compaction.""" diff --git a/tests/test_litellm/llms/azure/test_azure.py b/tests/test_litellm/llms/azure/test_azure.py index 6b6832f623c..86065c7adc6 100644 --- a/tests/test_litellm/llms/azure/test_azure.py +++ b/tests/test_litellm/llms/azure/test_azure.py @@ -1,10 +1,13 @@ """Tests for litellm/llms/azure/azure.py AzureChatCompletion handler behaviour.""" +import asyncio import time from typing import Final -from openai import AzureOpenAI +import pytest +from openai import AsyncAzureOpenAI, AzureOpenAI +import litellm from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.llms.azure.azure import AzureChatCompletion @@ -52,3 +55,25 @@ def test_sync_streaming_stamps_response_headers_on_the_logging_obj() -> None: ) assert logging_obj.model_call_details["response_headers"] == {"x-ms-is-spilled-over": "true"} + + +class _CancelledRawCompletions: + async def create(self, **kwargs): + raise asyncio.CancelledError() + + +@pytest.mark.asyncio +async def test_acompletion_propagates_cancelled_error() -> None: + client = AsyncAzureOpenAI( + api_key="fake-key", + api_version="2024-02-01", + azure_endpoint="https://fake-resource.openai.azure.com", + ) + client.chat.completions.with_raw_response = _CancelledRawCompletions() + + with pytest.raises(asyncio.CancelledError): + await litellm.acompletion( + model="azure/fake-deployment", + messages=[{"role": "user", "content": "hi"}], + client=client, + ) diff --git a/tests/test_litellm/llms/azure/test_azure_common_utils.py b/tests/test_litellm/llms/azure/test_azure_common_utils.py index 83ec85f1176..caf941ebd19 100644 --- a/tests/test_litellm/llms/azure/test_azure_common_utils.py +++ b/tests/test_litellm/llms/azure/test_azure_common_utils.py @@ -597,7 +597,8 @@ async def test_ensure_initialize_azure_sdk_client_always_used(call_type): "litellm.files.main.azure_files_instance.initialize_azure_sdk_client" ) elif ( - call_type == CallTypes.avideo_content + call_type == CallTypes.avideo_generation + or call_type == CallTypes.avideo_content or call_type == CallTypes.avideo_list or call_type == CallTypes.avideo_remix or call_type == CallTypes.avideo_create_character diff --git a/tests/test_litellm/llms/bedrock/test_bedrock_common_utils.py b/tests/test_litellm/llms/bedrock/test_bedrock_common_utils.py index df042ce5902..87321cc2e65 100644 --- a/tests/test_litellm/llms/bedrock/test_bedrock_common_utils.py +++ b/tests/test_litellm/llms/bedrock/test_bedrock_common_utils.py @@ -926,3 +926,31 @@ def test_every_bedrock_config_get_error_class_keeps_provider_headers(config): def test_bedrock_get_error_class_audit_covers_every_surface(): assert len(_bedrock_configs_with_get_error_class()) >= 30 + + +def test_s3_static_key_pair_returns_the_pair_when_both_keys_are_set(): + from litellm.llms.bedrock.common_utils import s3_static_key_pair + + assert s3_static_key_pair( + { + "aws_access_key_id": "bedrock-key", + "aws_secret_access_key": "bedrock-secret", + "s3_access_key_id": "s3-key", + "s3_secret_access_key": "s3-secret", + } + ) == ("s3-key", "s3-secret") + + +@pytest.mark.parametrize( + "partial_s3_pair", + [ + {}, + {"s3_access_key_id": "s3-key"}, + {"s3_secret_access_key": "s3-secret"}, + {"s3_access_key_id": "", "s3_secret_access_key": ""}, + ], +) +def test_s3_static_key_pair_is_none_without_a_full_pair(partial_s3_pair): + from litellm.llms.bedrock.common_utils import s3_static_key_pair + + assert s3_static_key_pair({"aws_access_key_id": "bedrock-key", **partial_s3_pair}) is None diff --git a/tests/test_litellm/llms/bedrock/test_mantle.py b/tests/test_litellm/llms/bedrock/test_mantle.py index 09be2118001..37cf49a85ec 100644 --- a/tests/test_litellm/llms/bedrock/test_mantle.py +++ b/tests/test_litellm/llms/bedrock/test_mantle.py @@ -447,6 +447,63 @@ async def test_mantle_anthropic_messages_sends_workspace_header_and_clean_body() assert "aws_bedrock_project_id" not in requests[0]["body"] +async def _send_anthropic_messages_with_betas(**request_params: object) -> dict: + import litellm + + requests = [] + + async def mock_post(self, url, data=None, headers=None, **kwargs): + requests.append(_capture_request(url=url, headers=headers or {}, data=data)) + return _anthropic_response(url) + + try: + with patch( + "litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post", + new=mock_post, + ): + await litellm.anthropic_messages( + model="bedrock/mantle/anthropic.claude-mythos-preview", + messages=[{"role": "user", "content": "hello"}], + max_tokens=10, + aws_access_key_id="fake-key", + aws_secret_access_key="fake-secret", + aws_region_name="us-east-1", + **request_params, + ) + finally: + await litellm.close_litellm_async_clients() + + assert len(requests) == 1 + return requests[0] + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("local_beta_headers_config") +async def test_mantle_anthropic_messages_sends_every_beta_in_the_header_not_the_body(): + sent = await _send_anthropic_messages_with_betas( + extra_headers={"anthropic-beta": "context-1m-2025-08-07,interleaved-thinking-2025-05-14"}, + context_management={"edits": [{"type": "clear_tool_uses_20250919"}]}, + ) + + assert ( + sent["headers"]["anthropic-beta"] + == "context-1m-2025-08-07,context-management-2025-06-27,interleaved-thinking-2025-05-14" + ) + assert sent["headers"]["anthropic-version"] == "2023-06-01" + assert sent["body"]["context_management"] == {"edits": [{"type": "clear_tool_uses_20250919"}]} + assert "anthropic_beta" not in sent["body"] + assert "anthropic_version" not in sent["body"] + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("local_beta_headers_config") +async def test_mantle_anthropic_messages_drops_the_beta_header_when_mantle_rejects_every_value(): + sent = await _send_anthropic_messages_with_betas(extra_headers={"anthropic-beta": "code-execution-2025-08-25"}) + + assert "anthropic-beta" not in sent["headers"] + assert "anthropic_beta" not in sent["body"] + + def _usageless_anthropic_response(url: str) -> httpx.Response: return httpx.Response( status_code=200, diff --git a/tests/test_litellm/llms/custom_httpx/test_llm_http_handler.py b/tests/test_litellm/llms/custom_httpx/test_llm_http_handler.py index fa4b7439dd8..67a8d045036 100644 --- a/tests/test_litellm/llms/custom_httpx/test_llm_http_handler.py +++ b/tests/test_litellm/llms/custom_httpx/test_llm_http_handler.py @@ -4051,3 +4051,83 @@ async def test_responses_agentic_followup_does_not_repeat_request_params_from_pl assert followup_calls[0]["prompt_cache_key"] == "thread-1" assert followup_calls[0]["metadata"] == {"user": "u1"} assert followup_calls[0]["_agentic_loop_depth"] == 1 + + +@pytest.mark.asyncio +async def test_responses_agentic_followup_sends_the_plans_request_param_over_a_stale_kwargs_copy(monkeypatch): + from litellm.integrations.custom_logger import CustomLogger + from litellm.types.integrations.custom_logger import AgenticLoopPlan, AgenticLoopRequestPatch + + followup_calls: list[dict[str, object]] = [] + + async def fake_aresponses(**kwargs: object) -> str: + followup_calls.append(kwargs) + return "followup-response" + + monkeypatch.setattr(litellm, "aresponses", fake_aresponses) + + await BaseLLMHTTPHandler()._execute_responses_agentic_plan( + plan=AgenticLoopPlan( + run_agentic_loop=True, + request_patch=AgenticLoopRequestPatch( + model="gpt-5", + messages=[{"role": "user", "content": "x"}], + optional_params={"prompt_cache_key": "from-plan-params"}, + kwargs={"prompt_cache_key": "stale-copy"}, + ), + ), + model="gpt-5", + response_api_optional_request_params={"prompt_cache_key": "from-request"}, + logging_obj=Mock(litellm_call_id="call-1"), + kwargs={}, + depth=0, + max_loops=3, + fingerprints=[], + fingerprint="fp", + callback=CustomLogger(), + ) + + assert followup_calls[0]["prompt_cache_key"] == "from-plan-params" + + +@pytest.mark.asyncio +async def test_chat_completion_agentic_followup_does_not_repeat_request_params_from_plan_kwargs(monkeypatch): + """A plan whose kwargs repeat a request param, or the explicitly passed model, must not crash the chat follow-up with a duplicate keyword""" + from litellm.types.integrations.custom_logger import AgenticLoopPlan, AgenticLoopRequestPatch + + followup_calls: list[dict[str, object]] = [] + + async def fake_acompletion(**kwargs: object) -> str: + followup_calls.append(kwargs) + return "followup-response" + + monkeypatch.setattr(litellm, "acompletion", fake_acompletion) + request_kwargs: Final = {"temperature": 0.2, "api_base": "https://a", "model": "gpt-5"} + plan: Final = AgenticLoopPlan( + run_agentic_loop=True, + request_patch=AgenticLoopRequestPatch( + model="gpt-5", + messages=[{"role": "user", "content": "x"}], + optional_params={"temperature": 0.2}, + kwargs=dict(request_kwargs), + ), + ) + + response: Final = await BaseLLMHTTPHandler()._execute_chat_completion_agentic_plan( + plan=plan, + model="gpt-5", + messages=[{"role": "user", "content": "x"}], + optional_params={"temperature": 0.2}, + kwargs=dict(request_kwargs), + custom_llm_provider="openai", + depth=0, + max_loops=3, + fingerprints=[], + fingerprint="fp", + ) + + assert response == "followup-response" + assert len(followup_calls) == 1 + assert followup_calls[0]["temperature"] == 0.2 + assert followup_calls[0]["api_base"] == "https://a" + assert followup_calls[0]["model"] == "openai/gpt-5" diff --git a/tests/test_litellm/llms/databricks/chat/__init__.py b/tests/test_litellm/llms/databricks/chat/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/test_litellm/llms/databricks/chat/test_databricks_chat_transformation.py b/tests/test_litellm/llms/databricks/chat/test_databricks_chat_transformation.py new file mode 100644 index 00000000000..a3391a2c585 --- /dev/null +++ b/tests/test_litellm/llms/databricks/chat/test_databricks_chat_transformation.py @@ -0,0 +1,79 @@ +import json +from typing import Final + +import httpx +import respx + +import litellm + + +def test_completion_merges_leading_system_and_developer_messages_for_chat_template_models( + respx_mock: respx.MockRouter, +): + upstream: Final = respx_mock.post("https://example.databricks.test/serving-endpoints/chat/completions").mock( + return_value=httpx.Response( + status_code=200, + json={ + "id": "chatcmpl-123", + "object": "chat.completion", + "created": 1677652288, + "model": "my-custom-model", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "Answer"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 9, "completion_tokens": 1, "total_tokens": 10}, + }, + ) + ) + + response: Final = litellm.completion( + model="databricks/my-custom-model", + messages=[ + {"role": "system", "content": "You are terse."}, + {"role": "developer", "content": "Skills: none."}, + {"role": "user", "content": "Hello"}, + ], + api_base="https://example.databricks.test/serving-endpoints", + api_key="fake-databricks-api-key", + num_retries=0, + ) + + assert upstream.call_count == 1 + request_body: Final = json.loads(upstream.calls[0].request.read()) + assert request_body["messages"] == [ + {"role": "system", "content": "You are terse.\n\nSkills: none."}, + {"role": "user", "content": "Hello"}, + ] + assert response.choices[0].message.content == "Answer" + + +def test_completion_merges_system_messages_when_one_has_empty_content(respx_mock: respx.MockRouter): + upstream: Final = respx_mock.post("https://example.databricks.test/serving-endpoints/chat/completions").mock( + return_value=httpx.Response( + status_code=200, + json={ + "id": "chatcmpl-123", + "object": "chat.completion", + "created": 1677652288, + "model": "my-custom-model", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "Answer"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 9, "completion_tokens": 1, "total_tokens": 10}, + }, + ) + ) + + litellm.completion( + model="databricks/my-custom-model", + messages=[ + {"role": "system", "content": "You are terse."}, + {"role": "system", "content": ""}, + {"role": "user", "content": "Hello"}, + ], + api_base="https://example.databricks.test/serving-endpoints", + api_key="fake-databricks-api-key", + num_retries=0, + ) + + request_body: Final = json.loads(upstream.calls[0].request.read()) + assert request_body["messages"] == [ + {"role": "system", "content": "You are terse."}, + {"role": "user", "content": "Hello"}, + ] diff --git a/tests/test_litellm/llms/fal_ai/chat/test_fal_ai_chat_transformation.py b/tests/test_litellm/llms/fal_ai/chat/test_fal_ai_chat_transformation.py new file mode 100644 index 00000000000..41e8fc0c8c5 --- /dev/null +++ b/tests/test_litellm/llms/fal_ai/chat/test_fal_ai_chat_transformation.py @@ -0,0 +1,358 @@ +import httpx +import pytest + +import litellm +from litellm.llms.fal_ai.chat.transformation import FalAIChatConfig, FalAIError +from litellm.types.utils import LlmProviders, ModelResponse +from litellm.utils import ProviderConfigManager + +MODEL = "fal-ai/moondream3-preview/query" + + +@pytest.fixture(autouse=True) +def _use_local_model_cost_map(monkeypatch): + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") + monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url="")) + litellm.get_model_info.cache_clear() + yield + litellm.get_model_info.cache_clear() + + +def _messages(*content): + return [ + { + "role": "user", + "content": [{"type": "text", "text": text} for text in content[:1]] + + [{"type": "image_url", "image_url": {"url": c}} for c in content[1:]], + } + ] + + +def test_provider_config_manager_resolves_fal_ai_chat_config(): + config = ProviderConfigManager.get_provider_chat_config(model=MODEL, provider=LlmProviders.FAL_AI) + assert isinstance(config, FalAIChatConfig) + + +def test_get_complete_url_targets_fal_endpoint(): + assert ( + FalAIChatConfig().get_complete_url( + api_base=None, api_key=None, model=MODEL, optional_params={}, litellm_params={} + ) + == "https://fal.run/fal-ai/moondream3-preview/query" + ) + + +def test_get_complete_url_strips_fal_ai_model_prefix(): + assert ( + FalAIChatConfig().get_complete_url( + api_base=None, api_key=None, model=f"fal_ai/{MODEL}", optional_params={}, litellm_params={} + ) + == "https://fal.run/fal-ai/moondream3-preview/query" + ) + + +def test_validate_environment_uses_fal_key_scheme(): + headers = FalAIChatConfig().validate_environment( + headers={}, model=MODEL, messages=[], optional_params={}, litellm_params={}, api_key="secret" + ) + assert headers["Authorization"] == "Key secret" + + +def test_transform_request_joins_text_parts_and_extracts_image_url(): + body = FalAIChatConfig().transform_request( + model=MODEL, + messages=[ + { + "role": "user", + "content": [ + {"type": "text", "text": "what is"}, + {"type": "text", "text": "in this image?"}, + {"type": "image_url", "image_url": {"url": "https://example.com/pic.png"}}, + ], + } + ], + optional_params={"temperature": 0.2, "top_p": 0.9, "reasoning": False}, + litellm_params={}, + headers={}, + ) + assert body == { + "prompt": "what is\nin this image?", + "image_url": "https://example.com/pic.png", + "temperature": 0.2, + "top_p": 0.9, + "reasoning": False, + } + + +def test_transform_request_passes_data_url_through(): + body = FalAIChatConfig().transform_request( + model=MODEL, + messages=[ + { + "role": "user", + "content": [ + {"type": "text", "text": "describe"}, + {"type": "image_url", "image_url": {"url": "data:image/png;base64,AAAA"}}, + ], + } + ], + optional_params={}, + litellm_params={}, + headers={}, + ) + assert body["image_url"] == "data:image/png;base64,AAAA" + + +def test_transform_request_accepts_single_user_message(): + body = FalAIChatConfig().transform_request( + model=MODEL, + messages=[ + { + "role": "user", + "content": [{"type": "text", "text": "describe"}, {"type": "image_url", "image_url": "https://a"}], + } + ], + optional_params={}, + litellm_params={}, + headers={}, + ) + assert body["prompt"] == "describe" + assert body["image_url"] == "https://a" + + +def test_transform_request_rejects_system_message(): + with pytest.raises(FalAIError, match="exactly one user message") as exc_info: + FalAIChatConfig().transform_request( + model=MODEL, + messages=[ + {"role": "system", "content": "be terse"}, + { + "role": "user", + "content": [{"type": "text", "text": "describe"}, {"type": "image_url", "image_url": "https://a"}], + }, + ], + optional_params={}, + litellm_params={}, + headers={}, + ) + assert exc_info.value.status_code == 400 + + +def test_transform_request_rejects_multi_turn_history(): + with pytest.raises(FalAIError, match="exactly one user message") as exc_info: + FalAIChatConfig().transform_request( + model=MODEL, + messages=[ + { + "role": "user", + "content": [{"type": "text", "text": "first"}, {"type": "image_url", "image_url": "https://a"}], + }, + {"role": "assistant", "content": "an answer"}, + { + "role": "user", + "content": [{"type": "text", "text": "second"}, {"type": "image_url", "image_url": "https://b"}], + }, + ], + optional_params={}, + litellm_params={}, + headers={}, + ) + assert exc_info.value.status_code == 400 + + +def test_transform_request_rejects_zero_images(): + with pytest.raises(FalAIError, match="exactly one image_url"): + FalAIChatConfig().transform_request( + model=MODEL, + messages=[{"role": "user", "content": [{"type": "text", "text": "describe"}]}], + optional_params={}, + litellm_params={}, + headers={}, + ) + + +def test_transform_request_rejects_two_images(): + with pytest.raises(FalAIError, match="exactly one image_url"): + FalAIChatConfig().transform_request( + model=MODEL, + messages=[ + { + "role": "user", + "content": [ + {"type": "text", "text": "compare"}, + {"type": "image_url", "image_url": {"url": "https://a"}}, + {"type": "image_url", "image_url": {"url": "https://b"}}, + ], + } + ], + optional_params={}, + litellm_params={}, + headers={}, + ) + + +def test_transform_request_rejects_missing_text(): + with pytest.raises(FalAIError, match="require text"): + FalAIChatConfig().transform_request( + model=MODEL, + messages=[{"role": "user", "content": [{"type": "image_url", "image_url": {"url": "https://a"}}]}], + optional_params={}, + litellm_params={}, + headers={}, + ) + + +def test_transform_request_rejects_streaming(): + with pytest.raises(FalAIError, match="streaming"): + FalAIChatConfig().transform_request( + model=MODEL, + messages=[ + { + "role": "user", + "content": [ + {"type": "text", "text": "describe"}, + {"type": "image_url", "image_url": {"url": "https://a"}}, + ], + } + ], + optional_params={"stream": True}, + litellm_params={}, + headers={}, + ) + + +def test_completion_dispatch_rejects_streaming(): + with pytest.raises(litellm.BadRequestError): + litellm.completion( + model=MODEL, + custom_llm_provider="fal_ai", + stream=True, + messages=[{"role": "user", "content": "describe"}], + ) + + +@pytest.mark.parametrize( + "effort,expected", + [("none", False), ("minimal", False), ("low", True), ("medium", True), ("high", True)], +) +def test_map_openai_params_maps_reasoning_effort(effort, expected): + mapped = FalAIChatConfig().map_openai_params( + non_default_params={"reasoning_effort": effort}, optional_params={}, model=MODEL, drop_params=False + ) + assert mapped["reasoning"] is expected + + +def test_map_openai_params_drops_unknown_reasoning_effort_when_dropping(): + mapped = FalAIChatConfig().map_openai_params( + non_default_params={"reasoning_effort": "extreme"}, optional_params={}, model=MODEL, drop_params=True + ) + assert "reasoning" not in mapped + + +def test_map_openai_params_maps_sampling_params(): + mapped = FalAIChatConfig().map_openai_params( + non_default_params={"temperature": 0.5, "top_p": 0.7, "max_tokens": 10}, + optional_params={}, + model=MODEL, + drop_params=False, + ) + assert mapped == {"temperature": 0.5, "top_p": 0.7} + + +def test_transform_response_maps_output_reasoning_usage_and_finish_reason(): + raw = httpx.Response( + 200, + json={ + "output": "a red circle", + "reasoning": "looked at shapes", + "finish_reason": "stop", + "usage_info": { + "input_tokens": 11, + "output_tokens": 4, + "prefill_time_ms": 1.0, + "decode_time_ms": 2.0, + "ttft_ms": 1.5, + }, + }, + ) + response = FalAIChatConfig().transform_response( + model=MODEL, + raw_response=raw, + model_response=ModelResponse(), + logging_obj=None, + request_data={}, + messages=[], + optional_params={}, + litellm_params={}, + encoding=None, + ) + assert response.choices[0].message.content == "a red circle" + assert response.choices[0].message.reasoning_content == "looked at shapes" + assert response.choices[0].finish_reason == "stop" + assert response.usage.prompt_tokens == 11 + assert response.usage.completion_tokens == 4 + assert response.usage.total_tokens == 15 + assert response.model == MODEL + + +def test_transform_response_omits_reasoning_when_null(): + raw = httpx.Response( + 200, + json={ + "output": "a red circle", + "reasoning": None, + "finish_reason": "stop", + "usage_info": {"input_tokens": 3, "output_tokens": 2}, + }, + ) + response = FalAIChatConfig().transform_response( + model=MODEL, + raw_response=raw, + model_response=ModelResponse(), + logging_obj=None, + request_data={}, + messages=[], + optional_params={}, + litellm_params={}, + encoding=None, + ) + assert response.choices[0].message.content == "a red circle" + assert getattr(response.choices[0].message, "reasoning_content", None) is None + assert response.usage.total_tokens == 5 + + +def test_transform_response_rejects_body_missing_output(): + raw = httpx.Response( + 200, + json={"reasoning": "looked", "usage_info": {"input_tokens": 3, "output_tokens": 2}}, + ) + with pytest.raises(FalAIError) as exc_info: + FalAIChatConfig().transform_response( + model=MODEL, + raw_response=raw, + model_response=ModelResponse(), + logging_obj=None, + request_data={}, + messages=[], + optional_params={}, + litellm_params={}, + encoding=None, + ) + assert exc_info.value.status_code == 422 + + +def test_transform_response_rejects_body_missing_usage_info(): + raw = httpx.Response(200, json={"output": "a red circle"}) + with pytest.raises(FalAIError) as exc_info: + FalAIChatConfig().transform_response( + model=MODEL, + raw_response=raw, + model_response=ModelResponse(), + logging_obj=None, + request_data={}, + messages=[], + optional_params={}, + litellm_params={}, + encoding=None, + ) + assert exc_info.value.status_code == 422 diff --git a/tests/test_litellm/llms/fal_ai/image_edit/test_fal_ai_flux_lora_depth_transformation.py b/tests/test_litellm/llms/fal_ai/image_edit/test_fal_ai_flux_lora_depth_transformation.py new file mode 100644 index 00000000000..d99701db9e8 --- /dev/null +++ b/tests/test_litellm/llms/fal_ai/image_edit/test_fal_ai_flux_lora_depth_transformation.py @@ -0,0 +1,116 @@ +import base64 +import io + +import pytest + +import litellm +from litellm.litellm_core_utils.llm_cost_calc.utils import CostCalculatorUtils +from litellm.llms.fal_ai.image_edit import ( + FalAIFluxLoraDepthEditConfig, + FalAIImageEditConfig, + get_fal_ai_image_edit_config, +) +from litellm.types.images.main import ImageEditOptionalRequestParams +from litellm.types.router import GenericLiteLLMParams +from litellm.types.utils import ImageObject, ImageResponse, LlmProviders +from litellm.utils import ProviderConfigManager + +PNG_BYTES = b"\x89PNG\r\n\x1a\n" + b"\x00" * 16 +MODEL = "fal-ai/flux-lora-depth" + + +@pytest.fixture(autouse=True) +def _use_local_model_cost_map(monkeypatch): + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") + monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url="")) + litellm.get_model_info.cache_clear() + yield + litellm.get_model_info.cache_clear() + + +@pytest.mark.parametrize("model", ["fal-ai/flux-lora-depth", "flux-lora-depth", "fal_ai/fal-ai/flux-lora-depth"]) +def test_dispatch_selects_flux_lora_depth_config(model): + assert isinstance(get_fal_ai_image_edit_config(model), FalAIFluxLoraDepthEditConfig) + + +def test_dispatch_keeps_gpt_image_config_for_openai_edit_models(): + config = get_fal_ai_image_edit_config("openai/gpt-image-2.5/flare/edit") + assert type(config) is FalAIImageEditConfig + + +def test_provider_config_manager_resolves_flux_lora_depth(): + config = ProviderConfigManager.get_provider_image_edit_config(model=MODEL, provider=LlmProviders.FAL_AI) + assert isinstance(config, FalAIFluxLoraDepthEditConfig) + + +@pytest.mark.parametrize("model", ["fal-ai/flux-lora-depth", "flux-lora-depth"]) +def test_get_complete_url_targets_endpoint_without_edit_suffix(model): + url = FalAIFluxLoraDepthEditConfig().get_complete_url(model=model, api_base=None, litellm_params={}) + assert url == "https://fal.run/fal-ai/flux-lora-depth" + + +def test_get_supported_openai_params_excludes_quality_mask_background(): + params = FalAIFluxLoraDepthEditConfig().get_supported_openai_params(model=MODEL) + assert "quality" not in params + assert "mask" not in params + assert "background" not in params + + +def test_map_openai_params_translates_n_and_size(): + mapped = FalAIFluxLoraDepthEditConfig().map_openai_params( + image_edit_optional_params=ImageEditOptionalRequestParams(n=2, size="1024x1536", quality="high"), + model=MODEL, + drop_params=False, + ) + assert mapped == {"num_images": 2, "image_size": {"width": 1024, "height": 1536}} + + +def test_transform_request_sends_single_image_url_as_data_url(): + body, files = FalAIFluxLoraDepthEditConfig().transform_image_edit_request( + model=MODEL, + prompt="follow the depth map", + image=io.BytesIO(PNG_BYTES), + image_edit_optional_request_params={"num_images": 1}, + litellm_params=GenericLiteLLMParams(), + headers={}, + ) + assert files == () + assert body["prompt"] == "follow the depth map" + assert body["image_url"] == "data:image/png;base64," + base64.b64encode(PNG_BYTES).decode() + assert "image_urls" not in body + assert body["num_images"] == 1 + + +def test_transform_request_passes_remote_url_through_untouched(): + body, _ = FalAIFluxLoraDepthEditConfig().transform_image_edit_request( + model=MODEL, + prompt="follow the depth map", + image="https://example.com/depth.png", + image_edit_optional_request_params={}, + litellm_params=GenericLiteLLMParams(), + headers={}, + ) + assert body["image_url"] == "https://example.com/depth.png" + + +def test_transform_request_rejects_two_images(): + with pytest.raises(ValueError, match="exactly one control image"): + FalAIFluxLoraDepthEditConfig().transform_image_edit_request( + model=MODEL, + prompt="follow the depth map", + image=["https://example.com/a.png", "https://example.com/b.png"], + image_edit_optional_request_params={}, + litellm_params=GenericLiteLLMParams(), + headers={}, + ) + + +def test_image_edit_cost_uses_flat_output_cost_per_image(): + cost = CostCalculatorUtils.route_image_generation_cost_calculator( + model=MODEL, + completion_response=ImageResponse(data=[ImageObject(url="https://example.com/out.png")]), + custom_llm_provider="fal_ai", + optional_params={}, + call_type="aimage_edit", + ) + assert cost == litellm.model_cost[f"fal_ai/{MODEL}"]["output_cost_per_image"] > 0 diff --git a/tests/test_litellm/llms/fal_ai/test_cost_calculator.py b/tests/test_litellm/llms/fal_ai/test_cost_calculator.py index 6fb34d9f88e..56dcba04b5c 100644 --- a/tests/test_litellm/llms/fal_ai/test_cost_calculator.py +++ b/tests/test_litellm/llms/fal_ai/test_cost_calculator.py @@ -1,11 +1,12 @@ +from typing import Final + import pytest import litellm from litellm.litellm_core_utils.llm_cost_calc.utils import CostCalculatorUtils -from litellm.llms.fal_ai.cost_calculator import cost_calculator +from litellm.llms.fal_ai.cost_calculator import cost_calculator, fal_ai_passthrough_cost from litellm.types.utils import ImageObject, ImageResponse - @pytest.fixture(autouse=True) def _use_local_model_cost_map(monkeypatch): monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") @@ -78,14 +79,27 @@ def test_gpt_image_response_dimensions_override_request_size(): assert cost == expected -def test_gpt_image_response_dimensions_fall_back_to_request_size_when_unpriced(): +def test_gpt_image_response_dimensions_use_nearest_keyed_row_when_unpriced(): model = "fal_ai/openai/gpt-image-2.5/flare/text-to-image" cost = cost_calculator( model=model, image_response=_image_response_with_dimensions(((777, 888),)), optional_params={"quality": "low", "image_size": {"width": 1024, "height": 1536}}, ) - expected = litellm.model_cost[f"fal_ai/low/1024-x-1536/{model.removeprefix('fal_ai/')}"]["output_cost_per_image"] + expected = litellm.model_cost[f"fal_ai/low/1024-x-768/{model.removeprefix('fal_ai/')}"]["output_cost_per_image"] + assert cost == expected + + +def test_gpt_image_25_noncanonical_response_uses_nearest_keyed_row(): + model: Final = "fal_ai/openai/gpt-image-2.5/flare/text-to-image" + cost: Final = cost_calculator( + model=model, + image_response=_image_response_with_dimensions(((1536, 1024),)), + optional_params={"quality": "low", "image_size": {"width": 1536, "height": 1024}}, + ) + expected: Final = litellm.model_cost[ + "fal_ai/low/1024-x-1536/openai/gpt-image-2.5/flare/text-to-image" + ]["output_cost_per_image"] assert cost == expected @@ -160,3 +174,32 @@ def test_image_edit_call_type_routes_to_fal_keyed_pricing(): call_type="aimage_edit", ) assert cost == litellm.model_cost[f"fal_ai/medium/1024-x-1024/{model}"]["output_cost_per_image"] > 0 + + +def test_passthrough_trellis_charges_flat_rate(): + assert ( + fal_ai_passthrough_cost("fal-ai/trellis", {}) + == litellm.model_cost["fal_ai/fal-ai/trellis"]["output_cost_per_image"] + > 0 + ) + + +@pytest.mark.parametrize("resolution", [512, 1024, 1536]) +def test_passthrough_trellis_2_resolution_picks_keyed_tier(resolution): + assert ( + fal_ai_passthrough_cost("fal-ai/trellis-2", {"resolution": resolution}) + == litellm.model_cost["fal_ai/fal-ai/trellis-2"][f"output_cost_per_image_{resolution}"] + > 0 + ) + + +def test_passthrough_trellis_2_without_resolution_falls_back_to_default_rate(): + assert ( + fal_ai_passthrough_cost("fal-ai/trellis-2", {"image_url": "https://a"}) + == litellm.model_cost["fal_ai/fal-ai/trellis-2"]["output_cost_per_image"] + > 0 + ) + + +def test_passthrough_unknown_model_returns_none(): + assert fal_ai_passthrough_cost("fal-ai/no-such-model", {"resolution": 512}) is None diff --git a/tests/test_litellm/llms/openai/responses/test_openai_responses_guardrail_handler.py b/tests/test_litellm/llms/openai/responses/test_openai_responses_guardrail_handler.py index b45cd2ec299..a6b930db7a9 100644 --- a/tests/test_litellm/llms/openai/responses/test_openai_responses_guardrail_handler.py +++ b/tests/test_litellm/llms/openai/responses/test_openai_responses_guardrail_handler.py @@ -2297,6 +2297,56 @@ class TestStructuredMessagesWriteBack: } assert result["input"][3] == {"role": "user", "content": "What is the codename?"} + @pytest.mark.asyncio + async def test_codex_custom_tool_items_survive_tool_output_compression(self): + handler = OpenAIResponsesHandler() + additional_tools_item = { + "type": "additional_tools", + "tools": [{"type": "custom", "name": "exec", "description": "Run a JavaScript snippet"}], + } + reasoning_item = { + "id": "rs_456", + "type": "reasoning", + "summary": [], + "encrypted_content": "gAAAAA-signed-reasoning", + } + custom_tool_call_item = { + "id": "ctc_456", + "type": "custom_tool_call", + "call_id": "call_exec", + "name": "exec", + "input": 'const r = await tools.exec_command({"cmd": "cat memo.txt"});\ntext(r.output);', + "status": "completed", + } + data = { + "model": "gpt-5.6", + "input": [ + additional_tools_item, + {"role": "user", "content": "What is the codename?"}, + reasoning_item, + custom_tool_call_item, + { + "type": "custom_tool_call_output", + "call_id": "call_exec", + "output": [ + {"type": "input_text", "text": "Script completed\nOutput:\n"}, + {"type": "input_text", "text": "memo " * 400}, + ], + }, + ], + } + + result = await handler.process_input_messages(data, ToolOutputRewriteGuardrail()) + + assert result["input"][0] is additional_tools_item + assert result["input"][1] == {"role": "user", "content": "What is the codename?"} + assert result["input"][2] is reasoning_item + assert result["input"][3] is custom_tool_call_item + assert result["input"][4]["type"] == "custom_tool_call_output" + assert result["input"][4]["call_id"] == "call_exec" + assert COMPRESSED_MARKER in str(result["input"][4]["output"]) + assert len(result["input"]) == 5 + @pytest.mark.asyncio async def test_web_search_call_item_preserved_verbatim(self): handler = OpenAIResponsesHandler() diff --git a/tests/test_litellm/llms/xai/test_xai_chat_transformation.py b/tests/test_litellm/llms/xai/test_xai_chat_transformation.py index 290cd3dcb3a..3fd666e4f50 100644 --- a/tests/test_litellm/llms/xai/test_xai_chat_transformation.py +++ b/tests/test_litellm/llms/xai/test_xai_chat_transformation.py @@ -90,6 +90,16 @@ class TestXAIReasoningTokenFolding: assert response.usage.total_tokens == 999 +def test_max_completion_tokens_is_accepted_and_mapped_to_max_tokens() -> None: + optional_params = litellm.get_optional_params( + model="grok-4.20", + custom_llm_provider="xai", + max_completion_tokens=64, + ) + assert optional_params["max_tokens"] == 64, optional_params + assert "max_completion_tokens" not in optional_params, optional_params + + class TestXAIParallelToolCalls: """Test suite for XAI parallel tool calls functionality.""" diff --git a/tests/test_litellm/llms/xai/test_xai_key_fallback.py b/tests/test_litellm/llms/xai/test_xai_key_fallback.py index 092e4951547..cbc507c5ee3 100644 --- a/tests/test_litellm/llms/xai/test_xai_key_fallback.py +++ b/tests/test_litellm/llms/xai/test_xai_key_fallback.py @@ -12,6 +12,9 @@ from litellm.types.router import GenericLiteLLMParams class FakeLogging: + def __init__(self) -> None: + self.litellm_params: dict = {} + def update_from_kwargs(self, **kwargs): pass diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/auth/test_user_api_key_auth_mcp.py b/tests/test_litellm/proxy/_experimental/mcp_server/auth/test_user_api_key_auth_mcp.py index 087c5a03498..5c885a3168d 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/auth/test_user_api_key_auth_mcp.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/auth/test_user_api_key_auth_mcp.py @@ -12,6 +12,7 @@ from starlette.datastructures import Headers from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import ( MCPRequestHandler, UnloadableEntitlementError, + _agent_capped_servers, _is_mcp_admitted_user_subject, ) from litellm.proxy._types import ( @@ -21,6 +22,7 @@ from litellm.proxy._types import ( SpecialMCPServerNames, UserAPIKeyAuth, ) +from litellm.types.agents import AgentCaller @pytest.mark.asyncio @@ -4169,10 +4171,114 @@ async def test_get_allowed_mcp_servers_for_key_prefers_in_memory_permission(): global_mcp_server_manager.registry.pop("direct-server", None) +@pytest.mark.parametrize( + ("agent_servers", "group_ceiling", "expected"), + [ + ([], frozenset({"server_1"}), ("server_1",)), + ([], frozenset({"server_1", "server_2", "server_3"}), ("server_1", "server_2")), + ([], frozenset(), ()), + (["server_2"], frozenset({"server_1", "server_2"}), ("server_2",)), + (["server_1"], frozenset({"server_2"}), ()), + (["server_1"], None, ("server_1",)), + ], +) +def test_agent_capped_servers_intersects_agent_config_and_access_groups(agent_servers, group_ceiling, expected): + """The agent's attached access groups cap the key/team servers alongside its own + object_permission; groups naming no server deny all.""" + assert _agent_capped_servers(["server_1", "server_2"], agent_servers, group_ceiling) == expected + + +def test_agent_capped_servers_without_agent_restrictions_is_uncapped(): + assert _agent_capped_servers(["server_1", "server_2"], [], None) is None + + @pytest.mark.asyncio class TestAgentMCPPermissions: """Test agent-level MCP server and tool permission intersection.""" + @staticmethod + def _agent_key_acting_for(user_id: str, team_id: str | None) -> UserAPIKeyAuth: + agent_key = UserAPIKeyAuth(api_key="agent-key", user_id="agent-owner", team_id="agent-team", agent_id="agent-1") + agent_key.agent_caller = AgentCaller(user_id=user_id, team_id=team_id) + return agent_key + + @staticmethod + def _team_servers(grants: dict[str, list[str]]) -> AsyncMock: + async def by_team(user_api_key_auth: UserAPIKeyAuth | None = None) -> list[str]: + assert user_api_key_auth is not None + return grants.get(user_api_key_auth.team_id or "", []) + + return AsyncMock(side_effect=by_team) + + @staticmethod + def _user_servers(grants: dict[str, list[str] | None]) -> AsyncMock: + async def by_user(user_api_key_auth: UserAPIKeyAuth | None = None) -> list[str] | None: + assert user_api_key_auth is not None + return grants.get(user_api_key_auth.user_id or "", []) + + return AsyncMock(side_effect=by_user) + + async def test_agent_key_acting_for_a_user_is_capped_at_the_invoking_teams_servers(self): + """LIT-8014: the agent's own key reaches server_1 and server_2, but the human who invoked it + belongs to a team granted only server_2, so on their behalf the agent reaches only server_2.""" + agent_key = self._agent_key_acting_for(user_id="alice", team_id="callers") + + with ( + patch.object( # test-quality-ok: the level resolvers read proxy_server globals with no injection seam + MCPRequestHandler, "_get_allowed_mcp_servers_for_key", AsyncMock(return_value=["server_1", "server_2"]) + ), + patch.object( # test-quality-ok: same seam, keyed by which team is being asked about + MCPRequestHandler, + "_get_allowed_mcp_servers_for_team", + self._team_servers({"callers": ["server_2", "server_3"]}), + ), + patch.object( # test-quality-ok: agent object_permission lookup hits the DB, not under test here + MCPRequestHandler, "_get_allowed_mcp_servers_for_agent", AsyncMock(return_value=[]) + ), + patch.object( # test-quality-ok: neither the agent's owner nor the caller has a personal grant + MCPRequestHandler, "_get_allowed_mcp_servers_for_user", self._user_servers({}) + ), + ): + assert await MCPRequestHandler.get_allowed_mcp_servers(user_api_key_auth=agent_key) == ["server_2"] + + async def test_agent_key_acting_for_a_teamless_user_is_capped_at_that_users_servers(self): + agent_key = self._agent_key_acting_for(user_id="alice", team_id=None) + + with ( + patch.object( # test-quality-ok: the level resolvers read proxy_server globals with no injection seam + MCPRequestHandler, "_get_allowed_mcp_servers_for_key", AsyncMock(return_value=["server_1", "server_2"]) + ), + patch.object( # test-quality-ok: same seam + MCPRequestHandler, "_get_allowed_mcp_servers_for_team", self._team_servers({}) + ), + patch.object( # test-quality-ok: agent object_permission lookup hits the DB, not under test here + MCPRequestHandler, "_get_allowed_mcp_servers_for_agent", AsyncMock(return_value=[]) + ), + patch.object( # test-quality-ok: same seam, keyed by which user is being asked about + MCPRequestHandler, "_get_allowed_mcp_servers_for_user", self._user_servers({"alice": ["server_1"]}) + ), + ): + assert await MCPRequestHandler.get_allowed_mcp_servers(user_api_key_auth=agent_key) == ["server_1"] + + async def test_agent_key_acting_for_a_caller_whose_entitlement_is_unreadable_reaches_nothing(self): + agent_key = self._agent_key_acting_for(user_id="alice", team_id=None) + + with ( + patch.object( # test-quality-ok: the level resolvers read proxy_server globals with no injection seam + MCPRequestHandler, "_get_allowed_mcp_servers_for_key", AsyncMock(return_value=["server_1"]) + ), + patch.object( # test-quality-ok: same seam + MCPRequestHandler, "_get_allowed_mcp_servers_for_team", self._team_servers({}) + ), + patch.object( # test-quality-ok: agent object_permission lookup hits the DB, not under test here + MCPRequestHandler, "_get_allowed_mcp_servers_for_agent", AsyncMock(return_value=[]) + ), + patch.object( # test-quality-ok: None is the resolver's own "entitlement unresolvable" signal + MCPRequestHandler, "_get_allowed_mcp_servers_for_user", self._user_servers({"alice": None}) + ), + ): + assert await MCPRequestHandler.get_allowed_mcp_servers(user_api_key_auth=agent_key) == [] + async def test_get_allowed_mcp_servers_agent_intersection(self): """Key/team allow [server_1, server_2]; agent allows [server_1]. Result = [server_1].""" user_api_key_auth = UserAPIKeyAuth( @@ -4208,6 +4314,46 @@ class TestAgentMCPPermissions: assert sorted(result) == ["server_1", "server_2"] mock_agent.assert_called_once_with(user_api_key_auth) + async def test_agent_access_group_server_ceiling_expands_group_servers(self): + from litellm.proxy._experimental.mcp_server.mcp_server_manager import global_mcp_server_manager + from litellm.proxy.agent_endpoints.auth.agent_access_groups import AgentAccessGroupCeiling + from litellm.types.mcp import MCPTransport + from litellm.types.mcp_server.mcp_server_manager import MCPServer + + asked: list[str] = [] + + async def resolve(agent_id: str) -> AgentAccessGroupCeiling | None: + asked.append(agent_id) + return AgentAccessGroupCeiling( + access_group_ids=("ag-1",), + models=frozenset(), + mcp_server_ids=frozenset({"aliased-server"}), + agent_ids=frozenset(), + ) + + global_mcp_server_manager.registry["ag-server-id"] = MCPServer( + server_id="ag-server-id", + name="ag-server", + server_name="ag-server", + alias="aliased-server", + url="https://ag-server.example.com", + transport=MCPTransport.http, + ) + try: + result = await MCPRequestHandler._get_agent_access_group_server_ceiling( + UserAPIKeyAuth(api_key="test-key", agent_id="agent-ag"), resolve + ) + finally: + global_mcp_server_manager.registry.pop("ag-server-id", None) + + assert result == frozenset({"ag-server-id"}) + assert asked == ["agent-ag"] + assert ( + await MCPRequestHandler._get_agent_access_group_server_ceiling(UserAPIKeyAuth(api_key="k"), resolve) + is None + ) + assert asked == ["agent-ag"] + async def test_get_allowed_mcp_servers_key_team_agent_intersection(self): """Key allows [1, 2], agent allows [2, 3]. Result = [2].""" user_api_key_auth = UserAPIKeyAuth( diff --git a/tests/test_litellm/proxy/agent_endpoints/auth/test_agent_access_groups.py b/tests/test_litellm/proxy/agent_endpoints/auth/test_agent_access_groups.py new file mode 100644 index 00000000000..e744e84d671 --- /dev/null +++ b/tests/test_litellm/proxy/agent_endpoints/auth/test_agent_access_groups.py @@ -0,0 +1,146 @@ +from typing import Final + +import pytest +from fastapi import HTTPException + +from litellm.models.access_group import LiteLLM_AccessGroupTable +from litellm.proxy.agent_endpoints.auth.agent_access_groups import ( + AgentAccessGroupCeiling, + resolve_agent_access_group_ceiling, +) +from litellm.types.agents import AgentResponse + +_CARD: Final = {"name": "agent", "url": "http://localhost:9999", "version": "1.0.0"} + + +def _agent(access_group_ids: list[str] | None) -> AgentResponse: + return AgentResponse( + agent_id="agent-1", agent_name="agent", agent_card_params=_CARD, access_group_ids=access_group_ids + ) + + +def _group( + group_id: str, + models: tuple[str, ...] = (), + mcp_servers: tuple[str, ...] = (), + agents: tuple[str, ...] = (), +) -> LiteLLM_AccessGroupTable: + return LiteLLM_AccessGroupTable( + access_group_id=group_id, + access_group_name=group_id, + access_model_names=list(models), + access_mcp_server_ids=list(mcp_servers), + access_agent_ids=list(agents), + ) + + +def _loaders(agent: AgentResponse | None, groups: dict[str, LiteLLM_AccessGroupTable]): + async def load_agent(agent_id: str) -> tuple[str, ...]: + return tuple(agent.access_group_ids or ()) if agent is not None else () + + async def load_group(group_id: str) -> LiteLLM_AccessGroupTable | None: + return groups.get(group_id) + + return load_agent, load_group + + +@pytest.mark.asyncio +@pytest.mark.parametrize("access_group_ids", [None, []]) +async def test_agent_without_access_groups_has_no_ceiling(access_group_ids: list[str] | None): + load_agent, load_group = _loaders(_agent(access_group_ids), {"g1": _group("g1", models=("gpt-5",))}) + + assert await resolve_agent_access_group_ceiling("agent-1", load_agent, load_group) is None + + +@pytest.mark.asyncio +async def test_unknown_agent_has_no_ceiling(): + load_agent, load_group = _loaders(None, {}) + + assert await resolve_agent_access_group_ceiling("missing", load_agent, load_group) is None + + +@pytest.mark.asyncio +async def test_ceiling_is_the_union_of_every_attached_group(): + load_agent, load_group = _loaders( + _agent(["g1", "g2"]), + { + "g1": _group("g1", models=("gpt-5",), mcp_servers=("mcp-a",), agents=("agent-b",)), + "g2": _group("g2", models=("claude-sonnet",), mcp_servers=("mcp-b",), agents=("agent-c",)), + }, + ) + + ceiling: Final = await resolve_agent_access_group_ceiling("agent-1", load_agent, load_group) + + assert ceiling == AgentAccessGroupCeiling( + access_group_ids=("g1", "g2"), + models=frozenset({"gpt-5", "claude-sonnet"}), + mcp_server_ids=frozenset({"mcp-a", "mcp-b"}), + agent_ids=frozenset({"agent-b", "agent-c"}), + ) + + +@pytest.mark.asyncio +async def test_unloadable_group_contributes_nothing_but_the_ceiling_still_applies(): + load_agent, load_group = _loaders(_agent(["g1", "gone"]), {"g1": _group("g1", models=("gpt-5",))}) + + ceiling: Final = await resolve_agent_access_group_ceiling("agent-1", load_agent, load_group) + + assert ceiling == AgentAccessGroupCeiling( + access_group_ids=("g1", "gone"), + models=frozenset({"gpt-5"}), + mcp_server_ids=frozenset(), + agent_ids=frozenset(), + ) + + +@pytest.mark.asyncio +async def test_only_unloadable_groups_is_an_empty_ceiling_not_unrestricted(): + load_agent, load_group = _loaders(_agent(["gone"]), {}) + + ceiling: Final = await resolve_agent_access_group_ceiling("agent-1", load_agent, load_group) + + assert ceiling is not None + assert ceiling.models == frozenset() + assert ceiling.mcp_server_ids == frozenset() + assert ceiling.agent_ids == frozenset() + + +@pytest.mark.asyncio +async def test_default_agent_loader_reads_the_attached_groups_from_the_registry(): + from litellm.proxy.agent_endpoints.agent_registry import global_agent_registry + + _, load_group = _loaders(None, {"g1": _group("g1", models=("gpt-5",))}) + global_agent_registry.register_agent(_agent(["g1"])) + try: + ceiling: Final = await resolve_agent_access_group_ceiling("agent-1", load_access_group=load_group) + finally: + global_agent_registry.deregister_agent("agent") + + assert ceiling == AgentAccessGroupCeiling( + access_group_ids=("g1",), models=frozenset({"gpt-5"}), mcp_server_ids=frozenset(), agent_ids=frozenset() + ) + + +@pytest.mark.asyncio +async def test_default_loader_treats_a_missing_group_as_unreadable(monkeypatch: pytest.MonkeyPatch): + from litellm.proxy import proxy_server + from litellm.proxy.agent_endpoints.auth.agent_access_groups import _load_access_group + from litellm.proxy.auth import auth_checks + + async def missing_group(**_: object) -> LiteLLM_AccessGroupTable: + raise HTTPException(status_code=404, detail={"error": "Access group doesn't exist in db."}) + + monkeypatch.setattr(proxy_server, "prisma_client", object()) + monkeypatch.setattr(auth_checks, "get_access_object", missing_group) + + assert await _load_access_group("gone") is None + + +@pytest.mark.asyncio +async def test_default_loader_returns_nothing_without_a_db(monkeypatch: pytest.MonkeyPatch): + from litellm.proxy import proxy_server + from litellm.proxy.agent_endpoints.auth.agent_access_groups import _load_access_group + + monkeypatch.setattr(proxy_server, "prisma_client", None) + + assert await _load_access_group("ag-1") is None diff --git a/tests/test_litellm/proxy/agent_endpoints/auth/test_agent_caller.py b/tests/test_litellm/proxy/agent_endpoints/auth/test_agent_caller.py new file mode 100644 index 00000000000..b08964503c8 --- /dev/null +++ b/tests/test_litellm/proxy/agent_endpoints/auth/test_agent_caller.py @@ -0,0 +1,57 @@ +from typing import Final + +import pytest + +from litellm.proxy._types import UserAPIKeyAuth +from litellm.proxy.agent_endpoints.auth.agent_caller import agent_caller_auth, agent_caller_from_headers +from litellm.types.agents import AgentCaller + +_AGENT_KEY: Final = UserAPIKeyAuth(api_key="agent-key", user_id="agent-owner", team_id="agent-team", agent_id="agent-1") + + +def test_agent_key_echoing_both_ids_acts_for_that_user_and_team() -> None: + headers: Final = {"X-LiteLLM-User-Id": " alice ", "x-litellm-team-id": "callers"} + + assert agent_caller_from_headers(headers, _AGENT_KEY) == AgentCaller(user_id="alice", team_id="callers") + + +def test_agent_key_echoing_only_a_user_id_acts_for_a_teamless_user() -> None: + assert agent_caller_from_headers({"x-litellm-user-id": "alice"}, _AGENT_KEY) == AgentCaller(user_id="alice") + + +@pytest.mark.parametrize("headers", [{}, {"x-litellm-user-id": " ", "x-litellm-team-id": ""}]) +def test_agent_key_echoing_no_caller_acts_for_itself(headers: dict[str, str]) -> None: + assert agent_caller_from_headers(headers, _AGENT_KEY) is None + + +def test_caller_headers_on_a_key_without_an_agent_are_ignored() -> None: + plain_key: Final = UserAPIKeyAuth(api_key="plain-key", user_id="bob") + + assert agent_caller_from_headers({"x-litellm-user-id": "alice", "x-litellm-team-id": "callers"}, plain_key) is None + + +def test_caller_auth_stands_for_the_invoking_user_not_the_agent() -> None: + agent_key: Final = UserAPIKeyAuth( + api_key="agent-key", user_id="agent-owner", team_id="agent-team", agent_id="agent-1" + ) + agent_key.agent_caller = AgentCaller(user_id="alice", team_id="callers") + + caller_auth: Final = agent_caller_auth(agent_key) + + assert caller_auth is not None + assert (caller_auth.user_id, caller_auth.team_id, caller_auth.agent_id, caller_auth.api_key) == ( + "alice", + "callers", + None, + None, + ) + assert agent_caller_auth(_AGENT_KEY) is None + + +def test_agent_caller_cannot_be_set_from_a_request_payload() -> None: + forged: Final = UserAPIKeyAuth.model_validate( + {"api_key": "agent-key", "agent_id": "agent-1", "agent_caller": {"user_id": "alice", "team_id": "callers"}} + ) + + assert forged.agent_caller is None + assert "agent_caller" not in forged.model_dump() diff --git a/tests/test_litellm/proxy/agent_endpoints/auth/test_agent_permission_handler.py b/tests/test_litellm/proxy/agent_endpoints/auth/test_agent_permission_handler.py index 383b72e5c58..a87716375e8 100644 --- a/tests/test_litellm/proxy/agent_endpoints/auth/test_agent_permission_handler.py +++ b/tests/test_litellm/proxy/agent_endpoints/auth/test_agent_permission_handler.py @@ -9,10 +9,10 @@ from unittest.mock import AsyncMock, patch import pytest - from litellm.constants import UI_SESSION_TOKEN_TEAM_ID -from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth +from litellm.proxy._types import LiteLLM_ObjectPermissionTable, LitellmUserRoles, UserAPIKeyAuth from litellm.proxy.agent_endpoints.agent_registry import AgentRegistry +from litellm.proxy.agent_endpoints.auth.agent_access_groups import AgentAccessGroupCeiling, CeilingResolver from litellm.proxy.agent_endpoints.auth.agent_permission_handler import ( AgentAccess, AgentRequestHandler, @@ -20,6 +20,7 @@ from litellm.proxy.agent_endpoints.auth.agent_permission_handler import ( UnrestrictedAgentAccess, accessible_agents, ) +from litellm.types.agents import AgentCaller def _registry_with(*agent_names: str) -> AgentRegistry: @@ -157,6 +158,130 @@ class TestAgentRequestHandler: is False ), agent_id + @staticmethod + def _ceiling_resolver(agent_ids: frozenset[str] | None) -> tuple[CeilingResolver, list[str]]: + """A resolver that records the agent ids it was asked about and answers with a fixed + ceiling, or None when the agent has no access groups attached.""" + asked: Final[list[str]] = [] + + async def resolve(agent_id: str) -> AgentAccessGroupCeiling | None: + asked.append(agent_id) + if agent_ids is None: + return None + return AgentAccessGroupCeiling( + access_group_ids=("ag-1",), models=frozenset(), mcp_server_ids=frozenset(), agent_ids=agent_ids + ) + + return resolve, asked + + @staticmethod + def _key_granting(agent_ids: list[str], agent_id: str | None) -> UserAPIKeyAuth: + return UserAPIKeyAuth( + api_key="test-key", + user_id="test-user", + agent_id=agent_id, + object_permission=LiteLLM_ObjectPermissionTable(object_permission_id="obj-1", agents=agent_ids), + ) + + async def test_agent_access_groups_cap_an_otherwise_unrestricted_key(self): + """A key with no agent grant of its own may still only reach the agents its + agent's attached access groups name.""" + agent_key: Final = UserAPIKeyAuth(api_key="test-key", user_id="test-user", agent_id="caller-agent") + resolve, asked = self._ceiling_resolver(frozenset({"agent-beta"})) + + assert await AgentRequestHandler.resolve_agent_access(agent_key, resolve) == RestrictedAgentAccess( + frozenset({"agent-beta"}) + ) + assert await AgentRequestHandler.is_agent_allowed("agent-beta", agent_key, resolve) is True + assert await AgentRequestHandler.is_agent_allowed("agent-alpha", agent_key, resolve) is False + assert asked == ["caller-agent"] * 3 + + @staticmethod + def _team_grants(grants: dict[str, AgentAccess]) -> AsyncMock: + async def by_team(user_api_key_auth: UserAPIKeyAuth | None = None) -> AgentAccess: + assert user_api_key_auth is not None + return grants.get(user_api_key_auth.team_id or "", UnrestrictedAgentAccess()) + + return AsyncMock(side_effect=by_team) + + async def test_agent_key_acting_for_a_user_is_capped_at_the_invoking_teams_agents(self): + """LIT-8014: the agent's key and access groups reach alpha and beta, but the human who + invoked it belongs to a team granted only beta, so on their behalf the agent reaches only beta.""" + agent_key: Final = self._key_granting(["agent-alpha", "agent-beta"], agent_id="caller-agent") + agent_key.agent_caller = AgentCaller(user_id="alice", team_id="callers") + resolve, _ = self._ceiling_resolver(frozenset({"agent-alpha", "agent-beta", "agent-gamma"})) + + with patch.object( # test-quality-ok: the team resolver reads proxy_server globals with no injection seam + AgentRequestHandler, + "_get_allowed_agents_for_team", + self._team_grants({"callers": RestrictedAgentAccess(frozenset({"agent-beta", "agent-gamma"}))}), + ) as mock_team: + assert await AgentRequestHandler.resolve_agent_access(agent_key, resolve) == RestrictedAgentAccess( + frozenset({"agent-beta"}) + ) + assert await AgentRequestHandler.is_agent_allowed("agent-alpha", agent_key, resolve) is False + + assert {call.args[0].team_id for call in mock_team.call_args_list} == {None, "callers"} + + async def test_agent_key_acting_for_a_user_whose_team_grants_no_agent_reaches_none(self): + agent_key: Final = UserAPIKeyAuth(api_key="test-key", user_id="test-user", agent_id="caller-agent") + agent_key.agent_caller = AgentCaller(user_id="alice", team_id="callers") + resolve, _ = self._ceiling_resolver(None) + + with patch.object( # test-quality-ok: the team resolver reads proxy_server globals with no injection seam + AgentRequestHandler, + "_get_allowed_agents_for_team", + self._team_grants({"callers": RestrictedAgentAccess(frozenset())}), + ): + assert await AgentRequestHandler.resolve_agent_access(agent_key, resolve) == RestrictedAgentAccess( + frozenset() + ) + + async def test_agent_key_acting_for_an_ungranted_caller_keeps_its_own_agents(self): + agent_key: Final = self._key_granting(["agent-alpha"], agent_id="caller-agent") + agent_key.agent_caller = AgentCaller(user_id="alice", team_id="callers") + resolve, _ = self._ceiling_resolver(None) + + with patch.object( # test-quality-ok: the team resolver reads proxy_server globals with no injection seam + AgentRequestHandler, "_get_allowed_agents_for_team", self._team_grants({}) + ): + assert await AgentRequestHandler.resolve_agent_access(agent_key, resolve) == RestrictedAgentAccess( + frozenset({"agent-alpha"}) + ) + + + async def test_agent_access_groups_intersect_with_key_grants(self): + agent_key: Final = self._key_granting(["agent-alpha", "agent-beta"], agent_id="caller-agent") + resolve, _ = self._ceiling_resolver(frozenset({"agent-beta", "agent-gamma"})) + + assert await AgentRequestHandler.resolve_agent_access(agent_key, resolve) == RestrictedAgentAccess( + frozenset({"agent-beta"}) + ) + assert await AgentRequestHandler.is_agent_allowed("agent-gamma", agent_key, resolve) is False + + async def test_agent_access_groups_naming_no_agent_deny_every_agent(self): + agent_key: Final = UserAPIKeyAuth(api_key="test-key", user_id="test-user", agent_id="caller-agent") + resolve, _ = self._ceiling_resolver(frozenset()) + + assert await AgentRequestHandler.resolve_agent_access(agent_key, resolve) == RestrictedAgentAccess(frozenset()) + assert await AgentRequestHandler.is_agent_allowed("agent-alpha", agent_key, resolve) is False + + async def test_agent_without_access_groups_keeps_key_grants(self): + agent_key: Final = self._key_granting(["agent-alpha"], agent_id="caller-agent") + resolve, asked = self._ceiling_resolver(None) + + assert await AgentRequestHandler.resolve_agent_access(agent_key, resolve) == RestrictedAgentAccess( + frozenset({"agent-alpha"}) + ) + assert asked == ["caller-agent"] + + async def test_key_without_agent_never_consults_agent_access_groups(self): + plain_key: Final = UserAPIKeyAuth(api_key="test-key", user_id="test-user") + resolve, asked = self._ceiling_resolver(frozenset()) + + assert await AgentRequestHandler.resolve_agent_access(plain_key, resolve) == UnrestrictedAgentAccess() + assert asked == [] + async def test_empty_access_group_denies_every_agent(self): """LIT-5143: a key restricted to an access group that resolves to no agents is restricted to nothing, not unrestricted. A failed group lookup still fails open.""" diff --git a/tests/test_litellm/proxy/agent_endpoints/test_a2a_endpoints.py b/tests/test_litellm/proxy/agent_endpoints/test_a2a_endpoints.py index 441e9640ef9..b9a260f5b14 100644 --- a/tests/test_litellm/proxy/agent_endpoints/test_a2a_endpoints.py +++ b/tests/test_litellm/proxy/agent_endpoints/test_a2a_endpoints.py @@ -16,6 +16,7 @@ from unittest.mock import AsyncMock, MagicMock, patch import pytest from litellm.proxy._types import UserAPIKeyAuth +from litellm.types.agents import AgentCaller AddLiteLLMData = Callable[..., Awaitable[dict[str, object]]] @@ -511,6 +512,24 @@ async def test_message_methods_forward_caller_identity_headers(method: str): assert forwarded_headers.get("X-LiteLLM-Team-Id") == "team-xyz" +@pytest.mark.asyncio +@pytest.mark.parametrize("method", ["message/send", "message/stream"]) +async def test_agent_calling_another_agent_forwards_the_human_who_invoked_it(method: str): + """LIT-8014: an agent acting for alice calls a second agent through the proxy. That hop must + carry alice, not the first agent's owner, so the chain stays capped at what alice may reach.""" + mock_request = _make_request_mock(method, _HELLO_MESSAGE_PARAMS) + agent_key = UserAPIKeyAuth(api_key="sk-agent", user_id="agent-owner", team_id="agent-team", agent_id="agent-1") + agent_key.agent_caller = AgentCaller(user_id="alice", team_id="callers") + + captured = await _invoke_message_method(method, mock_request, agent_key) + + forwarded_headers = captured.agent_extra_headers or {} + assert (forwarded_headers.get("X-LiteLLM-User-Id"), forwarded_headers.get("X-LiteLLM-Team-Id")) == ( + "alice", + "callers", + ) + + @pytest.mark.asyncio @pytest.mark.parametrize("method", ["message/send", "message/stream"]) async def test_message_methods_send_the_entra_bearer_for_azure_agents(method: str): diff --git a/tests/test_litellm/proxy/agent_endpoints/test_agent_registry.py b/tests/test_litellm/proxy/agent_endpoints/test_agent_registry.py index 231626c7eb5..b036e0dac4d 100644 --- a/tests/test_litellm/proxy/agent_endpoints/test_agent_registry.py +++ b/tests/test_litellm/proxy/agent_endpoints/test_agent_registry.py @@ -15,6 +15,7 @@ from litellm.proxy.agent_endpoints.agent_registry import ( _restore_redacted_litellm_params, redact_sensitive_agent_litellm_params, ) +from litellm.types.agents import PatchAgentRequest # Obviously-fake stand-ins for a real AWS credential pair (LIT-6736 regression # fixtures) -- never a real key shape, and must never appear in any response. @@ -990,3 +991,138 @@ async def test_patch_agent_in_db_preserves_secret_when_echoed_back_redacted(): stored_params: Final = json.loads(mock_update.call_args.kwargs["data"]["litellm_params"]) assert stored_params["aws_secret_access_key"] == SENTINEL_AWS_SECRET_ACCESS_KEY assert stored_params["is_public"] is True + + +def _agent_row_mock(access_group_ids: list[str]) -> MagicMock: + row: Final = MagicMock() + row.model_dump.return_value = { + "agent_id": "agent-123", + "agent_name": "Test Agent", + "agent_card_params": _sample_agent_card_params(), + "litellm_params": {}, + "object_permission": None, + "access_group_ids": access_group_ids, + } + row.object_permission = None + return row + + +@pytest.mark.asyncio +async def test_add_agent_to_db_persists_deduplicated_access_group_ids(): + registry: Final = AgentRegistry() + mock_prisma: Final = MagicMock() + mock_create = AsyncMock(return_value=_agent_row_mock(["ag-1", "ag-2"])) + mock_prisma.db.litellm_agentstable.create = mock_create + + result: Final = await registry.add_agent_to_db( + agent={ + "agent_name": "Test Agent", + "agent_card_params": _sample_agent_card_params(), + "access_group_ids": ["ag-1", "ag-2", "ag-1"], + }, + prisma_client=mock_prisma, + created_by="test-user", + ) + + assert tuple(mock_create.call_args.kwargs["data"]["access_group_ids"]) == ("ag-1", "ag-2") + assert result.access_group_ids == ["ag-1", "ag-2"] + + +@pytest.mark.asyncio +async def test_add_agent_to_db_without_access_group_ids_leaves_column_to_its_default(): + registry: Final = AgentRegistry() + mock_prisma: Final = MagicMock() + mock_create = AsyncMock(return_value=_agent_row_mock([])) + mock_prisma.db.litellm_agentstable.create = mock_create + + await registry.add_agent_to_db( + agent={"agent_name": "Test Agent", "agent_card_params": _sample_agent_card_params()}, + prisma_client=mock_prisma, + created_by="test-user", + ) + + assert "access_group_ids" not in mock_create.call_args.kwargs["data"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("patch_body", "expected"), + [ + ({"access_group_ids": ["ag-2", "ag-3"]}, ["ag-2", "ag-3"]), + ({"access_group_ids": []}, []), + ({"access_group_ids": None}, []), + ], +) +async def test_patch_agent_in_db_replaces_access_group_ids_when_provided( + patch_body: PatchAgentRequest, expected: list[str] +): + registry: Final = AgentRegistry() + mock_prisma: Final = MagicMock() + mock_prisma.db.litellm_agentstable.find_unique = AsyncMock( + return_value={ + "agent_id": "agent-123", + "agent_name": "Test Agent", + "litellm_params": {}, + "object_permission_id": None, + "access_group_ids": ["ag-1"], + } + ) + mock_update = AsyncMock(return_value=_agent_row_mock(expected)) + mock_prisma.db.litellm_agentstable.update = mock_update + + await registry.patch_agent_in_db( + agent_id="agent-123", agent=patch_body, prisma_client=mock_prisma, updated_by="test-user" + ) + + assert tuple(mock_update.call_args.kwargs["data"]["access_group_ids"]) == tuple(expected) + + +@pytest.mark.asyncio +async def test_patch_agent_in_db_keeps_access_group_ids_when_omitted(): + registry: Final = AgentRegistry() + mock_prisma: Final = MagicMock() + mock_prisma.db.litellm_agentstable.find_unique = AsyncMock( + return_value={ + "agent_id": "agent-123", + "agent_name": "Old Name", + "litellm_params": {}, + "object_permission_id": None, + "access_group_ids": ["ag-1"], + } + ) + mock_update = AsyncMock(return_value=_agent_row_mock(["ag-1"])) + mock_prisma.db.litellm_agentstable.update = mock_update + + await registry.patch_agent_in_db( + agent_id="agent-123", agent={"agent_name": "New Name"}, prisma_client=mock_prisma, updated_by="test-user" + ) + + assert "access_group_ids" not in mock_update.call_args.kwargs["data"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("body_access_group_ids", "expected"), + [(["ag-9", "ag-9"], ["ag-9"]), (None, []), ("omitted", [])], +) +async def test_update_agent_in_db_always_writes_access_group_ids(body_access_group_ids, expected: list[str]): + """PUT is a full replacement: omitting the field clears any previously attached groups.""" + registry: Final = AgentRegistry() + mock_prisma: Final = MagicMock() + mock_prisma.db.litellm_agentstable.find_unique = AsyncMock( + return_value=SimpleNamespace(litellm_params={}, object_permission_id=None, access_group_ids=["ag-1"]) + ) + mock_update = AsyncMock(return_value=_agent_row_mock(expected)) + mock_prisma.db.litellm_agentstable.update = mock_update + body: Final = { + "agent_name": "Test Agent", + "agent_card_params": _sample_agent_card_params(), + "litellm_params": {"model": "bedrock/agentcore/my-agent"}, + **({} if body_access_group_ids == "omitted" else {"access_group_ids": body_access_group_ids}), + } + + await registry.update_agent_in_db( + agent_id="agent-123", agent=body, prisma_client=mock_prisma, updated_by="test-user" + ) + + assert tuple(mock_update.call_args.kwargs["data"]["access_group_ids"]) == tuple(expected) diff --git a/tests/test_litellm/proxy/anthropic_endpoints/test_endpoints.py b/tests/test_litellm/proxy/anthropic_endpoints/test_endpoints.py index 9a9ccd9a213..801f61aa498 100644 --- a/tests/test_litellm/proxy/anthropic_endpoints/test_endpoints.py +++ b/tests/test_litellm/proxy/anthropic_endpoints/test_endpoints.py @@ -198,6 +198,62 @@ class TestProxyExceptionAnthropicEnvelope: assert fallback.status_code == 500 assert json.loads(fallback.body)["error"]["type"] == "api_error" + @staticmethod + def _call_id_error_response(general_settings, provider_specific_fields=None): + import litellm.proxy.anthropic_endpoints.endpoints as ep + from litellm.proxy._types import ProxyException + + request = MagicMock() + request.headers = {} + exc = ProxyException( + message="Rate limit exceeded", + type="rate_limit_error", + param=None, + code=429, + headers={"x-litellm-call-id": "call-8302"}, + provider_specific_fields=provider_specific_fields, + ) + with patch("litellm.proxy.proxy_server.general_settings", general_settings): + return ep._anthropic_error_json_response(exc, request) + + def test_anthropic_error_copies_the_call_id_into_the_error_when_opted_in(self): + """With include_call_id_in_error_body on, error.litellm_call_id is byte-identical to + the x-litellm-call-id header and lives inside the error object, which is what the + Anthropic SDK keeps as e.body.""" + response = self._call_id_error_response({"include_call_id_in_error_body": True}) + + assert response.headers["x-litellm-call-id"] == "call-8302" + assert json.loads(response.body) == { + "type": "error", + "error": { + "type": "rate_limit_error", + "message": "Rate limit exceeded", + "litellm_call_id": "call-8302", + }, + } + + def test_anthropic_error_keeps_provider_specific_fields_next_to_the_call_id(self): + response = self._call_id_error_response( + {"include_call_id_in_error_body": True}, + provider_specific_fields={"guardrail": "keyword-block"}, + ) + + assert json.loads(response.body)["error"] == { + "type": "rate_limit_error", + "message": "Rate limit exceeded", + "provider_specific_fields": {"guardrail": "keyword-block"}, + "litellm_call_id": "call-8302", + } + + def test_anthropic_error_leaves_the_envelope_alone_when_opted_out(self): + response = self._call_id_error_response({}) + + assert response.headers["x-litellm-call-id"] == "call-8302" + assert json.loads(response.body) == { + "type": "error", + "error": {"type": "rate_limit_error", "message": "Rate limit exceeded"}, + } + class TestHttpExceptionDictDetail: @pytest.mark.asyncio diff --git a/tests/test_litellm/proxy/auth/test_auth_checks.py b/tests/test_litellm/proxy/auth/test_auth_checks.py index 44c1d6a3c6b..0cab6535f3d 100644 --- a/tests/test_litellm/proxy/auth/test_auth_checks.py +++ b/tests/test_litellm/proxy/auth/test_auth_checks.py @@ -34,20 +34,26 @@ from litellm.proxy._types import ( UserAPIKeyAuth, WebhookEvent, ) +from litellm.proxy.agent_endpoints.auth.agent_access_groups import AgentAccessGroupCeiling, CeilingResolver +from litellm.types.agents import AgentCaller from litellm.proxy.auth.auth_checks import ( ExperimentalUIJWTToken, _cache_management_object, _can_object_call_model, _can_object_call_vector_stores, + _check_agent_access_group_model_access, _check_end_user_budget, _check_team_member_budget, _fetch_key_object_from_db_with_reconnect, _get_fuzzy_user_object, + CallerTeamLoader, + CallerUserLoader, _get_team_db_check, _log_budget_lookup_failure, _tag_max_budget_check, _team_max_budget_check, _virtual_key_max_budget_alert_check, + _check_agent_caller_model_access, _virtual_key_max_budget_check, _virtual_key_soft_budget_check, get_key_object, @@ -8930,6 +8936,69 @@ def test_request_skips_budget_checks_extends_route_rule_with_zero_cost_models() assert request_skips_budget_checks(route="/v1/chat/completions", model=None, llm_router=None) is False +def _agent_model_ceiling_resolver( + models: frozenset[str] | None, +) -> tuple[CeilingResolver, list[str]]: + """Resolver that records the agent ids it was asked about and answers with a fixed model + ceiling, or None when the agent has no access groups attached.""" + asked: Final[list[str]] = [] + + async def resolve(agent_id: str) -> AgentAccessGroupCeiling | None: + asked.append(agent_id) + if models is None: + return None + return AgentAccessGroupCeiling( + access_group_ids=("ag-1",), models=models, mcp_server_ids=frozenset(), agent_ids=frozenset() + ) + + return resolve, asked + + +@pytest.mark.asyncio +async def test_agent_access_groups_cap_models_even_when_key_allows_them(): + agent_key: Final = UserAPIKeyAuth(token="agent-token", agent_id="agent-1", models=["gpt-5", "claude-sonnet"]) + resolve, asked = _agent_model_ceiling_resolver(frozenset({"gpt-5"})) + + assert await _check_agent_access_group_model_access("gpt-5", agent_key, None, resolve) is True + + with pytest.raises(ProxyException) as exc_info: + await _check_agent_access_group_model_access("claude-sonnet", agent_key, None, resolve) + + assert exc_info.value.type == ProxyErrorTypes.agent_model_access_denied + assert exc_info.value.code == str(status.HTTP_403_FORBIDDEN) + assert asked == ["agent-1", "agent-1"] + + +@pytest.mark.asyncio +async def test_agent_access_groups_naming_no_model_deny_every_model(): + agent_key: Final = UserAPIKeyAuth(token="agent-token", agent_id="agent-1", models=[]) + resolve, _ = _agent_model_ceiling_resolver(frozenset()) + + with pytest.raises(ProxyException) as exc_info: + await _check_agent_access_group_model_access("gpt-5", agent_key, None, resolve) + + assert exc_info.value.type == ProxyErrorTypes.agent_model_access_denied + + +@pytest.mark.asyncio +async def test_agent_without_access_groups_adds_no_model_ceiling(): + agent_key: Final = UserAPIKeyAuth(token="agent-token", agent_id="agent-1", models=["gpt-5", "claude-sonnet"]) + resolve, asked = _agent_model_ceiling_resolver(None) + + assert await _check_agent_access_group_model_access("gpt-5", agent_key, None, resolve) is True + assert await _check_agent_access_group_model_access("claude-sonnet", agent_key, None, resolve) is True + assert asked == ["agent-1", "agent-1"] + + +@pytest.mark.asyncio +async def test_key_without_agent_never_consults_agent_access_groups(): + plain_key: Final = UserAPIKeyAuth(token="plain-token", models=["gpt-5"]) + resolve, asked = _agent_model_ceiling_resolver(frozenset()) + + assert await _check_agent_access_group_model_access("gpt-5", plain_key, None, resolve) is True + assert asked == [] + + @pytest.mark.asyncio async def test_team_member_budget_check_temp_budget_increase_extends_cap(): """Spend above max_budget but below max_budget + active temp increase @@ -9086,3 +9155,117 @@ async def test_team_member_budget_check_adds_temp_increase_to_live_team_default( proxy_logging_obj=ProxyLogging(user_api_key_cache=None), ) assert exc_info.value.max_budget == expected_cap + + +def _agent_key_acting_for(user_id: str | None, team_id: str | None) -> UserAPIKeyAuth: + agent_key: Final = UserAPIKeyAuth(token="agent-token", agent_id="agent-1", models=["gpt-5", "claude-sonnet"]) + agent_key.agent_caller = AgentCaller(user_id=user_id, team_id=team_id) + return agent_key + + +def _caller_loaders( + team: LiteLLM_TeamTable | None, + user: LiteLLM_UserTable | None, +) -> tuple[CallerTeamLoader, CallerUserLoader, list[str]]: + """Loaders that hand back fixed caller rows and record the agent_caller they were asked about.""" + asked: Final[list[str]] = [] + + async def load_team(valid_token: UserAPIKeyAuth) -> LiteLLM_TeamTable | None: + asked.append(f"team:{valid_token.agent_caller.team_id if valid_token.agent_caller else None}") + return team + + async def load_user(valid_token: UserAPIKeyAuth) -> LiteLLM_UserTable | None: + asked.append(f"user:{valid_token.agent_caller.user_id if valid_token.agent_caller else None}") + return user + + return load_team, load_user, asked + + +async def _cache_with_membership(user_id: str, team_id: str, allowed_models: list[str] | None) -> UserApiKeyCache: + from litellm.proxy._types import LiteLLM_TeamMembership + from litellm.proxy.common_utils.user_api_key_cache import team_membership_reservation_cache_key + + cache: Final = UserApiKeyCache() + await cache.async_set_cache( + key=team_membership_reservation_cache_key(user_id=user_id, team_id=team_id), + value=LiteLLM_TeamMembership( + user_id=user_id, + team_id=team_id, + litellm_budget_table=LiteLLM_BudgetTable(allowed_models=allowed_models) if allowed_models else None, + ), + model_type=LiteLLM_TeamMembership, + ) + return cache + + +async def _check_caller_models( + agent_key: UserAPIKeyAuth, + model: str, + load_team: CallerTeamLoader, + load_user: CallerUserLoader, + cache: UserApiKeyCache | None = None, +) -> None: + await _check_agent_caller_model_access( + model=model, + valid_token=agent_key, + llm_router=None, + prisma_client=None, + user_api_key_cache=cache or UserApiKeyCache(), + proxy_logging_obj=MagicMock(), + load_team=load_team, + load_user=load_user, + ) + + +@pytest.mark.asyncio +async def test_agent_key_acting_for_a_team_is_capped_at_that_teams_models(): + """LIT-8014: the invoking team may only call gpt-5, so the agent's own claude grant does not help.""" + agent_key: Final = _agent_key_acting_for(user_id="alice", team_id="team-a") + load_team, load_user, asked = _caller_loaders(LiteLLM_TeamTable(team_id="team-a", models=["gpt-5"]), None) + cache: Final = await _cache_with_membership("alice", "team-a", allowed_models=None) + + await _check_caller_models(agent_key, "gpt-5", load_team, load_user, cache) + with pytest.raises(ProxyException) as exc_info: + await _check_caller_models(agent_key, "claude-sonnet", load_team, load_user, cache) + + assert exc_info.value.type == ProxyErrorTypes.team_model_access_denied + assert exc_info.value.code == str(status.HTTP_403_FORBIDDEN) + assert asked == ["team:team-a", "team:team-a"] + + +@pytest.mark.asyncio +async def test_agent_key_acting_for_a_team_member_is_capped_at_the_members_scope(): + agent_key: Final = _agent_key_acting_for(user_id="alice", team_id="team-a") + load_team, load_user, _ = _caller_loaders( + LiteLLM_TeamTable(team_id="team-a", models=["gpt-5", "claude-sonnet"]), None + ) + cache: Final = await _cache_with_membership("alice", "team-a", allowed_models=["gpt-5"]) + + await _check_caller_models(agent_key, "gpt-5", load_team, load_user, cache) + with pytest.raises(ProxyException) as exc_info: + await _check_caller_models(agent_key, "claude-sonnet", load_team, load_user, cache) + + assert "User=alice, Team=team-a" in exc_info.value.internal_message + + +@pytest.mark.asyncio +async def test_agent_key_acting_for_a_teamless_user_is_capped_at_that_users_models(): + agent_key: Final = _agent_key_acting_for(user_id="alice", team_id=None) + load_team, load_user, asked = _caller_loaders(None, LiteLLM_UserTable(user_id="alice", models=["gpt-5"])) + + await _check_caller_models(agent_key, "gpt-5", load_team, load_user) + with pytest.raises(ProxyException) as exc_info: + await _check_caller_models(agent_key, "claude-sonnet", load_team, load_user) + + assert exc_info.value.type == ProxyErrorTypes.user_model_access_denied + assert asked == ["team:None", "user:alice", "team:None", "user:alice"] + + +@pytest.mark.asyncio +async def test_agent_key_without_an_echoed_caller_keeps_its_own_models(): + agent_key: Final = UserAPIKeyAuth(token="agent-token", agent_id="agent-1", models=["gpt-5", "claude-sonnet"]) + load_team, load_user, asked = _caller_loaders(LiteLLM_TeamTable(team_id="team-a", models=[]), None) + + await _check_caller_models(agent_key, "claude-sonnet", load_team, load_user) + + assert asked == [] diff --git a/tests/test_litellm/proxy/auth/test_handle_jwt.py b/tests/test_litellm/proxy/auth/test_handle_jwt.py index e768139f04a..cd14f630130 100644 --- a/tests/test_litellm/proxy/auth/test_handle_jwt.py +++ b/tests/test_litellm/proxy/auth/test_handle_jwt.py @@ -291,6 +291,106 @@ async def test_find_team_with_model_access_uses_request_method_for_passthrough_a assert "allowed_passthrough_routes" in exc_info.value.detail +_AUTH_ENFORCED_MODEL_HOST_ROUTES: Final = { + "test-uuid-1:subpath:/model-host/v1/extractor:GET,POST": { + "endpoint_id": "test-uuid-1", + "path": "/model-host/v1/extractor", + "type": "subpath", + "auth": True, + }, +} + + +@pytest.mark.asyncio +async def test_find_team_with_model_access_team_allowed_routes_wildcard_grants_auth_passthrough(): + jwt_handler = JWTHandler() + jwt_handler.litellm_jwtauth = LiteLLM_JWTAuth(team_allowed_routes=["openai_routes", "/model-host/*"]) + team_without_passthrough_allowlist = LiteLLM_TeamTable(team_id="team-a", models=["all-proxy-models"], metadata={}) + + with ( + patch( + "litellm.proxy.auth.handle_jwt.get_team_object", + new_callable=AsyncMock, + return_value=team_without_passthrough_allowlist, + ), + patch( + "litellm.proxy.pass_through_endpoints.pass_through_endpoints._registered_pass_through_routes", + _AUTH_ENFORCED_MODEL_HOST_ROUTES, + ), + patch("litellm.proxy.utils.get_server_root_path", return_value="/"), + ): + team_id, team_obj = await JWTAuthManager.find_team_with_model_access( + team_ids={"team-a"}, + requested_model=None, + route="/model-host/v1/extractor/predict", + jwt_handler=jwt_handler, + prisma_client=None, + user_api_key_cache=MagicMock(), + parent_otel_span=None, + proxy_logging_obj=MagicMock(), + request_method="POST", + ) + + assert team_id == "team-a" + assert team_obj == team_without_passthrough_allowlist + + +@pytest.mark.asyncio +async def test_auth_builder_header_team_allows_auth_passthrough_for_team_allowed_routes_wildcard(): + from litellm.proxy.utils import ProxyLogging + + jwt_handler = JWTHandler() + user_api_key_cache = DualCache() + jwt_handler.update_environment( + prisma_client=None, + user_api_key_cache=user_api_key_cache, + litellm_jwtauth=LiteLLM_JWTAuth( + team_ids_jwt_field="groups", + user_id_jwt_field="sub", + team_allowed_routes=["openai_routes", "/model-host/*"], + ), + ) + + with ( + patch.object(jwt_handler, "auth_jwt", new_callable=AsyncMock) as mock_auth_jwt, + patch.object(JWTAuthManager, "check_rbac_role", new_callable=AsyncMock), + patch.object(JWTAuthManager, "check_admin_access", new_callable=AsyncMock, return_value=None), + patch( + "litellm.proxy.auth.handle_jwt.get_team_object", + new_callable=AsyncMock, + return_value=LiteLLM_TeamTable(team_id="team-2", metadata={}), + ), + patch.object( + JWTAuthManager, + "get_objects", + new_callable=AsyncMock, + return_value=(None, None, None, None, "user-1"), + ), + patch( + "litellm.proxy.pass_through_endpoints.pass_through_endpoints._registered_pass_through_routes", + _AUTH_ENFORCED_MODEL_HOST_ROUTES, + ), + patch("litellm.proxy.utils.get_server_root_path", return_value="/"), + ): + mock_auth_jwt.return_value = {"sub": "user-1", "scope": "", "groups": ["team-1", "team-2"]} + + result = await JWTAuthManager.auth_builder( + api_key="jwt-token", + jwt_handler=jwt_handler, + request_data={}, + general_settings={}, + route="/model-host/v1/extractor/predict", + prisma_client=None, + user_api_key_cache=user_api_key_cache, + parent_otel_span=None, + proxy_logging_obj=ProxyLogging(user_api_key_cache=user_api_key_cache), + request_headers={"x-litellm-team-id": "team-2"}, + request_method="POST", + ) + + assert result["team_id"] == "team-2" + + @pytest.mark.asyncio async def test_auth_builder_proxy_admin_user_role(): """Test that is_proxy_admin is True when user_object.user_role is PROXY_ADMIN""" @@ -3250,15 +3350,26 @@ async def test_auth_builder_single_team_db_fallback_when_jwt_has_no_team( mock_get_membership.assert_not_called() -@pytest.mark.asyncio -async def test_auth_builder_single_team_fallback_membership_error_skips_no_raise(): - """ - get_team_object succeeds but get_team_membership raises — do not set team; no exception. - """ - from fastapi import HTTPException +class _UnreachableMembershipPrisma: + class db: + class litellm_teammembership: + @staticmethod + async def find_unique(where: dict[str, dict[str, str]], include: dict[str, bool]) -> None: + raise httpx.ConnectError("All connection attempts failed") - user_id = "u_mem_fail" - team_id_val = "team_mem_fail" + +@pytest.mark.asyncio +async def test_auth_builder_single_team_fallback_membership_outage_raises_instead_of_dropping_the_team(): + """ + get_team_object succeeds but the membership read hits a database outage: the + outage propagates (auth maps it to 503) instead of the team being dropped. + """ + from litellm.proxy.auth.auth_exception_handler import _as_proxy_exception + from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache + from litellm.proxy.utils import ProxyLogging + + user_id = "u_mem_outage" + team_id_val = "team_mem_outage" user_object = LiteLLM_UserTable( user_id=user_id, user_role=LitellmUserRoles.INTERNAL_USER, @@ -3267,6 +3378,7 @@ async def test_auth_builder_single_team_fallback_membership_error_skips_no_raise team_table = LiteLLM_TeamTable(team_id=team_id_val) jwt_handler = JWTHandler() jwt_handler.litellm_jwtauth = LiteLLM_JWTAuth() + cache = UserApiKeyCache() with ( patch.object(jwt_handler, "auth_jwt", new_callable=AsyncMock) as mock_auth_jwt, @@ -3316,34 +3428,26 @@ async def test_auth_builder_single_team_fallback_membership_error_skips_no_raise "litellm.proxy.auth.handle_jwt.get_team_object", new_callable=AsyncMock, ) as mock_get_team, - patch( - "litellm.proxy.auth.handle_jwt.get_team_membership", - new_callable=AsyncMock, - ) as mock_get_membership, ): mock_auth_jwt.return_value = {"sub": user_id, "scope": ""} mock_get_team.return_value = team_table - mock_get_membership.side_effect = HTTPException( - status_code=500, detail="membership lookup failed" - ) - result = await JWTAuthManager.auth_builder( - api_key="test_jwt_token", - jwt_handler=jwt_handler, - request_data={"model": "gpt-4"}, - general_settings={"enforce_rbac": False}, - route="/chat/completions", - prisma_client=None, - user_api_key_cache=None, - parent_otel_span=None, - proxy_logging_obj=None, - ) + with pytest.raises(httpx.ConnectError) as raised: + await JWTAuthManager.auth_builder( + api_key="test_jwt_token", + jwt_handler=jwt_handler, + request_data={"model": "gpt-4"}, + general_settings={"enforce_rbac": False}, + route="/chat/completions", + prisma_client=_UnreachableMembershipPrisma(), + user_api_key_cache=cache, + parent_otel_span=None, + proxy_logging_obj=ProxyLogging(user_api_key_cache=cache), + ) - assert result["team_id"] is None - assert result["team_object"] is None - assert result["team_membership"] is None - mock_get_team.assert_called() - mock_get_membership.assert_called_once() + mock_get_team.assert_called() + surfaced = _as_proxy_exception(raised.value) + assert (surfaced.code, surfaced.type) == ("503", ProxyErrorTypes.no_db_connection) # --------------------------------------------------------------------------- @@ -6463,6 +6567,90 @@ async def test_auth_builder_db_fallback_enforces_passthrough_route_access(): assert "passthrough route" in exc_info.value.detail +async def _auth_builder_via_db_team_fallback(team_allowed_routes: list[str]): + user_id = "u_passthrough" + user_object = LiteLLM_UserTable( + user_id=user_id, + user_role=LitellmUserRoles.INTERNAL_USER, + teams=["team_no_passthrough"], + ) + jwt_handler = JWTHandler() + jwt_handler.litellm_jwtauth = LiteLLM_JWTAuth(fallback_to_db_teams=True, team_allowed_routes=team_allowed_routes) + + async def fake_get_team(team_id, **kwargs): + return LiteLLM_TeamTable(team_id=team_id, metadata={}) + + with ( + patch.object(jwt_handler, "auth_jwt", new_callable=AsyncMock, return_value={"sub": user_id, "scope": ""}), + patch.object(JWTAuthManager, "check_rbac_role", new_callable=AsyncMock), + patch.object(jwt_handler, "get_rbac_role", return_value=None), + patch.object(jwt_handler, "get_scopes", return_value=[]), + patch.object(jwt_handler, "get_object_id", return_value=None), + patch.object( + JWTAuthManager, + "get_user_info", + new_callable=AsyncMock, + return_value=(user_id, "u@example.com", True), + ), + patch.object(jwt_handler, "get_org_id", return_value=None), + patch.object(jwt_handler, "get_end_user_id", return_value=None), + patch.object(JWTAuthManager, "check_admin_access", new_callable=AsyncMock, return_value=None), + patch.object( + JWTAuthManager, + "get_objects", + new_callable=AsyncMock, + return_value=(user_object, None, None, None, user_id), + ), + patch.object(JWTAuthManager, "map_user_to_teams", new_callable=AsyncMock), + patch.object(JWTAuthManager, "validate_object_id", return_value=True), + patch.object(JWTAuthManager, "sync_user_role_and_teams", new_callable=AsyncMock), + patch( + "litellm.proxy.auth.handle_jwt.get_team_object", + new_callable=AsyncMock, + side_effect=fake_get_team, + ), + patch( + "litellm.proxy.auth.handle_jwt.get_team_membership", + new_callable=AsyncMock, + return_value=None, + ), + patch( + "litellm.proxy.pass_through_endpoints.pass_through_endpoints._registered_pass_through_routes", + _AUTH_ENFORCED_MODEL_HOST_ROUTES, + ), + patch("litellm.proxy.utils.get_server_root_path", return_value="/"), + ): + return await JWTAuthManager.auth_builder( + api_key="test_jwt_token", + jwt_handler=jwt_handler, + request_data={}, + general_settings={"enforce_rbac": False}, + route="/model-host/v1/extractor/predict", + prisma_client=None, + user_api_key_cache=None, + parent_otel_span=None, + proxy_logging_obj=None, + request_headers=None, + request_method="POST", + ) + + +@pytest.mark.asyncio +async def test_auth_builder_db_fallback_team_allowed_routes_wildcard_grants_auth_passthrough(): + result = await _auth_builder_via_db_team_fallback(team_allowed_routes=["openai_routes", "/model-host/*"]) + + assert result["team_id"] == "team_no_passthrough" + + +@pytest.mark.asyncio +async def test_auth_builder_db_fallback_route_groups_alone_do_not_grant_auth_passthrough(): + with pytest.raises(HTTPException) as exc_info: + await _auth_builder_via_db_team_fallback(team_allowed_routes=["openai_routes", "mapped_pass_through_routes"]) + + assert exc_info.value.status_code == 403, exc_info.value.detail + assert "allowed_passthrough_routes" in exc_info.value.detail + + @pytest.mark.asyncio async def test_sync_user_role_and_teams_singular_claim_reconciles_memberships(): """When fallback_to_db_teams is on but the JWT carries a singular team claim diff --git a/tests/test_litellm/proxy/auth/test_login_utils.py b/tests/test_litellm/proxy/auth/test_login_utils.py index 55ece36252d..3786169c320 100644 --- a/tests/test_litellm/proxy/auth/test_login_utils.py +++ b/tests/test_litellm/proxy/auth/test_login_utils.py @@ -5,12 +5,15 @@ This module tests the refactored login logic that was moved from proxy_server.py to login_utils.py for better reusability. """ +import hashlib import os from collections.abc import Mapping from contextlib import ExitStack from typing import TYPE_CHECKING, Final +from datetime import datetime, timedelta, timezone from unittest.mock import AsyncMock, MagicMock, patch +import httpx import pytest if TYPE_CHECKING: @@ -34,6 +37,7 @@ def _unlimited_throttle(): from litellm.constants import LITELLM_PROXY_ADMIN_NAME +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler from litellm.proxy._types import ( LiteLLM_UserTable, LitellmUserRoles, @@ -46,8 +50,13 @@ from litellm.proxy.auth.login_utils import ( authenticate_user, get_ui_credentials, is_env_credential_login_enabled, + screen_login_password_for_breach, ) +# Successful DB-user logins schedule the background HIBP screen; disable it so +# no test ever does live network I/O to haveibeenpwned.com from CI. +_POLICY_NO_BREACH_CHECK = {"password_policy_check_breached_passwords": False} + def test_get_ui_credentials_prefers_explicit_password(): """The configured UI password should be returned when available.""" @@ -326,6 +335,7 @@ async def test_authenticate_user_email_case_insensitive_login(): master_key=master_key, prisma_client=mock_prisma_client, throttle=_unlimited_throttle(), + general_settings=_POLICY_NO_BREACH_CHECK, ) result_lower = await authenticate_user( username=stored_email, @@ -333,6 +343,7 @@ async def test_authenticate_user_email_case_insensitive_login(): master_key=master_key, prisma_client=mock_prisma_client, throttle=_unlimited_throttle(), + general_settings=_POLICY_NO_BREACH_CHECK, ) assert result_mixed.user_id == result_lower.user_id == "test-user-123" @@ -576,6 +587,7 @@ async def test_authenticate_user_database_login_with_non_ascii_password(): master_key=master_key, prisma_client=mock_prisma_client, throttle=_unlimited_throttle(), + general_settings=_POLICY_NO_BREACH_CHECK, ) assert isinstance(result, LoginResult) @@ -721,7 +733,12 @@ async def _db_login(throttle, username: str, password: str, *, correct: bool): ), ): return await authenticate_user( - username=username, password=password, master_key="sk-master", prisma_client=MagicMock(), throttle=throttle + username=username, + password=password, + master_key="sk-master", + prisma_client=MagicMock(), + throttle=throttle, + general_settings=_POLICY_NO_BREACH_CHECK, ) @@ -2064,3 +2081,265 @@ class TestIsEnvCredentialLoginEnabled: with ExitStack() as stack: _patch_sso_configured(stack, configured=False) assert is_env_credential_login_enabled({"disable_password_login_when_sso_enabled": True}) is True + + +def _db_user_row(*, password: str, password_reset_required: bool | None = None, last_breach_check_at=None): + hashed = hash_token(token=password) + row = MagicMock() + row.user_id = "reset-user-1" + row.user_email = "reset@example.com" + row.password = hashed + row.user_role = LitellmUserRoles.INTERNAL_USER + row.password_reset_required = password_reset_required + row.last_breach_check_at = last_breach_check_at + return row + + +def _prisma_with_user(row) -> MagicMock: + mock_prisma_client = MagicMock() + mock_prisma_client.db.litellm_usertable.find_first = AsyncMock(return_value=row) + mock_prisma_client.db.litellm_usertable.update = AsyncMock(return_value=row) + return mock_prisma_client + + +_DB_LOGIN_ENV = { + "DATABASE_URL": "postgresql://test:test@localhost/test", + "UI_USERNAME": "admin", + "UI_PASSWORD": "admin-password", +} + + +class TestPasswordResetRequiredSessionMinting: + """A user flagged `password_reset_required` must receive a UI session key + restricted to the change-password endpoint (server-side enforcement, so a + script driving the management API with the session key is blocked too); + an unflagged user must keep getting an unrestricted key.""" + + async def _login(self, mock_prisma_client) -> tuple[LoginResult, dict]: + with patch.dict(os.environ, _DB_LOGIN_ENV): + with patch( # test-quality-ok: asserting the minted key's restriction requires seeing its kwargs + "litellm.proxy.auth.login_utils.generate_key_helper_fn", + new_callable=AsyncMock, + return_value={"token": "session-token"}, + ) as mock_generate_key: + result = await authenticate_user( + username="reset@example.com", + password="Str0ng!Passw0rd", + master_key="sk-1234", + prisma_client=mock_prisma_client, + throttle=_unlimited_throttle(), + general_settings=_POLICY_NO_BREACH_CHECK, + ) + return result, mock_generate_key.call_args.kwargs + + @pytest.mark.asyncio + async def test_flagged_user_gets_key_restricted_to_change_password(self): + row = _db_user_row(password="Str0ng!Passw0rd", password_reset_required=True) + result, key_kwargs = await self._login(_prisma_with_user(row)) + + assert key_kwargs["allowed_routes"] == ["/user/password/change"] + assert key_kwargs["metadata"] == {"login_method": "username_password", "password_reset_required": True} + assert result.password_reset_required is True + + @pytest.mark.asyncio + async def test_unflagged_user_gets_unrestricted_key(self): + row = _db_user_row(password="Str0ng!Passw0rd", password_reset_required=None) + result, key_kwargs = await self._login(_prisma_with_user(row)) + + assert key_kwargs["allowed_routes"] is None + assert key_kwargs["metadata"] == {"login_method": "username_password"} + assert result.password_reset_required is False + + async def _login_with_screen_result(self, mock_prisma_client, breached: bool) -> tuple[LoginResult, dict, dict]: + with patch.dict(os.environ, _DB_LOGIN_ENV): + with patch( # test-quality-ok: asserting the minted key's restriction requires seeing its kwargs + "litellm.proxy.auth.login_utils.generate_key_helper_fn", + new_callable=AsyncMock, + return_value={"token": "session-token"}, + ) as mock_generate_key: + with ( + patch( # test-quality-ok: authenticate_user has no HIBP client seam; the screen itself is tested against MockTransport below + "litellm.proxy.auth.login_utils.screen_login_password_for_breach", + new_callable=AsyncMock, + return_value=breached, + ) as mock_screen + ): + result = await authenticate_user( + username="reset@example.com", + password="Str0ng!Passw0rd", + master_key="sk-1234", + prisma_client=mock_prisma_client, + throttle=_unlimited_throttle(), + general_settings=_POLICY_NO_BREACH_CHECK, + ) + return result, mock_generate_key.call_args.kwargs, mock_screen.call_args.kwargs + + @pytest.mark.asyncio + async def test_login_screens_with_row_state_before_minting(self): + """The login must hand the screen the row's recheck timestamp, or the + 24h throttle can never work.""" + checked_at = datetime.now(timezone.utc) - timedelta(hours=1) + row = _db_user_row(password="Str0ng!Passw0rd", last_breach_check_at=checked_at) + mock_prisma_client = _prisma_with_user(row) + + _, _, screen_kwargs = await self._login_with_screen_result(mock_prisma_client, breached=False) + + assert screen_kwargs["user_id"] == "reset-user-1" + assert screen_kwargs["password"] == "Str0ng!Passw0rd" + assert screen_kwargs["last_breach_check_at"] == checked_at + assert screen_kwargs["prisma_client"] is mock_prisma_client + + @pytest.mark.asyncio + async def test_fresh_breach_hit_restricts_the_current_session(self): + """A breach found during THIS login must restrict THIS session, not + just the next one.""" + row = _db_user_row(password="Str0ng!Passw0rd", password_reset_required=None) + mock_prisma_client = _prisma_with_user(row) + + result, key_kwargs, _ = await self._login_with_screen_result(mock_prisma_client, breached=True) + + assert key_kwargs["allowed_routes"] == ["/user/password/change"] + assert key_kwargs["metadata"] == {"login_method": "username_password", "password_reset_required": True} + assert result.password_reset_required is True + + +def _sha1_upper(password: str) -> str: + return hashlib.sha1(password.encode("utf-8"), usedforsecurity=False).hexdigest().upper() + + +def _client_with_transport(handler) -> AsyncHTTPHandler: + http_handler = AsyncHTTPHandler() + http_handler.client = httpx.AsyncClient(transport=httpx.MockTransport(handler)) + return http_handler + + +def _client_returning_breach_hit(password: str) -> AsyncHTTPHandler: + body = f"{_sha1_upper(password)[5:]}:42" + return _client_with_transport(lambda request: httpx.Response(200, text=body)) + + +def _client_returning_no_hit() -> AsyncHTTPHandler: + return _client_with_transport(lambda request: httpx.Response(200, text="0000000000000000000000000000000000A:3")) + + +def _client_never_called() -> AsyncHTTPHandler: + def handler(request: httpx.Request) -> httpx.Response: + raise AssertionError(f"unexpected HTTP call to {request.url}") + + return _client_with_transport(handler) + + +class TestScreenLoginPasswordForBreach: + """The awaited login-time screen: flags a breached password for a forced + reset, stamps the recheck timestamp, rechecks at most every 24h, returns + the breach verdict so the login can restrict the session it is minting, + and never raises into the login.""" + + @pytest.mark.asyncio + async def test_breached_password_sets_reset_flag_and_timestamp(self): + password = "Password123!" + mock_prisma_client = _prisma_with_user(None) + + breached = await screen_login_password_for_breach( + user_id="reset-user-1", + password=password, + last_breach_check_at=None, + general_settings={}, + prisma_client=mock_prisma_client, + client=_client_returning_breach_hit(password), + ) + + assert breached is True + update_kwargs = mock_prisma_client.db.litellm_usertable.update.call_args.kwargs + assert update_kwargs["where"] == {"user_id": "reset-user-1"} + assert update_kwargs["data"]["password_reset_required"] is True + assert isinstance(update_kwargs["data"]["last_breach_check_at"], datetime) + + @pytest.mark.asyncio + async def test_clean_password_stamps_timestamp_without_flag(self): + mock_prisma_client = _prisma_with_user(None) + + breached = await screen_login_password_for_breach( + user_id="reset-user-1", + password="Str0ng!Passw0rd", + last_breach_check_at=None, + general_settings={}, + prisma_client=mock_prisma_client, + client=_client_returning_no_hit(), + ) + + assert breached is False + update_kwargs = mock_prisma_client.db.litellm_usertable.update.call_args.kwargs + assert "password_reset_required" not in update_kwargs["data"] + assert isinstance(update_kwargs["data"]["last_breach_check_at"], datetime) + + @pytest.mark.asyncio + async def test_skips_hibp_when_checked_within_24_hours(self): + mock_prisma_client = _prisma_with_user(None) + + breached = await screen_login_password_for_breach( + user_id="reset-user-1", + password="Password123!", + last_breach_check_at=datetime.now(timezone.utc) - timedelta(hours=23), + general_settings={}, + prisma_client=mock_prisma_client, + client=_client_never_called(), + ) + + assert breached is False + mock_prisma_client.db.litellm_usertable.update.assert_not_called() + + @pytest.mark.asyncio + async def test_rechecks_when_last_check_is_older_than_24_hours(self): + password = "Password123!" + mock_prisma_client = _prisma_with_user(None) + + breached = await screen_login_password_for_breach( + user_id="reset-user-1", + password=password, + last_breach_check_at=datetime.now(timezone.utc) - timedelta(hours=25), + general_settings={}, + prisma_client=mock_prisma_client, + client=_client_returning_breach_hit(password), + ) + + assert breached is True + assert ( + mock_prisma_client.db.litellm_usertable.update.call_args.kwargs["data"]["password_reset_required"] is True + ) + + @pytest.mark.asyncio + async def test_skips_hibp_when_check_disabled(self): + mock_prisma_client = _prisma_with_user(None) + + breached = await screen_login_password_for_breach( + user_id="reset-user-1", + password="Password123!", + last_breach_check_at=None, + general_settings=_POLICY_NO_BREACH_CHECK, + prisma_client=mock_prisma_client, + client=_client_never_called(), + ) + + assert breached is False + mock_prisma_client.db.litellm_usertable.update.assert_not_called() + + @pytest.mark.asyncio + async def test_db_failure_never_raises_but_still_reports_the_breach(self): + """A failed flag write must not fail the login, but the breach verdict + still has to restrict the session being minted right now.""" + password = "Password123!" + mock_prisma_client = _prisma_with_user(None) + mock_prisma_client.db.litellm_usertable.update = AsyncMock(side_effect=RuntimeError("db down")) + + assert ( + await screen_login_password_for_breach( + user_id="reset-user-1", + password=password, + last_breach_check_at=None, + general_settings={}, + prisma_client=mock_prisma_client, + client=_client_returning_breach_hit(password), + ) + is True + ) diff --git a/tests/test_litellm/proxy/auth/test_onboarding.py b/tests/test_litellm/proxy/auth/test_onboarding.py index 524b655b465..0454aea1239 100644 --- a/tests/test_litellm/proxy/auth/test_onboarding.py +++ b/tests/test_litellm/proxy/auth/test_onboarding.py @@ -8,15 +8,20 @@ Covers the security behavior of: session key only after the password is written """ +import hashlib from datetime import timedelta from unittest.mock import AsyncMock, MagicMock, patch +import httpx import jwt import pytest +import respx from fastapi import HTTPException import litellm -from litellm.proxy._types import InvitationClaim +from litellm.proxy._types import InvitationClaim, ProxyException + +_POLICY_NO_BREACH_CHECK = {"password_policy_check_breached_passwords": False} # --------------------------------------------------------------------------- # Helpers @@ -386,7 +391,9 @@ async def test_claim_token_rejects_concurrent_reuse_before_password_write(): with ( patch("litellm.proxy.proxy_server.prisma_client", prisma), patch("litellm.proxy.proxy_server.master_key", "sk-test"), - patch("litellm.proxy.proxy_server.general_settings", {}), + patch( # test-quality-ok: claim_onboarding_link reads proxy_server module globals; no injection seam + "litellm.proxy.proxy_server.general_settings", _POLICY_NO_BREACH_CHECK + ), patch( "litellm.proxy.proxy_server.generate_key_helper_fn", new_callable=AsyncMock, @@ -426,7 +433,9 @@ async def test_claim_token_sets_accepted_at_after_password_written(): with ( patch("litellm.proxy.proxy_server.prisma_client", prisma), patch("litellm.proxy.proxy_server.master_key", "sk-test"), - patch("litellm.proxy.proxy_server.general_settings", {}), + patch( # test-quality-ok: claim_onboarding_link reads proxy_server module globals; no injection seam + "litellm.proxy.proxy_server.general_settings", _POLICY_NO_BREACH_CHECK + ), patch("litellm.proxy.proxy_server.premium_user", False), patch( "litellm.proxy.proxy_server.generate_key_helper_fn", @@ -454,6 +463,10 @@ async def test_claim_token_sets_accepted_at_after_password_written(): call_kwargs = prisma.db.litellm_usertable.update.call_args assert call_kwargs.kwargs["where"] == {"user_id": "user-123"} assert "password" in call_kwargs.kwargs["data"] + # A freshly claimed, policy-screened password lifts any pending forced + # reset and re-arms the login-time breach screen. + assert call_kwargs.kwargs["data"]["password_reset_required"] is False + assert call_kwargs.kwargs["data"]["last_breach_check_at"] is None # is_accepted was flipped to True on the invitation link prisma.db.litellm_invitationlink.update.assert_called_once() @@ -483,7 +496,9 @@ async def test_claim_token_rolls_back_invite_when_session_key_mint_fails(): with ( patch("litellm.proxy.proxy_server.prisma_client", prisma), patch("litellm.proxy.proxy_server.master_key", "sk-test"), - patch("litellm.proxy.proxy_server.general_settings", {}), + patch( # test-quality-ok: claim_onboarding_link reads proxy_server module globals; no injection seam + "litellm.proxy.proxy_server.general_settings", _POLICY_NO_BREACH_CHECK + ), patch( "litellm.proxy.proxy_server.generate_key_helper_fn", new_callable=AsyncMock, @@ -505,3 +520,124 @@ async def test_claim_token_rolls_back_invite_when_session_key_mint_fails(): } assert rollback_kwargs["data"]["accepted_at"] is None assert rollback_kwargs["data"]["is_accepted"] is False + + +# --------------------------------------------------------------------------- +# POST /onboarding/claim_token - password policy +# --------------------------------------------------------------------------- + + +def _hibp_url_for(password: str) -> str: + sha1 = hashlib.sha1(password.encode("utf-8"), usedforsecurity=False).hexdigest().upper() + return f"https://api.pwnedpasswords.com/range/{sha1[:5]}" + + +def _hibp_suffix_for(password: str) -> str: + return hashlib.sha1(password.encode("utf-8"), usedforsecurity=False).hexdigest().upper()[5:] + + +@pytest.mark.asyncio +async def test_claim_token_rejects_short_password_before_consuming_invite(): + """Default policy requires 12 characters; the invite must stay claimable.""" + from litellm.proxy.proxy_server import claim_onboarding_link + + invite = _make_invite(is_accepted=False) + prisma = _make_prisma(invite, _make_user()) + request = _make_claim_request(_make_onboarding_token()) + data = InvitationClaim( + invitation_link="invite-abc", + user_id="user-123", + password="Sh0rt!pw", + ) + + with ( + patch("litellm.proxy.proxy_server.prisma_client", prisma), # test-quality-ok: claim_onboarding_link reads proxy_server module globals; no injection seam + patch("litellm.proxy.proxy_server.master_key", "sk-test"), # test-quality-ok: same as above + patch("litellm.proxy.proxy_server.general_settings", {}), # test-quality-ok: same as above + ): + with pytest.raises(ProxyException) as exc_info: + await claim_onboarding_link(data=data, request=request) + + assert exc_info.value.code == "400" + assert "at least 12 characters" in exc_info.value.message + prisma.db.litellm_invitationlink.update_many.assert_not_called() + prisma.db.litellm_usertable.update.assert_not_called() + + +@pytest.mark.asyncio +@respx.mock +async def test_claim_token_rejects_breached_password_before_consuming_invite(): + """A password found in the HIBP corpus must be rejected and never stored.""" + from litellm.proxy.proxy_server import claim_onboarding_link + + password = "P@ssword123456" + respx.get(_hibp_url_for(password)).mock( + return_value=httpx.Response(200, text=f"{_hibp_suffix_for(password)}:1387") + ) + + invite = _make_invite(is_accepted=False) + prisma = _make_prisma(invite, _make_user()) + request = _make_claim_request(_make_onboarding_token()) + data = InvitationClaim( + invitation_link="invite-abc", + user_id="user-123", + password=password, + ) + + with ( + patch("litellm.proxy.proxy_server.prisma_client", prisma), # test-quality-ok: claim_onboarding_link reads proxy_server module globals; no injection seam + patch("litellm.proxy.proxy_server.master_key", "sk-test"), # test-quality-ok: same as above + patch("litellm.proxy.proxy_server.general_settings", {}), # test-quality-ok: same as above + ): + with pytest.raises(ProxyException) as exc_info: + await claim_onboarding_link(data=data, request=request) + + assert exc_info.value.code == "400" + assert "data breaches" in exc_info.value.message + prisma.db.litellm_invitationlink.update_many.assert_not_called() + prisma.db.litellm_usertable.update.assert_not_called() + + +@pytest.mark.asyncio +@respx.mock +async def test_claim_token_fails_open_when_hibp_unreachable(): + """An HIBP outage must never block onboarding: the claim proceeds.""" + from litellm.proxy.proxy_server import claim_onboarding_link + + password = "NewP@ssw0rd-2026" + respx.get(_hibp_url_for(password)).mock(side_effect=httpx.ConnectError("no route to host")) + + invite = _make_invite(is_accepted=False) + user = _make_user() + prisma = _make_prisma(invite, user) + request = _make_claim_request(_make_onboarding_token()) + data = InvitationClaim( + invitation_link="invite-abc", + user_id="user-123", + password=password, + ) + + with ( + patch("litellm.proxy.proxy_server.prisma_client", prisma), # test-quality-ok: claim_onboarding_link reads proxy_server module globals; no injection seam + patch("litellm.proxy.proxy_server.master_key", "sk-test"), # test-quality-ok: same as above + patch("litellm.proxy.proxy_server.general_settings", {}), # test-quality-ok: same as above + patch("litellm.proxy.proxy_server.premium_user", False), # test-quality-ok: same as above + patch( # test-quality-ok: same as above + "litellm.proxy.proxy_server.generate_key_helper_fn", + new_callable=AsyncMock, + return_value={"token": "sk-generated-key", "user_id": "user-123"}, + ), + patch( # test-quality-ok: same as above + "litellm.proxy.proxy_server.get_custom_url", + return_value="http://localhost:4000/", + ), + patch( # test-quality-ok: same as above + "litellm.proxy.proxy_server.get_disabled_non_admin_personal_key_creation", + return_value=False, + ), + patch("litellm.proxy.proxy_server.get_server_root_path", return_value=""), # test-quality-ok: same as above + ): + result = await claim_onboarding_link(data=data, request=request) + + assert "token" in result + prisma.db.litellm_usertable.update.assert_called_once() diff --git a/tests/test_litellm/proxy/auth/test_password_policy.py b/tests/test_litellm/proxy/auth/test_password_policy.py index f6e7d443907..f9f5025b57f 100644 --- a/tests/test_litellm/proxy/auth/test_password_policy.py +++ b/tests/test_litellm/proxy/auth/test_password_policy.py @@ -2,22 +2,56 @@ Tests for the configurable password-strength policy in `litellm.proxy.auth.password_policy`, enforced on every path that persists a new or changed password for a locally-managed user. + +The breach-check (HIBP) tests inject a real AsyncHTTPHandler wrapping an +httpx.MockTransport, so no network is touched and nothing is monkeypatched. """ +import asyncio +import hashlib + +import httpx import pytest +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler from litellm.proxy._types import ProxyErrorTypes, ProxyException from litellm.proxy.auth.password_policy import ( DEFAULT_MIN_LENGTH, MIN_ALLOWED_LENGTH, PasswordPolicy, get_password_policy, + validate_password_not_breached, validate_password_policy, + validate_passwords_bulk, ) STRONG_PASSWORD = "Str0ng!Passw0rd" +def _sha1_upper(password: str) -> str: + return hashlib.sha1(password.encode("utf-8"), usedforsecurity=False).hexdigest().upper() + + +def _client_with_transport(handler) -> AsyncHTTPHandler: + http_handler = AsyncHTTPHandler() + http_handler.client = httpx.AsyncClient(transport=httpx.MockTransport(handler)) + return http_handler + + +def _client_never_called() -> AsyncHTTPHandler: + def handler(request: httpx.Request) -> httpx.Response: + raise AssertionError(f"unexpected HTTP call to {request.url}") + + return _client_with_transport(handler) + + +def _client_returning(body: str, status_code: int = 200) -> AsyncHTTPHandler: + def handler(request: httpx.Request) -> httpx.Response: + return httpx.Response(status_code, text=body) + + return _client_with_transport(handler) + + def test_get_password_policy_defaults_to_pif_baseline(): policy = get_password_policy({}) assert policy == PasswordPolicy( @@ -134,3 +168,178 @@ def test_validate_password_policy_rejects_unicode_letter_as_special_character(): def test_validate_password_policy_accepts_real_special_character_with_unicode_letters(): """Same base password as the rejection test above, plus an actual symbol.""" assert validate_password_policy("Passwörd1234!", {}) is None + + +@pytest.mark.asyncio +async def test_breach_check_skipped_when_disabled(): + result = await validate_password_not_breached( + password="password12345", # breached in reality, but the check is off + general_settings={"password_policy_check_breached_passwords": False}, + client=_client_never_called(), + ) + assert result is None + + +@pytest.mark.asyncio +async def test_rejects_breached_password(): + password = "correct horse battery staple" + sha1 = _sha1_upper(password) + body = f"AAAA000000000000000000000000000000A:0\r\n{sha1[5:]}:42\r\nBBBB000000000000000000000000000000B:7" + + with pytest.raises(ProxyException) as exc_info: + await validate_password_not_breached(password=password, general_settings={}, client=_client_returning(body)) + assert exc_info.value.code == "400" + assert exc_info.value.type == ProxyErrorTypes.validation_error + assert exc_info.value.param == "password" + assert "data breaches" in exc_info.value.message + + +@pytest.mark.asyncio +async def test_only_sha1_prefix_leaves_the_proxy(): + password = "a very secret password" + sha1 = _sha1_upper(password) + captured_requests: list[httpx.Request] = [] + + def handler(request: httpx.Request) -> httpx.Response: + captured_requests.append(request) + return httpx.Response(200, text="0000000000000000000000000000000000A:1") + + result = await validate_password_not_breached( + password=password, general_settings={}, client=_client_with_transport(handler) + ) + assert result is None + + (request,) = captured_requests + assert request.url.path == f"/range/{sha1[:5]}" + assert sha1[5:] not in str(request.url) + assert request.headers["Add-Padding"] == "true" + assert "litellm" in request.headers["User-Agent"] + + +@pytest.mark.asyncio +async def test_ignores_padding_entries_with_zero_count(): + """HIBP padding entries (requested via Add-Padding) carry count 0 and must + not be treated as breaches when they collide with the password's suffix.""" + password = "a padded-away password" + sha1 = _sha1_upper(password) + + result = await validate_password_not_breached( + password=password, general_settings={}, client=_client_returning(f"{sha1[5:]}:0") + ) + assert result is None + + +@pytest.mark.asyncio +async def test_accepts_password_absent_from_breach_corpus(): + result = await validate_password_not_breached( + password="a genuinely novel password", + general_settings={}, + client=_client_returning("0018A45C4D1DEF81644B54AB7F969B88D65:1\r\n00D4F6E8FA6EECAD2A3AA415EEC418D38EC:2"), + ) + assert result is None + + +@pytest.mark.asyncio +async def test_breach_check_fails_open_on_network_error(): + def handler(request: httpx.Request) -> httpx.Response: + raise httpx.ConnectError("no route to host") + + result = await validate_password_not_breached( + password="password12345", # breached, but HIBP is unreachable + general_settings={}, + client=_client_with_transport(handler), + ) + assert result is None + + +@pytest.mark.asyncio +async def test_breach_check_fails_open_on_http_error_status(): + result = await validate_password_not_breached( + password="password12345", + general_settings={}, + client=_client_returning("service unavailable", status_code=503), + ) + assert result is None + + +@pytest.mark.asyncio +async def test_breach_check_fails_open_on_malformed_response_body(): + result = await validate_password_not_breached( + password="password12345", + general_settings={}, + client=_client_returning(f"{_sha1_upper('password12345')[5:]}:not-a-number"), + ) + assert result is None + + +@pytest.mark.asyncio +async def test_validate_passwords_bulk_screens_concurrently(): + """All HIBP lookups for a batch must be in flight at once: each handler + call stalls until every expected request has arrived, and a handler that + gives up waiting reports the password as breached. Serial awaiting (the + old per-user behavior) leaves each earlier request waiting forever for the + later ones, so every verdict comes back as a breach and the test fails.""" + passwords = ("Uniqu3!Passw0rd-a", "Uniqu3!Passw0rd-b", "Uniqu3!Passw0rd-c") + suffix_by_prefix = {_sha1_upper(p)[:5]: _sha1_upper(p)[5:] for p in passwords} + all_arrived = asyncio.Event() + arrivals: list[str] = [] + + async def handler(request: httpx.Request) -> httpx.Response: + arrivals.append(request.url.path) + if len(arrivals) == len(passwords): + all_arrived.set() + try: + await asyncio.wait_for(all_arrived.wait(), timeout=5) + except TimeoutError: + return httpx.Response(200, text=f"{suffix_by_prefix[request.url.path.rsplit('/', 1)[-1]]}:1") + return httpx.Response(200, text="0000000000000000000000000000000000A:1") + + verdicts = await validate_passwords_bulk(passwords, {}, client=_client_with_transport(handler)) + assert set(arrivals) == {f"/range/{prefix}" for prefix in suffix_by_prefix} + assert all(verdicts[p] is None for p in passwords) + + +@pytest.mark.asyncio +async def test_validate_passwords_bulk_deduplicates_lookups(): + """500 users sharing one password must cost exactly one HIBP lookup.""" + password = "Sh@red-Passw0rd!" + request_count = 0 + + def handler(request: httpx.Request) -> httpx.Response: + nonlocal request_count + request_count += 1 + return httpx.Response(200, text="0000000000000000000000000000000000A:1") + + verdicts = await validate_passwords_bulk((password,) * 500, {}, client=_client_with_transport(handler)) + assert request_count == 1 + assert verdicts == {password: None} + + +@pytest.mark.asyncio +async def test_validate_passwords_bulk_mixed_verdicts(): + """Weak passwords are rejected without an HIBP lookup; breached ones get + the breach error; acceptable ones map to None.""" + breached = "Br3ached!Passw0rd" + clean = "Cl3an!!Passw0rd42" + weak = "short1!" + breached_sha1 = _sha1_upper(breached) + looked_up_prefixes: list[str] = [] + + def handler(request: httpx.Request) -> httpx.Response: + looked_up_prefixes.append(request.url.path.rsplit("/", 1)[-1]) + if request.url.path == f"/range/{breached_sha1[:5]}": + return httpx.Response(200, text=f"{breached_sha1[5:]}:99") + return httpx.Response(200, text="0000000000000000000000000000000000A:1") + + verdicts = await validate_passwords_bulk((breached, clean, weak), {}, client=_client_with_transport(handler)) + assert _sha1_upper(weak)[:5] not in looked_up_prefixes + assert verdicts[clean] is None + assert "data breaches" in verdicts[breached].message + assert verdicts[breached].code == "400" + assert "12 characters" in verdicts[weak].message + + +@pytest.mark.asyncio +async def test_validate_passwords_bulk_empty_batch_makes_no_lookups(): + verdicts = await validate_passwords_bulk((), {}, client=_client_never_called()) + assert verdicts == {} diff --git a/tests/test_litellm/proxy/auth/test_route_checks.py b/tests/test_litellm/proxy/auth/test_route_checks.py index d1148ccf634..8ffe89710b2 100644 --- a/tests/test_litellm/proxy/auth/test_route_checks.py +++ b/tests/test_litellm/proxy/auth/test_route_checks.py @@ -3,7 +3,6 @@ from datetime import datetime from typing import Final from unittest.mock import MagicMock, patch - import pytest from fastapi import HTTPException, Request @@ -39,7 +38,7 @@ def test_non_admin_config_update_route_rejected(): request.query_params = {} # Test that calling /config/update route raises HTTPException with 403 status - with pytest.raises(Exception, match='Only proxy admin can be used to generate, delete, update') as exc_info: + with pytest.raises(Exception, match="Only proxy admin can be used to generate, delete, update") as exc_info: RouteChecks.non_proxy_admin_allowed_routes_check( user_obj=user_obj, _user_role=LitellmUserRoles.INTERNAL_USER.value, @@ -50,9 +49,8 @@ def test_non_admin_config_update_route_rejected(): ) # Verify the exception is raised with the correct message - assert ( - "Only proxy admin can be used to generate, delete, update info for new keys/users/teams" - in str(exc_info.value) + assert "Only proxy admin can be used to generate, delete, update info for new keys/users/teams" in str( + exc_info.value ) assert "Route=/config/update" in str(exc_info.value) assert "Your role=internal_user" in str(exc_info.value) @@ -158,7 +156,7 @@ def test_user_banner_update_rejected_for_non_admin(): request = MagicMock(spec=Request) request.query_params = {} - with pytest.raises(Exception, match='Only proxy admin can be used to generate, delete, update') as exc_info: + with pytest.raises(Exception, match="Only proxy admin can be used to generate, delete, update") as exc_info: RouteChecks.non_proxy_admin_allowed_routes_check( user_obj=user_obj, _user_role=LitellmUserRoles.INTERNAL_USER.value, @@ -735,9 +733,7 @@ def test_virtual_key_llm_api_route_includes_passthrough_prefix(route): valid_token = UserAPIKeyAuth(user_id="test_user", allowed_routes=["llm_api_routes"]) - result = RouteChecks.is_virtual_key_allowed_to_call_route( - route=route, valid_token=valid_token - ) + result = RouteChecks.is_virtual_key_allowed_to_call_route(route=route, valid_token=valid_token) assert result is True @@ -762,9 +758,7 @@ def test_virtual_key_llm_api_routes_allows_google_routes(route): valid_token = UserAPIKeyAuth(user_id="test_user", allowed_routes=["llm_api_routes"]) - result = RouteChecks.is_virtual_key_allowed_to_call_route( - route=route, valid_token=valid_token - ) + result = RouteChecks.is_virtual_key_allowed_to_call_route(route=route, valid_token=valid_token) assert result is True @@ -834,18 +828,14 @@ def test_google_routes_with_dynamic_model_names_accessible_to_internal_users(): ) # If no exception is raised, the test passes except Exception as e: - pytest.fail( - f"Internal user should be able to access Google generateContent route. Got error: {str(e)}" - ) + pytest.fail(f"Internal user should be able to access Google generateContent route. Got error: {e!s}") def test_virtual_key_allowed_routes_with_multiple_litellm_routes_member_names(): """Test that virtual key works with multiple LiteLLMRoutes member names in allowed_routes""" # Create a UserAPIKeyAuth with multiple LiteLLMRoutes member names - valid_token = UserAPIKeyAuth( - user_id="test_user", allowed_routes=["openai_routes", "info_routes"] - ) + valid_token = UserAPIKeyAuth(user_id="test_user", allowed_routes=["openai_routes", "info_routes"]) # Test that routes from both groups are allowed result1 = RouteChecks.is_virtual_key_allowed_to_call_route( @@ -899,13 +889,9 @@ def test_virtual_key_allowed_routes_with_no_member_names_only_explicit(): ) # Test that explicit routes are allowed - result1 = RouteChecks.is_virtual_key_allowed_to_call_route( - route="/chat/completions", valid_token=valid_token - ) + result1 = RouteChecks.is_virtual_key_allowed_to_call_route(route="/chat/completions", valid_token=valid_token) - result2 = RouteChecks.is_virtual_key_allowed_to_call_route( - route="/custom/route", valid_token=valid_token - ) + result2 = RouteChecks.is_virtual_key_allowed_to_call_route(route="/custom/route", valid_token=valid_token) assert result1 is True assert result2 is True @@ -1265,6 +1251,122 @@ def test_non_proxy_admin_allows_auth_pass_through_with_team_allowlist(): ) +@pytest.mark.parametrize( + "route, team_allowed_routes, expected", + [ + ("/model-host/v1/extractor/predict", ["/model-host/*"], True), + ("/model-host", ["/model-host/*"], False), + ("/model-host/v1/extractor", ["/model-host/v1/extractor"], True), + ("/model-host/v1/extractor/predict", ["/model-host/v1/extractor"], False), + ("/other/v1/extractor", ["/model-host/*"], False), + ("/model-host/v1/extractor", ["openai_routes", "llm_api_routes", "mapped_pass_through_routes"], False), + ("/model-host/v1/extractor", ["*"], False), + ("/model-host/v1/extractor", ["/*"], False), + ("/model-host/v1/extractor", [], False), + ], +) +def test_jwt_team_routes_grant_pass_through_only_for_explicit_paths(route, team_allowed_routes, expected): + assert ( + RouteChecks.jwt_team_routes_grant_pass_through(route=route, team_allowed_routes=team_allowed_routes) + is expected + ) + + +_AUTH_ENFORCED_MODEL_HOST_ROUTES: Final = { + "test-uuid-1:subpath:/model-host/v1/extractor:GET,POST": { + "endpoint_id": "test-uuid-1", + "path": "/model-host/v1/extractor", + "type": "subpath", + "auth": True, + }, +} + + +def _jwt_handler_with_team_allowed_routes(team_allowed_routes: list[str]): + from litellm.proxy._types import LiteLLM_JWTAuth + from litellm.proxy.auth.handle_jwt import JWTHandler + + jwt_handler: Final = JWTHandler() + jwt_handler.litellm_jwtauth = LiteLLM_JWTAuth(team_allowed_routes=team_allowed_routes) + return jwt_handler + + +def _check_model_host_route_as(valid_token: UserAPIKeyAuth, team_allowed_routes: list[str]) -> None: + with ( + patch( + "litellm.proxy.pass_through_endpoints.pass_through_endpoints._registered_pass_through_routes", + _AUTH_ENFORCED_MODEL_HOST_ROUTES, + ), + patch("litellm.proxy.utils.get_server_root_path", return_value="/"), + patch( + "litellm.proxy.proxy_server.jwt_handler", + _jwt_handler_with_team_allowed_routes(team_allowed_routes), + ), + ): + RouteChecks.non_proxy_admin_allowed_routes_check( + user_obj=None, + _user_role=LitellmUserRoles.INTERNAL_USER.value, + route="/model-host/v1/extractor/predict", + request=MagicMock(spec=Request), + valid_token=valid_token, + request_data={}, + ) + + +def test_non_proxy_admin_allows_auth_pass_through_for_jwt_team_allowed_routes_wildcard(): + jwt_token: Final = UserAPIKeyAuth( + user_id="test_user", + user_role=LitellmUserRoles.INTERNAL_USER.value, + team_id="team-a", + jwt_claims={"sub": "test_user"}, + ) + + _check_model_host_route_as(jwt_token, team_allowed_routes=["openai_routes", "/model-host/*"]) + + +def test_non_proxy_admin_denies_auth_pass_through_for_jwt_when_only_route_groups_configured(): + jwt_token: Final = UserAPIKeyAuth( + user_id="test_user", + user_role=LitellmUserRoles.INTERNAL_USER.value, + team_id="team-a", + jwt_claims={"sub": "test_user"}, + ) + + with pytest.raises(HTTPException) as exc_info: + _check_model_host_route_as(jwt_token, team_allowed_routes=["openai_routes", "mapped_pass_through_routes"]) + + assert exc_info.value.status_code == 403, exc_info.value.detail + assert "allowed_passthrough_routes" in exc_info.value.detail + + +@pytest.mark.parametrize( + "api_key, team_id, jwt_claims", + [ + ("sk-test-key", "team-a", None), + ("sk-test-key", "team-a", {"sub": "test_user"}), + (None, "team-a", None), + (None, None, {"sub": "test_user"}), + ], + ids=["plain_virtual_key", "jwt_mapped_virtual_key", "keyless_non_jwt_caller", "jwt_without_team"], +) +def test_non_proxy_admin_jwt_team_allowed_routes_grant_pass_through_only_to_jwt_team_callers( + api_key, team_id, jwt_claims +): + caller: Final = UserAPIKeyAuth( + api_key=api_key, + user_id="test_user", + user_role=LitellmUserRoles.INTERNAL_USER.value, + team_id=team_id, + jwt_claims=jwt_claims, + ) + + with pytest.raises(HTTPException) as exc_info: + _check_model_host_route_as(caller, team_allowed_routes=["openai_routes", "/model-host/*"]) + + assert exc_info.value.status_code == 403, exc_info.value.detail + assert "allowed_passthrough_routes" in exc_info.value.detail + + def test_virtual_key_without_llm_api_routes_cannot_access_pass_through(): """ Test that virtual keys without llm_api_routes permission cannot access registered pass-through endpoints. @@ -1303,9 +1405,7 @@ def test_virtual_key_without_llm_api_routes_cannot_access_pass_through(): ) assert exc_info.value.status_code == 403 - assert "Virtual key is not allowed to call this route" in str( - exc_info.value.detail - ) + assert "Virtual key is not allowed to call this route" in str(exc_info.value.detail) def test_check_passthrough_route_access_key_metadata_exact_match(): @@ -1764,9 +1864,7 @@ def test_videos_route_accessible_to_internal_users(): ) # If no exception is raised, the test passes except Exception as e: - pytest.fail( - f"Internal user should be able to access /v1/videos route. Got error: {str(e)}" - ) + pytest.fail(f"Internal user should be able to access /v1/videos route. Got error: {e!s}") def test_videos_route_with_virtual_key_llm_api_routes(): @@ -1788,12 +1886,8 @@ def test_videos_route_with_virtual_key_llm_api_routes(): ] for route in test_routes: - result = RouteChecks.is_virtual_key_allowed_to_call_route( - route=route, valid_token=valid_token - ) - assert ( - result is True - ), f"Virtual key with llm_api_routes should be able to access {route}" + result = RouteChecks.is_virtual_key_allowed_to_call_route(route=route, valid_token=valid_token) + assert result is True, f"Virtual key with llm_api_routes should be able to access {route}" def test_non_proxy_admin_wildcard_allowed_routes(): @@ -1864,9 +1958,7 @@ def test_proxy_admin_viewer_can_access_global_spend_tags(): ) # If no exception is raised, the test passes except Exception as e: - pytest.fail( - f"proxy_admin_viewer should be able to access /global/spend/tags route. Got error: {str(e)}" - ) + pytest.fail(f"proxy_admin_viewer should be able to access /global/spend/tags route. Got error: {e!s}") # Routes returning proxy-wide spend across every team / customer / api_key. @@ -1894,7 +1986,7 @@ def test_internal_user_blocked_from_global_spend_routes(route): request = MagicMock(spec=Request) request.query_params = {} - with pytest.raises(Exception, match='Only proxy admin can be used to generate, delete, update') as exc_info: + with pytest.raises(Exception, match="Only proxy admin can be used to generate, delete, update") as exc_info: RouteChecks.non_proxy_admin_allowed_routes_check( user_obj=user_obj, _user_role=LitellmUserRoles.INTERNAL_USER.value, @@ -1923,7 +2015,7 @@ def test_internal_user_view_only_blocked_from_global_spend_routes(route): request = MagicMock(spec=Request) request.query_params = {} - with pytest.raises(Exception, match='Only proxy admin can be used to generate, delete, update') as exc_info: + with pytest.raises(Exception, match="Only proxy admin can be used to generate, delete, update") as exc_info: RouteChecks.non_proxy_admin_allowed_routes_check( user_obj=user_obj, _user_role=LitellmUserRoles.INTERNAL_USER_VIEW_ONLY.value, @@ -2025,9 +2117,7 @@ def test_proxy_admin_viewer_can_access_audit_logs(route): request_data={}, ) except Exception as e: - pytest.fail( - f"proxy_admin_viewer should be able to access {route} route. Got error: {str(e)}" - ) + pytest.fail(f"proxy_admin_viewer should be able to access {route} route. Got error: {e!s}") # ── Admin Viewer parity: Logs page endpoints ────────────────────────────────── @@ -2090,9 +2180,7 @@ def test_proxy_admin_viewer_can_access_logs_page_endpoints(route): request_data={}, ) except Exception as e: - pytest.fail( - f"proxy_admin_viewer should be able to access {route}. Got error: {str(e)}" - ) + pytest.fail(f"proxy_admin_viewer should be able to access {route}. Got error: {e!s}") @pytest.mark.parametrize( @@ -2202,7 +2290,7 @@ def test_internal_user_blocked_from_admin_viewer_logs_routes(route): if route not in INTERNAL_USER_BLOCKED_SUBSET: return - with pytest.raises(Exception, match='Only proxy admin can be used to generate, delete, update') as exc_info: + with pytest.raises(Exception, match="Only proxy admin can be used to generate, delete, update") as exc_info: RouteChecks.non_proxy_admin_allowed_routes_check( user_obj=user_obj, _user_role=LitellmUserRoles.INTERNAL_USER.value, @@ -2278,9 +2366,7 @@ def test_proxy_admin_viewer_can_access_settings_read_endpoints(route): request_data={}, ) except Exception as e: - pytest.fail( - f"proxy_admin_viewer should be able to access {route}. Got error: {str(e)}" - ) + pytest.fail(f"proxy_admin_viewer should be able to access {route}. Got error: {e!s}") # ── Admin Viewer parity: default-allow GET semantics ───────────────────────── @@ -2479,9 +2565,7 @@ class TestModelsRouteExemptFromDisableLLMEndpoints: ) local_file = os.path.abspath(local_file) - spec = importlib.util.spec_from_file_location( - "local_enterprise_route_checks", local_file - ) + spec = importlib.util.spec_from_file_location("local_enterprise_route_checks", local_file) mod = importlib.util.module_from_spec(spec) spec.loader.exec_module(mod) return mod.EnterpriseRouteChecks @@ -2492,9 +2576,7 @@ class TestModelsRouteExemptFromDisableLLMEndpoints: EnterpriseRouteChecks = self._get_enterprise_route_checks() with ( - patch.object( - EnterpriseRouteChecks, "is_llm_api_route_disabled", return_value=True - ), + patch.object(EnterpriseRouteChecks, "is_llm_api_route_disabled", return_value=True), patch.object( EnterpriseRouteChecks, "is_management_routes_disabled", @@ -2510,9 +2592,7 @@ class TestModelsRouteExemptFromDisableLLMEndpoints: EnterpriseRouteChecks = self._get_enterprise_route_checks() with ( - patch.object( - EnterpriseRouteChecks, "is_llm_api_route_disabled", return_value=True - ), + patch.object(EnterpriseRouteChecks, "is_llm_api_route_disabled", return_value=True), patch.object( EnterpriseRouteChecks, "is_management_routes_disabled", @@ -2528,9 +2608,7 @@ class TestModelsRouteExemptFromDisableLLMEndpoints: EnterpriseRouteChecks = self._get_enterprise_route_checks() with ( - patch.object( - EnterpriseRouteChecks, "is_llm_api_route_disabled", return_value=True - ), + patch.object(EnterpriseRouteChecks, "is_llm_api_route_disabled", return_value=True), patch.object( EnterpriseRouteChecks, "is_management_routes_disabled", @@ -2541,9 +2619,7 @@ class TestModelsRouteExemptFromDisableLLMEndpoints: EnterpriseRouteChecks.should_call_route("/v1/chat/completions") assert exc_info.value.status_code == 403 - assert "LLM API routes are disabled for this instance." in str( - exc_info.value.detail - ) + assert "LLM API routes are disabled for this instance." in str(exc_info.value.detail) @patch("litellm.proxy.proxy_server.premium_user", True) def test_should_embeddings_still_blocked_when_llm_api_disabled(self): @@ -2551,9 +2627,7 @@ class TestModelsRouteExemptFromDisableLLMEndpoints: EnterpriseRouteChecks = self._get_enterprise_route_checks() with ( - patch.object( - EnterpriseRouteChecks, "is_llm_api_route_disabled", return_value=True - ), + patch.object(EnterpriseRouteChecks, "is_llm_api_route_disabled", return_value=True), patch.object( EnterpriseRouteChecks, "is_management_routes_disabled", @@ -2571,9 +2645,7 @@ class TestModelsRouteExemptFromDisableLLMEndpoints: EnterpriseRouteChecks = self._get_enterprise_route_checks() with ( - patch.object( - EnterpriseRouteChecks, "is_llm_api_route_disabled", return_value=False - ), + patch.object(EnterpriseRouteChecks, "is_llm_api_route_disabled", return_value=False), patch.object( EnterpriseRouteChecks, "is_management_routes_disabled", @@ -2592,9 +2664,7 @@ def test_route_in_additional_public_routes_wildcard_match(): from litellm.proxy.auth.auth_utils import route_in_additonal_public_routes with ( - patch( - "litellm.proxy.proxy_server.general_settings", {"public_routes": ["/api/*"]} - ), + patch("litellm.proxy.proxy_server.general_settings", {"public_routes": ["/api/*"]}), patch("litellm.proxy.proxy_server.premium_user", True), ): # Wildcard should match subpaths @@ -2686,7 +2756,7 @@ def test_non_admin_non_team_admin_cannot_access_config_update_but_can_attempt_re ) # /config/update is still blocked - with pytest.raises(Exception, match='Only proxy admin can be used to generate, delete, update') as exc_info: + with pytest.raises(Exception, match="Only proxy admin can be used to generate, delete, update") as exc_info: RouteChecks.non_proxy_admin_allowed_routes_check( user_obj=user_obj, _user_role=LitellmUserRoles.INTERNAL_USER.value, @@ -2774,8 +2844,6 @@ def test_available_roles_accessible_to_non_admin_users(user_role): # ── _user_is_org_admin tests ────────────────────────────────────────────────── - - def _make_org_admin_user(org_id: str) -> LiteLLM_UserTable: membership = LiteLLM_OrganizationMembershipTable( user_id="org-admin-user", @@ -2898,9 +2966,7 @@ async def test_add_team_org_context_noop_when_org_id_already_present(): raise AssertionError("must not resolve when organization_id is present") body = {"team_id": "team-1", "organization_id": "org-explicit"} - out = await add_team_org_context_to_request_body( - route="/team/update", request_body=body, fetch_team_org_id=fetch - ) + out = await add_team_org_context_to_request_body(route="/team/update", request_body=body, fetch_team_org_id=fetch) assert out == body @@ -2912,9 +2978,7 @@ async def test_add_team_org_context_noop_for_other_routes(): raise AssertionError("must not resolve for a non-opted-in route") body = {"team_id": "team-1"} - out = await add_team_org_context_to_request_body( - route="/team/delete", request_body=body, fetch_team_org_id=fetch - ) + out = await add_team_org_context_to_request_body(route="/team/delete", request_body=body, fetch_team_org_id=fetch) assert out == body @@ -2927,9 +2991,7 @@ async def test_add_team_org_context_noop_when_team_has_no_org(): return None body = {"team_id": "team-1"} - out = await add_team_org_context_to_request_body( - route="/team/update", request_body=body, fetch_team_org_id=fetch - ) + out = await add_team_org_context_to_request_body(route="/team/update", request_body=body, fetch_team_org_id=fetch) assert out == body @@ -3200,9 +3262,7 @@ async def test_initialize_pass_through_registers_wildcard_for_auth_subpath(): # Removing the endpoint should clean up openai_routes # remove_endpoint_routes takes endpoint_id (UUID portion of # the route key "{id}:exact:{path}:{methods}") - registered = ( - InitPassThroughEndpointHelpers.get_all_registered_pass_through_routes() - ) + registered = InitPassThroughEndpointHelpers.get_all_registered_pass_through_routes() endpoint_ids = {k.split(":")[0] for k in registered} for eid in endpoint_ids: InitPassThroughEndpointHelpers.remove_endpoint_routes(eid) @@ -3212,9 +3272,7 @@ async def test_initialize_pass_through_registers_wildcard_for_auth_subpath(): LiteLLMRoutes.openai_routes.value[:] = original_routes # Clean up any routes registered during this test to avoid # polluting the module-level _registered_pass_through_routes - registered = ( - InitPassThroughEndpointHelpers.get_all_registered_pass_through_routes() - ) + registered = InitPassThroughEndpointHelpers.get_all_registered_pass_through_routes() for k in registered: InitPassThroughEndpointHelpers.remove_endpoint_routes(k.split(":")[0]) @@ -3245,8 +3303,7 @@ def test_provider_name_substring_not_classified_as_llm_route(route): from litellm.proxy.auth.route_checks import RouteChecks assert RouteChecks.is_llm_api_route(route=route) is False, ( - f"{route!r} should NOT be classified as an LLM API route — " - "provider-name substring match bypass" + f"{route!r} should NOT be classified as an LLM API route — provider-name substring match bypass" ) @@ -3268,9 +3325,7 @@ def test_legitimate_passthrough_routes_still_classified_as_llm_route(route): """Legitimate passthrough routes must still pass is_llm_api_route.""" from litellm.proxy.auth.route_checks import RouteChecks - assert ( - RouteChecks.is_llm_api_route(route=route) is True - ), f"{route!r} should be classified as an LLM API route" + assert RouteChecks.is_llm_api_route(route=route) is True, f"{route!r} should be classified as an LLM API route" @pytest.mark.parametrize( @@ -3328,7 +3383,7 @@ def test_internal_user_blocked_from_search_tool_writes(route): request = MagicMock(spec=Request) request.query_params = {} - with pytest.raises(Exception, match='Only proxy admin can be used to generate, delete, update') as exc_info: + with pytest.raises(Exception, match="Only proxy admin can be used to generate, delete, update") as exc_info: RouteChecks.non_proxy_admin_allowed_routes_check( user_obj=user_obj, _user_role=LitellmUserRoles.INTERNAL_USER.value, @@ -3704,12 +3759,7 @@ def test_agent_inference_routes_stay_llm_api(route): def test_agent_routes_union_still_covers_both_halves(route): """Keys configured with allowed_routes=["agent_routes"] must keep both halves.""" - assert ( - RouteChecks.check_route_access( - route=route, allowed_routes=LiteLLMRoutes.agent_routes.value - ) - is True - ) + assert RouteChecks.check_route_access(route=route, allowed_routes=LiteLLMRoutes.agent_routes.value) is True @pytest.mark.parametrize("route", AGENT_MANAGEMENT_ROUTES) @@ -3763,6 +3813,136 @@ def test_agent_registry_route_gate_open_to_non_admin_roles(user_role, method, ro valid_token=valid_token, request_data={}, ) + + +def test_proxy_admin_viewer_user_update_password_param_rejected(): + """The self-service /user/update password carve-out is closed: non-admins + change their own password through /user/password/change, which verifies + the current password. Admin password sets don't pass through this check.""" + with pytest.raises(HTTPException) as exc_info: + RouteChecks._check_proxy_admin_viewer_access( + route="/user/update", + _user_role=LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY.value, + request_data={"password": "hunter2hunter2"}, + ) + assert exc_info.value.status_code == 403 + assert "password" in str(exc_info.value.detail) + + +def test_proxy_admin_viewer_user_update_user_email_still_allowed(): + request = MagicMock(spec=Request) + request.method = "POST" + + allowed = RouteChecks._check_proxy_admin_viewer_access( + route="/user/update", + _user_role=LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY.value, + request_data={"user_email": "viewer@example.com"}, + request=request, + ) + + assert allowed is None + + +def test_proxy_admin_viewer_can_change_own_password(): + request = MagicMock(spec=Request) + request.method = "POST" + + allowed = RouteChecks._check_proxy_admin_viewer_access( + route="/user/password/change", + _user_role=LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY.value, + request_data={"current_password": "a", "new_password": "b"}, + request=request, + ) + + assert allowed is None + + +@pytest.mark.parametrize( + "user_role", + [ + LitellmUserRoles.INTERNAL_USER.value, + LitellmUserRoles.INTERNAL_USER_VIEW_ONLY.value, + ], +) +def test_non_admin_roles_can_change_own_password(user_role): + valid_token = UserAPIKeyAuth(user_id="test_user", user_role=user_role) + request = MagicMock(spec=Request) + request.method = "POST" + request.query_params = {} + + allowed = RouteChecks.non_proxy_admin_allowed_routes_check( + user_obj=LiteLLM_UserTable(user_id="test_user", user_role=user_role), + _user_role=user_role, + route="/user/password/change", + request=request, + valid_token=valid_token, + request_data={"current_password": "a", "new_password": "b"}, + ) + + assert allowed is None + + +def _password_reset_session_token() -> UserAPIKeyAuth: + """The UI session key `authenticate_user` mints for a user flagged + `password_reset_required`.""" + return UserAPIKeyAuth( + user_id="flagged_user", + allowed_routes=["/user/password/change"], + metadata={"password_reset_required": True}, + ) + + +def test_password_reset_session_can_reach_change_password(): + result = RouteChecks.is_virtual_key_allowed_to_call_route( + route="/user/password/change", + valid_token=_password_reset_session_token(), + ) + + assert result is True + + +@pytest.mark.parametrize( + "route", + [ + "/user/info", + "/key/generate", + "/user/update", + "/chat/completions", + ], +) +def test_password_reset_session_is_blocked_everywhere_else_with_reset_message(route): + """Server-side enforcement of the forced reset: a script that logs in via + /v2/login and drives the management API with the session key must get a 403 + naming the remediation endpoint, on every route but the change-password one.""" + with pytest.raises(HTTPException) as exc_info: + RouteChecks.is_virtual_key_allowed_to_call_route( + route=route, + valid_token=_password_reset_session_token(), + ) + + assert exc_info.value.status_code == 403 + assert "password must be changed" in str(exc_info.value.detail) + assert "/user/password/change" in str(exc_info.value.detail) + + +def test_restricted_key_without_reset_marker_keeps_generic_message(): + """The reset-specific 403 must not leak onto ordinary allowed_routes keys.""" + valid_token = UserAPIKeyAuth( + user_id="test_user", + allowed_routes=["/chat/completions"], + ) + + with pytest.raises(HTTPException) as exc_info: + RouteChecks.is_virtual_key_allowed_to_call_route( + route="/user/info", + valid_token=valid_token, + ) + + assert exc_info.value.status_code == 403 + assert "password must be changed" not in str(exc_info.value.detail) + assert "not allowed to call this route" in str(exc_info.value.detail) + + TEAM_CALLBACK_ROUTES = ( "/team/06bda574-5ca9-43d3-beb8-3b23c2f17112/callback", "/team/06bda574-5ca9-43d3-beb8-3b23c2f17112/callback/langfuse", diff --git a/tests/test_litellm/proxy/auth/test_user_api_key_auth.py b/tests/test_litellm/proxy/auth/test_user_api_key_auth.py index da36071a5b4..f03abe8f124 100644 --- a/tests/test_litellm/proxy/auth/test_user_api_key_auth.py +++ b/tests/test_litellm/proxy/auth/test_user_api_key_auth.py @@ -59,6 +59,7 @@ from litellm.proxy.auth.user_api_key_auth import ( _user_api_key_auth_builder, get_api_key, user_api_key_auth, + user_api_key_auth_websocket_for_model, ) from litellm.proxy.spend_tracking.carried_budget_state import carried_budget_metadata @@ -9043,3 +9044,90 @@ async def test_router_settings_model_group_alias_authorizes_target_for_team(monk await authorize() assert (await request.json())["model"] == target assert get_client_requested_model(request) == "AgentX-LLM" + + +@pytest.mark.asyncio +async def test_reserve_budget_after_common_checks_hands_the_reservation_to_the_request_state(): + from fastapi import Request + + request = Request(scope={"type": "http"}) + user_api_key_auth_obj = UserAPIKeyAuth(token="test_token") + reservation = {"reserved_cost": 0.5, "entries": [], "finalized": False, "callback_bound": False} + + with patch( + "litellm.proxy.spend_tracking.budget_reservation.reserve_budget_for_request", + new=AsyncMock(return_value=reservation), + ): + await _reserve_budget_after_common_checks( + user_api_key_auth_obj=user_api_key_auth_obj, + request_data={"model": "gpt-4o"}, + route="/v1/batches/batch_123/cancel", + llm_router=None, + team_object=None, + user_object=None, + prisma_client=None, + user_api_key_cache=MagicMock(), + proxy_logging_obj=MagicMock(), + skip_budget_checks=False, + general_settings={}, + request=request, + ) + + assert user_api_key_auth_obj.budget_reservation is reservation + assert request.state.budget_reservation is reservation + assert request.scope["state"]["budget_reservation"] is reservation + + +@pytest.mark.asyncio +async def test_reserve_budget_after_common_checks_clears_the_request_state_when_budget_checks_skip(): + from fastapi import Request + + request = Request(scope={"type": "http", "state": {"budget_reservation": {"reserved_cost": 0.5}}}) + + await _reserve_budget_after_common_checks( + user_api_key_auth_obj=UserAPIKeyAuth(token="test_token"), + request_data={"model": "free-model"}, + route="/v1/chat/completions", + llm_router=None, + team_object=None, + user_object=None, + prisma_client=None, + user_api_key_cache=MagicMock(), + proxy_logging_obj=MagicMock(), + skip_budget_checks=True, + general_settings={}, + request=request, + ) + + assert request.state.budget_reservation is None + + +@pytest.mark.asyncio +async def test_websocket_auth_hands_the_reservation_to_the_socket_state(): + from fastapi import WebSocket + + reservation = {"reserved_cost": 0.5, "entries": [], "finalized": False, "callback_bound": False} + websocket = WebSocket( + scope={ + "type": "websocket", + "path": "/v1/realtime", + "headers": [(b"authorization", b"Bearer sk-1234")], + "query_string": b"model=gpt-realtime", + }, + receive=AsyncMock(), + send=AsyncMock(), + ) + + async def auth_that_reserves(request, api_key): + request.state.budget_reservation = reservation + return UserAPIKeyAuth(token="hashed", budget_reservation=reservation) + + with patch( + "litellm.proxy.auth.user_api_key_auth.user_api_key_auth", + new=AsyncMock(side_effect=auth_that_reserves), + ): + result = await user_api_key_auth_websocket_for_model(websocket, model="gpt-realtime") + + assert result.budget_reservation == reservation + assert websocket.state.budget_reservation is reservation + assert websocket.scope["state"]["budget_reservation"] is reservation diff --git a/tests/test_litellm/proxy/client/cli/test_claude_settings.py b/tests/test_litellm/proxy/client/cli/test_claude_settings.py index a48c64eb4a0..9353c149d15 100644 --- a/tests/test_litellm/proxy/client/cli/test_claude_settings.py +++ b/tests/test_litellm/proxy/client/cli/test_claude_settings.py @@ -5,13 +5,16 @@ import shlex import stat import sys import time +from concurrent.futures import ThreadPoolExecutor, TimeoutError as FutureTimeoutError from pathlib import Path +from threading import Event +from typing import Final from unittest.mock import patch import pytest from click.testing import CliRunner -from litellm.litellm_core_utils.private_json import commit_staged_json +from litellm.litellm_core_utils.private_json import commit_staged_json, write_private_bytes from litellm.proxy.client.cli.commands.claude_settings import ( ANTHROPIC_DEFAULT_MODEL_ENV_KEYS, AUTOROUTE_BACKUP_PATH, @@ -773,7 +776,7 @@ class TestStatusLine: script = tmp_path / "lite" / "statusline.py" command = install_statusline_script(script) - assert script.read_bytes() == pathlib.Path(statusline_script.__file__).read_bytes() + assert script.read_bytes().split(b"\n", 1)[1] == pathlib.Path(statusline_script.__file__).read_bytes() assert shlex.split(command) == [sys.executable, str(script)] assert command == statusline_command(script) assert stat.S_IMODE(script.stat().st_mode) == 0o600 @@ -783,15 +786,13 @@ class TestStatusLine: def test_a_reinstall_replaces_the_script_in_one_step_and_a_refused_one_leaves_the_old_script_whole(self, tmp_path): # Claude Code may be running the script at the moment `lite` reinstalls it; the file it has open # must stay complete, and a reinstall that cannot land must not leave a truncated script behind. - from litellm.proxy.client.cli.commands import statusline_script - script = tmp_path / "lite" / "statusline.py" install_statusline_script(script) - bundled = pathlib.Path(statusline_script.__file__).read_bytes() + bundled = script.read_bytes() with script.open("rb") as running: install_statusline_script(script) assert running.read() == bundled - assert [child.name for child in script.parent.iterdir()] == ["statusline.py"] + assert {child.name for child in script.parent.iterdir()} <= {"statusline.py", "statusline.py.lock"} if os.geteuid() != 0: script.parent.chmod(0o500) @@ -802,6 +803,141 @@ class TestStatusLine: script.parent.chmod(0o700) assert script.read_bytes() == bundled + @pytest.mark.parametrize( + ("installed_version", "older_version"), + (("2.10.0", "2.9.0"), ("2.1.0", "2.1.0rc1"), ("2.1.0rc1", "2.1.0.dev2"), ("2.1.0.post1", "2.1.0")), + ) + def test_an_older_cli_preserves_the_newer_footer( + self, tmp_path: Path, capsys: pytest.CaptureFixture[str], installed_version: str, older_version: str + ) -> None: + script: Final = tmp_path / "statusline.py" + command: Final = install_statusline_script(script, package_version=installed_version) + installed: Final = script.read_bytes() + modified: Final = script.stat().st_mtime_ns + + assert install_statusline_script(script, package_version=older_version) == command + + assert script.read_bytes() == installed + assert script.stat().st_mtime_ns == modified + assert f"Keeping the status line from LiteLLM {installed_version}" in capsys.readouterr().err + + @pytest.mark.parametrize( + "old_header", (b"", b"# litellm-statusline-version: invalid\n", b"# litellm-statusline-version: \xff\n") + ) + def test_a_legacy_or_damaged_version_marker_is_repaired(self, tmp_path: Path, old_header: bytes) -> None: + from litellm.proxy.client.cli.commands import statusline_script + + script: Final = tmp_path / "statusline.py" + script.write_bytes(old_header + b"print('old footer')\n") + + install_statusline_script(script, package_version="2.1.0") + + assert script.read_bytes() == ( + b"# litellm-statusline-version: 2.1.0\n" + Path(statusline_script.__file__).read_bytes() + ) + + @pytest.mark.parametrize("next_version", ("2.1.0", "2.2.0")) + def test_an_equal_or_newer_cli_refreshes_the_footer(self, tmp_path: Path, next_version: str) -> None: + from litellm.proxy.client.cli.commands import statusline_script + + script: Final = tmp_path / "statusline.py" + script.write_bytes(b"# litellm-statusline-version: 2.1.0\nprint('old footer')\n") + + install_statusline_script(script, package_version=next_version) + + assert script.read_bytes() == ( + f"# litellm-statusline-version: {next_version}\n".encode() + Path(statusline_script.__file__).read_bytes() + ) + + def test_configure_keeps_a_newer_footer_while_updating_settings( + self, tmp_path: Path, capsys: pytest.CaptureFixture[str] + ) -> None: + script: Final = tmp_path / "statusline.py" + script.write_bytes(b"# litellm-statusline-version: 999999.0.0\nprint('newer footer')\n") + installed: Final = script.read_bytes() + rig: Final = _Rig(tmp_path, {"theme": "dark"}) + + rig.configure(script_path=script) + + assert script.read_bytes() == installed + assert rig.read()["statusLine"]["command"] == statusline_command(script) + assert rig.read()["env"]["ANTHROPIC_BASE_URL"] == PROXY + assert "Keeping the status line" in capsys.readouterr().err + + @pytest.mark.parametrize("package_version", ("unknown", "", "invalid-version")) + @pytest.mark.parametrize("existing", (None, b"print('legacy footer')\n", b"# litellm-statusline-version: invalid\n")) + def test_an_unknown_cli_version_can_install_and_refresh_an_unversioned_footer( + self, tmp_path: Path, package_version: str, existing: bytes | None + ) -> None: + from litellm.proxy.client.cli.commands import statusline_script + + script: Final = tmp_path / "statusline.py" + if existing is not None: + script.write_bytes(existing) + + assert install_statusline_script(script, package_version=package_version) == statusline_command(script) + assert script.read_bytes() == Path(statusline_script.__file__).read_bytes() + + def test_an_unknown_cli_version_preserves_a_versioned_footer( + self, tmp_path: Path, capsys: pytest.CaptureFixture[str] + ) -> None: + script: Final = tmp_path / "statusline.py" + command: Final = install_statusline_script(script, package_version="2.1.0") + installed: Final = script.read_bytes() + + assert install_statusline_script(script, package_version="unknown") == command + assert script.read_bytes() == installed + assert "Keeping the status line from LiteLLM 2.1.0" in capsys.readouterr().err + + @pytest.mark.parametrize(("first_version", "second_version"), (("2.0", "3.0"), ("3.0", "2.0"))) + def test_overlapping_installs_keep_the_newest_footer( + self, tmp_path: Path, first_version: str, second_version: str + ) -> None: + from litellm.proxy.client.cli.commands import statusline_script + + script: Final = tmp_path / "statusline.py" + first_writing: Final = Event() + release_first: Final = Event() + second_started: Final = Event() + + def paused_write(path: str, data: bytes) -> None: + first_writing.set() + assert release_first.wait(5), "First installer was never released" + write_private_bytes(path, data) + + def second_install() -> str: + second_started.set() + return install_statusline_script(script, package_version=second_version) + + with ThreadPoolExecutor(max_workers=2) as pool: + first: Final = pool.submit(install_statusline_script, script, package_version=first_version, write=paused_write) + try: + assert first_writing.wait(5), "First installer did not reach the write" + second: Final = pool.submit(second_install) + assert second_started.wait(5), "Second installer did not start" + with pytest.raises(FutureTimeoutError): + second.result(timeout=0.5) + finally: + release_first.set() + assert first.result(timeout=5) == statusline_command(script) + assert second.result(timeout=5) == statusline_command(script) + + assert script.read_bytes() == b"# litellm-statusline-version: 3.0\n" + Path(statusline_script.__file__).read_bytes() + + def test_a_failed_install_keeps_the_footer_and_releases_the_lock(self, tmp_path: Path) -> None: + script: Final = tmp_path / "statusline.py" + install_statusline_script(script, package_version="2.0") + installed: Final = script.read_bytes() + + def failed_write(path: str, data: bytes) -> None: + raise OSError("disk full") + + with pytest.raises(ClaudeSettingsError, match="disk full"): + install_statusline_script(script, package_version="3.0", write=failed_write) + assert script.read_bytes() == installed + assert install_statusline_script(script, package_version="3.0") == statusline_command(script) + assert script.read_bytes().startswith(b"# litellm-statusline-version: 3.0\n") + def test_configure_installs_it_and_unconfigure_removes_only_ours(self, tmp_path): rig = _Rig(tmp_path, {"theme": "dark"}) script = tmp_path / "statusline.py" diff --git a/tests/test_litellm/proxy/common_utils/test_error_body_call_id.py b/tests/test_litellm/proxy/common_utils/test_error_body_call_id.py new file mode 100644 index 00000000000..8872b5de397 --- /dev/null +++ b/tests/test_litellm/proxy/common_utils/test_error_body_call_id.py @@ -0,0 +1,35 @@ +import pytest + +from litellm.proxy._types import ConfigGeneralSettings +from litellm.proxy.common_utils.error_body_call_id import error_body_call_id, with_call_id + + +@pytest.mark.parametrize( + "general_settings, call_id, expected", + [ + ({"include_call_id_in_error_body": True}, "call-1", "call-1"), + ({"include_call_id_in_error_body": True}, None, None), + ({"include_call_id_in_error_body": True}, "", None), + ({"include_call_id_in_error_body": False}, "call-1", None), + ({"include_call_id_in_error_body": "true"}, "call-1", None), + ({}, "call-1", None), + ], +) +def test_only_the_boolean_opt_in_with_a_real_id_yields_a_body_call_id(general_settings, call_id, expected): + """The setting is off by default and only a literal true turns it on; without an id + there is nothing to copy, so the body must never get a fabricated one.""" + assert error_body_call_id(general_settings, call_id) == expected + + +def test_with_call_id_appends_the_key_without_touching_the_input(): + error = {"message": "bad input", "type": "invalid_request_error", "param": None, "code": "400"} + + assert with_call_id(error, "call-1") == {**error, "litellm_call_id": "call-1"} + assert with_call_id(error, None) == error + assert "litellm_call_id" not in error + + +def test_the_setting_name_is_a_config_general_settings_field(): + """The yaml key the docs name and the key the runtime reads must be the same field.""" + assert ConfigGeneralSettings.model_validate({"include_call_id_in_error_body": True}).include_call_id_in_error_body + assert ConfigGeneralSettings.model_validate({}).include_call_id_in_error_body is None diff --git a/tests/test_litellm/proxy/common_utils/test_registry_read_through.py b/tests/test_litellm/proxy/common_utils/test_registry_read_through.py index 6f7c20166c5..ca2ff8bcce1 100644 --- a/tests/test_litellm/proxy/common_utils/test_registry_read_through.py +++ b/tests/test_litellm/proxy/common_utils/test_registry_read_through.py @@ -207,7 +207,9 @@ async def test_get_agent_with_read_through_recovers_agent_by_name(clean_agent_re @pytest.mark.asyncio -async def test_get_agent_with_read_through_returns_none_for_unknown_agent(clean_agent_registry, monkeypatch): +async def test_get_agent_with_read_through_returns_none_for_unknown_agent( + clean_agent_registry, fresh_agent_read_through, monkeypatch +): from unittest.mock import AsyncMock, MagicMock import litellm.proxy.proxy_server as proxy_server diff --git a/tests/test_litellm/proxy/conftest.py b/tests/test_litellm/proxy/conftest.py index 8ef5017a952..49dc8d02bdb 100644 --- a/tests/test_litellm/proxy/conftest.py +++ b/tests/test_litellm/proxy/conftest.py @@ -273,3 +273,12 @@ def create_proxy_test_client( # Initialize proxy asyncio.run(initialize(config=config_fp, debug=init_options.get("debug", False))) return TestClient(app) + + +@pytest.fixture +def fresh_agent_read_through(monkeypatch): + from litellm.proxy.common_utils import registry_read_through + + read_through = registry_read_through.RegistryReadThrough(resync=registry_read_through._resync_agents) + monkeypatch.setattr(registry_read_through, "agent_registry_read_through", read_through) + return read_through diff --git a/tests/test_litellm/proxy/db/test_autorouter_session_rollup.py b/tests/test_litellm/proxy/db/test_autorouter_session_rollup.py index acd3dc18b54..c61a489f894 100644 --- a/tests/test_litellm/proxy/db/test_autorouter_session_rollup.py +++ b/tests/test_litellm/proxy/db/test_autorouter_session_rollup.py @@ -56,6 +56,31 @@ def _build(payload: dict | None = None, metadata: dict | None = None): class TestBuildTransaction: + @pytest.mark.parametrize( + "api_key, user_id, included", + [ + ("hashed-key", "canonical-user", True), + ("hashed-key", None, True), + ("hashed-key", "", True), + ("", "canonical-user", True), + ("", None, False), + ("", "", False), + ], + ) + def test_attribution_uses_the_canonical_user_even_without_a_key( + self, api_key: str, user_id: str | None, included: bool + ) -> None: + transaction: Final = _build( + payload=_payload(api_key=api_key, user=user_id), + metadata=_metadata(user="client-user", user_api_key_user_id="metadata-user"), + ) + if not included: + assert transaction is None + return + assert transaction is not None + assert transaction.api_key == api_key + assert transaction.user_id == (user_id or "") + def test_successful_auto_routed_turn_builds_every_field(self): transaction = _build( metadata=_metadata( @@ -205,23 +230,43 @@ class TestBuildTransaction: class _FakeDB: - def __init__(self, failures: "list[Exception] | None" = None, poison_session: str | None = None): + def __init__( + self, + failures: "list[Exception] | None" = None, + poison_session: str | None = None, + poison_user: str | None = None, + commit_then_error_users: frozenset[str] = frozenset(), + ): self.calls: list[tuple] = [] + self.attempts: list[tuple[str, tuple[object, ...]]] = [] self._failures = list(failures or []) self._poison_session = poison_session + self._poison_user = poison_user + self._commit_then_error_users = commit_then_error_users async def execute_raw(self, sql: str, *params: object) -> int: + self.attempts.append((sql, params)) if self._poison_session is not None and params[1] == self._poison_session: raise RuntimeError("index row size exceeds btree maximum") + if self._poison_user is not None and params[19] == self._poison_user: + raise RuntimeError("index row size exceeds btree maximum") if self._failures: raise self._failures.pop(0) self.calls.append((sql, params)) + if params[19] in self._commit_then_error_users: + raise RuntimeError("commit succeeded but acknowledgement was lost") return 1 class _FakeClient: - def __init__(self, failures: "list[Exception] | None" = None, poison_session: str | None = None): - self.db = _FakeDB(failures, poison_session) + def __init__( + self, + failures: "list[Exception] | None" = None, + poison_session: str | None = None, + poison_user: str | None = None, + commit_then_error_users: frozenset[str] = frozenset(), + ): + self.db = _FakeDB(failures, poison_session, poison_user, commit_then_error_users) def _transaction( @@ -229,9 +274,11 @@ def _transaction( at: datetime = datetime(2026, 8, 1, 12, 0, 0), tier: str | None = "medium", baseline_model: str | None = "anthropic/claude-opus-5", + api_key: str = "k1", + user_id: str = "", ) -> AutoRouterTurnTransaction: return AutoRouterTurnTransaction( - api_key="k1", + api_key=api_key, session_id=session_id, router_name="live-auto", router_type="complexity", @@ -247,6 +294,7 @@ def _transaction( cache_touched=False, tier=tier, baseline_model=baseline_model, + user_id=user_id, ) @@ -261,7 +309,7 @@ class TestFlush: def test_params_marshal_in_statement_order(self): client = _FakeClient() - asyncio.run(flush_autorouter_turn_transactions(client, [_transaction()])) + asyncio.run(flush_autorouter_turn_transactions(client, [_transaction(user_id="canonical-user")])) sql, params = client.db.calls[0] assert sql == UPSERT_AUTOROUTER_SESSION_SQL assert params == ( @@ -284,8 +332,65 @@ class TestFlush: 0, 0.0, 0.0, + "canonical-user", ) + def test_a_keys_turns_stay_chronological_when_its_canonical_user_changes(self) -> None: + client: Final = _FakeClient() + earlier: Final = _transaction(user_id="z-user", at=datetime(2026, 8, 1, 12, 0, 0)) + later: Final = _transaction(user_id="a-user", at=datetime(2026, 8, 1, 12, 0, 10)) + asyncio.run(flush_autorouter_turn_transactions(client, [later, earlier])) + assert [(params[5], params[19]) for _, params in client.db.calls] == [ + ("2026-08-01T12:00:00", "z-user"), + ("2026-08-01T12:00:10", "a-user"), + ] + + def test_one_keyless_users_failed_session_does_not_drop_another_users_turn(self) -> None: + client: Final = _FakeClient(poison_user="a-user") + failed: Final = _transaction(api_key="", user_id="a-user") + other: Final = _transaction(api_key="", user_id="b-user", at=datetime(2026, 8, 1, 12, 0, 10)) + asyncio.run(flush_autorouter_turn_transactions(client, [other, failed])) + assert [(params[0], params[1], params[19]) for _, params in client.db.calls] == [("", "s1", "b-user")] + + def test_uncertain_commits_quarantine_only_the_key_and_each_failed_user(self) -> None: + client: Final = _FakeClient(commit_then_error_users=frozenset({"a-failed", "c-failed"})) + turns: Final = tuple( + _transaction(user_id=user, at=datetime(2026, 8, 1, 12, 0, second), api_key=key) + for user, second, key in ( + ("b-healthy", 0, "k1"), + ("a-failed", 1, "k1"), + ("b-healthy", 2, "k1"), + ("c-failed", 3, "k1"), + ("b-healthy", 4, "k1"), + ("d-healthy", 5, "k1"), + ("c-failed", 6, "k1"), + ("d-healthy", 7, "k1"), + ("a-failed", 8, "k1"), + ("", 9, "k1"), + ("z-other", 10, "k2"), + ) + ) + asyncio.run(flush_autorouter_turn_transactions(client, tuple(reversed(turns)))) + + assert client.db.attempts == client.db.calls + assert [ + (params[0], params[19], params[5]) + for sql, params in client.db.calls + if sql == UPSERT_AUTOROUTER_SESSION_SQL + ] == [ + ("k1", "b-healthy", "2026-08-01T12:00:00"), + ("k1", "a-failed", "2026-08-01T12:00:01"), + ("k2", "z-other", "2026-08-01T12:00:10"), + ] + assert [params[19] for _, params in client.db.attempts].count("a-failed") == 1 + assert [params[19] for _, params in client.db.attempts].count("c-failed") == 1 + for user, seconds in (("b-healthy", (2, 4)), ("c-failed", (3,)), ("d-healthy", (5, 7))): + assert [ + (params[0], params[5]) + for sql, params in client.db.calls + if sql != UPSERT_AUTOROUTER_SESSION_SQL and params[19] == user + ] == [("k1", f"2026-08-01T12:00:{second:02d}") for second in seconds] + def test_a_connect_error_retries_the_same_statement(self): client = _FakeClient(failures=[httpx.ConnectError("boom")]) asyncio.run(flush_autorouter_turn_transactions(client, [_transaction()])) diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/unified_guardrails/test_unified_guardrail.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/unified_guardrails/test_unified_guardrail.py index d1d22d0d7c2..c90f88ec110 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/unified_guardrails/test_unified_guardrail.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/unified_guardrails/test_unified_guardrail.py @@ -2,7 +2,7 @@ import logging from types import SimpleNamespace -from typing import Final +from typing import TYPE_CHECKING, Final, Literal import pytest @@ -13,7 +13,7 @@ from litellm.integrations.custom_guardrail import ( log_guardrail_information, ) from litellm.litellm_core_utils.api_route_to_call_types import get_call_types_for_route -from litellm.llms import load_guardrail_translation_mappings +from litellm.llms import discover_guardrail_translation_mappings, load_guardrail_translation_mappings from litellm.llms.base_llm.guardrail_translation.base_translation import BaseTranslation from litellm.llms.base_llm.guardrail_translation.utils import ( effective_skip_system_message_for_guardrail, @@ -41,7 +41,10 @@ from litellm.proxy.guardrails.guardrail_hooks.unified_guardrail.unified_guardrai ) from litellm.types.guardrails import GuardrailEventHooks from litellm.types.llms.openai import ResponsesAPIResponse -from litellm.types.utils import CallTypes, Delta, ModelResponseStream, StreamingChoices +from litellm.types.utils import CallTypes, Delta, GenericGuardrailAPIInputs, ModelResponseStream, StreamingChoices + +if TYPE_CHECKING: + from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj class RecordingGuardrail(CustomGuardrail): @@ -61,6 +64,18 @@ class RecordingGuardrail(CustomGuardrail): return {"texts": inputs.get("texts", [])} +class RewritingGuardrail(RecordingGuardrail): + async def apply_guardrail( + self, + inputs: GenericGuardrailAPIInputs, + request_data: dict, # mutable-ok: CustomGuardrail.apply_guardrail contract + input_type: Literal["request", "response"], + logging_obj: "LiteLLMLoggingObj | None" = None, + ) -> GenericGuardrailAPIInputs: + recorded: Final = await super().apply_guardrail(inputs, request_data, input_type, logging_obj=logging_obj) + return GenericGuardrailAPIInputs(texts=[f"{text} [GUARDRAILED]" for text in recorded["texts"]]) + + class _NoopTranslation(BaseTranslation): """Test translation handler that simply echoes input/output.""" @@ -115,9 +130,7 @@ class TestUnifiedLLMGuardrails: assert msgs[0]["content"] == "sys" def test_effective_skip_respects_per_guardrail_over_global(self, monkeypatch): - monkeypatch.setattr( - litellm, "skip_system_message_in_guardrail", True, raising=False - ) + monkeypatch.setattr(litellm, "skip_system_message_in_guardrail", True, raising=False) class G: skip_system_message_in_guardrail = False @@ -130,21 +143,15 @@ class TestUnifiedLLMGuardrails: assert effective_skip_system_message_for_guardrail(G2()) is True @pytest.mark.asyncio - async def test_openai_handler_skips_system_in_guardrail_inputs( - self, monkeypatch - ): - monkeypatch.setattr( - litellm, "skip_system_message_in_guardrail", True, raising=False - ) + async def test_openai_handler_skips_system_in_guardrail_inputs(self, monkeypatch): + monkeypatch.setattr(litellm, "skip_system_message_in_guardrail", True, raising=False) captured = {} class MockGuardrail: skip_system_message_in_guardrail = None - async def apply_guardrail( - self, inputs, request_data, input_type, logging_obj=None - ): + async def apply_guardrail(self, inputs, request_data, input_type, logging_obj=None): captured["inputs"] = inputs return inputs @@ -169,21 +176,15 @@ class TestUnifiedLLMGuardrails: assert data["messages"][0]["content"] == "secret system" @pytest.mark.asyncio - async def test_openai_handler_per_guardrail_skip_false_overrides_global( - self, monkeypatch - ): - monkeypatch.setattr( - litellm, "skip_system_message_in_guardrail", True, raising=False - ) + async def test_openai_handler_per_guardrail_skip_false_overrides_global(self, monkeypatch): + monkeypatch.setattr(litellm, "skip_system_message_in_guardrail", True, raising=False) captured = {} class MockGuardrail: skip_system_message_in_guardrail = False - async def apply_guardrail( - self, inputs, request_data, input_type, logging_obj=None - ): + async def apply_guardrail(self, inputs, request_data, input_type, logging_obj=None): captured["inputs"] = inputs return inputs @@ -201,10 +202,7 @@ class TestUnifiedLLMGuardrails: ) assert "sys" in captured["inputs"]["texts"] - roles = { - m.get("role") - for m in (captured["inputs"].get("structured_messages") or []) - } + roles = {m.get("role") for m in (captured["inputs"].get("structured_messages") or [])} assert "system" in roles class TestSkipToolMessageForChatCompletions: @@ -229,12 +227,8 @@ class TestUnifiedLLMGuardrails: assert all(m["role"] != "tool" for m in out) assert msgs[2]["content"] == "tool result" - def test_effective_skip_tool_respects_per_guardrail_over_global( - self, monkeypatch - ): - monkeypatch.setattr( - litellm, "skip_tool_message_in_guardrail", True, raising=False - ) + def test_effective_skip_tool_respects_per_guardrail_over_global(self, monkeypatch): + monkeypatch.setattr(litellm, "skip_tool_message_in_guardrail", True, raising=False) class G: skip_tool_message_in_guardrail = False @@ -248,18 +242,14 @@ class TestUnifiedLLMGuardrails: @pytest.mark.asyncio async def test_openai_handler_skips_tool_in_guardrail_inputs(self, monkeypatch): - monkeypatch.setattr( - litellm, "skip_tool_message_in_guardrail", True, raising=False - ) + monkeypatch.setattr(litellm, "skip_tool_message_in_guardrail", True, raising=False) captured = {} class MockGuardrail: skip_tool_message_in_guardrail = None - async def apply_guardrail( - self, inputs, request_data, input_type, logging_obj=None - ): + async def apply_guardrail(self, inputs, request_data, input_type, logging_obj=None): captured["inputs"] = inputs return inputs @@ -299,21 +289,15 @@ class TestUnifiedLLMGuardrails: assert data["messages"][2]["content"] == "secret tool result" @pytest.mark.asyncio - async def test_openai_handler_per_guardrail_skip_tool_false_overrides_global( - self, monkeypatch - ): - monkeypatch.setattr( - litellm, "skip_tool_message_in_guardrail", True, raising=False - ) + async def test_openai_handler_per_guardrail_skip_tool_false_overrides_global(self, monkeypatch): + monkeypatch.setattr(litellm, "skip_tool_message_in_guardrail", True, raising=False) captured = {} class MockGuardrail: skip_tool_message_in_guardrail = False - async def apply_guardrail( - self, inputs, request_data, input_type, logging_obj=None - ): + async def apply_guardrail(self, inputs, request_data, input_type, logging_obj=None): captured["inputs"] = inputs return inputs @@ -331,10 +315,7 @@ class TestUnifiedLLMGuardrails: ) assert "tr" in captured["inputs"]["texts"] - roles = { - m.get("role") - for m in (captured["inputs"].get("structured_messages") or []) - } + roles = {m.get("role") for m in (captured["inputs"].get("structured_messages") or [])} assert "tool" in roles class TestAsyncPreCallHook: @@ -360,6 +341,38 @@ class TestUnifiedLLMGuardrails: assert guardrail.event_history == [GuardrailEventHooks.pre_mcp_call] + @pytest.mark.asyncio + @pytest.mark.parametrize( + "call_type", + ["avideo_generation", "acreate_video", "avideo_remix", "avideo_edit", "avideo_extension"], + ) + async def test_video_routes_scan_prompt_and_keep_rewrite(self, monkeypatch, call_type: str) -> None: + """LIT-6685: /v1/videos dispatches call_type="avideo_generation", which the + hook once swallowed as an unknown CallTypes value and returned unscanned. + Runs against the discovered handler map so the video package must really exist.""" + _patch_translation_mappings(monkeypatch, discover_guardrail_translation_mappings()) + handler = UnifiedLLMGuardrails() + guardrail = RewritingGuardrail() + data = { + "guardrail_to_apply": guardrail, + "model": "veo-3.1-fast", + "prompt": "a paper boat on a stream", + "seconds": "4", + } + + result = await handler.async_pre_call_hook( + user_api_key_dict=UserAPIKeyAuth(api_key="test-key"), + cache=DualCache(), + data=data, + call_type=call_type, + ) + + assert guardrail.event_history == [GuardrailEventHooks.pre_call] + assert [call["inputs"]["texts"] for call in guardrail.apply_calls] == [["a paper boat on a stream"]] + assert guardrail.apply_calls[0]["inputs"]["model"] == "veo-3.1-fast" + assert result["prompt"] == "a paper boat on a stream [GUARDRAILED]" + assert result["seconds"] == "4" + class TestAsyncModerationHook: @pytest.mark.asyncio async def test_uses_mcp_event_type(self): @@ -424,7 +437,9 @@ class TestUnifiedLLMGuardrails: async def process_input_messages(self, data, guardrail_to_apply, litellm_logging_obj=None): # type: ignore[override] return data - async def process_output_response(self, response, guardrail_to_apply, litellm_logging_obj=None, user_api_key_dict=None): # type: ignore[override] + async def process_output_response( + self, response, guardrail_to_apply, litellm_logging_obj=None, user_api_key_dict=None + ): # type: ignore[override] return response async def process_output_streaming_response( @@ -493,9 +508,7 @@ class TestUnifiedLLMGuardrails: response=mock_stream(), request_data=request_data, ): - content = ( - item.choices[0].delta.content if item.choices[0].delta else None - ) + content = item.choices[0].delta.content if item.choices[0].delta else None yielded_contents.append(content) # Every chunk should have non-empty content @@ -546,23 +559,18 @@ class TestUnifiedLLMGuardrails: ], ) @pytest.mark.asyncio - async def test_post_call_scans_output_on_every_registered_alias( - self, request_route: str - ) -> None: + async def test_post_call_scans_output_on_every_registered_alias(self, request_route: str) -> None: handler = UnifiedLLMGuardrails() guardrail = RecordingGuardrail() await handler.async_post_call_success_hook( data={"guardrail_to_apply": guardrail, "model": "gpt-4o"}, - user_api_key_dict=UserAPIKeyAuth( - api_key="test-key", request_route=request_route - ), + user_api_key_dict=UserAPIKeyAuth(api_key="test-key", request_route=request_route), response=self._responses_api_response(), ) assert guardrail.apply_calls, ( - f"guardrail never ran for request_route={request_route!r}; model " - f"output reached the client unscanned" + f"guardrail never ran for request_route={request_route!r}; model output reached the client unscanned" ) assert guardrail.apply_calls[0]["input_type"] == "response" assert guardrail.apply_calls[0]["inputs"]["texts"] == ["Paris"] @@ -592,18 +600,14 @@ class TestUnifiedLLMGuardrails: assert CallTypes.responses in mappings @pytest.mark.asyncio - async def test_unresolvable_route_skips_scanning_and_says_so( - self, caplog: pytest.LogCaptureFixture - ) -> None: + async def test_unresolvable_route_skips_scanning_and_says_so(self, caplog: pytest.LogCaptureFixture) -> None: handler = UnifiedLLMGuardrails() guardrail = RecordingGuardrail() with caplog.at_level(logging.WARNING): result = await handler.async_post_call_success_hook( data={"guardrail_to_apply": guardrail, "model": "gpt-4o"}, - user_api_key_dict=UserAPIKeyAuth( - api_key="test-key", request_route="/cursor/chat/completions" - ), + user_api_key_dict=UserAPIKeyAuth(api_key="test-key", request_route="/cursor/chat/completions"), response=self._responses_api_response(), ) @@ -622,9 +626,7 @@ class TestUnifiedLLMGuardrails: with caplog.at_level(logging.WARNING): await handler.async_post_call_success_hook( data={"guardrail_to_apply": guardrail, "model": "gpt-4o"}, - user_api_key_dict=UserAPIKeyAuth( - api_key="test-key", request_route="/v1/chat/completions" - ), + user_api_key_dict=UserAPIKeyAuth(api_key="test-key", request_route="/v1/chat/completions"), response=self._responses_api_response(), ) @@ -734,15 +736,10 @@ class TestUnifiedLLMGuardrails: assert guardrail.event_history == [GuardrailEventHooks.pre_call] assert len(guardrail.apply_calls) == 1 assert guardrail.apply_calls[0]["input_type"] == "request" - assert ( - "https://arxiv.org/pdf/2201.04234" - in guardrail.apply_calls[0]["inputs"]["texts"] - ) + assert "https://arxiv.org/pdf/2201.04234" in guardrail.apply_calls[0]["inputs"]["texts"] # Data should be returned with document intact - assert ( - result["document"]["document_url"] == "https://arxiv.org/pdf/2201.04234" - ) + assert result["document"]["document_url"] == "https://arxiv.org/pdf/2201.04234" @pytest.mark.asyncio async def test_moderation_hook_invokes_ocr_handler(self): @@ -770,10 +767,7 @@ class TestUnifiedLLMGuardrails: assert guardrail.event_history == [GuardrailEventHooks.during_call] assert len(guardrail.apply_calls) == 1 - assert ( - "https://example.com/scan.png" - in guardrail.apply_calls[0]["inputs"]["texts"] - ) + assert "https://example.com/scan.png" in guardrail.apply_calls[0]["inputs"]["texts"] @pytest.mark.asyncio async def test_post_call_success_hook_guardrails_ocr_output(self): @@ -789,9 +783,7 @@ class TestUnifiedLLMGuardrails: def should_run_guardrail(self, data, event_type): # type: ignore[override] return True - async def apply_guardrail( - self, inputs, request_data, input_type, **kwargs - ): + async def apply_guardrail(self, inputs, request_data, input_type, **kwargs): texts = inputs.get("texts", []) return {"texts": [t.replace("SECRET", "[REDACTED]") for t in texts]} @@ -1538,9 +1530,7 @@ class TestStreamingTransform: # And the redacted text ("SECRET") reached the wire on some non-tool # chunk (i.e. the text terminator). transformed = "".join( - item.choices[0].delta.content or "" - for item in out - if item.choices and not item.choices[0].delta.tool_calls + item.choices[0].delta.content or "" for item in out if item.choices and not item.choices[0].delta.tool_calls ) assert "SECRET" in transformed assert "secret" not in transformed @@ -1685,7 +1675,9 @@ class TestStreamingTransform: _stream_chunk("went home."), ModelResponseStream( choices=[ - StreamingChoices(index=0, delta=Delta(content=None, role="assistant", tool_calls=None), finish_reason=None), + StreamingChoices( + index=0, delta=Delta(content=None, role="assistant", tool_calls=None), finish_reason=None + ), StreamingChoices( index=1, delta=Delta( @@ -1985,9 +1977,7 @@ class TestStreamingHttpErrorFrames: guardrail = _EosHttpBlockingGuardrail() chunks = _anthropic_message_chunks(["hello ", "world"]) - out = await _drive_stream( - UnifiedLLMGuardrails(), guardrail, chunks, request_route="/v1/messages" - ) + out = await _drive_stream(UnifiedLLMGuardrails(), guardrail, chunks, request_route="/v1/messages") raw = b"".join(c for c in out if isinstance(c, bytes)).decode() assert "hello " in raw @@ -2011,9 +2001,7 @@ class TestStreamingHttpErrorFrames: }, ] - out = await _drive_stream( - UnifiedLLMGuardrails(), guardrail, chunks, request_route="/v1/responses" - ) + out = await _drive_stream(UnifiedLLMGuardrails(), guardrail, chunks, request_route="/v1/responses") assert chunks[0] in out and chunks[1] in out assert chunks[2] not in out @@ -2076,9 +2064,7 @@ class TestStreamingGuardrailInformationBucket: for chunk in chunks: yield chunk - user_api_key_dict = UserAPIKeyAuth( - api_key="test-key", user_id="user-1", request_route="/v1/chat/completions" - ) + user_api_key_dict = UserAPIKeyAuth(api_key="test-key", user_id="user-1", request_route="/v1/chat/completions") request_data = {"guardrail_to_apply": guardrail, "model": "gpt-4", "metadata": {}} out = [] @@ -2407,7 +2393,5 @@ class TestTranslationMappingsAreReadLive: assert len(guardrail.apply_calls) == 1 assert not [ - name - for name, value in vars(unified_module).items() - if isinstance(value, dict) and CallTypes.aocr in value + name for name, value in vars(unified_module).items() if isinstance(value, dict) and CallTypes.aocr in value ] diff --git a/tests/test_litellm/proxy/management_endpoints/test_access_group_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_access_group_endpoints.py index d687f8d1c8d..13e57408bd0 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_access_group_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_access_group_endpoints.py @@ -17,7 +17,9 @@ from litellm.proxy._types import ( LitellmUserRoles, UserAPIKeyAuth, ) +from litellm.proxy.agent_endpoints.agent_registry import global_agent_registry from litellm.proxy.proxy_server import app +from litellm.types.agents import AgentResponse def _make_access_group_record( @@ -126,6 +128,7 @@ def client_and_mocks(monkeypatch): mock_agents_table = MagicMock() mock_agents_table.find_many = AsyncMock(return_value=[]) + mock_agents_table.update = AsyncMock(return_value=None) @asynccontextmanager async def mock_tx(): @@ -133,6 +136,7 @@ def client_and_mocks(monkeypatch): litellm_accessgrouptable=mock_access_group_table, litellm_teamtable=mock_team_table, litellm_verificationtoken=mock_key_table, + litellm_agentstable=mock_agents_table, ) yield tx @@ -158,15 +162,9 @@ def client_and_mocks(monkeypatch): mock_proxy_logging = MagicMock() mock_proxy_logging.internal_usage_cache = MagicMock() mock_proxy_logging.internal_usage_cache.dual_cache = MagicMock() - mock_proxy_logging.internal_usage_cache.dual_cache.async_delete_cache = AsyncMock( - return_value=None - ) - mock_proxy_logging.internal_usage_cache.dual_cache.async_get_cache = AsyncMock( - return_value=None - ) - mock_proxy_logging.internal_usage_cache.dual_cache.async_set_cache = AsyncMock( - return_value=None - ) + mock_proxy_logging.internal_usage_cache.dual_cache.async_delete_cache = AsyncMock(return_value=None) + mock_proxy_logging.internal_usage_cache.dual_cache.async_get_cache = AsyncMock(return_value=None) + mock_proxy_logging.internal_usage_cache.dual_cache.async_set_cache = AsyncMock(return_value=None) monkeypatch.setattr(ps, "proxy_logging_obj", mock_proxy_logging) admin_user = UserAPIKeyAuth( @@ -239,9 +237,7 @@ def test_create_access_group_duplicate_name_conflict(client_and_mocks): "unique constraint violation", ], ) -def test_create_access_group_race_condition_returns_409( - client_and_mocks, error_message -): +def test_create_access_group_race_condition_returns_409(client_and_mocks, error_message): """Create race condition: Prisma unique constraint surfaces as 409, not 500.""" client, _, mock_table, *_ = client_and_mocks @@ -288,9 +284,7 @@ def test_create_access_group_500_on_non_constraint_prisma_error(client_and_mocks # Use raise_server_exceptions=False so unhandled exceptions become 500 responses test_client = TestClient(app, raise_server_exceptions=False) - resp = test_client.post( - "/v1/access_group", json={"access_group_name": "test-group"} - ) + resp = test_client.post("/v1/access_group", json={"access_group_name": "test-group"}) assert resp.status_code == 500 @@ -558,9 +552,7 @@ def test_update_access_group_empty_body(client_and_mocks): """Update with empty body succeeds; only updated_by is set.""" client, _, mock_table, *_ = client_and_mocks - existing = _make_access_group_record( - access_group_id="ag-update", access_group_name="unchanged" - ) + existing = _make_access_group_record(access_group_id="ag-update", access_group_name="unchanged") mock_table.find_unique = AsyncMock(return_value=existing) resp = client.put("/v1/access_group/ag-update", json={}) @@ -576,14 +568,10 @@ def test_update_access_group_name_success(client_and_mocks): """Update access_group_name succeeds when new name is unique.""" client, _, mock_table, *_ = client_and_mocks - existing = _make_access_group_record( - access_group_id="ag-update", access_group_name="old-name" - ) + existing = _make_access_group_record(access_group_id="ag-update", access_group_name="old-name") mock_table.find_unique = AsyncMock(return_value=existing) - resp = client.put( - "/v1/access_group/ag-update", json={"access_group_name": "new-name"} - ) + resp = client.put("/v1/access_group/ag-update", json={"access_group_name": "new-name"}) assert resp.status_code == 200 mock_table.update.assert_awaited_once() call_kwargs = mock_table.update.call_args.kwargs @@ -594,19 +582,13 @@ def test_update_access_group_name_duplicate_conflict(client_and_mocks): """Update access_group_name to existing name returns 409 (unique constraint).""" client, _, mock_table, *_ = client_and_mocks - existing = _make_access_group_record( - access_group_id="ag-update", access_group_name="old-name" - ) + existing = _make_access_group_record(access_group_id="ag-update", access_group_name="old-name") mock_table.find_unique = AsyncMock(return_value=existing) mock_table.update = AsyncMock( - side_effect=Exception( - "Unique constraint failed on the fields: (`access_group_name`)" - ) + side_effect=Exception("Unique constraint failed on the fields: (`access_group_name`)") ) - resp = client.put( - "/v1/access_group/ag-update", json={"access_group_name": "taken-name"} - ) + resp = client.put("/v1/access_group/ag-update", json={"access_group_name": "taken-name"}) assert resp.status_code == 409 assert "already exists" in resp.json()["detail"] mock_table.update.assert_awaited_once() @@ -620,21 +602,15 @@ def test_update_access_group_name_duplicate_conflict(client_and_mocks): "unique constraint violation", ], ) -def test_update_access_group_name_unique_constraint_returns_409( - client_and_mocks, error_message -): +def test_update_access_group_name_unique_constraint_returns_409(client_and_mocks, error_message): """Update access_group_name: Prisma unique constraint surfaces as 409.""" client, _, mock_table, *_ = client_and_mocks - existing = _make_access_group_record( - access_group_id="ag-update", access_group_name="old-name" - ) + existing = _make_access_group_record(access_group_id="ag-update", access_group_name="old-name") mock_table.find_unique = AsyncMock(return_value=existing) mock_table.update = AsyncMock(side_effect=Exception(error_message)) - resp = client.put( - "/v1/access_group/ag-update", json={"access_group_name": "race-name"} - ) + resp = client.put("/v1/access_group/ag-update", json={"access_group_name": "race-name"}) assert resp.status_code == 409 assert "already exists" in resp.json()["detail"] @@ -690,9 +666,7 @@ def test_delete_access_group_forbidden_non_admin(client_and_mocks, user_role): def test_delete_access_group_cleans_up_teams_and_keys(client_and_mocks): """Delete removes access_group_id from teams and keys before deleting the group.""" - client, mock_prisma, mock_access_group_table, mock_cache, mock_proxy_logging = ( - client_and_mocks - ) + client, mock_prisma, mock_access_group_table, mock_cache, mock_proxy_logging = client_and_mocks mock_team_table = mock_prisma.db.litellm_teamtable mock_key_table = mock_prisma.db.litellm_verificationtoken @@ -722,10 +696,61 @@ def test_delete_access_group_cleans_up_teams_and_keys(client_and_mocks): where={"token": "key-token-1"}, data={"access_group_ids": []}, ) - mock_access_group_table.delete.assert_awaited_once_with( - where={"access_group_id": "ag-to-delete"} + mock_access_group_table.delete.assert_awaited_once_with(where={"access_group_id": "ag-to-delete"}) + + +def test_delete_access_group_detaches_group_from_agents(client_and_mocks): + """Delete strips the group from every agent that had it attached, so agents are not left + pointing at a group that no longer exists (which would deny them every model, server and agent).""" + client, mock_prisma, mock_access_group_table, _mock_cache, _mock_proxy_logging = client_and_mocks + mock_agents_table = mock_prisma.db.litellm_agentstable + + existing = _make_access_group_record(access_group_id="ag-to-delete") + mock_access_group_table.find_unique = AsyncMock(return_value=existing) + + agent_with_group = MagicMock() + agent_with_group.agent_id = "agent-1" + agent_with_group.access_group_ids = ["ag-keep", "ag-to-delete"] + mock_agents_table.find_many = AsyncMock(return_value=[agent_with_group]) + global_agent_registry.register_agent( + AgentResponse( + agent_id="agent-1", + agent_name="detach-test-agent", + agent_card_params={"name": "detach-test-agent", "url": "http://localhost:9", "version": "1"}, + access_group_ids=["ag-keep", "ag-to-delete"], + ) ) + try: + resp = client.delete("/v1/access_group/ag-to-delete") + assert resp.status_code == 204 + + mock_agents_table.update.assert_awaited_once_with( + where={"agent_id": "agent-1"}, + data={"access_group_ids": ("ag-keep",)}, + ) + mock_access_group_table.delete.assert_awaited_once_with(where={"access_group_id": "ag-to-delete"}) + registered = global_agent_registry.get_agent_by_id("agent-1") + assert registered is not None + assert tuple(registered.access_group_ids or ()) == ("ag-keep",) + finally: + global_agent_registry.deregister_agent("detach-test-agent") + + +def test_delete_access_group_without_attached_agents_leaves_agents_untouched(client_and_mocks): + client, mock_prisma, mock_access_group_table, _mock_cache, _mock_proxy_logging = client_and_mocks + mock_agents_table = mock_prisma.db.litellm_agentstable + + mock_access_group_table.find_unique = AsyncMock( + return_value=_make_access_group_record(access_group_id="ag-to-delete") + ) + + resp = client.delete("/v1/access_group/ag-to-delete") + assert resp.status_code == 204 + + mock_agents_table.find_many.assert_awaited_once_with(where={"access_group_ids": {"hasSome": ("ag-to-delete",)}}) + mock_agents_table.update.assert_not_awaited() + @pytest.mark.parametrize( "team_cache_group_ids,key_cache_group_ids,expected_team_ids_after,expected_key_ids_after", @@ -792,9 +817,7 @@ def test_delete_access_group_patches_cached_team_and_key( """Delete patches cached team/key objects to remove the deleted access_group_id.""" from litellm.proxy._types import LiteLLM_TeamTableCachedObj - client, mock_prisma, mock_access_group_table, mock_cache, mock_proxy_logging = ( - client_and_mocks - ) + client, mock_prisma, mock_access_group_table, mock_cache, mock_proxy_logging = client_and_mocks mock_team_table = mock_prisma.db.litellm_teamtable mock_key_table = mock_prisma.db.litellm_verificationtoken @@ -820,13 +843,9 @@ def test_delete_access_group_patches_cached_team_and_key( team_id="team-1", access_group_ids=list(team_cache_group_ids), ) - mock_proxy_logging.internal_usage_cache.dual_cache.async_get_cache = AsyncMock( - return_value=cached_team - ) + mock_proxy_logging.internal_usage_cache.dual_cache.async_get_cache = AsyncMock(return_value=cached_team) else: - mock_proxy_logging.internal_usage_cache.dual_cache.async_get_cache = AsyncMock( - return_value=None - ) + mock_proxy_logging.internal_usage_cache.dual_cache.async_get_cache = AsyncMock(return_value=None) # user_api_key_cache is queried both for teams (fallback after dual_cache) and # hashed keys — return the right stub per ``key``. A single AsyncMock(return_value=key) @@ -834,9 +853,7 @@ def test_delete_access_group_patches_cached_team_and_key( # Use a synchronous side_effect (not async def): AsyncMock awaits coroutine side_effects # inconsistently across Python/unittest versions; sync returns are awaited as immediate results. def user_cache_get_side_effect(*args, **kwargs): - cache_key = ( - kwargs.get("key") if "key" in kwargs else (args[0] if args else None) - ) + cache_key = kwargs.get("key") if "key" in kwargs else (args[0] if args else None) if cache_key == "team_id:team-1": if team_cache_group_ids is None: return None @@ -868,14 +885,11 @@ def test_delete_access_group_patches_cached_team_and_key( team_set_calls = [ c for c in mock_cache.async_set_cache.call_args_list - if c.kwargs.get("key", "") == "team_id:team-1" - or (len(c.args) >= 1 and c.args[0] == "team_id:team-1") + if c.kwargs.get("key", "") == "team_id:team-1" or (len(c.args) >= 1 and c.args[0] == "team_id:team-1") ] assert len(team_set_calls) >= 1, "Expected team cache to be patched" # The cached team object should have the updated access_group_ids - written_team = ( - team_set_calls[0].kwargs.get("value") or team_set_calls[0].args[1] - ) + written_team = team_set_calls[0].kwargs.get("value") or team_set_calls[0].args[1] if isinstance(written_team, LiteLLM_TeamTableCachedObj): assert written_team.access_group_ids == expected_team_ids_after else: @@ -883,8 +897,7 @@ def test_delete_access_group_patches_cached_team_and_key( team_set_calls = [ c for c in mock_cache.async_set_cache.call_args_list - if c.kwargs.get("key", "") == "team_id:team-1" - or (len(c.args) >= 1 and c.args[0] == "team_id:team-1") + if c.kwargs.get("key", "") == "team_id:team-1" or (len(c.args) >= 1 and c.args[0] == "team_id:team-1") ] assert len(team_set_calls) == 0, "Should not patch team cache when not cached" @@ -892,8 +905,7 @@ def test_delete_access_group_patches_cached_team_and_key( key_set_calls = [ c for c in mock_cache.async_set_cache.call_args_list - if c.kwargs.get("key", "") == "hashed-key-1" - or (len(c.args) >= 1 and c.args[0] == "hashed-key-1") + if c.kwargs.get("key", "") == "hashed-key-1" or (len(c.args) >= 1 and c.args[0] == "hashed-key-1") ] assert len(key_set_calls) >= 1, "Expected key cache to be patched" written_key = key_set_calls[0].kwargs.get("value") or key_set_calls[0].args[1] @@ -903,17 +915,14 @@ def test_delete_access_group_patches_cached_team_and_key( key_set_calls = [ c for c in mock_cache.async_set_cache.call_args_list - if c.kwargs.get("key", "") == "hashed-key-1" - or (len(c.args) >= 1 and c.args[0] == "hashed-key-1") + if c.kwargs.get("key", "") == "hashed-key-1" or (len(c.args) >= 1 and c.args[0] == "hashed-key-1") ] assert len(key_set_calls) == 0, "Should not patch key cache when not cached" def test_delete_access_group_patches_key_cached_as_dict(client_and_mocks): """Delete patches key cache — mock returns UserAPIKeyAuth (what UserApiKeyCache emits after deserialize).""" - client, mock_prisma, mock_access_group_table, mock_cache, mock_proxy_logging = ( - client_and_mocks - ) + client, mock_prisma, mock_access_group_table, mock_cache, mock_proxy_logging = client_and_mocks mock_team_table = mock_prisma.db.litellm_teamtable mock_key_table = mock_prisma.db.litellm_verificationtoken @@ -929,9 +938,7 @@ def test_delete_access_group_patches_key_cached_as_dict(client_and_mocks): mock_key_table.find_unique = AsyncMock(return_value=key_with_group) # No team in cache - mock_proxy_logging.internal_usage_cache.dual_cache.async_get_cache = AsyncMock( - return_value=None - ) + mock_proxy_logging.internal_usage_cache.dual_cache.async_get_cache = AsyncMock(return_value=None) # Serialized shape from Redis dict; UserApiKeyCache.async_get_cache(model_type=...) yields a model — simulate that. cached_key_payload = { @@ -940,18 +947,14 @@ def test_delete_access_group_patches_key_cached_as_dict(client_and_mocks): } def user_cache_get_dict_when_key_matches(*args, **kwargs): - cache_key = ( - kwargs.get("key") if "key" in kwargs else (args[0] if args else None) - ) + cache_key = kwargs.get("key") if "key" in kwargs else (args[0] if args else None) if cache_key == "team_id:team-1": return None if cache_key == "hashed-key-dict": return UserAPIKeyAuth.model_validate(cached_key_payload) return None - mock_cache.async_get_cache = AsyncMock( - side_effect=user_cache_get_dict_when_key_matches - ) + mock_cache.async_get_cache = AsyncMock(side_effect=user_cache_get_dict_when_key_matches) resp = client.delete("/v1/access_group/ag-to-delete") assert resp.status_code == 204 @@ -960,8 +963,7 @@ def test_delete_access_group_patches_key_cached_as_dict(client_and_mocks): key_set_calls = [ c for c in mock_cache.async_set_cache.call_args_list - if c.kwargs.get("key", "") == "hashed-key-dict" - or (len(c.args) >= 1 and c.args[0] == "hashed-key-dict") + if c.kwargs.get("key", "") == "hashed-key-dict" or (len(c.args) >= 1 and c.args[0] == "hashed-key-dict") ] assert len(key_set_calls) >= 1, "Expected key cache to be patched" written_key = key_set_calls[0].kwargs.get("value") or key_set_calls[0].args[1] @@ -988,9 +990,7 @@ def test_delete_access_group_404_on_p2025_or_record_not_found(client_and_mocks): existing = _make_access_group_record(access_group_id="ag-to-delete") mock_table.find_unique = AsyncMock(return_value=existing) - mock_table.delete = AsyncMock( - side_effect=Exception("P2025: Record to delete does not exist") - ) + mock_table.delete = AsyncMock(side_effect=Exception("P2025: Record to delete does not exist")) resp = client.delete("/v1/access_group/ag-to-delete") assert resp.status_code == 404 @@ -1039,9 +1039,7 @@ def test_delete_access_group_500_on_generic_exception(client_and_mocks): ("delete", "/v1/unified_access_group/ag-123", lambda: {}), ], ) -def test_access_group_endpoints_db_not_connected( - client_and_mocks, monkeypatch, method, url, factory -): +def test_access_group_endpoints_db_not_connected(client_and_mocks, monkeypatch, method, url, factory): """All endpoints return 500 when DB is not connected.""" client, *_ = client_and_mocks @@ -1049,9 +1047,7 @@ def test_access_group_endpoints_db_not_connected( resp = getattr(client, method)(url, **factory()) assert resp.status_code == 500 - assert ( - resp.json()["detail"]["error"] == CommonProxyErrors.db_not_connected_error.value - ) + assert resp.json()["detail"]["error"] == CommonProxyErrors.db_not_connected_error.value # --------------------------------------------------------------------------- @@ -1107,9 +1103,7 @@ def test_attached_team_ids_by_group_keeps_column_order_then_appends_unmirrored_t def test_create_access_group_syncs_assigned_teams(client_and_mocks): """Create adds access_group_id to each assigned team's access_group_ids in DB.""" - client, mock_prisma, mock_access_group_table, mock_cache, mock_proxy_logging = ( - client_and_mocks - ) + client, mock_prisma, mock_access_group_table, mock_cache, mock_proxy_logging = client_and_mocks mock_team_table = mock_prisma.db.litellm_teamtable team_record = _make_team_record("team-1") @@ -1132,9 +1126,7 @@ def test_create_access_group_syncs_assigned_teams(client_and_mocks): def test_create_access_group_syncs_assigned_keys(client_and_mocks): """Create adds access_group_id to each assigned key's access_group_ids in DB.""" - client, mock_prisma, mock_access_group_table, mock_cache, mock_proxy_logging = ( - client_and_mocks - ) + client, mock_prisma, mock_access_group_table, mock_cache, mock_proxy_logging = client_and_mocks mock_key_table = mock_prisma.db.litellm_verificationtoken key_record = MagicMock() @@ -1148,9 +1140,7 @@ def test_create_access_group_syncs_assigned_keys(client_and_mocks): ) assert resp.status_code == 201 - mock_key_table.find_unique.assert_awaited_once_with( - where={"token": "hashed-token-1"} - ) + mock_key_table.find_unique.assert_awaited_once_with(where={"token": "hashed-token-1"}) mock_key_table.update.assert_awaited_once() call_kwargs = mock_key_table.update.call_args.kwargs assert call_kwargs["where"] == {"token": "hashed-token-1"} @@ -1200,14 +1190,10 @@ def test_create_access_group_idempotent_team_sync(client_and_mocks): def test_update_access_group_syncs_added_teams(client_and_mocks): """Update adds access_group_id to newly assigned teams.""" - client, mock_prisma, mock_access_group_table, mock_cache, mock_proxy_logging = ( - client_and_mocks - ) + client, mock_prisma, mock_access_group_table, mock_cache, mock_proxy_logging = client_and_mocks mock_team_table = mock_prisma.db.litellm_teamtable - existing = _make_access_group_record( - access_group_id="ag-update", assigned_team_ids=["team-existing"] - ) + existing = _make_access_group_record(access_group_id="ag-update", assigned_team_ids=["team-existing"]) mock_access_group_table.find_unique = AsyncMock(return_value=existing) team_record = _make_team_record("team-new") @@ -1248,14 +1234,10 @@ def test_update_access_group_rejects_nonexistent_team(client_and_mocks): def test_update_access_group_syncs_removed_teams(client_and_mocks): """Update removes access_group_id from de-assigned teams.""" - client, mock_prisma, mock_access_group_table, mock_cache, mock_proxy_logging = ( - client_and_mocks - ) + client, mock_prisma, mock_access_group_table, mock_cache, mock_proxy_logging = client_and_mocks mock_team_table = mock_prisma.db.litellm_teamtable - existing = _make_access_group_record( - access_group_id="ag-update", assigned_team_ids=["team-keep", "team-remove"] - ) + existing = _make_access_group_record(access_group_id="ag-update", assigned_team_ids=["team-keep", "team-remove"]) mock_access_group_table.find_unique = AsyncMock(return_value=existing) team_to_remove = _make_team_record("team-remove", ["ag-update"]) @@ -1268,9 +1250,7 @@ def test_update_access_group_syncs_removed_teams(client_and_mocks): ) assert resp.status_code == 200 - mock_team_table.find_unique.assert_awaited_once_with( - where={"team_id": "team-remove"} - ) + mock_team_table.find_unique.assert_awaited_once_with(where={"team_id": "team-remove"}) mock_team_table.update.assert_awaited_once() call_kwargs = mock_team_table.update.call_args.kwargs assert call_kwargs["where"] == {"team_id": "team-remove"} @@ -1296,19 +1276,15 @@ def test_update_access_group_detaches_team_the_mirror_missed(client_and_mocks): mock_team_table.update.assert_awaited_once() call_kwargs = mock_team_table.update.call_args.kwargs assert call_kwargs["where"] == {"team_id": "team-unmirrored"} - assert call_kwargs["data"]["access_group_ids"] == ["ag-other"] + assert tuple(call_kwargs["data"]["access_group_ids"]) == ("ag-other",) def test_update_access_group_no_team_sync_when_ids_not_in_payload(client_and_mocks): """Update does not sync teams when assigned_team_ids is absent from the payload.""" - client, mock_prisma, mock_access_group_table, mock_cache, mock_proxy_logging = ( - client_and_mocks - ) + client, mock_prisma, mock_access_group_table, mock_cache, mock_proxy_logging = client_and_mocks mock_team_table = mock_prisma.db.litellm_teamtable - existing = _make_access_group_record( - access_group_id="ag-update", assigned_team_ids=["team-1"] - ) + existing = _make_access_group_record(access_group_id="ag-update", assigned_team_ids=["team-1"]) mock_access_group_table.find_unique = AsyncMock(return_value=existing) resp = client.put("/v1/access_group/ag-update", json={"description": "new desc"}) @@ -1320,14 +1296,10 @@ def test_update_access_group_no_team_sync_when_ids_not_in_payload(client_and_moc def test_update_access_group_syncs_added_keys(client_and_mocks): """Update adds access_group_id to newly assigned keys.""" - client, mock_prisma, mock_access_group_table, mock_cache, mock_proxy_logging = ( - client_and_mocks - ) + client, mock_prisma, mock_access_group_table, mock_cache, mock_proxy_logging = client_and_mocks mock_key_table = mock_prisma.db.litellm_verificationtoken - existing = _make_access_group_record( - access_group_id="ag-update", assigned_key_ids=["old-token"] - ) + existing = _make_access_group_record(access_group_id="ag-update", assigned_key_ids=["old-token"]) mock_access_group_table.find_unique = AsyncMock(return_value=existing) key_record = MagicMock() @@ -1350,14 +1322,10 @@ def test_update_access_group_syncs_added_keys(client_and_mocks): def test_update_access_group_syncs_removed_keys(client_and_mocks): """Update removes access_group_id from de-assigned keys.""" - client, mock_prisma, mock_access_group_table, mock_cache, mock_proxy_logging = ( - client_and_mocks - ) + client, mock_prisma, mock_access_group_table, mock_cache, mock_proxy_logging = client_and_mocks mock_key_table = mock_prisma.db.litellm_verificationtoken - existing = _make_access_group_record( - access_group_id="ag-update", assigned_key_ids=["keep-token", "remove-token"] - ) + existing = _make_access_group_record(access_group_id="ag-update", assigned_key_ids=["keep-token", "remove-token"]) mock_access_group_table.find_unique = AsyncMock(return_value=existing) key_to_remove = MagicMock() @@ -1385,9 +1353,7 @@ def test_update_access_group_syncs_removed_keys(client_and_mocks): def test_delete_access_group_handles_out_of_sync_assigned_teams(client_and_mocks): """Delete includes teams from assigned_team_ids even when not found by hasSome query.""" - client, mock_prisma, mock_access_group_table, mock_cache, mock_proxy_logging = ( - client_and_mocks - ) + client, mock_prisma, mock_access_group_table, mock_cache, mock_proxy_logging = client_and_mocks mock_team_table = mock_prisma.db.litellm_teamtable # Access group has assigned_team_ids but the team's access_group_ids is not synced @@ -1409,18 +1375,14 @@ def test_delete_access_group_handles_out_of_sync_assigned_teams(client_and_mocks assert resp.status_code == 204 # find_unique is called for the out-of-sync team (included via union with assigned_team_ids) - mock_team_table.find_unique.assert_awaited_once_with( - where={"team_id": "team-out-of-sync"} - ) + mock_team_table.find_unique.assert_awaited_once_with(where={"team_id": "team-out-of-sync"}) # No update needed since team's access_group_ids doesn't contain "ag-to-delete" mock_team_table.update.assert_not_awaited() def test_delete_access_group_handles_out_of_sync_assigned_keys(client_and_mocks): """Delete includes keys from assigned_key_ids even when not found by hasSome query.""" - client, mock_prisma, mock_access_group_table, mock_cache, mock_proxy_logging = ( - client_and_mocks - ) + client, mock_prisma, mock_access_group_table, mock_cache, mock_proxy_logging = client_and_mocks mock_key_table = mock_prisma.db.litellm_verificationtoken existing = _make_access_group_record( @@ -1439,9 +1401,7 @@ def test_delete_access_group_handles_out_of_sync_assigned_keys(client_and_mocks) resp = client.delete("/v1/access_group/ag-to-delete") assert resp.status_code == 204 - mock_key_table.find_unique.assert_awaited_once_with( - where={"token": "token-out-of-sync"} - ) + mock_key_table.find_unique.assert_awaited_once_with(where={"token": "token-out-of-sync"}) mock_key_table.update.assert_not_awaited() @@ -1536,10 +1496,16 @@ def test_list_access_groups_resolves_names_with_one_query_per_table(client_and_m mock_table.find_many = AsyncMock( return_value=[ _make_access_group_record( - access_group_id="ag-1", access_mcp_server_ids=["mcp-a"], access_agent_ids=["agent-a"], assigned_key_ids=["key-a"] + access_group_id="ag-1", + access_mcp_server_ids=["mcp-a"], + access_agent_ids=["agent-a"], + assigned_key_ids=["key-a"], ), _make_access_group_record( - access_group_id="ag-2", access_mcp_server_ids=["mcp-b"], access_agent_ids=["agent-b"], assigned_key_ids=["key-b"] + access_group_id="ag-2", + access_mcp_server_ids=["mcp-b"], + access_agent_ids=["agent-b"], + assigned_key_ids=["key-b"], ), ] ) @@ -1573,7 +1539,10 @@ def test_list_access_groups_skips_lookups_when_nothing_to_resolve(client_and_moc """Groups with no MCP servers, agents or keys must not trigger an empty IN () query per table.""" client, mock_prisma, mock_table, *_ = client_and_mocks mock_table.find_many = AsyncMock( - return_value=[_make_access_group_record(access_group_id="ag-1"), _make_access_group_record(access_group_id="ag-2")] + return_value=[ + _make_access_group_record(access_group_id="ag-1"), + _make_access_group_record(access_group_id="ag-2"), + ] ) resp = client.get("/v1/access_group") diff --git a/tests/test_litellm/proxy/management_endpoints/test_auto_router_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_auto_router_endpoints.py index 931531441d3..a282ae731dd 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_auto_router_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_auto_router_endpoints.py @@ -5,6 +5,7 @@ Unit tests for auto router management endpoints from collections.abc import Mapping, Sequence from functools import partial from pathlib import Path +from types import SimpleNamespace from typing import Final from unittest.mock import AsyncMock, MagicMock @@ -781,17 +782,43 @@ class TestAutoRouterBenchmarks: assert _summed_agg_row([complexity, quality]).tier_turns == {} @pytest.mark.asyncio - async def test_non_admin_roles_cannot_read_benchmarks(self): + @pytest.mark.parametrize("user_id", [None, "own-user", "other-user"]) + async def test_non_admin_roles_cannot_read_benchmarks(self, user_id: str | None): from litellm.proxy.management_endpoints.auto_router_endpoints import get_auto_router_benchmarks with pytest.raises(HTTPException) as err: await get_auto_router_benchmarks( - user_api_key_dict=UserAPIKeyAuth(user_role=LitellmUserRoles.INTERNAL_USER, api_key="sk-x"), + user_api_key_dict=UserAPIKeyAuth( + user_role=LitellmUserRoles.INTERNAL_USER, api_key="sk-x", user_id="own-user" + ), start_date="2026-08-01", end_date="2026-08-02", + user_id=user_id, ) assert err.value.status_code == 403 + @pytest.mark.asyncio + async def test_an_empty_user_filter_is_rejected_before_querying_deployment_data( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + import httpx + from fastapi import FastAPI + + from litellm.proxy import proxy_server + from litellm.proxy.auth.user_api_key_auth import user_api_key_auth + from litellm.proxy.management_endpoints.auto_router_endpoints import get_auto_router_benchmarks + + query: Final = AsyncMock(return_value=[]) + monkeypatch.setattr(proxy_server, "prisma_client", SimpleNamespace(db=SimpleNamespace(query_raw=query))) + app: Final = FastAPI() + app.get("/auto_router/benchmarks")(get_auto_router_benchmarks) + app.dependency_overrides[user_api_key_auth] = lambda: ADMIN + async with httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url="http://test") as client: + response: Final = await client.get("/auto_router/benchmarks", params={"user_id": ""}) + + assert response.status_code == 422 + query.assert_not_awaited() + @pytest.mark.asyncio async def test_a_reversed_window_is_rejected(self, monkeypatch: pytest.MonkeyPatch): from litellm.proxy import proxy_server @@ -807,7 +834,11 @@ class TestAutoRouterBenchmarks: assert err.value.status_code == 400 @pytest.mark.asyncio - async def test_endpoint_returns_groups_and_totals_from_the_rollup(self, monkeypatch: pytest.MonkeyPatch): + @pytest.mark.parametrize("role", [LitellmUserRoles.PROXY_ADMIN, LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY]) + @pytest.mark.parametrize("user_id", [None, "selected-user"]) + async def test_endpoint_returns_groups_and_totals_from_the_rollup( + self, monkeypatch: pytest.MonkeyPatch, role: LitellmUserRoles, user_id: str | None + ): from litellm.proxy import proxy_server from litellm.proxy.management_endpoints.auto_router_endpoints import get_auto_router_benchmarks @@ -822,12 +853,13 @@ class TestAutoRouterBenchmarks: monkeypatch.setattr(proxy_server, "prisma_client", type("P", (), {"db": _DB()})()) response = await get_auto_router_benchmarks( - user_api_key_dict=ADMIN, + user_api_key_dict=UserAPIKeyAuth(user_role=role, api_key="sk-admin", user_id="viewer"), start_date="2026-07-01", end_date="2026-08-01", api_key="key-hash", + user_id=user_id, ) - assert captured["params"] == ("2026-07-01T00:00:00", "2026-08-02T00:00:00", "key-hash") + assert captured["params"] == ("2026-07-01T00:00:00", "2026-08-02T00:00:00", "key-hash", user_id) assert response.routers_in_scope == 1 assert response.groups[0].router_name == "live-auto" assert response.groups[0].saved_pct == response.totals.saved_pct == 75.0 diff --git a/tests/test_litellm/proxy/management_endpoints/test_internal_user_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_internal_user_endpoints.py index 008f703f38a..fb3ea7febd3 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_internal_user_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_internal_user_endpoints.py @@ -1,13 +1,18 @@ +import hashlib import json from datetime import datetime, timezone from types import SimpleNamespace from typing import Final +import httpx import pytest +import respx from fastapi import HTTPException from fastapi.testclient import TestClient from pytest_mock import MockerFixture +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler + from litellm.proxy._types import ( LiteLLM_UserTableFiltered, LitellmUserRoles, @@ -67,9 +72,7 @@ async def test_ui_view_users_with_null_email(mocker, caplog): # Proxy admin: no org filter, no get_user_object call response = await ui_view_users( - user_api_key_dict=UserAPIKeyAuth( - user_id="test_user", user_role=LitellmUserRoles.PROXY_ADMIN - ), + user_api_key_dict=UserAPIKeyAuth(user_id="test_user", user_role=LitellmUserRoles.PROXY_ADMIN), user_id="test_user", user_email=None, team_id=None, @@ -77,9 +80,7 @@ async def test_ui_view_users_with_null_email(mocker, caplog): page_size=50, ) - assert response == [ - LiteLLM_UserTableFiltered(user_id="test-user-null-email", user_email=None) - ] + assert response == [LiteLLM_UserTableFiltered(user_id="test-user-null-email", user_email=None)] @pytest.mark.asyncio @@ -103,9 +104,7 @@ async def test_ui_view_users_proxy_admin_no_org_filter(mocker): mocker.patch("litellm.proxy.proxy_server.prisma_client", mock_prisma_client) await ui_view_users( - user_api_key_dict=UserAPIKeyAuth( - user_id="admin", user_role=LitellmUserRoles.PROXY_ADMIN - ), + user_api_key_dict=UserAPIKeyAuth(user_id="admin", user_role=LitellmUserRoles.PROXY_ADMIN), user_id=None, user_email="foo", team_id=None, @@ -128,9 +127,7 @@ async def test_ui_view_users_org_admin_filtered_by_org(mocker): async def mock_find_many(*args, **kwargs): where = kwargs.get("where") or {} assert "organization_memberships" in where - assert where["organization_memberships"] == { - "some": {"organization_id": {"in": [org_id]}} - } + assert where["organization_memberships"] == {"some": {"organization_id": {"in": [org_id]}}} return [] mock_prisma_client.db.litellm_usertable.find_many = mock_find_many @@ -268,9 +265,7 @@ async def test_ui_view_users_flag_on_team_admin_org_team(mocker): async def mock_find_many(*args, **kwargs): where = kwargs.get("where") or {} assert "organization_memberships" in where - assert where["organization_memberships"] == { - "some": {"organization_id": {"in": [org_id]}} - } + assert where["organization_memberships"] == {"some": {"organization_id": {"in": [org_id]}}} return [] mock_prisma_client.db.litellm_usertable.find_many = mock_find_many @@ -401,9 +396,7 @@ async def test_ui_view_users_flag_on_team_admin_org_member_no_team_id(mocker): async def mock_find_many(*args, **kwargs): where = kwargs.get("where") or {} assert "organization_memberships" in where - assert where["organization_memberships"] == { - "some": {"organization_id": {"in": [org_id]}} - } + assert where["organization_memberships"] == {"some": {"organization_id": {"in": [org_id]}}} return [] mock_prisma_client.db.litellm_usertable.find_many = mock_find_many @@ -462,9 +455,7 @@ async def test_ui_view_users_flag_on_team_admin_not_in_org_resolves_via_key_team async def mock_find_many(*args, **kwargs): where = kwargs.get("where") or {} assert "organization_memberships" in where - assert where["organization_memberships"] == { - "some": {"organization_id": {"in": [org_id]}} - } + assert where["organization_memberships"] == {"some": {"organization_id": {"in": [org_id]}}} return [] mock_prisma_client.db.litellm_usertable.find_many = mock_find_many @@ -507,9 +498,7 @@ async def test_ui_view_users_flag_on_team_admin_not_in_org_resolves_via_key_team # No team_id query param, but team_id on the API key response = await ui_view_users( - user_api_key_dict=UserAPIKeyAuth( - user_id="team-admin-no-org", user_role=None, team_id=tid - ), + user_api_key_dict=UserAPIKeyAuth(user_id="team-admin-no-org", user_role=None, team_id=tid), user_id=None, user_email="u", team_id=None, @@ -538,13 +527,9 @@ def test_user_daily_activity_types(): # Assert all fields in SpendMetrics are reported in DailySpendMetadata as "total_" for field in spend_metrics.__dict__: if field.startswith("total_"): - assert hasattr( - daily_spend_metadata, field - ), f"Field {field} is not reported in DailySpendMetadata" + assert hasattr(daily_spend_metadata, field), f"Field {field} is not reported in DailySpendMetadata" else: - assert not hasattr( - daily_spend_metadata, field - ), f"Field {field} is reported in DailySpendMetadata" + assert not hasattr(daily_spend_metadata, field), f"Field {field} is reported in DailySpendMetadata" @pytest.mark.asyncio @@ -591,9 +576,7 @@ async def test_get_users_includes_timestamps(mocker): # Call get_users function directly with proxy admin auth admin_key = UserAPIKeyAuth(user_id="admin", user_role=LitellmUserRoles.PROXY_ADMIN) - response = await get_users( - page=1, page_size=1, user_api_key_dict=admin_key, organization_ids=None - ) + response = await get_users(page=1, page_size=1, user_api_key_dict=admin_key, organization_ids=None) print("user /list response: ", response) @@ -654,14 +637,10 @@ async def test_get_users_redacts_scim_enterprise_metadata(mocker): ) admin_key = UserAPIKeyAuth(user_id="admin", user_role=LitellmUserRoles.PROXY_ADMIN) - response = await get_users( - page=1, page_size=1, user_api_key_dict=admin_key, organization_ids=None - ) + response = await get_users(page=1, page_size=1, user_api_key_dict=admin_key, organization_ids=None) listed = response["users"][0] - assert listed.metadata == { - "scim_metadata": {"givenName": "Jane", "familyName": "Doe"} - } + assert listed.metadata == {"scim_metadata": {"givenName": "Jane", "familyName": "Doe"}} assert "scim_enterprise" not in (listed.metadata or {}) @@ -853,9 +832,7 @@ async def test_new_user_license_over_limit(mocker): mocker.patch("litellm.proxy.proxy_server._license_check", mock_license_check) # Create test request data - user_request = NewUserRequest( - user_email="test@example.com", user_role="internal_user" - ) + user_request = NewUserRequest(user_email="test@example.com", user_role="internal_user") # Mock user_api_key_dict mock_user_api_key_dict = UserAPIKeyAuth(user_id="test_admin") @@ -916,9 +893,7 @@ async def test_new_user_license_gate_counts_only_billable_users(mocker): request = NewUserRequest(user_role="internal_user") # 2 active + 3 deactivated -> billable 2, not over max_users 2: gate passes - mocker.patch( - "litellm.proxy.proxy_server.prisma_client", _prisma(total=5, deactivated=3) - ) + mocker.patch("litellm.proxy.proxy_server.prisma_client", _prisma(total=5, deactivated=3)) with pytest.raises(ProxyException) as passed: await new_user(data=request, user_api_key_dict=admin) assert key_gen.call_count == 1 @@ -926,9 +901,7 @@ async def test_new_user_license_gate_counts_only_billable_users(mocker): # 3 active, 0 deactivated -> billable 3, over max_users 2: gate blocks key_gen.reset_mock() - mocker.patch( - "litellm.proxy.proxy_server.prisma_client", _prisma(total=3, deactivated=0) - ) + mocker.patch("litellm.proxy.proxy_server.prisma_client", _prisma(total=3, deactivated=0)) with pytest.raises(ProxyException) as blocked: await new_user(data=request, user_api_key_dict=admin) assert blocked.value.code == 403 or blocked.value.code == "403" @@ -978,14 +951,10 @@ async def test_new_user_non_admin_cannot_create_admin(mocker): mocker.patch("litellm.proxy.proxy_server._license_check", mock_license_check) # Test Case 1: INTERNAL_USER trying to create PROXY_ADMIN - user_request = NewUserRequest( - user_email="admin@example.com", user_role=LitellmUserRoles.PROXY_ADMIN - ) + user_request = NewUserRequest(user_email="admin@example.com", user_role=LitellmUserRoles.PROXY_ADMIN) # Mock user_api_key_dict with non-admin role - mock_user_api_key_dict = UserAPIKeyAuth( - user_id="test_internal_user", user_role=LitellmUserRoles.INTERNAL_USER - ) + mock_user_api_key_dict = UserAPIKeyAuth(user_id="test_internal_user", user_role=LitellmUserRoles.INTERNAL_USER) # Call new_user function and expect ProxyException with pytest.raises(ProxyException) as exc_info: @@ -993,9 +962,7 @@ async def test_new_user_non_admin_cannot_create_admin(mocker): # Verify the exception details assert exc_info.value.code == 403 or exc_info.value.code == "403" - assert "Only proxy admins can create administrative users" in str( - exc_info.value.message - ) + assert "Only proxy admins can create administrative users" in str(exc_info.value.message) assert "proxy_admin" in str(exc_info.value.message) assert "proxy_admin_viewer" in str(exc_info.value.message) assert str(LitellmUserRoles.PROXY_ADMIN) in str(exc_info.value.message) @@ -1008,15 +975,11 @@ async def test_new_user_non_admin_cannot_create_admin(mocker): ) with pytest.raises(ProxyException) as exc_info2: - await new_user( - data=user_request_viewer, user_api_key_dict=mock_user_api_key_dict - ) + await new_user(data=user_request_viewer, user_api_key_dict=mock_user_api_key_dict) # Verify the exception details assert exc_info2.value.code == 403 or exc_info2.value.code == "403" - assert "Only proxy admins can create administrative users" in str( - exc_info2.value.message - ) + assert "Only proxy admins can create administrative users" in str(exc_info2.value.message) assert str(LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY) in str(exc_info2.value.message) @@ -1055,9 +1018,7 @@ async def test_new_user_non_admin_permissions_non_empty_rejected(mocker): user_role=LitellmUserRoles.INTERNAL_USER, permissions={"get_spend_routes": True}, ) - caller = UserAPIKeyAuth( - user_id="org-admin", user_role=LitellmUserRoles.ORG_ADMIN - ) + caller = UserAPIKeyAuth(user_id="org-admin", user_role=LitellmUserRoles.ORG_ADMIN) with pytest.raises(ProxyException) as exc_info: await new_user(data=data, user_api_key_dict=caller) @@ -1101,9 +1062,7 @@ async def test_new_user_non_admin_permissions_explicit_empty_rejected(mocker): permissions={}, ) assert "permissions" in data.model_fields_set - caller = UserAPIKeyAuth( - user_id="org-admin", user_role=LitellmUserRoles.ORG_ADMIN - ) + caller = UserAPIKeyAuth(user_id="org-admin", user_role=LitellmUserRoles.ORG_ADMIN) with pytest.raises(ProxyException) as exc_info: await new_user(data=data, user_api_key_dict=caller) @@ -1156,9 +1115,7 @@ async def test_new_user_non_admin_omits_permissions_succeeds(mocker): user_role=LitellmUserRoles.INTERNAL_USER, ) assert "permissions" not in data.model_fields_set - caller = UserAPIKeyAuth( - user_id="org-admin", user_role=LitellmUserRoles.ORG_ADMIN - ) + caller = UserAPIKeyAuth(user_id="org-admin", user_role=LitellmUserRoles.ORG_ADMIN) result = await new_user(data=data, user_api_key_dict=caller) assert result is not None @@ -1232,14 +1189,10 @@ async def test_update_single_user_non_admin_permissions_rejected(mocker): user_id="alice", permissions={"get_spend_routes": True}, ) - caller = UserAPIKeyAuth( - user_id="org-admin", user_role=LitellmUserRoles.ORG_ADMIN - ) + caller = UserAPIKeyAuth(user_id="org-admin", user_role=LitellmUserRoles.ORG_ADMIN) with pytest.raises(HTTPException) as exc_info: - await _update_single_user_helper( - user_request=data, user_api_key_dict=caller - ) + await _update_single_user_helper(user_request=data, user_api_key_dict=caller) assert exc_info.value.status_code == 403 assert "permissions" in str(exc_info.value.detail) @@ -1261,14 +1214,10 @@ async def test_update_single_user_non_admin_permissions_explicit_empty_rejected( data = UpdateUserRequest(user_id="alice", permissions={}) assert "permissions" in data.model_fields_set - caller = UserAPIKeyAuth( - user_id="org-admin", user_role=LitellmUserRoles.ORG_ADMIN - ) + caller = UserAPIKeyAuth(user_id="org-admin", user_role=LitellmUserRoles.ORG_ADMIN) with pytest.raises(HTTPException) as exc_info: - await _update_single_user_helper( - user_request=data, user_api_key_dict=caller - ) + await _update_single_user_helper(user_request=data, user_api_key_dict=caller) assert exc_info.value.status_code == 403 assert "permissions" in str(exc_info.value.detail) @@ -1324,15 +1273,11 @@ async def test_user_info_url_encoding_plus_character(mocker): mock_request.url.query = "user_id=machine-user+alp-air-admin-b58-b@tempus.com" # Mock user_api_key_dict - mock_user_api_key_dict = UserAPIKeyAuth( - user_id="test_admin", user_role="proxy_admin" - ) + mock_user_api_key_dict = UserAPIKeyAuth(user_id="test_admin", user_role="proxy_admin") # Call user_info function with the URL-decoded user_id (as FastAPI would pass it) # FastAPI would normally convert + to space, but our fix should handle this - decoded_user_id = ( - "machine-user alp-air-admin-b58-b@tempus.com" # What FastAPI gives us - ) + decoded_user_id = "machine-user alp-air-admin-b58-b@tempus.com" # What FastAPI gives us expected_user_id = "machine-user+alp-air-admin-b58-b@tempus.com" response = await user_info( @@ -1383,9 +1328,7 @@ async def test_user_info_nonexistent_user(mocker): mock_request = mocker.MagicMock(spec=Request) # Mock user_api_key_dict - mock_user_api_key_dict = UserAPIKeyAuth( - user_id="test_admin", user_role="proxy_admin" - ) + mock_user_api_key_dict = UserAPIKeyAuth(user_id="test_admin", user_role="proxy_admin") # Call user_info function with a non-existent user_id nonexistent_user_id = "nonexistent-user@example.com" @@ -1423,14 +1366,10 @@ async def test_user_info_no_user_id_view_only_admin_gets_proxy_admin_payload(moc mock_get_user_info_for_proxy_admin, ) - viewer = UserAPIKeyAuth( - user_id="viewer", user_role=LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY.value - ) + viewer = UserAPIKeyAuth(user_id="viewer", user_role=LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY.value) mock_request = mocker.MagicMock(spec=Request) - response = await user_info( - user_id=None, user_api_key_dict=viewer, request=mock_request - ) + response = await user_info(user_id=None, user_api_key_dict=viewer, request=mock_request) mock_get_user_info_for_proxy_admin.assert_awaited_once_with(user_api_key_dict=viewer) assert response is admin_payload @@ -1457,9 +1396,7 @@ async def test_new_user_default_teams_flow(mocker): mock_prisma_client.db.litellm_usertable.count = mock_count persisted_user_row = mocker.MagicMock() persisted_user_row.teams = ["96fed65b-0182-4ff4-8429-2721cd7d42af"] - mock_prisma_client.db.litellm_usertable.find_unique = mocker.AsyncMock( - return_value=persisted_user_row - ) + mock_prisma_client.db.litellm_usertable.find_unique = mocker.AsyncMock(return_value=persisted_user_row) # Mock duplicate checks to pass async def mock_check_duplicate_user_email(*args, **kwargs): @@ -1527,26 +1464,20 @@ async def test_new_user_default_teams_flow(mocker): ) # Create test request data WITHOUT teams (teams should come from defaults) - user_request = NewUserRequest( - user_email="test@example.com", user_role="internal_user" - ) + user_request = NewUserRequest(user_email="test@example.com", user_role="internal_user") # Mock user_api_key_dict mock_user_api_key_dict = UserAPIKeyAuth(user_id="test_admin") # Call new_user function - response = await new_user( - data=user_request, user_api_key_dict=mock_user_api_key_dict - ) + response = await new_user(data=user_request, user_api_key_dict=mock_user_api_key_dict) # Verify generate_key_helper_fn was called WITHOUT teams mock_generate_key_helper_fn.assert_called_once() call_kwargs = mock_generate_key_helper_fn.call_args.kwargs # Teams should be removed from the data passed to generate_key_helper_fn - assert ( - "teams" not in call_kwargs - ), "Teams should not be passed to generate_key_helper_fn" + assert "teams" not in call_kwargs, "Teams should not be passed to generate_key_helper_fn" assert call_kwargs["request_type"] == "user" assert call_kwargs["user_email"] == "test@example.com" assert call_kwargs["user_role"] == "internal_user" @@ -1591,24 +1522,16 @@ def test_update_internal_new_user_params_proxy_admin_role(): try: # Create test data with PROXY_ADMIN role - data = NewUserRequest( - user_email="admin@example.com", user_role=LitellmUserRoles.PROXY_ADMIN.value - ) + data = NewUserRequest(user_email="admin@example.com", user_role=LitellmUserRoles.PROXY_ADMIN.value) data_json = data.model_dump(exclude_unset=True) # Call the function result = _update_internal_new_user_params(data_json=data_json, data=data) # Assertions - default params should NOT be applied for PROXY_ADMIN - assert ( - "max_budget" not in result - ), "Default max_budget should NOT be applied to PROXY_ADMIN" - assert ( - "models" not in result - ), "Default models should NOT be applied to PROXY_ADMIN" - assert ( - "tpm_limit" not in result - ), "Default tpm_limit should NOT be applied to PROXY_ADMIN" + assert "max_budget" not in result, "Default max_budget should NOT be applied to PROXY_ADMIN" + assert "models" not in result, "Default models should NOT be applied to PROXY_ADMIN" + assert "tpm_limit" not in result, "Default tpm_limit should NOT be applied to PROXY_ADMIN" # These should still work assert result["user_email"] == "admin@example.com" @@ -1722,15 +1645,9 @@ async def test_check_duplicate_user_email_case_insensitive(mocker): user_email_clause = where_clause.get("user_email", {}) # Check that the query structure is correct for case insensitive search - assert ( - "equals" in user_email_clause - ), "Query should use 'equals' for case insensitive search" - assert ( - user_email_clause.get("mode") == "insensitive" - ), "Query should use 'insensitive' mode" - assert ( - user_email_clause.get("equals") == "user@example.com" - ), "Query should search for the provided email" + assert "equals" in user_email_clause, "Query should use 'equals' for case insensitive search" + assert user_email_clause.get("mode") == "insensitive", "Query should use 'insensitive' mode" + assert user_email_clause.get("equals") == "user@example.com", "Query should search for the provided email" return mock_existing_user # Return existing user to simulate duplicate @@ -1741,9 +1658,7 @@ async def test_check_duplicate_user_email_case_insensitive(mocker): await _check_duplicate_user_email("user@example.com", mock_prisma_client) assert exc_info.value.status_code == 409 - assert "User with email User@Example.com already exists" in str( - exc_info.value.detail - ) + assert "User with email User@Example.com already exists" in str(exc_info.value.detail) # Test Case 2: No duplicate found async def mock_find_first_no_duplicate(*args, **kwargs): @@ -1768,9 +1683,7 @@ async def test_check_duplicate_user_email_case_insensitive(mocker): pytest.fail(f"Should not raise exception when no duplicate found, but got: {e}") # Test Case 3: None email should not cause issues - await _check_duplicate_user_email( - None, mock_prisma_client - ) # Should not raise exception + await _check_duplicate_user_email(None, mock_prisma_client) # Should not raise exception @pytest.mark.asyncio @@ -1880,9 +1793,7 @@ def test_process_keys_for_user_info_filters_dashboard_keys(monkeypatch): # Verify dashboard key is not in results result_team_ids = [key.get("team_id") for key in result] - assert ( - UI_SESSION_TOKEN_TEAM_ID not in result_team_ids - ), "Dashboard key should be filtered out" + assert UI_SESSION_TOKEN_TEAM_ID not in result_team_ids, "Dashboard key should be filtered out" # Verify regular keys are included assert "regular-team" in result_team_ids, "Regular team key should be included" @@ -1892,9 +1803,7 @@ def test_process_keys_for_user_info_filters_dashboard_keys(monkeypatch): result_tokens = [key.get("token") for key in result] assert "sk-regular-token" in result_tokens, "Regular key should be included" assert "sk-no-team-token" in result_tokens, "No-team key should be included" - assert ( - "sk-dashboard-token" not in result_tokens - ), "Dashboard key should not be included" + assert "sk-dashboard-token" not in result_tokens, "Dashboard key should not be included" def test_process_keys_for_user_info_handles_none_keys(monkeypatch): @@ -2419,9 +2328,7 @@ async def test_get_user_daily_activity_non_admin_cannot_view_other_users(monkeyp ) assert exc_info.value.status_code == 403 - assert "Non-admin users can only view their own spend data" in str( - exc_info.value.detail - ) + assert "Non-admin users can only view their own spend data" in str(exc_info.value.detail) # Case 2: Non-admin omits user_id — should default to their own user_id mock_response = MagicMock() @@ -2708,14 +2615,10 @@ async def test_delete_user_cleans_up_created_by_invitation_links(mocker): async def mock_find_unique(*args, **kwargs): return mock_user_row - mock_prisma_client.db.litellm_usertable.find_unique = mocker.AsyncMock( - side_effect=mock_find_unique - ) + mock_prisma_client.db.litellm_usertable.find_unique = mocker.AsyncMock(side_effect=mock_find_unique) # Mock find_many for teams (no teams) - mock_prisma_client.db.litellm_teamtable.find_many = mocker.AsyncMock( - return_value=[] - ) + mock_prisma_client.db.litellm_teamtable.find_many = mocker.AsyncMock(return_value=[]) # Mock all delete_many calls mock_prisma_client.db.litellm_verificationtoken.find_many = mocker.AsyncMock( @@ -2741,9 +2644,7 @@ async def test_delete_user_cleans_up_created_by_invitation_links(mocker): # Call delete_user data = DeleteUserRequest(user_ids=["admin-creator"]) - user_api_key_dict = UserAPIKeyAuth( - user_id="proxy-admin", user_role=LitellmUserRoles.PROXY_ADMIN - ) + user_api_key_dict = UserAPIKeyAuth(user_id="proxy-admin", user_role=LitellmUserRoles.PROXY_ADMIN) await delete_user(data=data, user_api_key_dict=user_api_key_dict) @@ -2752,9 +2653,7 @@ async def test_delete_user_cleans_up_created_by_invitation_links(mocker): call_kwargs = mock_prisma_client.db.litellm_invitationlink.delete_many.call_args where_clause = call_kwargs.kwargs.get("where") or call_kwargs[1].get("where") - assert ( - "OR" in where_clause - ), "Should use OR to match user_id, created_by, and updated_by" + assert "OR" in where_clause, "Should use OR to match user_id, created_by, and updated_by" or_conditions = where_clause["OR"] assert len(or_conditions) == 3, "Should have 3 OR conditions" @@ -2875,9 +2774,7 @@ async def test_delete_user_rejects_org_admin_deleting_outside_scope(mocker): async def mock_find_unique(*args, **kwargs): return mock_target_user - mock_prisma_client.db.litellm_usertable.find_unique = mocker.AsyncMock( - side_effect=mock_find_unique - ) + mock_prisma_client.db.litellm_usertable.find_unique = mocker.AsyncMock(side_effect=mock_find_unique) # Caller (org_admin_user) administers org-A. caller_membership = mocker.MagicMock() @@ -2903,16 +2800,12 @@ async def test_delete_user_rejects_org_admin_deleting_outside_scope(mocker): return [caller_membership] return [] - mock_prisma_client.db.litellm_organizationmembership.find_many = mocker.AsyncMock( - side_effect=mock_find_memberships - ) + mock_prisma_client.db.litellm_organizationmembership.find_many = mocker.AsyncMock(side_effect=mock_find_memberships) mocker.patch("litellm.proxy.proxy_server.prisma_client", mock_prisma_client) data = DeleteUserRequest(user_ids=["victim"]) - user_api_key_dict = UserAPIKeyAuth( - user_id="org_admin_user", user_role=LitellmUserRoles.ORG_ADMIN - ) + user_api_key_dict = UserAPIKeyAuth(user_id="org_admin_user", user_role=LitellmUserRoles.ORG_ADMIN) with pytest.raises(HTTPException) as exc: await delete_user(data=data, user_api_key_dict=user_api_key_dict) @@ -2920,11 +2813,8 @@ async def test_delete_user_rejects_org_admin_deleting_outside_scope(mocker): # Critical: no delete_many calls should have executed. assert ( - not hasattr( - mock_prisma_client.db.litellm_verificationtoken.delete_many, "mock_calls" - ) - or len(mock_prisma_client.db.litellm_verificationtoken.delete_many.mock_calls) - == 0 + not hasattr(mock_prisma_client.db.litellm_verificationtoken.delete_many, "mock_calls") + or len(mock_prisma_client.db.litellm_verificationtoken.delete_many.mock_calls) == 0 ) @@ -2943,9 +2833,7 @@ async def test_user_update_rejects_silent_create_for_non_proxy_admin(mocker): mock_prisma_client = mocker.MagicMock() # user_email lookup yields None → would silently create pre-fix. - mock_prisma_client.db.litellm_usertable.find_first = mocker.AsyncMock( - return_value=None - ) + mock_prisma_client.db.litellm_usertable.find_first = mocker.AsyncMock(return_value=None) mocker.patch("litellm.proxy.proxy_server.prisma_client", mock_prisma_client) user_request = UpdateUserRequest( @@ -2959,9 +2847,7 @@ async def test_user_update_rejects_silent_create_for_non_proxy_admin(mocker): ) with pytest.raises(HTTPException) as exc: - await _update_single_user_helper( - user_request=user_request, user_api_key_dict=org_admin - ) + await _update_single_user_helper(user_request=user_request, user_api_key_dict=org_admin) assert exc.value.status_code == 404 @@ -3005,17 +2891,13 @@ async def test_user_info_v2_proxy_admin_can_query_any_user(mocker): return mock_user_row return None - mock_prisma_client.db.litellm_usertable.find_unique = mocker.AsyncMock( - side_effect=mock_find_unique - ) + mock_prisma_client.db.litellm_usertable.find_unique = mocker.AsyncMock(side_effect=mock_find_unique) mocker.patch("litellm.proxy.proxy_server.prisma_client", mock_prisma_client) mock_request = mocker.MagicMock(spec=Request) - admin_key = UserAPIKeyAuth( - user_id="admin-user", user_role=LitellmUserRoles.PROXY_ADMIN - ) + admin_key = UserAPIKeyAuth(user_id="admin-user", user_role=LitellmUserRoles.PROXY_ADMIN) response = await user_info_v2( request=mock_request, @@ -3069,17 +2951,13 @@ async def test_user_info_v2_redacts_scim_enterprise_metadata(mocker): return mock_user_row return None - mock_prisma_client.db.litellm_usertable.find_unique = mocker.AsyncMock( - side_effect=mock_find_unique - ) + mock_prisma_client.db.litellm_usertable.find_unique = mocker.AsyncMock(side_effect=mock_find_unique) mocker.patch("litellm.proxy.proxy_server.prisma_client", mock_prisma_client) mock_request = mocker.MagicMock(spec=Request) - admin_key = UserAPIKeyAuth( - user_id="admin-user", user_role=LitellmUserRoles.PROXY_ADMIN - ) + admin_key = UserAPIKeyAuth(user_id="admin-user", user_role=LitellmUserRoles.PROXY_ADMIN) response = await user_info_v2( request=mock_request, @@ -3088,9 +2966,7 @@ async def test_user_info_v2_redacts_scim_enterprise_metadata(mocker): ) assert isinstance(response, UserInfoV2Response) - assert response.metadata == { - "scim_metadata": {"givenName": "Jane", "familyName": "Doe"} - } + assert response.metadata == {"scim_metadata": {"givenName": "Jane", "familyName": "Doe"}} assert "scim_enterprise" not in (response.metadata or {}) @@ -3159,17 +3035,13 @@ async def test_user_info_v2_internal_user_can_query_self(mocker): return mock_user_row return None - mock_prisma_client.db.litellm_usertable.find_unique = mocker.AsyncMock( - side_effect=mock_find_unique - ) + mock_prisma_client.db.litellm_usertable.find_unique = mocker.AsyncMock(side_effect=mock_find_unique) mocker.patch("litellm.proxy.proxy_server.prisma_client", mock_prisma_client) mock_request = mocker.MagicMock(spec=Request) - user_key = UserAPIKeyAuth( - user_id="self-user", user_role=LitellmUserRoles.INTERNAL_USER - ) + user_key = UserAPIKeyAuth(user_id="self-user", user_role=LitellmUserRoles.INTERNAL_USER) response = await user_info_v2( request=mock_request, @@ -3204,17 +3076,13 @@ async def test_user_info_v2_internal_user_cannot_query_other(mocker): return mock_caller_row return None - mock_prisma_client.db.litellm_usertable.find_unique = mocker.AsyncMock( - side_effect=mock_find_unique - ) + mock_prisma_client.db.litellm_usertable.find_unique = mocker.AsyncMock(side_effect=mock_find_unique) mocker.patch("litellm.proxy.proxy_server.prisma_client", mock_prisma_client) mock_request = mocker.MagicMock(spec=Request) - user_key = UserAPIKeyAuth( - user_id="caller-user", user_role=LitellmUserRoles.INTERNAL_USER - ) + user_key = UserAPIKeyAuth(user_id="caller-user", user_role=LitellmUserRoles.INTERNAL_USER) with pytest.raises(ProxyException) as exc_info: await user_info_v2( @@ -3261,17 +3129,13 @@ async def test_user_info_v2_no_user_id_defaults_to_self(mocker): return mock_user_row return None - mock_prisma_client.db.litellm_usertable.find_unique = mocker.AsyncMock( - side_effect=mock_find_unique - ) + mock_prisma_client.db.litellm_usertable.find_unique = mocker.AsyncMock(side_effect=mock_find_unique) mocker.patch("litellm.proxy.proxy_server.prisma_client", mock_prisma_client) mock_request = mocker.MagicMock(spec=Request) - user_key = UserAPIKeyAuth( - user_id="my-user-id", user_role=LitellmUserRoles.INTERNAL_USER - ) + user_key = UserAPIKeyAuth(user_id="my-user-id", user_role=LitellmUserRoles.INTERNAL_USER) # Call without user_id response = await user_info_v2( @@ -3299,17 +3163,13 @@ async def test_user_info_v2_nonexistent_user_returns_404(mocker): async def mock_find_unique(*args, **kwargs): return None - mock_prisma_client.db.litellm_usertable.find_unique = mocker.AsyncMock( - side_effect=mock_find_unique - ) + mock_prisma_client.db.litellm_usertable.find_unique = mocker.AsyncMock(side_effect=mock_find_unique) mocker.patch("litellm.proxy.proxy_server.prisma_client", mock_prisma_client) mock_request = mocker.MagicMock(spec=Request) - admin_key = UserAPIKeyAuth( - user_id="admin-user", user_role=LitellmUserRoles.PROXY_ADMIN - ) + admin_key = UserAPIKeyAuth(user_id="admin-user", user_role=LitellmUserRoles.PROXY_ADMIN) with pytest.raises(ProxyException) as exc_info: await user_info_v2( @@ -3357,17 +3217,13 @@ async def test_user_info_v2_response_shape(mocker): async def mock_find_unique(*args, **kwargs): return mock_user_row - mock_prisma_client.db.litellm_usertable.find_unique = mocker.AsyncMock( - side_effect=mock_find_unique - ) + mock_prisma_client.db.litellm_usertable.find_unique = mocker.AsyncMock(side_effect=mock_find_unique) mocker.patch("litellm.proxy.proxy_server.prisma_client", mock_prisma_client) mock_request = mocker.MagicMock(spec=Request) - admin_key = UserAPIKeyAuth( - user_id="admin-user", user_role=LitellmUserRoles.PROXY_ADMIN - ) + admin_key = UserAPIKeyAuth(user_id="admin-user", user_role=LitellmUserRoles.PROXY_ADMIN) response = await user_info_v2( request=mock_request, @@ -3402,9 +3258,7 @@ async def test_user_info_v2_response_shape(mocker): # The dashboard's user edit form hydrates its per-model budget rows from # these two, so dropping them makes a save replace the user's budgets. - assert response_dict["model_max_budget"] == { - "gpt-3.5-turbo": {"budget_limit": 5.0, "time_period": "30d"} - } + assert response_dict["model_max_budget"] == {"gpt-3.5-turbo": {"budget_limit": 5.0, "time_period": "30d"}} assert response_dict["model_max_budget_usage"] == { "gpt-3.5-turbo": {"current_spend": 0.0, "budget_limit": 5.0, "time_period": "30d"} } @@ -3463,9 +3317,7 @@ async def test_user_info_v2_team_admin_can_query_team_member(mocker): return mock_target return None - mock_prisma_client.db.litellm_usertable.find_unique = mocker.AsyncMock( - side_effect=mock_find_unique - ) + mock_prisma_client.db.litellm_usertable.find_unique = mocker.AsyncMock(side_effect=mock_find_unique) # Mock team with caller as admin mock_team = mocker.MagicMock() @@ -3482,17 +3334,13 @@ async def test_user_info_v2_team_admin_can_query_team_member(mocker): async def mock_find_many_teams(*args, **kwargs): return [mock_team] - mock_prisma_client.db.litellm_teamtable.find_many = mocker.AsyncMock( - side_effect=mock_find_many_teams - ) + mock_prisma_client.db.litellm_teamtable.find_many = mocker.AsyncMock(side_effect=mock_find_many_teams) mocker.patch("litellm.proxy.proxy_server.prisma_client", mock_prisma_client) mock_request = mocker.MagicMock(spec=Request) - team_admin_key = UserAPIKeyAuth( - user_id="team-admin-user", user_role=LitellmUserRoles.INTERNAL_USER - ) + team_admin_key = UserAPIKeyAuth(user_id="team-admin-user", user_role=LitellmUserRoles.INTERNAL_USER) response = await user_info_v2( request=mock_request, @@ -3532,9 +3380,7 @@ async def test_user_info_v2_team_admin_cannot_query_non_team_member(mocker): return mock_target return None - mock_prisma_client.db.litellm_usertable.find_unique = mocker.AsyncMock( - side_effect=mock_find_unique - ) + mock_prisma_client.db.litellm_usertable.find_unique = mocker.AsyncMock(side_effect=mock_find_unique) # Mock team where caller is admin mock_team = mocker.MagicMock() @@ -3550,17 +3396,13 @@ async def test_user_info_v2_team_admin_cannot_query_non_team_member(mocker): async def mock_find_many_teams(*args, **kwargs): return [mock_team] - mock_prisma_client.db.litellm_teamtable.find_many = mocker.AsyncMock( - side_effect=mock_find_many_teams - ) + mock_prisma_client.db.litellm_teamtable.find_many = mocker.AsyncMock(side_effect=mock_find_many_teams) mocker.patch("litellm.proxy.proxy_server.prisma_client", mock_prisma_client) mock_request = mocker.MagicMock(spec=Request) - team_admin_key = UserAPIKeyAuth( - user_id="team-admin-user", user_role=LitellmUserRoles.INTERNAL_USER - ) + team_admin_key = UserAPIKeyAuth(user_id="team-admin-user", user_role=LitellmUserRoles.INTERNAL_USER) with pytest.raises(ProxyException) as exc_info: await user_info_v2( @@ -3610,18 +3452,14 @@ async def test_user_info_v2_url_encoding_plus_character(mocker): return mock_user_row return None - mock_prisma_client.db.litellm_usertable.find_unique = mocker.AsyncMock( - side_effect=mock_find_unique - ) + mock_prisma_client.db.litellm_usertable.find_unique = mocker.AsyncMock(side_effect=mock_find_unique) mocker.patch("litellm.proxy.proxy_server.prisma_client", mock_prisma_client) mock_request = mocker.MagicMock(spec=Request) mock_request.url.query = f"user_id={expected_user_id}" - admin_key = UserAPIKeyAuth( - user_id="admin-user", user_role=LitellmUserRoles.PROXY_ADMIN - ) + admin_key = UserAPIKeyAuth(user_id="admin-user", user_role=LitellmUserRoles.PROXY_ADMIN) # Simulate FastAPI converting + to space decoded_user_id = "machine-user admin@example.com" @@ -3718,9 +3556,7 @@ def test_enforce_user_info_access_admin_bypass(): _enforce_user_info_access, ) - admin = UserAPIKeyAuth( - user_id="admin", user_role=LitellmUserRoles.PROXY_ADMIN.value - ) + admin = UserAPIKeyAuth(user_id="admin", user_role=LitellmUserRoles.PROXY_ADMIN.value) # Should not raise even when querying a different user _enforce_user_info_access(user_id="someone_else", user_api_key_dict=admin) @@ -3759,9 +3595,7 @@ def test_enforce_user_info_access_owner_allowed(): _enforce_user_info_access, ) - user = UserAPIKeyAuth( - user_id="alice", user_role=LitellmUserRoles.INTERNAL_USER.value - ) + user = UserAPIKeyAuth(user_id="alice", user_role=LitellmUserRoles.INTERNAL_USER.value) _enforce_user_info_access(user_id="alice", user_api_key_dict=user) @@ -3773,9 +3607,7 @@ def test_enforce_user_info_access_no_user_id_allowed(): _enforce_user_info_access, ) - user = UserAPIKeyAuth( - user_id="alice", user_role=LitellmUserRoles.INTERNAL_USER.value - ) + user = UserAPIKeyAuth(user_id="alice", user_role=LitellmUserRoles.INTERNAL_USER.value) _enforce_user_info_access(user_id=None, user_api_key_dict=user) @@ -3832,9 +3664,7 @@ async def test_ghsa_wvg4_non_admin_cannot_self_escalate_max_budget(mocker, budge "max_budget": 100, } existing_user.user_id = "user-1" - mock_prisma_client.db.litellm_usertable.find_first = mocker.AsyncMock( - return_value=existing_user - ) + mock_prisma_client.db.litellm_usertable.find_first = mocker.AsyncMock(return_value=existing_user) mocker.patch("litellm.proxy.proxy_server.prisma_client", mock_prisma_client) user_request = UpdateUserRequest.model_validate({"user_id": "user-1", budget_field: budget_value}) @@ -3844,9 +3674,7 @@ async def test_ghsa_wvg4_non_admin_cannot_self_escalate_max_budget(mocker, budge ) with pytest.raises(HTTPException) as exc: - await _update_single_user_helper( - user_request=user_request, user_api_key_dict=caller - ) + await _update_single_user_helper(user_request=user_request, user_api_key_dict=caller) assert exc.value.status_code == 403 assert budget_field in str(exc.value.detail) mock_prisma_client.update_data.assert_not_called() @@ -3868,9 +3696,7 @@ async def test_ghsa_wvg4_non_admin_cannot_self_escalate_spend(mocker): "spend": 50.0, } existing_user.user_id = "user-1" - mock_prisma_client.db.litellm_usertable.find_first = mocker.AsyncMock( - return_value=existing_user - ) + mock_prisma_client.db.litellm_usertable.find_first = mocker.AsyncMock(return_value=existing_user) mocker.patch("litellm.proxy.proxy_server.prisma_client", mock_prisma_client) user_request = UpdateUserRequest( @@ -3883,9 +3709,7 @@ async def test_ghsa_wvg4_non_admin_cannot_self_escalate_spend(mocker): ) with pytest.raises(HTTPException) as exc: - await _update_single_user_helper( - user_request=user_request, user_api_key_dict=caller - ) + await _update_single_user_helper(user_request=user_request, user_api_key_dict=caller) assert exc.value.status_code == 403 assert "spend" in str(exc.value.detail) @@ -3904,12 +3728,8 @@ async def test_ghsa_wvg4_proxy_admin_can_update_user_budget(mocker): "max_budget": 100, } existing_user.user_id = "target-user" - mock_prisma_client.db.litellm_usertable.find_first = mocker.AsyncMock( - return_value=existing_user - ) - mock_prisma_client.update_data = mocker.AsyncMock( - return_value={"user_id": "target-user", "max_budget": 500} - ) + mock_prisma_client.db.litellm_usertable.find_first = mocker.AsyncMock(return_value=existing_user) + mock_prisma_client.update_data = mocker.AsyncMock(return_value={"user_id": "target-user", "max_budget": 500}) mock_prisma_client.jsonify_object = mocker.MagicMock(side_effect=lambda x: x) mocker.patch("litellm.proxy.proxy_server.prisma_client", mock_prisma_client) mocker.patch("litellm.proxy.proxy_server.litellm_proxy_admin_name", "admin") @@ -3923,9 +3743,7 @@ async def test_ghsa_wvg4_proxy_admin_can_update_user_budget(mocker): user_role=LitellmUserRoles.PROXY_ADMIN, ) - result = await _update_single_user_helper( - user_request=user_request, user_api_key_dict=admin_caller - ) + result = await _update_single_user_helper(user_request=user_request, user_api_key_dict=admin_caller) assert result is not None @@ -3941,12 +3759,8 @@ async def test_admin_user_update_spend_invalidates_counter(mocker): existing_user = mocker.MagicMock() existing_user.model_dump.return_value = {"user_id": "target-user", "spend": 50.0} existing_user.user_id = "target-user" - mock_prisma_client.db.litellm_usertable.find_first = mocker.AsyncMock( - return_value=existing_user - ) - mock_prisma_client.update_data = mocker.AsyncMock( - return_value={"user_id": "target-user", "spend": -25.0} - ) + mock_prisma_client.db.litellm_usertable.find_first = mocker.AsyncMock(return_value=existing_user) + mock_prisma_client.update_data = mocker.AsyncMock(return_value={"user_id": "target-user", "spend": -25.0}) mock_prisma_client.jsonify_object = mocker.MagicMock(side_effect=lambda x: x) mocker.patch("litellm.proxy.proxy_server.prisma_client", mock_prisma_client) mocker.patch("litellm.proxy.proxy_server.litellm_proxy_admin_name", "admin") @@ -3961,13 +3775,9 @@ async def test_admin_user_update_spend_invalidates_counter(mocker): # without raising the recurring budget ceiling. Future changes should # continue allowing negative spend counters. user_request = UpdateUserRequest(user_id="target-user", spend=-25) - admin_caller = UserAPIKeyAuth( - user_id="admin-1", user_role=LitellmUserRoles.PROXY_ADMIN - ) + admin_caller = UserAPIKeyAuth(user_id="admin-1", user_role=LitellmUserRoles.PROXY_ADMIN) - await _update_single_user_helper( - user_request=user_request, user_api_key_dict=admin_caller - ) + await _update_single_user_helper(user_request=user_request, user_api_key_dict=admin_caller) mock_invalidate.assert_awaited_once_with(counter_key="spend:user:target-user") @@ -3984,9 +3794,7 @@ async def test_user_update_rejects_non_finite_spend(mocker): existing_user = mocker.MagicMock() existing_user.model_dump.return_value = {"user_id": "target-user", "spend": 50.0} existing_user.user_id = "target-user" - mock_prisma_client.db.litellm_usertable.find_first = mocker.AsyncMock( - return_value=existing_user - ) + mock_prisma_client.db.litellm_usertable.find_first = mocker.AsyncMock(return_value=existing_user) mock_prisma_client.update_data = mocker.AsyncMock() mocker.patch("litellm.proxy.proxy_server.prisma_client", mock_prisma_client) mocker.patch("litellm.proxy.proxy_server.litellm_proxy_admin_name", "admin") @@ -3996,14 +3804,10 @@ async def test_user_update_rejects_non_finite_spend(mocker): ) user_request = UpdateUserRequest(user_id="target-user", spend=float("nan")) - admin_caller = UserAPIKeyAuth( - user_id="admin-1", user_role=LitellmUserRoles.PROXY_ADMIN - ) + admin_caller = UserAPIKeyAuth(user_id="admin-1", user_role=LitellmUserRoles.PROXY_ADMIN) with pytest.raises(HTTPException) as exc: - await _update_single_user_helper( - user_request=user_request, user_api_key_dict=admin_caller - ) + await _update_single_user_helper(user_request=user_request, user_api_key_dict=admin_caller) assert exc.value.status_code == 400 mock_prisma_client.update_data.assert_not_called() mock_invalidate.assert_not_awaited() @@ -4023,9 +3827,7 @@ async def test_resolve_user_email_metadata_maps_page_user_ids_to_email(mocker): mock_prisma_client = mocker.MagicMock() find_many = mocker.AsyncMock( return_value=[ - SimpleNamespace( - user_id="u1", user_email="alice@example.com", user_alias="Alice" - ), + SimpleNamespace(user_id="u1", user_email="alice@example.com", user_alias="Alice"), SimpleNamespace(user_id="u2", user_email=None, user_alias="bob-alias"), ] ) @@ -4224,19 +4026,13 @@ def _object_permission_mocks(mocker, existing_object_permission_id=None): } existing_user.user_id = "target-user" existing_user.object_permission_id = existing_object_permission_id - mock_prisma_client.db.litellm_usertable.find_first = mocker.AsyncMock( - return_value=existing_user - ) - mock_prisma_client.db.litellm_objectpermissiontable.find_unique = mocker.AsyncMock( - return_value=None - ) + mock_prisma_client.db.litellm_usertable.find_first = mocker.AsyncMock(return_value=existing_user) + mock_prisma_client.db.litellm_objectpermissiontable.find_unique = mocker.AsyncMock(return_value=None) mock_prisma_client.db.litellm_objectpermissiontable.upsert = mocker.AsyncMock( return_value=SimpleNamespace(object_permission_id="perm-new") ) mock_prisma_client.db.litellm_mcpservertable.find_many = mocker.AsyncMock(return_value=[]) - mock_prisma_client.update_data = mocker.AsyncMock( - return_value={"user_id": "target-user"} - ) + mock_prisma_client.update_data = mocker.AsyncMock(return_value={"user_id": "target-user"}) mock_prisma_client.jsonify_object = mocker.MagicMock(side_effect=lambda x: x) mocker.patch("litellm.proxy.proxy_server.prisma_client", mock_prisma_client) mocker.patch("litellm.proxy.proxy_server.litellm_proxy_admin_name", "admin") @@ -4272,9 +4068,7 @@ async def test_user_update_persists_mcp_entitlement_and_links_it(mocker): "mcp_tool_permissions": {"github": ["list_issues"]}, }, ), - user_api_key_dict=UserAPIKeyAuth( - user_id="admin-1", user_role=LitellmUserRoles.PROXY_ADMIN - ), + user_api_key_dict=UserAPIKeyAuth(user_id="admin-1", user_role=LitellmUserRoles.PROXY_ADMIN), ) upsert_kwargs = mock_prisma_client.db.litellm_objectpermissiontable.upsert.call_args.kwargs @@ -4308,9 +4102,7 @@ async def test_user_update_invalidates_the_cached_entitlement(mocker): user_id="target-user", object_permission={"mcp_tool_permissions": {"github": []}}, ), - user_api_key_dict=UserAPIKeyAuth( - user_id="admin-1", user_role=LitellmUserRoles.PROXY_ADMIN - ), + user_api_key_dict=UserAPIKeyAuth(user_id="admin-1", user_role=LitellmUserRoles.PROXY_ADMIN), ) deleted = {call.kwargs["key"] for call in cache.async_delete_cache.call_args_list} @@ -4343,9 +4135,7 @@ async def test_admin_can_clear_a_users_mcp_entitlement(mocker): await _update_single_user_helper( user_request=UpdateUserRequest(user_id="target-user", object_permission={}), - user_api_key_dict=UserAPIKeyAuth( - user_id="admin-1", user_role=LitellmUserRoles.PROXY_ADMIN - ), + user_api_key_dict=UserAPIKeyAuth(user_id="admin-1", user_role=LitellmUserRoles.PROXY_ADMIN), ) written = mock_prisma_client.update_data.call_args.kwargs["data"] @@ -4382,9 +4172,7 @@ async def test_user_update_invalidates_both_the_old_and_new_permission_rows(mock user_id="target-user", object_permission={"mcp_tool_permissions": {"github": ["list_issues"]}}, ), - user_api_key_dict=UserAPIKeyAuth( - user_id="admin-1", user_role=LitellmUserRoles.PROXY_ADMIN - ), + user_api_key_dict=UserAPIKeyAuth(user_id="admin-1", user_role=LitellmUserRoles.PROXY_ADMIN), ) deleted = {call.kwargs["key"] for call in cache.async_delete_cache.call_args_list} @@ -4415,9 +4203,7 @@ async def test_non_admin_cannot_clear_their_own_mcp_entitlement(mocker): with pytest.raises(HTTPException) as exc: await _update_single_user_helper( user_request=UpdateUserRequest(user_id="target-user", object_permission={}), - user_api_key_dict=UserAPIKeyAuth( - user_id="target-user", user_role=LitellmUserRoles.INTERNAL_USER - ), + user_api_key_dict=UserAPIKeyAuth(user_id="target-user", user_role=LitellmUserRoles.INTERNAL_USER), ) assert exc.value.status_code == 403 @@ -4445,9 +4231,7 @@ async def test_non_admin_cannot_rewrite_their_own_mcp_entitlement(mocker): user_id="target-user", object_permission={"mcp_servers": [], "mcp_tool_permissions": {}}, ), - user_api_key_dict=UserAPIKeyAuth( - user_id="target-user", user_role=LitellmUserRoles.INTERNAL_USER - ), + user_api_key_dict=UserAPIKeyAuth(user_id="target-user", user_role=LitellmUserRoles.INTERNAL_USER), ) assert exc.value.status_code == 403 @@ -4464,9 +4248,7 @@ async def test_new_user_persists_the_requested_mcp_entitlement(mocker): return_value=SimpleNamespace(object_permission_id="perm-created") ) mock_prisma_client.db.litellm_mcpservertable.find_many = mocker.AsyncMock(return_value=[]) - mock_prisma_client.db.litellm_usertable.find_first = mocker.AsyncMock( - return_value=None - ) + mock_prisma_client.db.litellm_usertable.find_first = mocker.AsyncMock(return_value=None) mock_prisma_client.db.litellm_usertable.count = mocker.AsyncMock(return_value=0) mocker.patch("litellm.proxy.proxy_server.prisma_client", mock_prisma_client) mocker.patch( @@ -4475,9 +4257,7 @@ async def test_new_user_persists_the_requested_mcp_entitlement(mocker): ) mock_generate = mocker.patch( "litellm.proxy.management_endpoints.internal_user_endpoints.generate_key_helper_fn", - new=mocker.AsyncMock( - return_value={"user_id": "new-human", "token": "sk-x", "expires": None} - ), + new=mocker.AsyncMock(return_value={"user_id": "new-human", "token": "sk-x", "expires": None}), ) mocker.patch( "litellm.proxy.hooks.user_management_event_hooks.UserManagementEventHooks.async_user_created_hook", @@ -4489,9 +4269,7 @@ async def test_new_user_persists_the_requested_mcp_entitlement(mocker): user_id="new-human", object_permission={"mcp_tool_permissions": {"github": ["list_issues"]}}, ), - user_api_key_dict=UserAPIKeyAuth( - user_id="admin-1", user_role=LitellmUserRoles.PROXY_ADMIN - ), + user_api_key_dict=UserAPIKeyAuth(user_id="admin-1", user_role=LitellmUserRoles.PROXY_ADMIN), ) created = mock_prisma_client.db.litellm_objectpermissiontable.create.call_args.kwargs["data"] @@ -4533,16 +4311,12 @@ async def test_user_info_v2_returns_the_mcp_entitlement(mocker): response = await user_info_v2( request=SimpleNamespace(query_params={}), user_id="human-1", - user_api_key_dict=UserAPIKeyAuth( - user_id="admin-1", user_role=LitellmUserRoles.PROXY_ADMIN - ), + user_api_key_dict=UserAPIKeyAuth(user_id="admin-1", user_role=LitellmUserRoles.PROXY_ADMIN), ) assert response.object_permission is not None assert response.object_permission.mcp_servers == ["github"] - assert response.object_permission.mcp_tool_permissions == { - "github": ["list_issues"] - } + assert response.object_permission.mcp_tool_permissions == {"github": ["list_issues"]} @pytest.mark.asyncio @@ -4558,9 +4332,7 @@ async def test_user_info_v2_returns_the_mcp_entitlement(mocker): ], ids=["supplied", "omitted", "empty"], ) -async def test_user_new_persists_model_max_budget( - monkeypatch, model_max_budget, expected_written -): +async def test_user_new_persists_model_max_budget(monkeypatch, model_max_budget, expected_written): """ /user/new used to echo model_max_budget back while writing {} to the user row, so a per-model budget looked configured and was read by nothing. @@ -4674,6 +4446,11 @@ async def test_user_update_hashes_and_persists_strong_password(_admin_prisma, mo _update_single_user_helper, ) + mocker.patch( # test-quality-ok: same module-global mocking every test in this file already uses + "litellm.proxy.proxy_server.general_settings", + {"password_policy_check_breached_passwords": False}, + ) + mock_prisma_client = _admin_prisma existing_user = mocker.MagicMock() existing_user.model_dump.return_value = {"user_id": "target-user"} @@ -4691,6 +4468,152 @@ async def test_user_update_hashes_and_persists_strong_password(_admin_prisma, mo written_data = mock_prisma_client.update_data.call_args.kwargs["data"] assert written_data.get("password") is not None assert written_data["password"] != strong_password + # An admin-set password is known to the admin, so the user must be forced + # to change it at next login and the breach screen re-armed. + assert written_data["password_reset_required"] is True + assert written_data["last_breach_check_at"] is None + + +@pytest.mark.asyncio +@respx.mock +async def test_user_update_rejects_breached_password(_admin_prisma): + """A strength-passing password found in the HIBP corpus must be rejected + before it ever reaches the DB write.""" + from litellm.proxy.management_endpoints.internal_user_endpoints import ( + _update_single_user_helper, + ) + + password = "Str0ng!Passw0rd" + sha1 = hashlib.sha1(password.encode("utf-8"), usedforsecurity=False).hexdigest().upper() + respx.get(f"https://api.pwnedpasswords.com/range/{sha1[:5]}").mock( + return_value=httpx.Response(200, text=f"{sha1[5:]}:1387") + ) + + user_request = UpdateUserRequest(user_id="target-user", password=password) + admin_caller = UserAPIKeyAuth(user_id="admin-1", user_role=LitellmUserRoles.PROXY_ADMIN) + + with pytest.raises(ProxyException) as exc_info: + await _update_single_user_helper(user_request=user_request, user_api_key_dict=admin_caller) + + assert exc_info.value.code == "400" + assert "data breaches" in exc_info.value.message + _admin_prisma.db.litellm_usertable.find_first.assert_not_called() + + +@pytest.mark.asyncio +async def test_bulk_update_all_users_rejects_a_password(_admin_prisma): + """The all_users fast path writes user_updates straight to update_many, + bypassing _update_single_user_helper. A password riding along would be + stored as unvalidated plaintext on every row, so it must be rejected + before any DB access.""" + from fastapi import HTTPException + + from litellm.proxy._types import UpdateUserRequestNoUserIDorEmail + from litellm.proxy.management_endpoints.internal_user_endpoints import bulk_user_update + from litellm.types.proxy.management_endpoints.internal_user_endpoints import ( + BulkUpdateUserRequest, + ) + + data = BulkUpdateUserRequest( + all_users=True, + user_updates=UpdateUserRequestNoUserIDorEmail(password="Str0ng!Passw0rd"), + ) + admin_caller = UserAPIKeyAuth(user_id="admin-1", user_role=LitellmUserRoles.PROXY_ADMIN) + + with pytest.raises(HTTPException) as exc_info: + await bulk_user_update(data=data, user_api_key_dict=admin_caller) + + assert exc_info.value.status_code == 400 + assert "not supported" in str(exc_info.value.detail) + _admin_prisma.db.litellm_usertable.find_many.assert_not_called() + _admin_prisma.db.litellm_usertable.update_many.assert_not_called() + + +def _hibp_client_with_handler(handler) -> AsyncHTTPHandler: + """A real AsyncHTTPHandler over httpx.MockTransport (the DI seam used + throughout test_password_policy.py), so no network is touched.""" + http_handler = AsyncHTTPHandler() + http_handler.client = httpx.AsyncClient(transport=httpx.MockTransport(handler)) + return http_handler + + +@pytest.mark.asyncio +async def test_bulk_update_breached_password_fails_only_that_user(_admin_prisma, mocker): + """In a bulk batch, a breached password fails only its own entry, before + any DB write for it; sibling entries with acceptable passwords persist.""" + from litellm.proxy.management_endpoints.internal_user_endpoints import ( + bulk_update_processed_users, + ) + + breached = "Br3ached!Passw0rd" + clean = "NewP@ssw0rd123" + breached_sha1 = hashlib.sha1(breached.encode("utf-8"), usedforsecurity=False).hexdigest().upper() + + def handler(request: httpx.Request) -> httpx.Response: + if request.url.path == f"/range/{breached_sha1[:5]}": + return httpx.Response(200, text=f"{breached_sha1[5:]}:1387") + return httpx.Response(200, text="0000000000000000000000000000000000A:1") + + mock_prisma_client = _admin_prisma + existing_user = mocker.MagicMock() + existing_user.model_dump.return_value = {"user_id": "user-clean"} + existing_user.user_id = "user-clean" + mock_prisma_client.db.litellm_usertable.find_first = mocker.AsyncMock(return_value=existing_user) + mock_prisma_client.update_data = mocker.AsyncMock(return_value={"user_id": "user-clean"}) + mock_prisma_client.jsonify_object = mocker.MagicMock(side_effect=lambda x: x) + + admin_caller = UserAPIKeyAuth(user_id="admin-1", user_role=LitellmUserRoles.PROXY_ADMIN) + response = await bulk_update_processed_users( + users_to_update=[ + UpdateUserRequest(user_id="user-breached", password=breached), + UpdateUserRequest(user_id="user-clean", password=clean), + ], + user_api_key_dict=admin_caller, + hibp_client=_hibp_client_with_handler(handler), + ) + + assert response.successful_updates == 1 + assert response.failed_updates == 1 + by_user = {r.user_id: r for r in response.results} + assert by_user["user-breached"].success is False + assert "data breaches" in by_user["user-breached"].error + assert by_user["user-clean"].success is True + (write_call,) = mock_prisma_client.update_data.call_args_list + assert write_call.kwargs["user_id"] == "user-clean" + + +@pytest.mark.asyncio +async def test_bulk_update_screens_shared_password_with_single_lookup(_admin_prisma, mocker): + """A batch where every user gets the same password costs one HIBP lookup, + not one per user (the serial per-user checks this regresses against).""" + from litellm.proxy.management_endpoints.internal_user_endpoints import ( + bulk_update_processed_users, + ) + + lookup_count = 0 + + def handler(request: httpx.Request) -> httpx.Response: + nonlocal lookup_count + lookup_count += 1 + return httpx.Response(200, text="0000000000000000000000000000000000A:1") + + mock_prisma_client = _admin_prisma + existing_user = mocker.MagicMock() + existing_user.model_dump.return_value = {"user_id": "user-0"} + existing_user.user_id = "user-0" + mock_prisma_client.db.litellm_usertable.find_first = mocker.AsyncMock(return_value=existing_user) + mock_prisma_client.update_data = mocker.AsyncMock(return_value={"user_id": "user-0"}) + mock_prisma_client.jsonify_object = mocker.MagicMock(side_effect=lambda x: x) + + admin_caller = UserAPIKeyAuth(user_id="admin-1", user_role=LitellmUserRoles.PROXY_ADMIN) + response = await bulk_update_processed_users( + users_to_update=[UpdateUserRequest(user_id=f"user-{i}", password="NewP@ssw0rd123") for i in range(5)], + user_api_key_dict=admin_caller, + hibp_client=_hibp_client_with_handler(handler), + ) + + assert response.successful_updates == 5 + assert lookup_count == 1 @pytest.mark.asyncio diff --git a/tests/test_litellm/proxy/management_endpoints/test_mcp_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_mcp_management_endpoints.py index 80773f314d8..47540eb5d6d 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_mcp_management_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_mcp_management_endpoints.py @@ -6,7 +6,7 @@ import logging from contextlib import ExitStack from datetime import datetime, timedelta from types import SimpleNamespace -from typing import List, Optional, cast +from typing import Final, List, Optional, cast from unittest.mock import AsyncMock, MagicMock, patch import pytest @@ -1614,6 +1614,60 @@ class TestTeamScopedMCPServerAccess: result = await fetch_all_mcp_servers(user_api_key_dict=mock_user_auth, team_id="any-team-id") assert len(result) == 1 + +class TestFetchAllMCPServersOrdering: + def test_display_order_is_case_insensitive_name_then_id(self) -> None: + servers: Final = ( + LiteLLM_MCPServerTable(server_id="s-2", server_name="GitHub", alias="aaa", transport=MCPTransport.http), + LiteLLM_MCPServerTable(server_id="s-1", alias="github", transport=MCPTransport.http), + LiteLLM_MCPServerTable(server_id="s-0", server_name="Slack", alias="zzz", transport=MCPTransport.http), + LiteLLM_MCPServerTable(server_id="confluence", server_name="", alias="", transport=MCPTransport.http), + ) + + ordered: Final = sorted(servers, key=mgmt_endpoints._mcp_server_display_order) + assert [s.server_id for s in ordered] == ["confluence", "s-1", "s-2", "s-0"] + + @pytest.mark.parametrize("team_id", [None, "team-1"]) + @pytest.mark.parametrize("reverse", [False, True]) + @pytest.mark.asyncio + async def test_list_is_sorted_by_display_name_regardless_of_resolution_order( + self, team_id: str | None, reverse: bool + ) -> None: + mock_user_auth: Final = generate_mock_user_api_key_auth( + user_role=LitellmUserRoles.PROXY_ADMIN, + user_id="admin_user", + ) + servers: Final = ( + generate_mock_mcp_server_db_record(server_id="s-zeta", alias="zeta"), + generate_mock_mcp_server_db_record(server_id="s-alpha", alias="Alpha"), + generate_mock_mcp_server_db_record(server_id="s-mid", alias="mid"), + ) + resolved: Final = list(reversed(servers) if reverse else servers) + mock_manager: Final = MagicMock() + mock_manager.get_all_allowed_mcp_servers = AsyncMock(return_value=resolved) + with ( + patch( # test-quality-ok: the route reads a module-global manager with no injection seam + "litellm.proxy.management_endpoints.mcp_management_endpoints.global_mcp_server_manager", + mock_manager, + ), + patch( # test-quality-ok: admin view is derived from module-global proxy settings + "litellm.proxy.management_endpoints.mcp_management_endpoints._user_has_admin_view", + return_value=True, + ), + patch( # test-quality-ok: auth contexts need a live prisma client + "litellm.proxy.management_endpoints.mcp_management_endpoints.build_effective_auth_contexts", + AsyncMock(return_value=[mock_user_auth]), + ), + patch( # test-quality-ok: isolate the route's ordering from team database resolution + "litellm.proxy.management_endpoints.mcp_management_endpoints._get_team_scoped_mcp_server_list", + AsyncMock(return_value=resolved), + ), + ): + result: Final = await mgmt_endpoints.fetch_all_mcp_servers( + user_api_key_dict=mock_user_auth, team_id=team_id + ) + assert [s.server_id for s in result] == ["s-alpha", "s-mid", "s-zeta"] + @pytest.mark.asyncio async def test_restricted_virtual_key_cannot_use_team_id_filter(self): """Restricted virtual keys must not bypass access limits via team_id.""" @@ -7824,3 +7878,25 @@ class TestDeleteMCPGatewaySessions: assert result.terminated_sessions == 2 assert {s.user_id for s in result.sessions} == {"bob"} assert "sk-live-bob" not in result.model_dump_json() + + +class TestGetMcpToolsWireShape: + @pytest.mark.asyncio + async def test_get_mcp_tools_returns_each_tool_in_mcp_wire_spelling(self): + from mcp.types import ListToolsResult, Tool + + add_schema = {"type": "object", "properties": {"a": {"type": "integer"}}, "required": ["a"]} + listed = ListToolsResult( + tools=[Tool(name="add", description="Add", inputSchema=add_schema, outputSchema={"type": "integer"})] + ) + with patch( + "litellm.proxy._experimental.mcp_server.server._list_mcp_tools", + AsyncMock(return_value=listed), + ): + result = await mgmt_endpoints.get_mcp_tools(user_api_key_dict=generate_mock_user_api_key_auth()) + + (tool,) = result["tools"] + assert tool["inputSchema"] == add_schema + assert tool["outputSchema"] == {"type": "integer"} + assert "_meta" in tool + assert not {"input_schema", "output_schema", "meta"} & tool.keys() diff --git a/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py index e6b5fb25c3e..bd252169131 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py @@ -1284,11 +1284,11 @@ class TestUpdateModel: "litellm.proxy.management_endpoints.model_management_endpoints.ModelManagementAuthChecks.can_user_make_model_call", new=AsyncMock(return_value=None), ), - patch( + patch( # test-quality-ok: [TQ008] isolate persistence from encryption implementation "litellm.proxy.management_endpoints.model_management_endpoints.encrypt_value_helper", side_effect=lambda value: value, ), - patch( + patch( # test-quality-ok: [TQ008] isolate persistence from router reload implementation "litellm.proxy.management_endpoints.model_management_endpoints.clear_cache", new=AsyncMock( return_value=ReconcileOutcome(still_desired=None, live_after=None) @@ -4063,6 +4063,294 @@ class TestModelInfoServerDerivedPricingFilter: assert written["access_groups"] == ["prod"] +class TestModelInfoCostMapEchoFilter: + """LIT-5534. ``/model/info`` fills a deployment's ``model_info`` from the cost map (context + limits, mode, provider, supported params, capability flags), and the Admin UI edit form sends + that whole blob back on any save. Only values that still equal the cost-map entry are the + echo; a value the operator changed is a real override and stays.""" + + def test_echoed_cost_map_metadata_is_not_persisted(self): + import litellm + + from litellm.proxy.management_endpoints.model_management_endpoints import update_db_model + from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo + + entry = litellm.get_model_info("openai/gpt-5.6") + echo = {**entry, "id": "dep-echo-0", "db_model": True, "access_groups": ["prod"]} + db_model = Deployment( + model_name="gpt-5.6", + litellm_params=LiteLLM_Params(model="openai/gpt-5.6"), + model_info=ModelInfo(id="dep-echo-0"), + ) + + result = update_db_model( + db_model=db_model, + updated_patch=updateDeployment(model_info=ModelInfo(**echo)), + ) + + info = json.loads(result["model_info"]) + assert info["access_groups"] == ["prod"] + assert set(info).isdisjoint(entry) + assert "max_input_tokens" not in info and "mode" not in info and "supports_vision" not in info, ( + "cost-map metadata must not be persisted from an unchanged /model/info echo" + ) + + def test_an_edited_value_survives_the_echo_filter(self): + import litellm + + from litellm.proxy.management_endpoints.model_management_endpoints import update_db_model + from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo + + entry = litellm.get_model_info("openai/gpt-5.6") + echo = { + **entry, + "id": "dep-echo-1", + "db_model": True, + "access_groups": ["prod"], + "max_input_tokens": entry["max_input_tokens"] + 1, + "mode": "completion" if entry["mode"] != "completion" else "chat", + } + db_model = Deployment( + model_name="gpt-5.6", + litellm_params=LiteLLM_Params(model="openai/gpt-5.6"), + model_info=ModelInfo(id="dep-echo-1"), + ) + + result = update_db_model( + db_model=db_model, + updated_patch=updateDeployment(model_info=ModelInfo(**echo)), + ) + + info = json.loads(result["model_info"]) + assert info["max_input_tokens"] == echo["max_input_tokens"] + assert info["mode"] == echo["mode"] + assert "litellm_provider" not in info + assert "supported_openai_params" not in info + + def test_metadata_without_a_cost_map_key_is_persisted(self): + import litellm + + from litellm.proxy.management_endpoints.model_management_endpoints import update_db_model + from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo + from litellm.types.utils import echoed_cost_map_fields + + entry = litellm.get_model_info("openai/gpt-5.6") + assert echoed_cost_map_fields({"max_input_tokens": entry["max_input_tokens"]}, entry) == () + db_model = Deployment( + model_name="gpt-5.6", + litellm_params=LiteLLM_Params(model="openai/gpt-5.6"), + model_info=ModelInfo(id="dep-echo-2"), + ) + + result = update_db_model( + db_model=db_model, + updated_patch=updateDeployment( + model_info=ModelInfo( + id="dep-echo-2", + max_input_tokens=entry["max_input_tokens"], + mode=entry["mode"], + ) + ), + ) + + info = json.loads(result["model_info"]) + assert info["max_input_tokens"] == entry["max_input_tokens"] + assert info["mode"] == entry["mode"] + + def test_a_stored_mode_survives_an_echoed_save(self): + import litellm + + from litellm.proxy.management_endpoints.model_management_endpoints import update_db_model + from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo + + entry = litellm.get_model_info("openai/gpt-5.6") + db_model = Deployment( + model_name="gpt-5.6", + litellm_params=LiteLLM_Params(model="openai/gpt-5.6"), + model_info=ModelInfo(id="dep-echo-3", mode=entry["mode"]), + ) + echo = {**entry, "id": "dep-echo-3", "db_model": True, "access_groups": ["prod"]} + + result = update_db_model( + db_model=db_model, + updated_patch=updateDeployment(model_info=ModelInfo(**echo)), + ) + + info = json.loads(result["model_info"]) + assert info["mode"] == entry["mode"] + assert "max_input_tokens" not in info + + def test_resetting_an_override_to_the_cost_map_value_removes_it(self): + import litellm + + from litellm.proxy.management_endpoints.model_management_endpoints import update_db_model + from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo + + entry = litellm.get_model_info("openai/gpt-5.6") + db_model = Deployment( + model_name="gpt-5.6", + litellm_params=LiteLLM_Params(model="openai/gpt-5.6"), + model_info=ModelInfo(id="dep-echo-4", mode="chat", max_input_tokens=2048), + ) + echo = {**entry, "id": "dep-echo-4", "db_model": True, "access_groups": ["staging"]} + + result = update_db_model( + db_model=db_model, + updated_patch=updateDeployment(model_info=ModelInfo(**echo)), + ) + + info = json.loads(result["model_info"]) + assert "max_input_tokens" not in info + assert info["mode"] == "chat" + assert info["access_groups"] == ["staging"] + + def test_reset_is_recognised_after_the_router_registered_the_override(self, monkeypatch: pytest.MonkeyPatch): + import litellm + + from litellm.proxy.management_endpoints.model_management_endpoints import update_db_model + from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo + + pristine = litellm.get_model_info("openai/gpt-5.6") + polluted = {**pristine, "max_input_tokens": 2048} + monkeypatch.setattr(litellm, "get_model_info", lambda model, **_: polluted) + db_model = Deployment( + model_name="gpt-5.6", + litellm_params=LiteLLM_Params(model="openai/gpt-5.6"), + model_info=ModelInfo(id="dep-echo-8", max_input_tokens=2048), + ) + echo = {**pristine, "id": "dep-echo-8", "db_model": True} + + result = update_db_model( + db_model=db_model, + updated_patch=updateDeployment(model_info=ModelInfo(**echo)), + ) + + info = json.loads(result["model_info"]) + assert "max_input_tokens" not in info, info + + def test_reset_to_a_remote_catalog_value_that_differs_from_the_bundled_one(self, monkeypatch: pytest.MonkeyPatch): + from types import MappingProxyType + + import litellm + + from litellm.proxy.management_endpoints.model_management_endpoints import update_db_model + from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo + + bundled = litellm.get_model_info("openai/gpt-5.6") + remote = {**bundled, "max_input_tokens": bundled["max_input_tokens"] + 1} + remote_catalog = MappingProxyType({remote["key"]: MappingProxyType(remote)}) + monkeypatch.setattr(litellm, "get_model_info", lambda model, **_: {**remote, "max_input_tokens": 2048}) + db_model = Deployment( + model_name="gpt-5.6", + litellm_params=LiteLLM_Params(model="openai/gpt-5.6"), + model_info=ModelInfo(id="dep-echo-9", max_input_tokens=2048), + ) + + result = update_db_model( + db_model=db_model, + updated_patch=updateDeployment(model_info=ModelInfo(**{**remote, "id": "dep-echo-9", "db_model": True})), + loaded_catalog=lambda: remote_catalog, + ) + + info = json.loads(result["model_info"]) + assert "max_input_tokens" not in info, info + + def test_echo_is_compared_against_the_deployments_lookup_not_the_key(self): + import litellm + + from litellm.proxy.management_endpoints.model_management_endpoints import update_db_model + from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo + + lookup_pairs: Final = ( + ("openai/gpt-5.6", "gpt-5.6"), + ("openai/gpt-4.1-mini", "gpt-4.1-mini"), + ) + lookup_data: Final = tuple( + (deployment_model, deployment_entry, differing_fields) + for deployment_model, key_model in lookup_pairs + for deployment_entry in (litellm.get_model_info(deployment_model),) + for key_entry in (litellm.get_model_info(key_model),) + for differing_fields in ( + frozenset( + k for k in deployment_entry if k in key_entry and deployment_entry[k] != key_entry[k] + ), + ) + if differing_fields + ) + if not lookup_data: + pytest.skip("No deployment/key cost-map lookup differences are available") + + deployment_model, entry, differing_fields = lookup_data[0] + assert differing_fields + db_model = Deployment( + model_name=deployment_model, + litellm_params=LiteLLM_Params(model=deployment_model), + model_info=ModelInfo(id="dep-echo-5"), + ) + echo = {**entry, "id": "dep-echo-5", "db_model": True, "access_groups": ["prod"]} + + result = update_db_model( + db_model=db_model, + updated_patch=updateDeployment(model_info=ModelInfo(**echo)), + ) + + info = json.loads(result["model_info"]) + assert not frozenset(info).intersection(frozenset(entry) - frozenset(("mode",))) + + def test_base_model_wins_over_litellm_params_model_for_the_lookup(self): + import litellm + + from litellm.proxy.management_endpoints.model_management_endpoints import update_db_model + from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo + + entry = litellm.get_model_info("azure/gpt-5.6") + db_model = Deployment( + model_name="azure/my-deploy", + litellm_params=LiteLLM_Params(model="azure/my-deploy"), + model_info=ModelInfo(id="dep-echo-6", base_model="azure/gpt-5.6"), + ) + echo = { + **entry, + "id": "dep-echo-6", + "base_model": "azure/gpt-5.6", + "db_model": True, + } + + result = update_db_model( + db_model=db_model, + updated_patch=updateDeployment(model_info=ModelInfo(**echo)), + ) + + info = json.loads(result["model_info"]) + assert not frozenset(info).intersection(frozenset(entry) - frozenset(("mode",))) + assert info["base_model"] == "azure/gpt-5.6" + + def test_encrypted_stored_model_is_decrypted_for_the_lookup(self, monkeypatch): + import litellm + + from litellm.proxy.management_endpoints.model_management_endpoints import update_db_model + from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo + + monkeypatch.setenv("LITELLM_SALT_KEY", "sk-1234") + entry = litellm.get_model_info("openai/gpt-5.6") + db_model = Deployment( + model_name="gpt-5.6", + litellm_params=LiteLLM_Params(model=encrypt_value_helper(value="openai/gpt-5.6")), + model_info=ModelInfo(id="dep-echo-7", mode="chat"), + ) + echo = {**entry, "id": "dep-echo-7", "db_model": True, "access_groups": ["prod"]} + + result = update_db_model( + db_model=db_model, + updated_patch=updateDeployment(model_info=ModelInfo(**echo)), + ) + + info = json.loads(result["model_info"]) + assert not frozenset(info).intersection(frozenset(entry) - frozenset(("mode",))) + assert info["mode"] == "chat" + assert info["access_groups"] == ["prod"] + + class TestUpdateDBModelClearCacheControlInjectionPoints: def test_explicit_null_removes_stored_injection_points(self): from litellm.proxy.management_endpoints.model_management_endpoints import ( @@ -4615,7 +4903,7 @@ class TestPatchModelBlockedAuthGate: "litellm.proxy.management_endpoints.model_management_endpoints.ModelManagementAuthChecks.can_user_make_model_call", new=AsyncMock(return_value=None), ), - patch( + patch( # test-quality-ok: [TQ008] isolate persistence from router reload implementation "litellm.proxy.management_endpoints.model_management_endpoints.clear_cache", new=AsyncMock( return_value=ReconcileOutcome(still_desired=None, live_after=None) @@ -7284,3 +7572,337 @@ class TestTeamMemberAutoRouterWrites: assert json.loads(written["model_info"])["member_auto_router"] is True assert appended.await_args.kwargs["data"].models == ["new-personal-router"] assert appended.await_args.kwargs["data"].team_id == "member-team" + + +class TestModelManagementActorEdges: + @pytest.mark.asyncio + async def test_add_model_rejects_non_team_internal_user(self): + from litellm.proxy._types import ProxyException + from litellm.proxy.management_endpoints.model_management_endpoints import add_new_model + + actor: Final = UserAPIKeyAuth(user_id="internal-user", user_role=LitellmUserRoles.INTERNAL_USER) + prisma: Final = MagicMock() + deployment: Final = Deployment( + model_name="internal-model", + litellm_params=LiteLLM_Params(model="openai/test-model"), + model_info=ModelInfo(id="internal-model-id"), + ) + with ( + patch("litellm.proxy.proxy_server.prisma_client", prisma), # test-quality-ok: [TQ008] endpoint reads proxy-server state through its only test seam + patch("litellm.proxy.proxy_server.store_model_in_db", True), # test-quality-ok: [TQ008] endpoint reads proxy-server state through its only test seam + patch("litellm.proxy.proxy_server.premium_user", True), # test-quality-ok: [TQ008] endpoint reads proxy-server state through its only test seam + patch("litellm.proxy.proxy_server.general_settings", {}), # test-quality-ok: [TQ008] endpoint reads proxy-server state through its only test seam + ): + with pytest.raises(ProxyException) as exc_info: + await add_new_model(model_params=deployment, user_api_key_dict=actor) + + assert str(exc_info.value.code) == "403" + assert "permission" in str(exc_info.value).lower() + prisma.db.litellm_proxymodeltable.create.assert_not_called() + + @pytest.mark.asyncio + async def test_add_model_rejects_proxy_admin_viewer(self): + from litellm.proxy._types import ProxyException + from litellm.proxy.management_endpoints.model_management_endpoints import add_new_model + + actor: Final = UserAPIKeyAuth( + user_id="view-only-user", user_role=LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY + ) + prisma: Final = MagicMock() + deployment: Final = Deployment( + model_name="view-only-model", + litellm_params=LiteLLM_Params(model="openai/test-model"), + model_info=ModelInfo(id="view-only-model-id"), + ) + with ( + patch("litellm.proxy.proxy_server.prisma_client", prisma), # test-quality-ok: [TQ008] endpoint reads proxy-server state through its only test seam + patch("litellm.proxy.proxy_server.store_model_in_db", True), # test-quality-ok: [TQ008] endpoint reads proxy-server state through its only test seam + patch("litellm.proxy.proxy_server.premium_user", True), # test-quality-ok: [TQ008] endpoint reads proxy-server state through its only test seam + patch("litellm.proxy.proxy_server.general_settings", {}), # test-quality-ok: [TQ008] endpoint reads proxy-server state through its only test seam + ): + with pytest.raises(ProxyException) as exc_info: + await add_new_model(model_params=deployment, user_api_key_dict=actor) + + assert str(exc_info.value.code) == "403" + assert "view-only" in str(exc_info.value).lower() + prisma.db.litellm_proxymodeltable.create.assert_not_called() + + @pytest.mark.asyncio + async def test_add_model_requires_database_storage(self): + from litellm.proxy._types import ProxyException + from litellm.proxy.management_endpoints.model_management_endpoints import add_new_model + + actor: Final = UserAPIKeyAuth(user_id="admin", user_role=LitellmUserRoles.PROXY_ADMIN) + prisma: Final = MagicMock() + deployment: Final = Deployment( + model_name="database-disabled-model", + litellm_params=LiteLLM_Params(model="openai/test-model"), + model_info=ModelInfo(id="database-disabled-model-id"), + ) + with ( + patch("litellm.proxy.proxy_server.prisma_client", prisma), # test-quality-ok: [TQ008] endpoint reads proxy-server state through its only test seam + patch("litellm.proxy.proxy_server.store_model_in_db", False), # test-quality-ok: [TQ008] endpoint reads proxy-server state through its only test seam + patch("litellm.proxy.proxy_server.premium_user", True), # test-quality-ok: [TQ008] endpoint reads proxy-server state through its only test seam + patch("litellm.proxy.proxy_server.general_settings", {}), # test-quality-ok: [TQ008] endpoint reads proxy-server state through its only test seam + ): + with pytest.raises(ProxyException) as exc_info: + await add_new_model(model_params=deployment, user_api_key_dict=actor) + + assert str(exc_info.value.code) == "500" + assert "STORE_MODEL_IN_DB" in str(exc_info.value) + prisma.db.litellm_proxymodeltable.create.assert_not_called() + + @pytest.mark.asyncio + async def test_legacy_model_update_persists_changed_field(self): + from litellm.proxy.management_endpoints.model_management_endpoints import update_model + + model_id: Final = "legacy-update-model-id" + existing_row: Final = MagicMock() + existing_row.litellm_params = {"model": "openai/test-model", "timeout": 30} + existing_row.model_dump.return_value = { + "model_name": "legacy-update-model", + "litellm_params": existing_row.litellm_params, + "model_info": {"id": model_id}, + } + existing_row.model_dump_json.return_value = "{}" + updated_row: Final = MagicMock() + updated_row.model_dump_json.return_value = "{}" + prisma: Final = MagicMock() + prisma.db.litellm_proxymodeltable.find_unique = AsyncMock(return_value=existing_row) + prisma.db.litellm_proxymodeltable.update = AsyncMock(return_value=updated_row) + router: Final = MagicMock() + router.get_model_ids.return_value = [model_id] + actor: Final = UserAPIKeyAuth(user_id="admin", user_role=LitellmUserRoles.PROXY_ADMIN) + with ( + patch("litellm.proxy.proxy_server.prisma_client", prisma), # test-quality-ok: [TQ008] update endpoint reads proxy-server state through its only test seam + patch("litellm.proxy.proxy_server.llm_router", router), # test-quality-ok: [TQ008] update endpoint reads proxy-server state through its only test seam + patch("litellm.proxy.proxy_server.store_model_in_db", True), # test-quality-ok: [TQ008] update endpoint reads proxy-server state through its only test seam + patch("litellm.proxy.proxy_server.premium_user", True), # test-quality-ok: [TQ008] update endpoint reads proxy-server state through its only test seam + patch( # test-quality-ok: [TQ008] isolate persistence from encryption implementation + "litellm.proxy.management_endpoints.model_management_endpoints.encrypt_value_helper", + side_effect=lambda value: value, + ), + patch( # test-quality-ok: [TQ008] isolate persistence from router reload implementation + "litellm.proxy.management_endpoints.model_management_endpoints.clear_cache", + new=AsyncMock(return_value=ReconcileOutcome(still_desired=None, live_after=None)), + ), + ): + await update_model( + model_params=updateDeployment( + litellm_params=updateLiteLLMParams(timeout=42), + model_info=ModelInfo(id=model_id), + ), + user_api_key_dict=actor, + ) + + written: Final = json.loads( + prisma.db.litellm_proxymodeltable.update.await_args.kwargs["data"]["litellm_params"] + ) + assert written["timeout"] == 42 + assert written["model"] == "openai/test-model" + + @pytest.mark.asyncio + async def test_legacy_model_update_explicit_null_preserves_existing_field(self): + from litellm.proxy.management_endpoints.model_management_endpoints import update_model + + model_id: Final = "legacy-null-model-id" + existing_row: Final = MagicMock() + existing_row.litellm_params = {"model": "openai/test-model", "timeout": 30} + existing_row.model_dump.return_value = { + "model_name": "legacy-null-model", + "litellm_params": existing_row.litellm_params, + "model_info": {"id": model_id}, + } + existing_row.model_dump_json.return_value = "{}" + updated_row: Final = MagicMock() + updated_row.model_dump_json.return_value = "{}" + prisma: Final = MagicMock() + prisma.db.litellm_proxymodeltable.find_unique = AsyncMock(return_value=existing_row) + prisma.db.litellm_proxymodeltable.update = AsyncMock(return_value=updated_row) + router: Final = MagicMock() + router.get_model_ids.return_value = [model_id] + actor: Final = UserAPIKeyAuth(user_id="admin", user_role=LitellmUserRoles.PROXY_ADMIN) + with ( + patch("litellm.proxy.proxy_server.prisma_client", prisma), # test-quality-ok: [TQ008] update endpoint reads proxy-server state through its only test seam + patch("litellm.proxy.proxy_server.llm_router", router), # test-quality-ok: [TQ008] update endpoint reads proxy-server state through its only test seam + patch("litellm.proxy.proxy_server.store_model_in_db", True), # test-quality-ok: [TQ008] update endpoint reads proxy-server state through its only test seam + patch("litellm.proxy.proxy_server.premium_user", True), # test-quality-ok: [TQ008] update endpoint reads proxy-server state through its only test seam + patch( # test-quality-ok: [TQ008] isolate persistence from encryption implementation + "litellm.proxy.management_endpoints.model_management_endpoints.encrypt_value_helper", + side_effect=lambda value: value, + ), + patch( # test-quality-ok: [TQ008] isolate persistence from router reload implementation + "litellm.proxy.management_endpoints.model_management_endpoints.clear_cache", + new=AsyncMock(return_value=ReconcileOutcome(still_desired=None, live_after=None)), + ), + ): + await update_model( + model_params=updateDeployment( + litellm_params=updateLiteLLMParams(timeout=None), + model_info=ModelInfo(id=model_id), + ), + user_api_key_dict=actor, + ) + + written: Final = json.loads( + prisma.db.litellm_proxymodeltable.update.await_args.kwargs["data"]["litellm_params"] + ) + assert written["timeout"] == 30 + + @pytest.mark.asyncio + async def test_patch_model_rejects_config_file_model(self): + from litellm.proxy._types import ProxyException + from litellm.proxy.management_endpoints.model_management_endpoints import patch_model + + model_id: Final = "config-model-id" + prisma: Final = MagicMock() + prisma.db.litellm_proxymodeltable.find_unique = AsyncMock(return_value=None) + prisma.db.litellm_proxymodeltable.update = AsyncMock() + router: Final = MagicMock() + router.get_deployment.return_value = Deployment( + model_name="config-model", + litellm_params=LiteLLM_Params(model="openai/test-model"), + model_info=ModelInfo(id=model_id), + ) + actor: Final = UserAPIKeyAuth(user_id="admin", user_role=LitellmUserRoles.PROXY_ADMIN) + with ( + patch("litellm.proxy.proxy_server.prisma_client", prisma), # test-quality-ok: [TQ008] patch endpoint reads proxy-server state through its only test seam + patch("litellm.proxy.proxy_server.llm_router", router), # test-quality-ok: [TQ008] patch endpoint reads proxy-server state through its only test seam + patch("litellm.proxy.proxy_server.store_model_in_db", True), # test-quality-ok: [TQ008] patch endpoint reads proxy-server state through its only test seam + patch("litellm.proxy.proxy_server.premium_user", True), # test-quality-ok: [TQ008] patch endpoint reads proxy-server state through its only test seam + ): + with pytest.raises(ProxyException) as exc_info: + await patch_model( + model_id=model_id, + patch_data=updateDeployment( + litellm_params=updateLiteLLMParams(timeout=42), + model_info=ModelInfo(id=model_id), + ), + user_api_key_dict=actor, + ) + + assert str(exc_info.value.code) == "400" + assert "Cannot edit config-based model" in str(exc_info.value) + prisma.db.litellm_proxymodeltable.update.assert_not_awaited() + + @contextlib.contextmanager + def _client_for(self, actor: UserAPIKeyAuth) -> Iterator[TestClient]: + import litellm.proxy.proxy_server as proxy_server + from litellm.proxy.proxy_server import app + + app.dependency_overrides[proxy_server.user_api_key_auth] = lambda: actor + try: + yield TestClient(app) + finally: + app.dependency_overrides.pop(proxy_server.user_api_key_auth, None) + + def test_post_model_new_binds_to_actor_guard(self): + actor: Final = UserAPIKeyAuth(user_id="internal-user", user_role=LitellmUserRoles.INTERNAL_USER) + prisma: Final = MagicMock() + with ( + patch("litellm.proxy.proxy_server.prisma_client", prisma), # test-quality-ok: [TQ008] route reads proxy-server state through its only test seam + patch("litellm.proxy.proxy_server.store_model_in_db", True), # test-quality-ok: [TQ008] route reads proxy-server state through its only test seam + patch("litellm.proxy.proxy_server.premium_user", True), # test-quality-ok: [TQ008] route reads proxy-server state through its only test seam + patch("litellm.proxy.proxy_server.general_settings", {}), # test-quality-ok: [TQ008] route reads proxy-server state through its only test seam + self._client_for(actor) as client, + ): + response: Final = client.post( + "/model/new", + json={ + "model_name": "internal-model", + "litellm_params": {"model": "openai/test-model"}, + "model_info": {"id": "internal-model-id"}, + }, + ) + + assert response.status_code == 403 + assert "permission" in response.text.lower() + prisma.db.litellm_proxymodeltable.create.assert_not_called() + + def test_post_legacy_model_update_binds_to_persistence(self): + model_id: Final = "legacy-route-model-id" + existing_row: Final = LiteLLM_ProxyModelTable( + model_id=model_id, + model_name="legacy-route-model", + litellm_params={"model": "openai/test-model", "timeout": 30}, + model_info={"id": model_id}, + created_by="admin", + updated_by="admin", + ) + updated_row: Final = LiteLLM_ProxyModelTable( + model_id=model_id, + model_name="legacy-route-model", + litellm_params={"model": "openai/test-model", "timeout": 42}, + model_info={"id": model_id}, + created_by="admin", + updated_by="admin", + ) + prisma: Final = MagicMock() + prisma.db.litellm_proxymodeltable.find_unique = AsyncMock(return_value=existing_row) + prisma.db.litellm_proxymodeltable.update = AsyncMock(return_value=updated_row) + router: Final = MagicMock() + router.get_model_ids.return_value = [model_id] + actor: Final = UserAPIKeyAuth(user_id="admin", user_role=LitellmUserRoles.PROXY_ADMIN) + with ( + patch("litellm.proxy.proxy_server.prisma_client", prisma), # test-quality-ok: [TQ008] route reads proxy-server state through its only test seam + patch("litellm.proxy.proxy_server.llm_router", router), # test-quality-ok: [TQ008] route reads proxy-server state through its only test seam + patch("litellm.proxy.proxy_server.store_model_in_db", True), # test-quality-ok: [TQ008] route reads proxy-server state through its only test seam + patch("litellm.proxy.proxy_server.premium_user", True), # test-quality-ok: [TQ008] route reads proxy-server state through its only test seam + patch( # test-quality-ok: [TQ008] isolate persistence from encryption implementation + "litellm.proxy.management_endpoints.model_management_endpoints.encrypt_value_helper", + side_effect=lambda value: value, + ), + patch( # test-quality-ok: [TQ008] isolate persistence from router reload implementation + "litellm.proxy.management_endpoints.model_management_endpoints.clear_cache", + new=AsyncMock(return_value=ReconcileOutcome(still_desired=None, live_after=None)), + ), + patch( # test-quality-ok: [TQ008] audit logging is outside the persistence contract + "litellm.proxy.management_endpoints.model_management_endpoints.create_object_audit_log", + new=AsyncMock(return_value=None), + ), + self._client_for(actor) as client, + ): + response: Final = client.post( + "/model/update", + json={ + "litellm_params": {"timeout": 42}, + "model_info": {"id": model_id}, + }, + ) + + assert response.status_code == 200, response.text + written: Final = json.loads( + prisma.db.litellm_proxymodeltable.update.await_args.kwargs["data"]["litellm_params"] + ) + assert written["timeout"] == 42 + + def test_patch_config_model_binds_to_patch_route(self): + model_id: Final = "config-route-model-id" + prisma: Final = MagicMock() + prisma.db.litellm_proxymodeltable.find_unique = AsyncMock(return_value=None) + prisma.db.litellm_proxymodeltable.update = AsyncMock() + router: Final = MagicMock() + router.get_deployment.return_value = Deployment( + model_name="config-route-model", + litellm_params=LiteLLM_Params(model="openai/test-model"), + model_info=ModelInfo(id=model_id), + ) + actor: Final = UserAPIKeyAuth(user_id="admin", user_role=LitellmUserRoles.PROXY_ADMIN) + with ( + patch("litellm.proxy.proxy_server.prisma_client", prisma), # test-quality-ok: [TQ008] route reads proxy-server state through its only test seam + patch("litellm.proxy.proxy_server.llm_router", router), # test-quality-ok: [TQ008] route reads proxy-server state through its only test seam + patch("litellm.proxy.proxy_server.store_model_in_db", True), # test-quality-ok: [TQ008] route reads proxy-server state through its only test seam + patch("litellm.proxy.proxy_server.premium_user", True), # test-quality-ok: [TQ008] route reads proxy-server state through its only test seam + self._client_for(actor) as client, + ): + response: Final = client.patch( + f"/model/{model_id}/update", + json={ + "litellm_params": {"timeout": 42}, + "model_info": {"id": model_id}, + }, + ) + + assert response.status_code == 400 + assert "Cannot edit config-based model" in response.text + prisma.db.litellm_proxymodeltable.update.assert_not_awaited() diff --git a/tests/test_litellm/proxy/management_endpoints/test_password_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_password_endpoints.py new file mode 100644 index 00000000000..c04353fec99 --- /dev/null +++ b/tests/test_litellm/proxy/management_endpoints/test_password_endpoints.py @@ -0,0 +1,401 @@ +""" +Tests for POST /user/password/change (litellm/proxy/management_endpoints/password_endpoints.py). + +HIBP traffic is intercepted with respx; no test here touches the network. +""" + +import hashlib +from unittest.mock import AsyncMock, MagicMock, patch + +import httpx +import pytest +import respx +from fastapi import HTTPException + +from litellm.proxy._types import UI_TEAM_ID, LitellmTableNames, ProxyErrorTypes, ProxyException, UserAPIKeyAuth +from litellm.proxy.auth.login_utils import PASSWORD_SESSION_METADATA +from litellm.proxy.management_endpoints.password_endpoints import change_password +from litellm.proxy.utils import hash_password, verify_password + +CURRENT_PASSWORD = "OldP@ssw0rd-2026" +NEW_PASSWORD = "NewP@ssw0rd-2026" + +_POLICY_NO_BREACH_CHECK = {"password_policy_check_breached_passwords": False} + + +def _make_user_row(password: str | None) -> MagicMock: + user = MagicMock() + user.user_id = "user-123" + user.password = password + return user + + +def _make_prisma(user: MagicMock | None) -> MagicMock: + prisma = MagicMock() + prisma.db.litellm_usertable.find_first = AsyncMock(return_value=user) + prisma.db.litellm_usertable.update = AsyncMock(return_value=user) + return prisma + + +def _caller(user_id: str | None = "user-123") -> UserAPIKeyAuth: + return UserAPIKeyAuth(user_id=user_id, team_id=UI_TEAM_ID, metadata=dict(PASSWORD_SESSION_METADATA)) + + +def _sso_session_caller() -> UserAPIKeyAuth: + return UserAPIKeyAuth(user_id="user-123", team_id=UI_TEAM_ID, metadata={}) + + +def _virtual_key_caller() -> UserAPIKeyAuth: + return UserAPIKeyAuth(user_id="user-123", team_id="team-abc", metadata=dict(PASSWORD_SESSION_METADATA)) + + +def _hibp_url_for(password: str) -> str: + sha1 = hashlib.sha1(password.encode(), usedforsecurity=False).hexdigest().upper() + return f"https://api.pwnedpasswords.com/range/{sha1[:5]}" + + +def _hibp_suffix_for(password: str) -> str: + return hashlib.sha1(password.encode(), usedforsecurity=False).hexdigest().upper()[5:] + + +@pytest.mark.asyncio +async def test_change_password_success_writes_new_scrypt_hash(): + from litellm.proxy._types import ChangePasswordRequest + + prisma = _make_prisma(_make_user_row(hash_password(CURRENT_PASSWORD))) + + with ( + patch( # test-quality-ok: change_password reads proxy_server module globals; no injection seam + "litellm.proxy.proxy_server.prisma_client", prisma + ), + patch( # test-quality-ok: change_password reads proxy_server module globals; no injection seam + "litellm.proxy.proxy_server.general_settings", _POLICY_NO_BREACH_CHECK + ), + ): + response = await change_password( + data=ChangePasswordRequest(current_password=CURRENT_PASSWORD, new_password=NEW_PASSWORD), + user_api_key_dict=_caller(), + ) + + assert response.user_id == "user-123" + update_kwargs = prisma.db.litellm_usertable.update.call_args.kwargs + assert update_kwargs["where"] == {"user_id": "user-123"} + stored = update_kwargs["data"]["password"] + assert stored != NEW_PASSWORD + assert verify_password(NEW_PASSWORD, stored) + # A successful change lifts any pending forced reset and re-arms the + # login-time breach screen for the new password. + assert update_kwargs["data"]["password_reset_required"] is False + assert update_kwargs["data"]["last_breach_check_at"] is None + + +@pytest.mark.asyncio +async def test_change_password_rejects_wrong_current_password(): + from litellm.proxy._types import ChangePasswordRequest + + prisma = _make_prisma(_make_user_row(hash_password(CURRENT_PASSWORD))) + + with ( + patch( # test-quality-ok: change_password reads proxy_server module globals; no injection seam + "litellm.proxy.proxy_server.prisma_client", prisma + ), + patch( # test-quality-ok: change_password reads proxy_server module globals; no injection seam + "litellm.proxy.proxy_server.general_settings", _POLICY_NO_BREACH_CHECK + ), + ): + with pytest.raises(HTTPException) as exc_info: + await change_password( + data=ChangePasswordRequest(current_password="not-the-password", new_password=NEW_PASSWORD), + user_api_key_dict=_caller(), + ) + + assert exc_info.value.status_code == 400 + assert "Current password is incorrect" in exc_info.value.detail["error"] + prisma.db.litellm_usertable.update.assert_not_called() + + +@pytest.mark.asyncio +async def test_change_password_rejects_unchanged_password(): + from litellm.proxy._types import ChangePasswordRequest + + prisma = _make_prisma(_make_user_row(hash_password(CURRENT_PASSWORD))) + + with ( + patch( # test-quality-ok: change_password reads proxy_server module globals; no injection seam + "litellm.proxy.proxy_server.prisma_client", prisma + ), + patch( # test-quality-ok: change_password reads proxy_server module globals; no injection seam + "litellm.proxy.proxy_server.general_settings", _POLICY_NO_BREACH_CHECK + ), + ): + with pytest.raises(HTTPException) as exc_info: + await change_password( + data=ChangePasswordRequest(current_password=CURRENT_PASSWORD, new_password=CURRENT_PASSWORD), + user_api_key_dict=_caller(), + ) + + assert exc_info.value.status_code == 400 + assert "must be different from the current password" in exc_info.value.detail["error"] + prisma.db.litellm_usertable.update.assert_not_called() + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "caller", + [ + pytest.param(_sso_session_caller(), id="sso_dashboard_session"), + pytest.param(_virtual_key_caller(), id="virtual_key_with_forged_metadata"), + ], +) +async def test_change_password_rejects_non_password_login_session(caller: UserAPIKeyAuth): + """Only the session minted by a password login may change the password, so a + stolen virtual key or an SSO session cannot use the endpoint as a + current_password guessing oracle.""" + from litellm.proxy._types import ChangePasswordRequest + + prisma = _make_prisma(_make_user_row(hash_password(CURRENT_PASSWORD))) + + with ( + patch( # test-quality-ok: change_password reads proxy_server module globals; no injection seam + "litellm.proxy.proxy_server.prisma_client", prisma + ), + patch( # test-quality-ok: change_password reads proxy_server module globals; no injection seam + "litellm.proxy.proxy_server.general_settings", _POLICY_NO_BREACH_CHECK + ), + ): + with pytest.raises(HTTPException) as exc_info: + await change_password( + data=ChangePasswordRequest(current_password=CURRENT_PASSWORD, new_password=NEW_PASSWORD), + user_api_key_dict=caller, + ) + + assert exc_info.value.status_code == 403 + assert "logging in with a password" in exc_info.value.detail["error"] + prisma.db.litellm_usertable.find_first.assert_not_called() + prisma.db.litellm_usertable.update.assert_not_called() + + +@pytest.mark.asyncio +async def test_change_password_rejects_session_without_user(): + from litellm.proxy._types import ChangePasswordRequest + + prisma = _make_prisma(user=None) + + with ( + patch( # test-quality-ok: change_password reads proxy_server module globals; no injection seam + "litellm.proxy.proxy_server.prisma_client", prisma + ), + patch( # test-quality-ok: change_password reads proxy_server module globals; no injection seam + "litellm.proxy.proxy_server.general_settings", _POLICY_NO_BREACH_CHECK + ), + ): + with pytest.raises(HTTPException) as exc_info: + await change_password( + data=ChangePasswordRequest(current_password=CURRENT_PASSWORD, new_password=NEW_PASSWORD), + user_api_key_dict=_caller(user_id=None), + ) + + assert exc_info.value.status_code == 400 + prisma.db.litellm_usertable.find_first.assert_not_called() + prisma.db.litellm_usertable.update.assert_not_called() + + +@pytest.mark.asyncio +async def test_change_password_rejects_account_without_password(): + """SSO users and the env-credential admin have no DB password row to change.""" + from litellm.proxy._types import ChangePasswordRequest + + prisma = _make_prisma(_make_user_row(password=None)) + + with ( + patch( # test-quality-ok: change_password reads proxy_server module globals; no injection seam + "litellm.proxy.proxy_server.prisma_client", prisma + ), + patch( # test-quality-ok: change_password reads proxy_server module globals; no injection seam + "litellm.proxy.proxy_server.general_settings", _POLICY_NO_BREACH_CHECK + ), + ): + with pytest.raises(HTTPException) as exc_info: + await change_password( + data=ChangePasswordRequest(current_password=CURRENT_PASSWORD, new_password=NEW_PASSWORD), + user_api_key_dict=_caller(), + ) + + assert exc_info.value.status_code == 400 + assert "no password set" in exc_info.value.detail["error"] + prisma.db.litellm_usertable.update.assert_not_called() + + +@pytest.mark.asyncio +async def test_change_password_enforces_min_length(): + from litellm.proxy._types import ChangePasswordRequest + + prisma = _make_prisma(_make_user_row(hash_password(CURRENT_PASSWORD))) + + with ( + patch( # test-quality-ok: change_password reads proxy_server module globals; no injection seam + "litellm.proxy.proxy_server.prisma_client", prisma + ), + patch( # test-quality-ok: change_password reads proxy_server module globals; no injection seam + "litellm.proxy.proxy_server.general_settings", _POLICY_NO_BREACH_CHECK + ), + ): + with pytest.raises(ProxyException) as exc_info: + await change_password( + data=ChangePasswordRequest(current_password=CURRENT_PASSWORD, new_password="Short1!"), + user_api_key_dict=_caller(), + ) + + assert exc_info.value.code == "400" + assert exc_info.value.type == ProxyErrorTypes.validation_error + assert exc_info.value.param == "password" + assert "at least 12 characters" in exc_info.value.message + prisma.db.litellm_usertable.update.assert_not_called() + + +@pytest.mark.asyncio +@respx.mock +async def test_change_password_rejects_breached_password(): + """With the default policy, the new password is screened against HIBP.""" + from litellm.proxy._types import ChangePasswordRequest + + breached_password = "Password123!" + respx.get(_hibp_url_for(breached_password)).mock( + return_value=httpx.Response(200, text=f"{_hibp_suffix_for(breached_password)}:1") + ) + prisma = _make_prisma(_make_user_row(hash_password(CURRENT_PASSWORD))) + + with ( + patch( # test-quality-ok: change_password reads proxy_server module globals; no injection seam + "litellm.proxy.proxy_server.prisma_client", prisma + ), + patch( # test-quality-ok: change_password reads proxy_server module globals; no injection seam + "litellm.proxy.proxy_server.general_settings", {} + ), + ): + with pytest.raises(ProxyException) as exc_info: + await change_password( + data=ChangePasswordRequest(current_password=CURRENT_PASSWORD, new_password=breached_password), + user_api_key_dict=_caller(), + ) + + assert exc_info.value.code == "400" + assert exc_info.value.type == ProxyErrorTypes.validation_error + assert exc_info.value.param == "password" + assert "data breaches" in exc_info.value.message + prisma.db.litellm_usertable.update.assert_not_called() + + +@pytest.mark.asyncio +@respx.mock +async def test_change_password_verifies_current_password_before_hibp_lookup(): + """A caller who fails current-password verification must not trigger any + HIBP traffic. The HIBP check fails open on errors, so an unmocked lookup + could not prove ordering; instead the route is registered and asserted + uncalled.""" + from litellm.proxy._types import ChangePasswordRequest + + hibp_route = respx.get(_hibp_url_for(NEW_PASSWORD)).mock(return_value=httpx.Response(200, text="")) + prisma = _make_prisma(_make_user_row(hash_password(CURRENT_PASSWORD))) + + with ( + patch( # test-quality-ok: change_password reads proxy_server module globals; no injection seam + "litellm.proxy.proxy_server.prisma_client", prisma + ), + patch( # test-quality-ok: change_password reads proxy_server module globals; no injection seam + "litellm.proxy.proxy_server.general_settings", {} + ), + ): + with pytest.raises(HTTPException) as exc_info: + await change_password( + data=ChangePasswordRequest(current_password="not-the-password", new_password=NEW_PASSWORD), + user_api_key_dict=_caller(), + ) + + assert exc_info.value.status_code == 400 + assert "Current password is incorrect" in exc_info.value.detail["error"] + assert not hibp_route.called + prisma.db.litellm_usertable.update.assert_not_called() + + +@pytest.mark.asyncio +async def test_change_password_success_emits_redacted_audit_log(): + """A successful change must land in the audit trail as field names only; + the plaintext passwords must never reach the audit call.""" + from litellm.proxy._types import ChangePasswordRequest + + prisma = _make_prisma(_make_user_row(hash_password(CURRENT_PASSWORD))) + audit_mock = AsyncMock() + + with ( + patch( # test-quality-ok: change_password reads proxy_server module globals; no injection seam + "litellm.proxy.proxy_server.prisma_client", prisma + ), + patch( # test-quality-ok: change_password reads proxy_server module globals; no injection seam + "litellm.proxy.proxy_server.general_settings", _POLICY_NO_BREACH_CHECK + ), + patch( # test-quality-ok: audit sink is a module-level import; no injection seam + "litellm.proxy.management_endpoints.password_endpoints.create_object_audit_log", audit_mock + ), + ): + await change_password( + data=ChangePasswordRequest(current_password=CURRENT_PASSWORD, new_password=NEW_PASSWORD), + user_api_key_dict=_caller(), + ) + + audit_mock.assert_awaited_once() + audit_kwargs = audit_mock.await_args.kwargs + assert audit_kwargs["object_id"] == "user-123" + assert audit_kwargs["action"] == "updated" + assert audit_kwargs["table_name"] == LitellmTableNames.USER_TABLE_NAME + assert audit_kwargs["after_value"] == '{"fields_changed": ["password"]}' + assert CURRENT_PASSWORD not in str(audit_kwargs) + assert NEW_PASSWORD not in str(audit_kwargs) + + +@pytest.mark.asyncio +async def test_change_password_failure_emits_no_audit_log(): + from litellm.proxy._types import ChangePasswordRequest + + prisma = _make_prisma(_make_user_row(hash_password(CURRENT_PASSWORD))) + audit_mock = AsyncMock() + + with ( + patch( # test-quality-ok: change_password reads proxy_server module globals; no injection seam + "litellm.proxy.proxy_server.prisma_client", prisma + ), + patch( # test-quality-ok: change_password reads proxy_server module globals; no injection seam + "litellm.proxy.proxy_server.general_settings", _POLICY_NO_BREACH_CHECK + ), + patch( # test-quality-ok: audit sink is a module-level import; no injection seam + "litellm.proxy.management_endpoints.password_endpoints.create_object_audit_log", audit_mock + ), + ): + with pytest.raises(HTTPException): + await change_password( + data=ChangePasswordRequest(current_password="not-the-password", new_password=NEW_PASSWORD), + user_api_key_dict=_caller(), + ) + + audit_mock.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_change_password_requires_db(): + from litellm.proxy._types import ChangePasswordRequest + + with ( + patch( # test-quality-ok: change_password reads proxy_server module globals; no injection seam + "litellm.proxy.proxy_server.prisma_client", None + ), + patch( # test-quality-ok: change_password reads proxy_server module globals; no injection seam + "litellm.proxy.proxy_server.general_settings", _POLICY_NO_BREACH_CHECK + ), + ): + with pytest.raises(HTTPException) as exc_info: + await change_password( + data=ChangePasswordRequest(current_password=CURRENT_PASSWORD, new_password=NEW_PASSWORD), + user_api_key_dict=_caller(), + ) + + assert exc_info.value.status_code == 500 diff --git a/tests/test_litellm/proxy/management_endpoints/test_router_settings_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_router_settings_endpoints.py index 308f4d88f02..3fcda310435 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_router_settings_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_router_settings_endpoints.py @@ -4,6 +4,8 @@ Tests for router settings management endpoints. Tests the GET endpoints for router settings and router fields. """ +from collections.abc import Mapping +from typing import Any, Final from unittest.mock import AsyncMock, MagicMock, patch import pytest @@ -15,12 +17,23 @@ from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth from litellm.proxy.management_endpoints.router_settings_endpoints import ( get_router_settings, ) +from litellm.proxy.config_resolvers import SettingsStore from litellm.proxy.proxy_server import app from litellm.router import Router client = TestClient(app) +class _StubProxyConfig: + def __init__(self, router_settings: SettingsStore, config_router_settings: Mapping[str, Any]) -> None: + self.router_settings: Final = router_settings + self._config_router_settings: Final = dict(config_router_settings) + + async def get_config(self, config_file_path: str | None = None) -> dict[str, Any]: + del config_file_path + return {"router_settings": dict(self._config_router_settings)} + + class TestRouterSettingsEndpoints: """Test suite for router settings endpoints""" @@ -75,6 +88,31 @@ class TestRouterSettingsEndpoints: assert isinstance(routing_strategy_field["options"], list) assert len(routing_strategy_field["options"]) > 0 + @pytest.mark.asyncio + async def test_get_router_settings_reports_sources( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + store = SettingsStore("router_settings") + store.load_yaml({"routing_strategy": "simple-shuffle"}) + store.apply_db_row("router_settings", {"num_retries": 3}) + monkeypatch.setattr( + proxy_server, + "proxy_config", + _StubProxyConfig( + store, + {"routing_strategy": "simple-shuffle", "num_retries": 3}, + ), + ) + monkeypatch.setattr(proxy_server, "llm_router", None) + + admin_user = UserAPIKeyAuth( + user_role=LitellmUserRoles.PROXY_ADMIN, api_key="sk-x" + ) + response = await get_router_settings(user_api_key_dict=admin_user) + + assert response.source["routing_strategy"] == "config" + assert response.source["num_retries"] == "db" + @pytest.mark.asyncio async def test_get_router_settings_includes_routing_groups_from_live_router( self, monkeypatch @@ -102,12 +140,10 @@ class TestRouterSettingsEndpoints: ) monkeypatch.setattr(proxy_server, "llm_router", llm_router) - - async def fake_get_config(self, config_file_path=None): - return {} - monkeypatch.setattr( - proxy_server.ProxyConfig, "get_config", fake_get_config, raising=True + proxy_server, + "proxy_config", + _StubProxyConfig(SettingsStore("router_settings"), {}), ) admin_user = UserAPIKeyAuth( @@ -116,6 +152,8 @@ class TestRouterSettingsEndpoints: response = await get_router_settings(user_api_key_dict=admin_user) assert response.current_values.get("routing_groups") == groups + assert response.current_values["timeout"] is not None + assert response.source["timeout"] == "default" rg_field = next(f for f in response.fields if f.field_name == "routing_groups") assert rg_field.field_value == groups diff --git a/tests/test_litellm/proxy/middleware/test_budget_reservation_release_middleware.py b/tests/test_litellm/proxy/middleware/test_budget_reservation_release_middleware.py new file mode 100644 index 00000000000..f37a20dff8b --- /dev/null +++ b/tests/test_litellm/proxy/middleware/test_budget_reservation_release_middleware.py @@ -0,0 +1,349 @@ +""" +Tests for BudgetReservationReleaseMiddleware. + +Auth reserves budget before the handler runs and hands the reservation to the +request or socket state. A litellm call made through the async client wrapper +claims it for the cost callback that runs after the call; anything still unclaimed +when the response is done or the socket has closed would keep the spend counter +pinned until its TTL, so the middleware releases it. +""" + +import asyncio +from collections.abc import AsyncIterator, Awaitable, Callable, Mapping +from datetime import datetime +from typing import Final + +import pytest +from starlette.applications import Starlette +from starlette.requests import Request +from starlette.responses import JSONResponse, Response, StreamingResponse +from starlette.routing import Route +from starlette.types import ASGIApp, Message, Receive, Scope, Send +from starlette.websockets import WebSocket + +import litellm +from litellm.caching import DualCache +from litellm.proxy import proxy_server +from litellm.proxy._types import UserAPIKeyAuth +from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache +from litellm.proxy.middleware.budget_reservation_release_middleware import ( + BudgetReservationReleaseMiddleware, +) +from litellm.proxy.spend_tracking.budget_reservation import ( + reconcile_budget_reservation, + release_unbound_budget_reservation, + reserve_budget_for_request, +) +from litellm.proxy.utils import ProxyLogging +from litellm.utils import Rules, function_setup + +KEY_TOKEN: Final = "hashed-release-middleware-key" +COUNTER_KEY: Final = f"spend:key:{KEY_TOKEN}" +CHAT_BODY: Final = {"model": "gpt-4o", "messages": [{"role": "user", "content": "hello"}]} + + +@pytest.fixture +def spend_counter_cache(monkeypatch: pytest.MonkeyPatch) -> DualCache: + cache: Final = DualCache() + monkeypatch.setattr(proxy_server, "spend_counter_cache", cache) + monkeypatch.setattr(proxy_server, "prisma_client", None) + return cache + + +@pytest.fixture +def no_callbacks(monkeypatch: pytest.MonkeyPatch) -> None: + for callback_list_name in ( + "callbacks", + "success_callback", + "failure_callback", + "_async_success_callback", + "_async_failure_callback", + ): + monkeypatch.setattr(litellm, callback_list_name, []) + + +async def _reserve() -> dict: + reservation: Final = await reserve_budget_for_request( + request_body=CHAT_BODY, + route="/v1/chat/completions", + llm_router=None, + valid_token=UserAPIKeyAuth(token=KEY_TOKEN, max_budget=1.0, spend=0.0), + team_object=None, + user_object=None, + prisma_client=None, + user_api_key_cache=UserApiKeyCache(), + proxy_logging_obj=ProxyLogging(user_api_key_cache=UserApiKeyCache()), + ) + assert reservation is not None + assert reservation["reserved_cost"] > 0 + return reservation + + +async def _chat(reservation: dict, **kwargs: object) -> object: + return await litellm.acompletion( + **CHAT_BODY, + metadata={"user_api_key_budget_reservation": reservation}, + **kwargs, + ) + + +def _proxy_pre_call_setup(route_type: str, reservation: dict) -> None: + function_setup( + original_function=route_type, + rules_obj=Rules(), + start_time=datetime.now(), + **CHAT_BODY, + litellm_call_id="proxy-pre-call-setup", + metadata={"user_api_key_budget_reservation": reservation}, + ) + + +def _app( + handler: Callable[[Request], Awaitable[Response]], + release: Callable[[Mapping[str, object]], Awaitable[None]] = release_unbound_budget_reservation, +) -> Starlette: + app: Final = Starlette(routes=[Route("/", handler, methods=["POST"])]) + app.add_middleware(BudgetReservationReleaseMiddleware, release=release) + return app + + +async def _post(app: ASGIApp) -> None: + scope: Final = { + "type": "http", + "method": "POST", + "path": "/", + "raw_path": b"/", + "headers": [], + "query_string": b"", + "scheme": "http", + "server": ("testserver", 80), + "client": ("testclient", 1), + } + + body_delivered: Final = asyncio.Event() + client_never_disconnects: Final = asyncio.Event() + + async def receive() -> Message: + if body_delivered.is_set(): + await client_never_disconnects.wait() + body_delivered.set() + return {"type": "http.request", "body": b"", "more_body": False} + + async def send(message: Message) -> None: + return None + + await app(scope, receive, send) + + +def _counter(spend_counter_cache: DualCache) -> float | None: + return spend_counter_cache.in_memory_cache.get_cache(key=COUNTER_KEY) + + +@pytest.mark.asyncio +async def test_unbound_reservation_is_released_after_the_response(spend_counter_cache: DualCache): + reservation: Final = await _reserve() + assert _counter(spend_counter_cache) == pytest.approx(reservation["reserved_cost"]) + + async def handler(request: Request) -> Response: + request.state.budget_reservation = reservation + return JSONResponse({"id": "batch_123", "status": "cancelling"}) + + await _post(_app(handler)) + + assert _counter(spend_counter_cache) == pytest.approx(0.0) + assert reservation["finalized"] is True + + +@pytest.mark.asyncio +async def test_unbound_reservation_is_released_when_the_handler_raises(spend_counter_cache: DualCache): + reservation: Final = await _reserve() + + async def handler(request: Request) -> Response: + request.state.budget_reservation = reservation + raise RuntimeError("upstream refused the cancel") + + with pytest.raises(RuntimeError): + await _post(_app(handler)) + + assert _counter(spend_counter_cache) == pytest.approx(0.0) + assert reservation["finalized"] is True + + +@pytest.mark.asyncio +async def test_reservation_seen_only_by_the_proxy_pre_call_logging_object_is_released( + spend_counter_cache: DualCache, no_callbacks: None +): + reservation: Final = await _reserve() + + async def cancel_batch_without_a_client_wrapper() -> dict: + return {"id": "batch_123", "status": "cancelling"} + + async def handler(request: Request) -> Response: + request.state.budget_reservation = reservation + _proxy_pre_call_setup("acancel_batch", reservation) + return JSONResponse(await cancel_batch_without_a_client_wrapper()) + + await _post(_app(handler)) + + assert _counter(spend_counter_cache) == pytest.approx(0.0) + assert reservation["finalized"] is True + + +@pytest.mark.asyncio +async def test_reservation_of_a_failed_call_is_released_after_the_error_response( + spend_counter_cache: DualCache, no_callbacks: None +): + reservation: Final = await _reserve() + refused: Final = litellm.AuthenticationError(message="bad key", llm_provider="openai", model="gpt-4o") + + async def handler(request: Request) -> Response: + request.state.budget_reservation = reservation + _proxy_pre_call_setup("acompletion", reservation) + try: + await _chat(reservation, mock_response=refused) + except litellm.AuthenticationError: + return JSONResponse({"error": {"message": "bad key"}}, status_code=401) + raise AssertionError("the mocked call must fail") + + await _post(_app(handler)) + + assert _counter(spend_counter_cache) == pytest.approx(0.0) + assert reservation["finalized"] is True + + +@pytest.mark.asyncio +async def test_reservation_claimed_by_a_completed_call_is_left_for_the_callback( + spend_counter_cache: DualCache, no_callbacks: None +): + reservation: Final = await _reserve() + reserved_cost: Final = reservation["reserved_cost"] + + async def handler(request: Request) -> Response: + request.state.budget_reservation = reservation + _proxy_pre_call_setup("acompletion", reservation) + response: Final = await _chat(reservation, mock_response="ok") + return JSONResponse(response.model_dump()) + + await _post(_app(handler)) + + assert _counter(spend_counter_cache) == pytest.approx(reserved_cost) + assert reservation["finalized"] is False + + +@pytest.mark.asyncio +async def test_reservation_claimed_by_a_streaming_call_is_left_for_the_callback_that_finishes_after_the_response( + spend_counter_cache: DualCache, no_callbacks: None +): + reservation: Final = await _reserve() + reserved_cost: Final = reservation["reserved_cost"] + + async def handler(request: Request) -> Response: + request.state.budget_reservation = reservation + _proxy_pre_call_setup("acompletion", reservation) + stream: Final = await _chat(reservation, mock_response="ok", stream=True) + + async def sse() -> AsyncIterator[bytes]: + async for chunk in stream: + yield f"data: {chunk.model_dump_json()}\n\n".encode() + yield b"data: [DONE]\n\n" + + return StreamingResponse(sse(), media_type="text/event-stream") + + await _post(_app(handler)) + + assert _counter(spend_counter_cache) == pytest.approx(reserved_cost) + assert reservation["finalized"] is False + + actual_cost: Final = reserved_cost / 4 + await reconcile_budget_reservation(budget_reservation=reservation, actual_cost=actual_cost) + + assert _counter(spend_counter_cache) == pytest.approx(actual_cost) + assert reservation["finalized"] is True + + +@pytest.mark.asyncio +async def test_unbound_reservation_of_a_websocket_session_is_released_when_the_socket_closes( + spend_counter_cache: DualCache, +): + reservation: Final = await _reserve() + + async def listen_without_a_provider_key(scope: Scope, receive: Receive, send: Send) -> None: + websocket: Final = WebSocket(scope, receive, send) + websocket.state.budget_reservation = reservation + await websocket.close(code=1011, reason="Required 'DEEPGRAM_API_KEY' in environment") + + async def receive() -> Message: + return {"type": "websocket.connect"} + + async def send(message: Message) -> None: + return None + + middleware: Final = BudgetReservationReleaseMiddleware( + listen_without_a_provider_key, release=release_unbound_budget_reservation + ) + await middleware({"type": "websocket", "path": "/deepgram/v1/listen", "headers": []}, receive, send) + + assert _counter(spend_counter_cache) == pytest.approx(0.0) + assert reservation["finalized"] is True + + +@pytest.mark.asyncio +async def test_release_runs_once_per_request_with_the_stamped_reservation(): + released: Final = [] + reservation: Final = {"reserved_cost": 0.5, "entries": [], "finalized": False, "callback_bound": False} + + async def release(budget_reservation: Mapping[str, object]) -> None: + released.append(budget_reservation) + + async def handler(request: Request) -> Response: + request.state.budget_reservation = reservation + return JSONResponse({}) + + await _post(_app(handler, release=release)) + + assert released == [reservation] + assert released[0] is reservation + + +@pytest.mark.asyncio +async def test_request_without_a_reservation_releases_nothing(): + released: Final = [] + + async def release(budget_reservation: Mapping[str, object]) -> None: + released.append(budget_reservation) + + async def unauthenticated(request: Request) -> Response: + return JSONResponse({}) + + async def budget_checks_skipped(request: Request) -> Response: + request.state.budget_reservation = None + return JSONResponse({}) + + await _post(_app(unauthenticated, release=release)) + await _post(_app(budget_checks_skipped, release=release)) + + assert released == [] + + +@pytest.mark.asyncio +async def test_lifespan_scopes_pass_through(): + released: Final = [] + seen: Final = [] + + async def release(budget_reservation: Mapping[str, object]) -> None: + released.append(budget_reservation) + + async def inner(scope: Scope, receive: Receive, send: Send) -> None: + seen.append(scope["type"]) + + async def receive() -> Message: + return {"type": "lifespan.startup"} + + async def send(message: Message) -> None: + return None + + middleware: Final = BudgetReservationReleaseMiddleware(inner, release=release) + await middleware({"type": "lifespan", "state": {"budget_reservation": {"reserved_cost": 1.0}}}, receive, send) + + assert seen == ["lifespan"] + assert released == [] diff --git a/tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_fal_ai_passthrough_logging_handler.py b/tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_fal_ai_passthrough_logging_handler.py new file mode 100644 index 00000000000..1c945b9110b --- /dev/null +++ b/tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_fal_ai_passthrough_logging_handler.py @@ -0,0 +1,132 @@ +"""Fal AI pass-through: upstream URL to model extraction and resolution-keyed spend tracking.""" + +from datetime import datetime +from typing import Final + +import pytest + +import litellm +from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.proxy.pass_through_endpoints.llm_provider_handlers.fal_ai_passthrough_logging_handler import ( + FalAIPassthroughLoggingHandler, +) +from litellm.types.utils import ImageResponse + +pytestmark: Final = pytest.mark.usefixtures("local_model_cost_map") + +UPSTREAM_URL: Final = "https://queue.fal.run/fal-ai/trellis-2" + + +def _logging_obj(call_id: str = "call-fal") -> LiteLLMLoggingObj: + return LiteLLMLoggingObj( + model="unknown", + messages=[{"role": "user", "content": "passthrough"}], + stream=False, + call_type="pass_through_endpoint", + start_time=datetime.now(), + litellm_call_id=call_id, + function_id="passthrough", + ) + + +def test_is_fal_ai_route_matches_only_the_fal_ai_provider(): + assert FalAIPassthroughLoggingHandler.is_fal_ai_route(UPSTREAM_URL, "fal_ai") is True + assert FalAIPassthroughLoggingHandler.is_fal_ai_route(UPSTREAM_URL, "deepgram") is False + assert FalAIPassthroughLoggingHandler.is_fal_ai_route(UPSTREAM_URL, None) is False + + +def test_handler_extracts_model_urls_and_resolution_keyed_cost(): + upstream_body: Final = { + "model_glb": {"url": "https://fal.media/model.glb", "content_type": "model/gltf-binary"}, + "images": [{"url": "https://fal.media/preview.png"}], + "timings": {"inference": 1.2}, + } + logging_obj: Final = _logging_obj() + expected_cost: Final = litellm.model_cost["fal_ai/fal-ai/trellis-2"]["output_cost_per_image_1536"] + + handler_result = FalAIPassthroughLoggingHandler().fal_ai_passthrough_handler( + response_body=upstream_body, + request_body={"image_url": "https://example.com/in.png", "resolution": 1536}, + logging_obj=logging_obj, + url_route=UPSTREAM_URL, + kwargs={"litellm_params": {"metadata": {}}}, + ) + + result = handler_result["result"] + assert isinstance(result, ImageResponse) + assert [image.url for image in result.data or ()] == [ + "https://fal.media/model.glb", + "https://fal.media/preview.png", + ] + assert result._hidden_params["response_cost"] == pytest.approx(expected_cost) + assert handler_result["kwargs"]["model"] == "fal-ai/trellis-2" + assert handler_result["kwargs"]["custom_llm_provider"] == "fal_ai" + assert handler_result["kwargs"]["response_cost"] == pytest.approx(expected_cost) + assert handler_result["kwargs"]["litellm_params"] == {"metadata": {}} + assert logging_obj.model == "fal-ai/trellis-2" + assert logging_obj.model_call_details["model"] == "fal-ai/trellis-2" + assert logging_obj.model_call_details["custom_llm_provider"] == "fal_ai" + assert logging_obj.model_call_details["response_cost"] == pytest.approx(expected_cost) + + +def test_handler_charges_nothing_and_names_the_model_for_queue_status_and_result_polls(): + for upstream_url in ( + "https://queue.fal.run/fal-ai/trellis-2/requests/req-1/status", + "https://queue.fal.run/fal-ai/trellis-2/requests/req-1", + ): + logging_obj: Final = _logging_obj() + handler_result = FalAIPassthroughLoggingHandler().fal_ai_passthrough_handler( + response_body={"status": "COMPLETED"}, + request_body={}, + logging_obj=logging_obj, + url_route=upstream_url, + kwargs={}, + ) + assert handler_result["kwargs"]["model"] == "fal-ai/trellis-2" + assert handler_result["kwargs"]["response_cost"] is None + assert logging_obj.model_call_details["response_cost"] is None + + +def test_handler_charges_for_queue_submit(): + handler_result = FalAIPassthroughLoggingHandler().fal_ai_passthrough_handler( + response_body={"request_id": "req-1", "status": "IN_QUEUE"}, + request_body={"image_url": "https://example.com/in.png", "resolution": 1536}, + logging_obj=_logging_obj(), + url_route="https://queue.fal.run/fal-ai/trellis-2", + kwargs={}, + ) + assert handler_result["kwargs"]["model"] == "fal-ai/trellis-2" + assert handler_result["kwargs"]["response_cost"] == pytest.approx( + litellm.model_cost["fal_ai/fal-ai/trellis-2"]["output_cost_per_image_1536"] + ) + + +def test_handler_strips_queue_base_path_prefix(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setenv("FAL_AI_QUEUE_API_BASE", "https://gw.example/fal/queue") + handler_result = FalAIPassthroughLoggingHandler().fal_ai_passthrough_handler( + response_body={"request_id": "req-1", "status": "IN_QUEUE"}, + request_body={"image_url": "https://example.com/in.png", "resolution": 1536}, + logging_obj=_logging_obj(), + url_route="https://gw.example/fal/queue/fal-ai/trellis-2", + kwargs={}, + ) + + assert handler_result["kwargs"]["model"] == "fal-ai/trellis-2" + assert handler_result["kwargs"]["response_cost"] == pytest.approx( + litellm.model_cost["fal_ai/fal-ai/trellis-2"]["output_cost_per_image_1536"] + ) + + +def test_handler_without_url_values_returns_empty_image_response_and_no_cost(): + handler_result = FalAIPassthroughLoggingHandler().fal_ai_passthrough_handler( + response_body={"status": "COMPLETED"}, + request_body={}, + logging_obj=_logging_obj(), + url_route="https://queue.fal.run/fal-ai/no-such-model", + kwargs={}, + ) + + assert isinstance(handler_result["result"], ImageResponse) + assert not handler_result["result"].data + assert handler_result["kwargs"]["response_cost"] is None + assert handler_result["kwargs"]["model"] == "fal-ai/no-such-model" diff --git a/tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_typesafe_passthrough_logging_handler.py b/tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_typesafe_passthrough_logging_handler.py index 345eeeedc31..e0a5ef063e8 100644 --- a/tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_typesafe_passthrough_logging_handler.py +++ b/tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_typesafe_passthrough_logging_handler.py @@ -42,6 +42,7 @@ def _handler_result(response_body: dict, request_body: dict) -> dict: end_time=datetime.now(), cache_hit=False, request_body=request_body, + custom_llm_provider="typesafe", ) @@ -59,6 +60,7 @@ def test_uses_registry_pricing_and_standard_usage(): end_time=datetime.now(), cache_hit=False, request_body={"model": "jev-latest"}, + custom_llm_provider="typesafe", ) expected_cost = 312 * model_cost["input_cost_per_token"] + 48 * model_cost["output_cost_per_token"] @@ -105,6 +107,7 @@ def test_records_model_provider_and_cost_on_logging_details(): end_time=datetime.now(), cache_hit=False, request_body={"model": "jev-latest"}, + custom_llm_provider="typesafe", ) assert result["kwargs"]["model"] == "typesafe/jev-1.13.0" @@ -132,3 +135,76 @@ def test_success_handler_dispatches_to_typesafe_handler(): assert normalized["kwargs"]["custom_llm_provider"] == "typesafe" assert normalized["kwargs"]["model"] == "typesafe/jev-1.13.0" + + +def test_openrouter_decisions_response_is_priced_from_request_model_registry_row(): + logging_obj = _logging_obj() + model_cost = litellm.model_cost["openrouter/typesafe/jev-1.13"] + response = TypeSafePassthroughLoggingHandler.typesafe_passthrough_handler( + httpx_response=_response(), + response_body={ + "model": "typesafe/jev-1.13-20260917", + "usage": {"input_tokens": 282, "output_tokens": 20}, + }, + logging_obj=logging_obj, + url_route="https://openrouter.ai/api/alpha/decisions", + result='{"answers": {}}', + start_time=datetime.now(), + end_time=datetime.now(), + cache_hit=False, + request_body={"model": "typesafe/jev-1.13"}, + custom_llm_provider="openrouter", + ) + + expected_cost = 282 * model_cost["input_cost_per_token"] + 20 * model_cost["output_cost_per_token"] + assert response["kwargs"]["model"] == "openrouter/typesafe/jev-1.13-20260917" + assert response["kwargs"]["custom_llm_provider"] == "openrouter" + assert response["kwargs"]["response_cost"] == pytest.approx(expected_cost) + assert response["kwargs"]["combined_usage_object"].prompt_tokens == 282 + assert response["kwargs"]["combined_usage_object"].completion_tokens == 20 + assert response["kwargs"]["combined_usage_object"].total_tokens == 302 + + +def test_success_handler_dispatches_openrouter_to_the_shared_handler(): + logging_obj = _logging_obj() + normalized = PassThroughEndpointLogging().normalize_llm_passthrough_logging_payload( + httpx_response=_response(), + response_body={ + "model": "typesafe/jev-1.13-20260917", + "usage": {"input_tokens": 282, "output_tokens": 20}, + }, + request_body={"model": "typesafe/jev-1.13"}, + logging_obj=logging_obj, + url_route="https://openrouter.ai/api/alpha/decisions", + result="{}", + start_time=datetime.now(), + end_time=datetime.now(), + cache_hit=False, + custom_llm_provider="openrouter", + ) + + assert normalized["kwargs"]["custom_llm_provider"] == "openrouter" + assert normalized["kwargs"]["model"] == "openrouter/typesafe/jev-1.13-20260917" + + +def test_success_handler_skips_typesafe_pricing_for_non_decisions_openrouter_routes(): + logging_obj = _logging_obj() + normalized = PassThroughEndpointLogging().normalize_llm_passthrough_logging_payload( + httpx_response=_response(), + response_body={ + "model": "typesafe/jev-1.13-20260917", + "usage": {"input_tokens": 282, "output_tokens": 20}, + }, + request_body={"model": "typesafe/jev-1.13"}, + logging_obj=logging_obj, + url_route="https://openrouter.ai/api/v1/chat/completions", + result="{}", + start_time=datetime.now(), + end_time=datetime.now(), + cache_hit=False, + custom_llm_provider="openrouter", + ) + + assert normalized["standard_logging_response_object"] is None + assert "combined_usage_object" not in normalized["kwargs"] + assert normalized["kwargs"].get("model") != "openrouter/typesafe/jev-1.13-20260917" diff --git a/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py b/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py index 636980eb6e3..dcc3bd2b690 100644 --- a/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py +++ b/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py @@ -29,6 +29,7 @@ from litellm.constants import LITELLM_PROXY_MASTER_KEY_ALIAS from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import ( BaseOpenAIPassThroughHandler, RouteChecks, + _fal_target, _join_url_paths, _proxy_general_settings, anthropic_proxy_route, @@ -38,6 +39,7 @@ from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import ( bedrock_proxy_route, create_pass_through_route, cursor_proxy_route, + fal_ai_proxy_route, get_azure_ai_search_index_from_endpoint, get_vertex_base_url, is_azure_ai_search_service_level_index_create, @@ -47,6 +49,7 @@ from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import ( mistral_proxy_route, relay_nvidia_nim_request, openai_proxy_route, + openrouter_proxy_route, typesafe_proxy_route, vertex_discovery_proxy_route, vertex_proxy_route, @@ -7114,3 +7117,257 @@ class TestTypeSafePassthroughRoute: custom_llm_provider="typesafe", is_streaming_request=False, ) + + +class TestFalAIPassthroughRoute: + @pytest.fixture + def client(self, monkeypatch: pytest.MonkeyPatch) -> Iterator[TestClient]: + from litellm.proxy.proxy_server import app + + monkeypatch.setenv("FAL_AI_API_KEY", "fal-test-key") + monkeypatch.delenv("FAL_AI_QUEUE_API_BASE", raising=False) + monkeypatch.delenv("SERVER_ROOT_PATH", raising=False) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + litellm.in_memory_llm_clients_cache.flush_cache() + monkeypatch.setitem(app.dependency_overrides, user_api_key_auth, lambda: UserAPIKeyAuth(api_key="sk-virtual")) + yield TestClient(app) + + def test_submit_forwards_body_and_key_scheme_to_queue_fal_run(self, client: TestClient) -> None: + body: Final = {"image_url": "https://example.com/in.png", "resolution": 1536} + with respx.mock(assert_all_called=True) as upstream: + route = upstream.post("https://queue.fal.run/fal-ai/trellis-2").mock( + return_value=httpx.Response(200, json={"request_id": "req-1", "status": "IN_QUEUE"}) + ) + response = client.post("/fal_ai/fal-ai/trellis-2", json=body) + + assert response.status_code == 200, response.text + assert response.json() == {"request_id": "req-1", "status": "IN_QUEUE"} + sent = route.calls.last.request + assert sent.headers["authorization"] == "Key fal-test-key" + assert json.loads(sent.content or b"{}") == body + + def test_status_get_forwards_to_queue_fal_run(self, client: TestClient) -> None: + with respx.mock(assert_all_called=True) as upstream: + route = upstream.get("https://queue.fal.run/fal-ai/trellis-2/requests/req-1/status").mock( + return_value=httpx.Response(200, json={"status": "COMPLETED"}) + ) + response = client.get("/fal_ai/fal-ai/trellis-2/requests/req-1/status") + + assert response.status_code == 200, response.text + assert response.json() == {"status": "COMPLETED"} + assert route.calls.last.request.headers["authorization"] == "Key fal-test-key" + + def test_honours_fal_ai_queue_api_base_override(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("FAL_AI_API_KEY", "fal-test-key") + monkeypatch.setenv("FAL_AI_QUEUE_API_BASE", "https://queue.example/base") + endpoint_func = AsyncMock(return_value={"ok": True}) + create_route = Mock(return_value=endpoint_func) + monkeypatch.setattr( + "litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints.create_pass_through_route", + create_route, + ) + request = MagicMock(spec=Request) + request.method = "POST" + request.query_params = {} + request.json = AsyncMock(return_value={}) + + result = asyncio.run( + fal_ai_proxy_route( + endpoint="fal-ai/trellis", + request=request, + fastapi_response=MagicMock(spec=Response), + user_api_key_dict=UserAPIKeyAuth(api_key="virtual-key"), + ) + ) + + assert result == {"ok": True} + create_route.assert_called_once_with( + endpoint="fal-ai/trellis", + target="https://queue.example/base/fal-ai/trellis", + custom_headers={"Authorization": "Key fal-test-key"}, + custom_llm_provider="fal_ai", + is_streaming_request=False, + ) + + def test_submit_to_unpriced_endpoint_returns_400_without_upstream_call(self, client: TestClient) -> None: + with respx.mock(assert_all_called=False) as upstream: + route = upstream.post("https://queue.fal.run/fal-ai/unpriced-model").mock( + return_value=httpx.Response(200, json={"request_id": "req-1"}) + ) + response = client.post("/fal_ai/fal-ai/unpriced-model", json={"image_url": "https://example.com/in.png"}) + + assert response.status_code == 400, response.text + assert "no pricing entry" in response.text + assert not route.calls + + def test_status_get_on_unpriced_endpoint_forwards(self, client: TestClient) -> None: + with respx.mock(assert_all_called=True) as upstream: + upstream.get("https://queue.fal.run/fal-ai/unpriced-model/requests/req-9/status").mock( + return_value=httpx.Response(200, json={"status": "IN_PROGRESS"}) + ) + response = client.get("/fal_ai/fal-ai/unpriced-model/requests/req-9/status") + + assert response.status_code == 200, response.text + assert response.json() == {"status": "IN_PROGRESS"} + + def test_missing_fal_key_returns_401(self, client: TestClient, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("FAL_AI_API_KEY", raising=False) + response = client.post("/fal_ai/fal-ai/trellis", json={}) + assert response.status_code == 401 + + +class TestFalTargetSelection: + def test_endpoint_targets_queue_base(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("FAL_AI_QUEUE_API_BASE", raising=False) + assert str(_fal_target("fal-ai/trellis-2")) == "https://queue.fal.run/fal-ai/trellis-2" + + def test_status_path_targets_queue_base(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("FAL_AI_QUEUE_API_BASE", raising=False) + assert str(_fal_target("fal-ai/trellis-2/requests/req-1/status")) == ( + "https://queue.fal.run/fal-ai/trellis-2/requests/req-1/status" + ) + + def test_queue_base_override(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("FAL_AI_QUEUE_API_BASE", "https://queue.example/base") + assert str(_fal_target("fal-ai/trellis-2")) == "https://queue.example/base/fal-ai/trellis-2" + + +class TestOpenRouterPassthroughRoute: + @staticmethod + def _request(body: object, query_params: Mapping[str, str] | None = None) -> MagicMock: + request = MagicMock(spec=Request) + request.method = "POST" + request.query_params = query_params or {} + request.json = AsyncMock(return_value=body) + return request + + @pytest.fixture + def client(self, monkeypatch: pytest.MonkeyPatch) -> Iterator[TestClient]: + from litellm.proxy.proxy_server import app + + monkeypatch.setenv("OPENROUTER_API_KEY", "openrouter-test-key") + monkeypatch.setenv("OPENROUTER_API_BASE", "https://openrouter.example/base") + monkeypatch.delenv("SERVER_ROOT_PATH", raising=False) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + litellm.in_memory_llm_clients_cache.flush_cache() + monkeypatch.setitem(app.dependency_overrides, user_api_key_auth, lambda: UserAPIKeyAuth(api_key="sk-virtual")) + yield TestClient(app) + + @pytest.mark.parametrize( + "method, body", + [ + ("GET", None), + ("POST", {"state": "The sky is blue."}), + ("PUT", {"state": "The sky is blue."}), + ("DELETE", None), + ("PATCH", {"state": "The sky is blue."}), + ], + ) + def test_forwards_every_method_and_body_upstream( + self, client: TestClient, method: str, body: dict[str, str] | None + ) -> None: + with respx.mock(assert_all_called=True) as upstream: + route = upstream.request(method, "https://openrouter.example/base/alpha/decisions").mock( + return_value=httpx.Response(200, json={"id": "upstream_123"}) + ) + response = client.request(method, "/openrouter/alpha/decisions", json=body) + + assert (response.status_code, response.json()) == (200, {"id": "upstream_123"}) + sent: Final = route.calls.last.request + assert sent.headers["authorization"] == "Bearer openrouter-test-key" + assert json.loads(sent.content or b"{}") == (body or {}) + + @pytest.mark.asyncio + async def test_forwards_target_auth_provider_and_query(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("OPENROUTER_API_KEY", "openrouter-test-key") + monkeypatch.setenv("OPENROUTER_API_BASE", "https://openrouter.example/base") + + async def fake_upstream(request, *_args): + target: Final = create_route.call_args.kwargs["target"] + upstream_url: Final = httpx.URL(target).copy_merge_params(request.query_params) + return {"upstream_query": parse_qs(upstream_url.query.decode())} + + endpoint_func = AsyncMock(side_effect=fake_upstream) + create_route = Mock(return_value=endpoint_func) + monkeypatch.setattr( + "litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints.create_pass_through_route", + create_route, + ) + + request = self._request({"state": "The sky is blue."}, {"trace": "yes"}) + result = await openrouter_proxy_route( + endpoint="alpha/decisions", + request=request, + fastapi_response=MagicMock(spec=Response), + user_api_key_dict=UserAPIKeyAuth(api_key="virtual-key"), + ) + + assert result == {"upstream_query": {"trace": ["yes"]}} + endpoint_func.assert_awaited_once() + create_route.assert_called_once_with( + endpoint="alpha/decisions", + target="https://openrouter.example/base/alpha/decisions", + custom_headers={ + "Authorization": "Bearer openrouter-test-key", + "Content-Type": "application/json", + }, + custom_llm_provider="openrouter", + is_streaming_request=False, + ) + + @pytest.mark.asyncio + async def test_uses_default_target_when_base_is_unset(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("OPENROUTER_API_KEY", "openrouter-test-key") + monkeypatch.delenv("OPENROUTER_API_BASE", raising=False) + + endpoint_func = AsyncMock(return_value={"ok": True}) + create_route = Mock(return_value=endpoint_func) + monkeypatch.setattr( + "litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints.create_pass_through_route", + create_route, + ) + + await openrouter_proxy_route( + endpoint="alpha/decisions", + request=self._request({"state": "The sky is blue."}), + fastapi_response=MagicMock(spec=Response), + user_api_key_dict=UserAPIKeyAuth(api_key="virtual-key"), + ) + + assert create_route.call_args.kwargs["target"] == "https://openrouter.ai/api/alpha/decisions" + + @pytest.mark.asyncio + @pytest.mark.parametrize("endpoint", ["alpha/decisions", "v1/chat/completions"]) + @pytest.mark.parametrize( + "base_env, expected_root", + [ + (None, "https://openrouter.ai/api"), + ("https://openrouter.ai/api/v1", "https://openrouter.ai/api"), + ("https://openrouter.example/base", "https://openrouter.example/base"), + ("https://openrouter.example/base/v1/", "https://openrouter.example/base"), + ], + ) + async def test_derives_api_root_from_configured_base( + self, monkeypatch: pytest.MonkeyPatch, base_env: str | None, expected_root: str, endpoint: str + ) -> None: + monkeypatch.setenv("OPENROUTER_API_KEY", "openrouter-test-key") + if base_env is None: + monkeypatch.delenv("OPENROUTER_API_BASE", raising=False) + else: + monkeypatch.setenv("OPENROUTER_API_BASE", base_env) + + endpoint_func = AsyncMock(return_value={"ok": True}) + create_route = Mock(return_value=endpoint_func) + monkeypatch.setattr( + "litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints.create_pass_through_route", + create_route, + ) + + await openrouter_proxy_route( + endpoint=endpoint, + request=self._request({"state": "The sky is blue."}), + fastapi_response=MagicMock(spec=Response), + user_api_key_dict=UserAPIKeyAuth(api_key="virtual-key"), + ) + + assert create_route.call_args.kwargs["target"] == f"{expected_root}/{endpoint}" diff --git a/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py b/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py index fb89e3a6973..6ad850866b7 100644 --- a/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py +++ b/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py @@ -4256,6 +4256,112 @@ async def test_pass_through_request_non_streaming_success_unchanged(): mock_success_handler.assert_called_once() +@pytest.mark.asyncio +@pytest.mark.parametrize( + "upstream_status_code, claimed_by_the_success_handler", + [(200, True), (500, False)], + ids=["success-claims-the-reservation", "upstream-error-leaves-it-for-the-request-end-release"], +) +async def test_pass_through_request_claims_the_budget_reservation_only_when_its_success_handler_runs( + upstream_status_code: int, claimed_by_the_success_handler: bool +): + reservation: Final = {"reserved_cost": 0.5, "entries": [], "finalized": False, "callback_bound": False} + user_api_key_dict: Final = UserAPIKeyAuth(api_key="hashed") + user_api_key_dict.budget_reservation = reservation + upstream_response: Final = httpx.Response( + status_code=upstream_status_code, + headers={"content-type": "application/json"}, + content=b'{"status": "upstream"}', + request=httpx.Request("POST", "http://target-api.com/api/generate"), + ) + + with ( + patch("litellm.proxy.proxy_server.proxy_logging_obj") as mock_proxy_logging, + patch("litellm.proxy.pass_through_endpoints.pass_through_endpoints.get_async_httpx_client") as mock_get_client, + patch( + "litellm.proxy.pass_through_endpoints.pass_through_endpoints.ProxyBaseLLMRequestProcessing" + ) as mock_processing, + patch("litellm.proxy.pass_through_endpoints.pass_through_endpoints.GLOBAL_LOGGING_WORKER") as mock_worker, + ): + mock_proxy_logging.pre_call_hook = AsyncMock(return_value={}) + mock_proxy_logging.post_call_failure_hook = AsyncMock() + mock_proxy_logging.post_call_response_headers_hook = AsyncMock(return_value=None) + mock_processing.get_custom_headers.return_value = {} + mock_worker.ensure_initialized_and_enqueue = MagicMock(side_effect=lambda async_coroutine: async_coroutine.close()) + async_client = MagicMock() + async_client.build_request = MagicMock(return_value=MagicMock()) + async_client.send = AsyncMock(return_value=upstream_response) + mock_get_client.return_value = MagicMock(client=async_client) + mock_request = MagicMock(spec=Request) + mock_request.method = "POST" + mock_request.url = "http://test-proxy.com/mock-upstream/api/generate" + mock_request.body = AsyncMock(return_value=b'{"prompt": "hi"}') + mock_request.headers = Headers({"content-type": "application/json"}) + mock_request.query_params = QueryParams({}) + + response = await pass_through_request( + request=mock_request, + target="http://target-api.com/api/generate", + custom_headers={}, + user_api_key_dict=user_api_key_dict, + ) + + assert response.status_code == upstream_status_code + assert reservation["callback_bound"] is claimed_by_the_success_handler + assert mock_worker.ensure_initialized_and_enqueue.call_count == int(claimed_by_the_success_handler) + + +@pytest.mark.asyncio +async def test_pass_through_request_leaves_the_budget_reservation_for_the_request_end_release_when_its_success_handler_cannot_be_enqueued(): + reservation: Final = {"reserved_cost": 0.5, "entries": [], "finalized": False, "callback_bound": False} + user_api_key_dict: Final = UserAPIKeyAuth(api_key="hashed") + user_api_key_dict.budget_reservation = reservation + upstream_response: Final = httpx.Response( + status_code=200, + headers={"content-type": "application/json"}, + content=b'{"status": "upstream"}', + request=httpx.Request("POST", "http://target-api.com/api/generate"), + ) + + def refuse_to_enqueue(async_coroutine): + async_coroutine.close() + raise RuntimeError("logging worker is shutting down") + + with ( + patch("litellm.proxy.proxy_server.proxy_logging_obj") as mock_proxy_logging, + patch("litellm.proxy.pass_through_endpoints.pass_through_endpoints.get_async_httpx_client") as mock_get_client, + patch( + "litellm.proxy.pass_through_endpoints.pass_through_endpoints.ProxyBaseLLMRequestProcessing" + ) as mock_processing, + patch("litellm.proxy.pass_through_endpoints.pass_through_endpoints.GLOBAL_LOGGING_WORKER") as mock_worker, + ): + mock_proxy_logging.pre_call_hook = AsyncMock(return_value={}) + mock_proxy_logging.post_call_failure_hook = AsyncMock() + mock_proxy_logging.post_call_response_headers_hook = AsyncMock(return_value=None) + mock_processing.get_custom_headers.return_value = {} + mock_worker.ensure_initialized_and_enqueue = MagicMock(side_effect=refuse_to_enqueue) + async_client = MagicMock() + async_client.build_request = MagicMock(return_value=MagicMock()) + async_client.send = AsyncMock(return_value=upstream_response) + mock_get_client.return_value = MagicMock(client=async_client) + mock_request = MagicMock(spec=Request) + mock_request.method = "POST" + mock_request.url = "http://test-proxy.com/mock-upstream/api/generate" + mock_request.body = AsyncMock(return_value=b'{"prompt": "hi"}') + mock_request.headers = Headers({"content-type": "application/json"}) + mock_request.query_params = QueryParams({}) + + with pytest.raises(ProxyException): + await pass_through_request( + request=mock_request, + target="http://target-api.com/api/generate", + custom_headers={}, + user_api_key_dict=user_api_key_dict, + ) + + assert reservation["callback_bound"] is False + + @pytest.mark.asyncio async def test_pass_through_request_internal_failure_still_raises_proxy_exception(): """ @@ -4651,6 +4757,90 @@ async def test_pass_through_request_upstream_error_body_stays_buffered(): await fake_client.aclose() +_UPSTREAM_JSON_ERROR: Final = b'{"error": {"message": "bad request", "type": "invalid_request_error"}}' + + +async def _relay_upstream_through_pass_through_request( + general_settings, status_code, content_type, body, callback_headers=None +): + from litellm.proxy._types import UserAPIKeyAuth + + fake_client, cleanup = _inject_fake_passthrough_client( + _FakeUpstreamTransport( + status_code=status_code, + headers={"content-type": content_type}, + stream=_RecordingUpstreamByteStream((body,)), + ), + timeout=313.0, + ) + try: + with ExitStack() as stack: + mock_proxy_logging, _ = _enter_relay_logging_mocks(stack, {}) + mock_proxy_logging.post_call_response_headers_hook = AsyncMock(return_value=callback_headers) + stack.enter_context(patch("litellm.proxy.proxy_server.general_settings", general_settings)) + return await pass_through_request( + request=_relay_client_request(), + target="http://upstream.test/v1/messages", + custom_headers={}, + user_api_key_dict=UserAPIKeyAuth(api_key="sk-relay-test"), + timeout=313.0, + ) + finally: + cleanup() + await fake_client.aclose() + + +@pytest.mark.asyncio +async def test_pass_through_error_body_carries_the_call_id_when_opted_in(): + """With include_call_id_in_error_body on, a buffered upstream JSON error gets a top-level + litellm_call_id byte-identical to the x-litellm-call-id header, and content-length still + matches the rewritten body.""" + response = await _relay_upstream_through_pass_through_request( + {"include_call_id_in_error_body": True}, 400, "application/json", _UPSTREAM_JSON_ERROR + ) + + call_id = response.headers["x-litellm-call-id"] + assert response.status_code == 400 + assert json.loads(response.body) == {**json.loads(_UPSTREAM_JSON_ERROR), "litellm_call_id": call_id} + assert int(response.headers["content-length"]) == len(response.body) + + +@pytest.mark.asyncio +async def test_pass_through_error_body_call_id_follows_a_restamped_header(): + """A post_call_response_headers_hook that rewrites x-litellm-call-id wins in the header, so the + body copies the emitted header value rather than the id the proxy generated.""" + response = await _relay_upstream_through_pass_through_request( + {"include_call_id_in_error_body": True}, + 400, + "application/json", + _UPSTREAM_JSON_ERROR, + callback_headers={"x-litellm-call-id": "restamped-by-hook"}, + ) + + assert response.headers["x-litellm-call-id"] == "restamped-by-hook" + assert json.loads(response.body)["litellm_call_id"] == "restamped-by-hook" + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "general_settings, status_code, content_type, body", + [ + ({}, 400, "application/json", _UPSTREAM_JSON_ERROR), + ({"include_call_id_in_error_body": True}, 502, "text/plain", b"upstream exploded"), + ({"include_call_id_in_error_body": True}, 200, "application/json", b'{"id": "msg_1", "type": "message"}'), + ], +) +async def test_pass_through_body_stays_byte_identical_outside_the_opt_in( + general_settings, status_code, content_type, body +): + """Opted out, a non-JSON error, or a success body: the upstream bytes are relayed as-is.""" + response = await _relay_upstream_through_pass_through_request(general_settings, status_code, content_type, body) + + assert response.status_code == status_code + assert response.body == body + assert "x-litellm-call-id" in response.headers + + _PARTIAL_RELAY_WARNING_MARKER = "ended before upstream body was fully relayed" diff --git a/tests/test_litellm/proxy/pass_through_endpoints/test_streaming_handler_interrupt.py b/tests/test_litellm/proxy/pass_through_endpoints/test_streaming_handler_interrupt.py index ea6adc35b9a..88b82349c83 100644 --- a/tests/test_litellm/proxy/pass_through_endpoints/test_streaming_handler_interrupt.py +++ b/tests/test_litellm/proxy/pass_through_endpoints/test_streaming_handler_interrupt.py @@ -838,3 +838,72 @@ async def test_chunk_processor_bills_partial_google_usage_on_mid_stream_exceptio assert failure_payload["completion_tokens"] == 12 assert failure_payload["response_cost"] > 12 * 3.75e-06 assert isinstance(recorder.failure_kwargs[0]["exception"], httpx.ReadTimeout) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "deferred_dispatch_armed", + [False, True], + ids=["enqueued-at-end-of-stream", "parked-for-deferred-dispatch"], +) +async def test_chunk_processor_claims_the_budget_reservation_before_handing_it_to_the_cost_callback( + deferred_dispatch_armed: bool, +): + reservation = {"reserved_cost": 0.5, "entries": [], "finalized": False, "callback_bound": False} + response = _make_streaming_response([b"event-1", b"event-2"]) + logging_obj = _unarmed_logging_obj() + logging_obj.litellm_params = {"metadata": {"user_api_key_budget_reservation": reservation}} + if deferred_dispatch_armed: + logging_obj._on_deferred_stream_complete = AsyncMock() + claimed_when_the_callback_ran = [] + + async def cost_callback(**kwargs): + claimed_when_the_callback_ran.append(reservation["callback_bound"]) + + async for _ in PassThroughStreamingHandler.chunk_processor( + response=response, + request_body={"model": "claude-3-haiku"}, + litellm_logging_obj=logging_obj, + endpoint_type=EndpointType.GENERIC, + start_time=datetime.now(), + passthrough_success_handler_obj=MagicMock(), + url_route="/bedrock/model/claude/invoke-with-response-stream", + route_streaming_logging=cost_callback, + ): + pass + + if deferred_dispatch_armed: + (parked_cost_callback,) = logging_obj._deferred_stream_complete_args + await parked_cost_callback + else: + await GLOBAL_LOGGING_WORKER.flush() + + assert reservation["callback_bound"] is True + assert claimed_when_the_callback_ran == [True] + + +@pytest.mark.asyncio +async def test_chunk_processor_leaves_the_budget_reservation_for_the_request_end_release_when_the_cost_callback_cannot_be_enqueued(): + reservation = {"reserved_cost": 0.5, "entries": [], "finalized": False, "callback_bound": False} + response = _make_streaming_response([b"event-1", b"event-2"]) + logging_obj = _unarmed_logging_obj() + logging_obj.litellm_params = {"metadata": {"user_api_key_budget_reservation": reservation}} + + def refuse_to_enqueue(async_coroutine): + async_coroutine.close() + raise RuntimeError("logging worker is shutting down") + + with patch.object(GLOBAL_LOGGING_WORKER, "ensure_initialized_and_enqueue", side_effect=refuse_to_enqueue): + async for _ in PassThroughStreamingHandler.chunk_processor( + response=response, + request_body={"model": "claude-3-haiku"}, + litellm_logging_obj=logging_obj, + endpoint_type=EndpointType.GENERIC, + start_time=datetime.now(), + passthrough_success_handler_obj=MagicMock(), + url_route="/bedrock/model/claude/invoke-with-response-stream", + route_streaming_logging=AsyncMock(), + ): + pass + + assert reservation["callback_bound"] is False diff --git a/tests/test_litellm/proxy/proxy_server/test_exception_handlers.py b/tests/test_litellm/proxy/proxy_server/test_exception_handlers.py index 53ea761daa7..b3028ae71dd 100644 --- a/tests/test_litellm/proxy/proxy_server/test_exception_handlers.py +++ b/tests/test_litellm/proxy/proxy_server/test_exception_handlers.py @@ -95,6 +95,71 @@ async def test_openai_exception_handler_invalid_empty_code_defaults_to_500(): } +def _call_id_exception(headers): + return ProxyException( + message="bad input", + type="invalid_request_error", + param="model", + code=400, + headers=headers, + ) + + +@pytest.mark.asyncio +async def test_openai_exception_handler_copies_the_call_id_into_the_error_when_opted_in(monkeypatch): + """With include_call_id_in_error_body on, error.litellm_call_id is byte-identical to the + x-litellm-call-id header, so a pasted str(e) names the request to look up.""" + monkeypatch.setattr("litellm.proxy.proxy_server.general_settings", {"include_call_id_in_error_body": True}) + exc = _call_id_exception({"x-litellm-call-id": "call-8302"}) + + response = await openai_exception_handler(request=_make_request(), exc=exc) + body = json.loads(response.body) + + assert response.headers["x-litellm-call-id"] == "call-8302" + assert body == { + "error": { + "message": "bad input", + "type": "invalid_request_error", + "param": "model", + "code": "400", + "litellm_call_id": "call-8302", + } + } + + +@pytest.mark.asyncio +async def test_openai_exception_handler_leaves_the_error_alone_when_opted_out(monkeypatch): + monkeypatch.setattr("litellm.proxy.proxy_server.general_settings", {}) + exc = _call_id_exception({"x-litellm-call-id": "call-8302"}) + + response = await openai_exception_handler(request=_make_request(), exc=exc) + body = json.loads(response.body) + + assert response.headers["x-litellm-call-id"] == "call-8302" + assert body == { + "error": { + "message": "bad input", + "type": "invalid_request_error", + "param": "model", + "code": "400", + } + } + + +@pytest.mark.asyncio +async def test_openai_exception_handler_never_fabricates_a_call_id(monkeypatch): + """An error raised before a call id exists (auth failures, say) carries no header, + and the body must not invent one.""" + monkeypatch.setattr("litellm.proxy.proxy_server.general_settings", {"include_call_id_in_error_body": True}) + exc = _call_id_exception({}) + + response = await openai_exception_handler(request=_make_request(), exc=exc) + body = json.loads(response.body) + + assert "x-litellm-call-id" not in response.headers + assert "litellm_call_id" not in body["error"] + + # --------------------------------------------------------------------------- # _close_dangling_otel_server_span # --------------------------------------------------------------------------- diff --git a/tests/test_litellm/proxy/proxy_server/test_routes_login_sso.py b/tests/test_litellm/proxy/proxy_server/test_routes_login_sso.py index 88f8be4e49a..8daea8d0ad2 100644 --- a/tests/test_litellm/proxy/proxy_server/test_routes_login_sso.py +++ b/tests/test_litellm/proxy/proxy_server/test_routes_login_sso.py @@ -10,6 +10,7 @@ Routes covered: from __future__ import annotations +from datetime import datetime, timezone from unittest.mock import AsyncMock, MagicMock from .conftest import normalize @@ -510,6 +511,8 @@ def _db_user(monkeypatch, email: str): user.user_email = email user.user_role = "internal_user" user.password = "scrypt:stored" + user.password_reset_required = None + user.last_breach_check_at = datetime.now(timezone.utc) repo = MagicMock() repo.return_value.table.find_first = AsyncMock(return_value=user) monkeypatch.setattr(ps, "prisma_client", MagicMock()) diff --git a/tests/test_litellm/proxy/proxy_server/test_routes_model_metrics.py b/tests/test_litellm/proxy/proxy_server/test_routes_model_metrics.py index 246e2cbba54..b5536b7618c 100644 --- a/tests/test_litellm/proxy/proxy_server/test_routes_model_metrics.py +++ b/tests/test_litellm/proxy/proxy_server/test_routes_model_metrics.py @@ -11,13 +11,18 @@ Pins (PR2): from __future__ import annotations +from collections.abc import Callable, Mapping +from contextlib import AbstractContextManager from unittest.mock import AsyncMock, MagicMock import pytest +from fastapi.testclient import TestClient import litellm from litellm.proxy import proxy_server from litellm.proxy._types import LitellmUserRoles +from litellm.proxy.config_resolvers.settings_rules import JsonValue +from litellm.proxy.config_resolvers.settings_store import SettingsStore from .conftest import normalize # type: ignore[import-not-found] @@ -179,6 +184,177 @@ def test_model_settings_method_not_allowed(client, auth_as): # --------------------------------------------------------------------------- +def _alerting_client( + monkeypatch: pytest.MonkeyPatch, + *, + yaml_values: Mapping[str, JsonValue], + db_row: Mapping[str, JsonValue], + live_args: Mapping[str, JsonValue], +) -> "SettingsStore": + pc = MagicMock() + row = MagicMock() + row.param_value = db_row + pc.db.litellm_config.find_first = AsyncMock(return_value=row) + monkeypatch.setattr(proxy_server, "prisma_client", pc) + + logging_obj = MagicMock() + args_model = MagicMock() + args_model.model_dump = MagicMock(return_value=live_args) + logging_obj.slack_alerting_instance.alerting_args = args_model + monkeypatch.setattr(proxy_server, "proxy_logging_obj", logging_obj) + + store = SettingsStore("general_settings") + store.load_yaml(yaml_values) + store.apply_db_row("general_settings", db_row) + monkeypatch.setattr(proxy_server.proxy_config, "settings", store) + monkeypatch.setattr(proxy_server, "general_settings", store) + return store + + +def test_alerting_settings_reports_sources( + client: TestClient, + auth_as: Callable[..., AbstractContextManager[None]], + monkeypatch: pytest.MonkeyPatch, +) -> None: + _alerting_client( + monkeypatch, + yaml_values={ + "alerting": ["slack"], + "alerting_args": {"daily_report_frequency": 3, "report_check_interval": 300}, + }, + db_row={"alerting_args": {"daily_report_frequency": 7, "outage_alert_ttl": 4242}}, + live_args={"daily_report_frequency": 3}, + ) + + with auth_as(LitellmUserRoles.PROXY_ADMIN): + response = client.get("/alerting/settings") + + assert response.status_code == 200 + by_name = {entry["field_name"]: entry for entry in response.json()} + + assert by_name["slack_alerting"]["source"] == "config" + assert by_name["daily_report_frequency"]["source"] == "config" + assert by_name["report_check_interval"]["source"] == "config" + assert by_name["outage_alert_ttl"]["source"] == "default" + assert by_name["budget_alert_ttl"]["source"] == "default" + + +def test_alerting_settings_reports_db_source_when_the_file_omits_alerting_args( + client: TestClient, + auth_as: Callable[..., AbstractContextManager[None]], + monkeypatch: pytest.MonkeyPatch, +) -> None: + store = _alerting_client( + monkeypatch, + yaml_values={"alerting": ["slack"]}, + db_row={ + "alerting_args": { + "outage_alert_ttl": 4242, + "region_outage_alert_ttl": [], + "report_check_interval": None, + } + }, + live_args={"outage_alert_ttl": 4242}, + ) + + with auth_as(LitellmUserRoles.PROXY_ADMIN): + response = client.get("/alerting/settings") + + assert response.status_code == 200 + by_name = {entry["field_name"]: entry for entry in response.json()} + + assert store.source("alerting_args") == "db" + assert by_name["outage_alert_ttl"]["source"] == "db" + assert by_name["region_outage_alert_ttl"]["source"] == "db" + assert by_name["report_check_interval"]["source"] == "db" + assert by_name["budget_alert_ttl"]["source"] == "default" + + +def test_alerting_settings_reports_config_source_when_db_disagrees( + client: TestClient, + auth_as: Callable[..., AbstractContextManager[None]], + monkeypatch: pytest.MonkeyPatch, +) -> None: + from litellm.proxy.config_resolvers import SettingsStore + + db_alerting_args = {"daily_report_frequency": 7} + + pc = MagicMock() + row = MagicMock() + row.param_value = {"alerting_args": db_alerting_args} + pc.db.litellm_config.find_first = AsyncMock(return_value=row) + monkeypatch.setattr(proxy_server, "prisma_client", pc) + + logging_obj = MagicMock() + args_model = MagicMock() + args_model.model_dump = MagicMock(return_value={"daily_report_frequency": 3}) + logging_obj.slack_alerting_instance.alerting_args = args_model + monkeypatch.setattr(proxy_server, "proxy_logging_obj", logging_obj) + + store = SettingsStore("general_settings") + store.load_yaml({"alerting_args": {"daily_report_frequency": 3}}) + store.apply_db_row("general_settings", {"alerting_args": db_alerting_args}) + monkeypatch.setattr(proxy_server.proxy_config, "settings", store) + monkeypatch.setattr(proxy_server, "general_settings", store) + + with auth_as(LitellmUserRoles.PROXY_ADMIN): + response = client.get("/alerting/settings") + + assert response.status_code == 200 + by_name = {entry["field_name"]: entry for entry in response.json()} + assert store.source("alerting_args") == "config" + assert by_name["daily_report_frequency"]["field_value"] == 3 + assert by_name["daily_report_frequency"]["source"] == "config" + + +@pytest.mark.parametrize("db_alerting_args", [None, []]) +def test_alerting_settings_handles_empty_db_args( + client: TestClient, + auth_as: Callable[..., AbstractContextManager[None]], + monkeypatch: pytest.MonkeyPatch, + db_alerting_args: JsonValue, +) -> None: + from litellm.proxy.config_resolvers import SettingsStore + + pc = MagicMock() + row = MagicMock() + row.param_value = {"alerting_args": db_alerting_args} + pc.db.litellm_config.find_first = AsyncMock(return_value=row) + monkeypatch.setattr(proxy_server, "prisma_client", pc) + + logging_obj = MagicMock() + args_model = MagicMock() + args_model.model_dump = MagicMock(return_value={}) + logging_obj.slack_alerting_instance.alerting_args = args_model + monkeypatch.setattr(proxy_server, "proxy_logging_obj", logging_obj) + + store = SettingsStore("general_settings") + store.load_yaml({"alerting_args": {"report_check_interval": 300}}) + monkeypatch.setattr(proxy_server.proxy_config, "settings", store) + monkeypatch.setattr(proxy_server, "general_settings", store) + + with auth_as(LitellmUserRoles.PROXY_ADMIN): + response = client.get("/alerting/settings") + + assert response.status_code == 200 + by_name = {entry["field_name"]: entry for entry in response.json()} + assert by_name["report_check_interval"]["source"] == "config" + assert by_name["budget_alert_ttl"]["source"] == "default" + + +@pytest.mark.parametrize( + ("field_default", "expected"), + [(43200, "default"), (None, "unset")], +) +def test_nested_setting_source_without_a_config_or_db_value(field_default: JsonValue, expected: str) -> None: + store = SettingsStore("general_settings") + store.load_yaml({}) + + assert ( + proxy_server._nested_setting_source(store, {}, "alerting_args", "budget_alert_ttl", field_default) == expected + ) + + def test_alerting_settings_no_db_error(client, auth_as, no_prisma): """Pins ``GET /alerting/settings`` (error: db not connected).""" with auth_as(LitellmUserRoles.PROXY_ADMIN): diff --git a/tests/test_litellm/proxy/shutdown/test_scheduled_jobs.py b/tests/test_litellm/proxy/shutdown/test_scheduled_jobs.py new file mode 100644 index 00000000000..fbce38db39f --- /dev/null +++ b/tests/test_litellm/proxy/shutdown/test_scheduled_jobs.py @@ -0,0 +1,158 @@ +import asyncio +import logging +from collections.abc import AsyncIterator +from contextlib import asynccontextmanager +from datetime import datetime, timedelta + +import pytest +from apscheduler.schedulers.asyncio import AsyncIOScheduler + +from litellm.proxy.shutdown.scheduled_jobs import ( + AwaitableAsyncIOExecutor, + pause_scheduled_jobs, + stop_in_flight_scheduler_jobs, +) + + +class _Job: + """A scheduled job that blocks until cancelled, or for ``work_seconds``, and records what it observed""" + + def __init__(self, swallow_cancellation: bool = False, work_seconds: float | None = None) -> None: + self.started = asyncio.Event() + self.events: list[str] = [] + self.swallow_cancellation = swallow_cancellation + self.work_seconds = work_seconds + + async def run(self) -> None: + self.started.set() + try: + if self.work_seconds is None: + await asyncio.Event().wait() + else: + await asyncio.sleep(self.work_seconds) + self.events.append("committed") + except asyncio.CancelledError: + self.events.append("cancelled") + if self.swallow_cancellation: + await asyncio.Event().wait() + raise + finally: + self.events.append("finished") + + +@asynccontextmanager +async def _running_scheduler(*jobs: _Job) -> AsyncIterator[tuple[AsyncIOScheduler, AwaitableAsyncIOExecutor]]: + """A started scheduler with every job in flight, stopped on the way out whatever the test did""" + executor = AwaitableAsyncIOExecutor() + scheduler = AsyncIOScheduler(executors={"default": executor}) + for index, job in enumerate(jobs): + scheduler.add_job(job.run, id=f"job-{index}", next_run_time=datetime.now()) + scheduler.start() + try: + for job in jobs: + await asyncio.wait_for(job.started.wait(), timeout=5) + yield scheduler, executor + finally: + if scheduler.running: + scheduler.shutdown(wait=False) + stragglers = executor.in_flight_jobs() + for straggler in stragglers: + straggler.cancel() + await asyncio.gather(*stragglers, return_exceptions=True) + + +@pytest.mark.asyncio +async def test_in_flight_jobs_observe_cancellation_before_shutdown_returns(): + """The job's own CancelledError handler records how a run ended, so shutdown must wait for it""" + job = _Job() + async with _running_scheduler(job) as (scheduler, executor): + await stop_in_flight_scheduler_jobs(scheduler, executor) + + assert job.events == ["cancelled", "finished"] + assert scheduler.running is False + assert executor.in_flight_jobs() == () + + +@pytest.mark.asyncio +async def test_a_job_that_is_finishing_is_allowed_to_finish_rather_than_cancelled(): + """A spend write cancelled mid-commit drops the rows it popped, so short jobs get to finish first""" + write = _Job(work_seconds=0.2) + stuck = _Job() + async with _running_scheduler(write, stuck) as (scheduler, executor): + await stop_in_flight_scheduler_jobs(scheduler, executor, finish_timeout_seconds=2.0) + + assert write.events == ["committed", "finished"] + assert stuck.events == ["cancelled", "finished"] + assert scheduler.running is False + + +@pytest.mark.asyncio +async def test_every_in_flight_job_is_cancelled_not_only_the_first(): + first, second = _Job(), _Job() + async with _running_scheduler(first, second) as (scheduler, executor): + await stop_in_flight_scheduler_jobs(scheduler, executor) + + assert first.events == ["cancelled", "finished"] + assert second.events == ["cancelled", "finished"] + + +@pytest.mark.asyncio +async def test_a_job_that_ignores_cancellation_is_abandoned_after_the_timeout(caplog): + """A job that swallows CancelledError must not hold the pod past its termination grace period""" + job = _Job(swallow_cancellation=True) + async with _running_scheduler(job) as (scheduler, executor): + with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"): + await stop_in_flight_scheduler_jobs(scheduler, executor, cancel_timeout_seconds=0.05) + + assert job.events == ["cancelled"] + assert "1 scheduled job(s) did not finish within 0.05s of cancellation" in caplog.text + + +@pytest.mark.asyncio +async def test_shutdown_with_nothing_in_flight_still_stops_the_scheduler(): + async with _running_scheduler() as (scheduler, executor): + await stop_in_flight_scheduler_jobs(scheduler, executor) + await asyncio.sleep(0) + + assert scheduler.running is False + + +@pytest.mark.asyncio +async def test_a_scheduler_that_never_started_is_left_alone(): + """The proxy runs without a scheduler when it has no database""" + executor = AwaitableAsyncIOExecutor() + scheduler = AsyncIOScheduler(executors={"default": executor}) + + await stop_in_flight_scheduler_jobs(scheduler, executor) + + assert scheduler.running is False + + +@pytest.mark.asyncio +async def test_pausing_stops_new_jobs_from_starting_but_leaves_running_ones_alone(): + """A job due during the shutdown drain would only be cancelled, so it must not start at all""" + running = _Job() + async with _running_scheduler(running) as (scheduler, executor): + late = _Job() + scheduler.add_job(late.run, id="late", next_run_time=datetime.now() + timedelta(seconds=0.1)) + + pause_scheduled_jobs(scheduler) + await asyncio.sleep(0.3) + + assert late.started.is_set() is False + assert running.events == [] + assert scheduler.running is True + + await stop_in_flight_scheduler_jobs(scheduler, executor) + + assert running.events == ["cancelled", "finished"] + assert late.started.is_set() is False + + +@pytest.mark.asyncio +async def test_pausing_a_scheduler_that_never_started_is_a_no_op(): + scheduler = AsyncIOScheduler(executors={"default": AwaitableAsyncIOExecutor()}) + + pause_scheduled_jobs(scheduler) + + assert scheduler.running is False diff --git a/tests/test_litellm/proxy/spend_tracking/test_budget_reservation.py b/tests/test_litellm/proxy/spend_tracking/test_budget_reservation.py index 0e0025f5194..214b5cde7da 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_budget_reservation.py +++ b/tests/test_litellm/proxy/spend_tracking/test_budget_reservation.py @@ -26,6 +26,7 @@ from litellm.proxy.spend_tracking.budget_reservation import ( _get_team_member_budget_counter, count_request_input_tokens, estimate_request_max_cost, + release_unbound_budget_reservation, reserve_budget_for_request, ) from litellm.proxy.utils import ProxyLogging @@ -546,3 +547,41 @@ async def test_team_member_reservation_counter_adds_temp_increase_to_live_team_d assert counter is not None assert counter.max_budget == expected_max_budget assert counter.fallback_spend == 0.5 + + +@pytest.mark.asyncio +async def test_reservation_starts_unbound_to_any_callback(): + reservation: Final = await _reserve("/v1/responses") + + assert reservation is not None + assert reservation["callback_bound"] is False + + +@pytest.mark.asyncio +async def test_release_unbound_budget_reservation_frees_the_counter(spend_counter_cache: DualCache): + counter_key: Final = f"spend:key:{TINY_BUDGET_KEY_TOKEN}" + reservation: Final = await _reserve_for_tiny_budget_key( + "/v1/chat/completions", {"model": "gpt-4o", "messages": [{"role": "user", "content": "hello"}]} + ) + assert reservation is not None + assert spend_counter_cache.in_memory_cache.get_cache(key=counter_key) == pytest.approx(reservation["reserved_cost"]) + + await release_unbound_budget_reservation(reservation) + + assert spend_counter_cache.in_memory_cache.get_cache(key=counter_key) == pytest.approx(0.0) + assert reservation["finalized"] is True + + +@pytest.mark.asyncio +async def test_release_unbound_budget_reservation_leaves_a_bound_one_to_its_callback(spend_counter_cache: DualCache): + counter_key: Final = f"spend:key:{TINY_BUDGET_KEY_TOKEN}" + reservation: Final = await _reserve_for_tiny_budget_key( + "/v1/chat/completions", {"model": "gpt-4o", "messages": [{"role": "user", "content": "hello"}]} + ) + assert reservation is not None + reservation["callback_bound"] = True + + await release_unbound_budget_reservation(reservation) + + assert spend_counter_cache.in_memory_cache.get_cache(key=counter_key) == pytest.approx(reservation["reserved_cost"]) + assert reservation["finalized"] is False diff --git a/tests/test_litellm/proxy/test__types.py b/tests/test_litellm/proxy/test__types.py index 9e1486ce90f..f5abe0561db 100644 --- a/tests/test_litellm/proxy/test__types.py +++ b/tests/test_litellm/proxy/test__types.py @@ -5,11 +5,13 @@ from pydantic import ValidationError from litellm.proxy._types import ( ROLES_WITHIN_ORG, + ChangePasswordRequest, GenerateKeyRequest, KeyRequest, LiteLLM_AuditLogs, LiteLLM_TeamMembership, LitellmUserRoles, + NewUserRequest, OrganizationMemberUpdateRequest, ResetSpendRequest, UpdateKeyRequest, @@ -335,3 +337,43 @@ def test_virtual_key_mapping_counts_as_configured_when_any_issuer_sets_the_claim ) assert jwt_auth.is_virtual_key_mapping_configured() is is_configured + + +def test_new_user_request_loudly_rejects_a_password(): + """ + /user/new has never persisted a password (the field used to be silently + dropped). Sending one must now fail visibly so the dead path cannot be + revived without going through the password policy. + """ + with pytest.raises(ValidationError, match="invitation link"): + NewUserRequest(user_email="alice@example.com", password="hunter2hunter2") + + +def test_new_user_request_without_password_still_works(): + request = NewUserRequest(user_email="alice@example.com") + assert request.password is None + + +def test_update_user_request_accepts_a_password(): + """Admins set user passwords through /user/update; the value must survive + model validation so the endpoint can policy-check and hash it.""" + request = UpdateUserRequest(user_id="user-123", password="hunter2hunter2") + assert request.password == "hunter2hunter2" + + +def test_update_user_request_password_hidden_from_repr(): + """management_endpoint_wrapper string-formats endpoint kwargs into Slack + alerts, so the model's repr/str must never contain the plaintext password.""" + request = UpdateUserRequest(user_id="user-123", password="hunter2hunter2") + assert "hunter2hunter2" not in repr(request) + assert "hunter2hunter2" not in str(request) + + +def test_change_password_request_passwords_hidden_from_repr(): + """Any accidental str()/repr() of the request model (debug logs, exception + handlers, a future management_endpoint_wrapper) must never contain either + plaintext password.""" + request = ChangePasswordRequest(current_password="hunter2hunter2", new_password="NewP@ssw0rd-2026") + for rendered in (repr(request), str(request)): + assert "hunter2hunter2" not in rendered + assert "NewP@ssw0rd-2026" not in rendered diff --git a/tests/test_litellm/proxy/test_common_request_processing.py b/tests/test_litellm/proxy/test_common_request_processing.py index 0b872400be0..218d8246715 100644 --- a/tests/test_litellm/proxy/test_common_request_processing.py +++ b/tests/test_litellm/proxy/test_common_request_processing.py @@ -2343,6 +2343,39 @@ class TestCommonRequestProcessingHelpers: assert isinstance(response, JSONResponse) assert response.headers["x-litellm-model-id"] == "fallback-deployment" + @staticmethod + async def _first_chunk_error_response(**create_response_kwargs): + async def mock_generator(): + yield 'data: {"error": {"code": 403, "message": "forbidden"}}\n\n' + yield "data: [DONE]\n\n" + + return await create_response( + mock_generator(), + "text/event-stream", + {"x-litellm-call-id": "call-8302"}, + **create_response_kwargs, + ) + + async def test_create_response_first_chunk_error_carries_the_call_id_when_opted_in(self): + """A stream that fails on its first chunk answers as JSON, and with + include_call_id_in_error_body on that JSON names the request like the + non-streaming error path does, byte-identical to the header.""" + response = await self._first_chunk_error_response(general_settings={"include_call_id_in_error_body": True}) + + assert isinstance(response, JSONResponse) + assert response.status_code == 403 + assert response.headers["x-litellm-call-id"] == "call-8302" + assert json.loads(response.body) == { + "error": {"code": 403, "message": "forbidden", "litellm_call_id": "call-8302"} + } + + async def test_create_response_first_chunk_error_body_is_unchanged_by_default(self): + response = await self._first_chunk_error_response() + + assert isinstance(response, JSONResponse) + assert response.headers["x-litellm-call-id"] == "call-8302" + assert json.loads(response.body) == {"error": {"code": 403, "message": "forbidden"}} + async def test_create_streaming_response_disables_proxy_buffering(self): """Regression for #28384: every StreamingResponse create_response returns must carry the headers that stop nginx/ingress/Envoy from buffering the @@ -9122,6 +9155,62 @@ class TestStreamingResponseHeadersFollowFallback: assert result.status_code == 400 assert result.headers["x-litellm-applied-guardrails"] == "stream-blocker" + @pytest.mark.asyncio + async def test_streaming_first_chunk_error_carries_the_call_id_when_opted_in(self, monkeypatch): + """The opt-in reaches the streaming path through base_process_llm_request, so a stream + that fails on its first chunk answers with the call id inside its JSON error body, + byte-identical to the x-litellm-call-id header.""" + + def select_data_generator(**kwargs): + async def generator(): + yield 'data: {"error": {"code": 403, "message": "forbidden"}}\n\n' + yield "data: [DONE]\n\n" + + return generator() + + logging_obj = MagicMock() + logging_obj.litellm_call_id = "lit-8302-call" + logging_obj._defer_async_logging = False + logging_obj._on_deferred_stream_complete = None + logging_obj.cost_breakdown = None + processor = ProxyBaseLLMRequestProcessing( + data={"model": "oa", "stream": True, "litellm_logging_obj": logging_obj} + ) + + proxy_logging_obj = MagicMock(spec=ProxyLogging) + proxy_logging_obj.during_call_hook = AsyncMock(return_value=None) + proxy_logging_obj.update_request_status = AsyncMock(return_value=None) + proxy_logging_obj.post_call_success_hook = AsyncMock( + side_effect=lambda data, user_api_key_dict, response: response + ) + proxy_logging_obj.post_call_response_headers_hook = AsyncMock(return_value={}) + + async def fake_route_request(**kwargs): + async def call(): + return SimpleNamespace(_hidden_params={}, fallback_headers_adopted=False) + + return call() + + monkeypatch.setattr(litellm.proxy.common_request_processing, "route_request", fake_route_request) + + result = await processor.base_process_llm_request( + request=Request(scope={"type": "http", "headers": []}), + fastapi_response=Response(), + user_api_key_dict=ProxyUserAPIKeyAuth(api_key="sk-test"), + route_type="acompletion", + proxy_logging_obj=proxy_logging_obj, + general_settings={"include_call_id_in_error_body": True}, + proxy_config=MagicMock(spec=ProxyConfig), + select_data_generator=select_data_generator, + is_streaming_request=True, + skip_pre_call_logic=True, + ) + + assert isinstance(result, JSONResponse) + assert result.status_code == 403 + assert result.headers["x-litellm-call-id"] == "lit-8302-call" + assert json.loads(result.body)["error"]["litellm_call_id"] == "lit-8302-call" + class _MessagesFallbackStream: def __init__(self) -> None: diff --git a/tests/test_litellm/proxy/test_proxy_server.py b/tests/test_litellm/proxy/test_proxy_server.py index 950a6cc3c40..2792e176e0e 100644 --- a/tests/test_litellm/proxy/test_proxy_server.py +++ b/tests/test_litellm/proxy/test_proxy_server.py @@ -11766,6 +11766,66 @@ def test_prompt_caching_settings_propagate_on_config_reload(monkeypatch, field_n assert getattr(litellm, field_name) == db_value +@pytest.mark.asyncio +async def test_db_stored_datadog_redaction_settings_apply_before_logger_init(monkeypatch: pytest.MonkeyPatch): + """A DB-only litellm_settings row that pairs success_callback: ["datadog"] with + datadog_params.turn_off_message_logging: true must build the DataDogLogger redacted, the + same as the identical block in YAML. Regression for the redaction keys being absent from + the safe-override allowlist while the callback half of the row was honoured.""" + import litellm.proxy.proxy_server as ps + from litellm.integrations.datadog.datadog import DataDogLogger + from litellm.litellm_core_utils import litellm_logging + + monkeypatch.setenv("DD_API_KEY", "test-key") + monkeypatch.setenv("DD_SITE", "us5.datadoghq.com") + monkeypatch.setattr(litellm, "datadog_params", None) + monkeypatch.setattr(litellm, "turn_off_message_logging", False) + monkeypatch.setattr(litellm, "success_callback", []) + monkeypatch.setattr(litellm, "_async_success_callback", []) + monkeypatch.setattr(litellm, "failure_callback", []) + monkeypatch.setattr(litellm, "_async_failure_callback", []) + monkeypatch.setattr(litellm, "callbacks", []) + monkeypatch.setattr(litellm_logging, "_in_memory_loggers", []) + + db_row = { + "success_callback": ["datadog"], + "datadog_params": {"turn_off_message_logging": True}, + "turn_off_message_logging": True, + } + pc = ps.ProxyConfig() + pc._apply_litellm_settings_db_values(pc._prepared_db_settings_values("litellm_settings", db_row)) + pc._add_callbacks_from_db_config({"litellm_settings": db_row}) + + datadog_loggers = [cb for cb in litellm.success_callback if isinstance(cb, DataDogLogger)] + assert len(datadog_loggers) == 1 + assert datadog_loggers[0].turn_off_message_logging is True + assert litellm.turn_off_message_logging is True + + +@pytest.mark.parametrize( + "field_name", + [ + "datadog_params", + "datadog_llm_observability_params", + "newrelic_params", + "pointfive_params", + "aws_sqs_callback_params", + ], +) +def test_db_stored_callback_params_propagate_to_litellm_module(monkeypatch: pytest.MonkeyPatch, field_name: str): + """Every callback init params block stored in the DB litellm_settings row must land on the + litellm module before the matching logger is built, so the DB row behaves like YAML.""" + import litellm.proxy.proxy_server as ps + + monkeypatch.setattr(litellm, field_name, None) + db_value = {"turn_off_message_logging": True} + + pc = ps.ProxyConfig() + pc._apply_litellm_settings_db_values(pc._prepared_db_settings_values("litellm_settings", {field_name: db_value})) + + assert getattr(litellm, field_name) == db_value + + def test_get_config_list_marks_untouched_prompt_caching_flag_as_not_set(monkeypatch): """The flag defaults to False rather than None, so a plain 'is not None' check would report the default as 'In Config' and imply an admin had set it.""" diff --git a/tests/test_litellm/proxy/test_route_llm_request.py b/tests/test_litellm/proxy/test_route_llm_request.py index 6cbbc279748..0b51062dd66 100644 --- a/tests/test_litellm/proxy/test_route_llm_request.py +++ b/tests/test_litellm/proxy/test_route_llm_request.py @@ -1282,7 +1282,7 @@ async def test_route_request_routing_group_name_passes_model_gate(): @pytest.mark.asyncio -async def test_route_request_a2a_agent_miss_does_not_consume_model_read_through(monkeypatch): +async def test_route_request_a2a_agent_miss_does_not_consume_model_read_through(fresh_agent_read_through, monkeypatch): from types import SimpleNamespace from unittest.mock import AsyncMock diff --git a/tests/test_litellm/proxy/test_spend_log_cleanup.py b/tests/test_litellm/proxy/test_spend_log_cleanup.py index bf1538183ab..e333da03950 100644 --- a/tests/test_litellm/proxy/test_spend_log_cleanup.py +++ b/tests/test_litellm/proxy/test_spend_log_cleanup.py @@ -7,6 +7,7 @@ import math import time from contextlib import asynccontextmanager from datetime import datetime, timedelta, timezone +from typing import Final from unittest.mock import AsyncMock, MagicMock import pytest @@ -793,18 +794,20 @@ async def test_spend_logs_retention_alone_does_not_touch_the_session_rollup(): tables = [call[0][0] for call in client.db.execute_raw.call_args_list] assert any('"LiteLLM_SpendLogs"' in sql for sql in tables) assert not any('"LiteLLM_AutoRouterSession"' in sql for sql in tables) + assert not any('"LiteLLM_AutoRouterUserSession"' in sql for sql in tables) assert not any('"LiteLLM_HealthCheckTable"' in sql for sql in tables) @pytest.mark.asyncio -async def test_session_retention_alone_cleans_only_the_session_rollup(): - client = _mock_prisma_for_retention([0]) +async def test_session_retention_alone_cleans_both_session_rollups(): + client = _mock_prisma_for_retention([0, 0]) cleaner = SpendLogCleanup(general_settings={"maximum_autorouter_session_retention_period": "365d"}) cleaner.pod_lock_manager = None await cleaner.cleanup_old_spend_logs(client) tables = [call[0][0] for call in client.db.execute_raw.call_args_list] - assert len(tables) == 1 + assert len(tables) == 2 assert '"LiteLLM_AutoRouterSession"' in tables[0] + assert '"LiteLLM_AutoRouterUserSession"' in tables[1] @pytest.mark.asyncio @@ -825,7 +828,7 @@ async def test_health_check_retention_alone_cleans_only_the_health_check_table() @pytest.mark.asyncio async def test_each_retention_key_cuts_off_at_its_own_horizon(): - client = _mock_prisma_for_retention([0, 0, 0, 0]) + client = _mock_prisma_for_retention([0, 0, 0, 0, 0]) cleaner = SpendLogCleanup( general_settings={ "maximum_spend_logs_retention_period": "7d", @@ -839,6 +842,8 @@ async def test_each_retention_key_cuts_off_at_its_own_horizon(): ( "LiteLLM_AutoRouterSession" if '"LiteLLM_AutoRouterSession"' in call[0][0] + else "LiteLLM_AutoRouterUserSession" + if '"LiteLLM_AutoRouterUserSession"' in call[0][0] else "LiteLLM_HealthCheckTable" if '"LiteLLM_HealthCheckTable"' in call[0][0] else "logs" @@ -848,6 +853,7 @@ async def test_each_retention_key_cuts_off_at_its_own_horizon(): now = datetime.now(timezone.utc) assert (now - cutoffs["logs"]).days == 7 assert (now - cutoffs["LiteLLM_AutoRouterSession"]).days == 365 + assert cutoffs["LiteLLM_AutoRouterUserSession"] == cutoffs["LiteLLM_AutoRouterSession"] assert (now - cutoffs["LiteLLM_HealthCheckTable"]).days == 30 @@ -1417,3 +1423,125 @@ def test_the_reported_run_outcome_is_the_most_significant_reason_in_any_order(st """ results = tuple(TableCleanupResult(rows_deleted=0, stop_reason=reason) for reason in stop_reasons) assert SpendLogCleanup._run_outcome(results) == expected + + +_OTHER_OUTCOMES: Final = ("completed", "budget_exhausted", "batch_cap_reached", "skipped_locked", "skipped_disabled") + + +def _runs_recorded(outcome: str) -> float: + """The real ``litellm_spend_log_cleanup_runs_total`` sample for one outcome, 0 when unset""" + from prometheus_client import REGISTRY + + return REGISTRY.get_sample_value("litellm_spend_log_cleanup_runs_total", {"outcome": outcome}) or 0.0 + + +@pytest.mark.asyncio +async def test_a_cancelled_run_records_aborted_and_logs_its_progress_before_re_raising(monkeypatch): + """A run cut short by shutdown must leave its outcome and how far it got behind""" + import litellm.proxy.db.db_transaction_queue.spend_log_cleanup as cleanup_module + + mock_logger = MagicMock() + monkeypatch.setattr(cleanup_module, "verbose_proxy_logger", mock_logger) + aborted_runs_before = _runs_recorded("aborted") + other_runs_before = {outcome: _runs_recorded(outcome) for outcome in _OTHER_OUTCOMES} + + third_batch_reached = asyncio.Event() + + async def _execute_raw(sql, *args): + if third_batch_reached.is_set(): + raise AssertionError("no batch may be issued after the cancelled one") + if _execute_raw.calls < 2: + _execute_raw.calls += 1 + return 150 + third_batch_reached.set() + await asyncio.Event().wait() + + _execute_raw.calls = 0 + mock_prisma_client = MagicMock() + _wire_tx(mock_prisma_client.db) + mock_prisma_client.db.execute_raw = _execute_raw + + cleaner = SpendLogCleanup(general_settings={"maximum_spend_logs_retention_period": "7d"}) + cleaner.pod_lock_manager = MagicMock() + cleaner.pod_lock_manager.redis_cache = MagicMock() + cleaner.pod_lock_manager.acquire_lock = AsyncMock(return_value=True) + cleaner.pod_lock_manager.release_lock = AsyncMock() + + run = asyncio.ensure_future(cleaner.cleanup_old_spend_logs(mock_prisma_client)) + await asyncio.wait_for(third_batch_reached.wait(), timeout=5) + run.cancel() + with pytest.raises(asyncio.CancelledError): + await run + + assert _runs_recorded("aborted") == aborted_runs_before + 1 + assert {outcome: _runs_recorded(outcome) for outcome in _OTHER_OUTCOMES} == other_runs_before + cleaner.pod_lock_manager.release_lock.assert_awaited_once() + mock_logger.exception.assert_not_called() + (error_call,) = mock_logger.error.call_args_list + rendered = error_call[0][0] % error_call[0][1:] + assert rendered.startswith("Spend log cleanup cancelled after ") + assert "s (rows_deleted=300, batches=2)" in rendered + + +@pytest.mark.asyncio +async def test_progress_reported_for_a_cancelled_run_is_that_run_only(monkeypatch): + """The scheduler holds one cleaner for the life of the process, so progress must not carry over""" + import litellm.proxy.db.db_transaction_queue.spend_log_cleanup as cleanup_module + + mock_logger = MagicMock() + monkeypatch.setattr(cleanup_module, "verbose_proxy_logger", mock_logger) + + mock_prisma_client = MagicMock() + _wire_tx(mock_prisma_client.db) + mock_prisma_client.db.execute_raw = AsyncMock(side_effect=[150, 0, 0]) + cleaner = SpendLogCleanup(general_settings={"maximum_spend_logs_retention_period": "7d"}) + cleaner.pod_lock_manager = None + await cleaner.cleanup_old_spend_logs(mock_prisma_client) + + mock_prisma_client.db.execute_raw = AsyncMock(side_effect=[150, asyncio.CancelledError()]) + with pytest.raises(asyncio.CancelledError): + await cleaner.cleanup_old_spend_logs(mock_prisma_client) + + (error_call,) = mock_logger.error.call_args_list + rendered = error_call[0][0] % error_call[0][1:] + assert "(rows_deleted=150, batches=1)" in rendered + + +@pytest.mark.asyncio +async def test_progress_reported_by_an_overlapping_run_is_its_own(monkeypatch): + """With APSCHEDULER_MAX_INSTANCES above one, two runs share the cleaner but not their progress""" + import litellm.proxy.db.db_transaction_queue.spend_log_cleanup as cleanup_module + + mock_logger = MagicMock() + monkeypatch.setattr(cleanup_module, "verbose_proxy_logger", mock_logger) + + first_batch_done = asyncio.Event() + second_run_done = asyncio.Event() + + async def _slow_execute_raw(sql, *args): + first_batch_done.set() + await second_run_done.wait() + return 100 + + slow_client = MagicMock() + _wire_tx(slow_client.db) + slow_client.db.execute_raw = _slow_execute_raw + fast_client = MagicMock() + _wire_tx(fast_client.db) + fast_client.db.execute_raw = AsyncMock(side_effect=[150, 150, 0, 0]) + + cleaner = SpendLogCleanup(general_settings={"maximum_spend_logs_retention_period": "7d"}) + cleaner.pod_lock_manager = None + + slow_run = asyncio.ensure_future(cleaner.cleanup_old_spend_logs(slow_client)) + await asyncio.wait_for(first_batch_done.wait(), timeout=5) + await cleaner.cleanup_old_spend_logs(fast_client) + second_run_done.set() + await asyncio.sleep(0) + slow_run.cancel() + with pytest.raises(asyncio.CancelledError): + await slow_run + + (error_call,) = mock_logger.error.call_args_list + rendered = error_call[0][0] % error_call[0][1:] + assert "(rows_deleted=100, batches=1)" in rendered diff --git a/tests/test_litellm/proxy/ui_crud_endpoints/test_proxy_setting_endpoints.py b/tests/test_litellm/proxy/ui_crud_endpoints/test_proxy_setting_endpoints.py index 0f57af7f82c..eecee2fd0f1 100644 --- a/tests/test_litellm/proxy/ui_crud_endpoints/test_proxy_setting_endpoints.py +++ b/tests/test_litellm/proxy/ui_crud_endpoints/test_proxy_setting_endpoints.py @@ -1342,6 +1342,48 @@ class TestProxySettingEndpoints: where={"id": "ui_settings"} ) + def test_get_ui_settings_reports_sources(self, monkeypatch: pytest.MonkeyPatch) -> None: + from unittest.mock import AsyncMock, MagicMock + + from litellm.proxy import proxy_server + from litellm.proxy.config_resolvers import SettingsStore + + mock_prisma = MagicMock() + mock_db_record = MagicMock() + mock_db_record.ui_settings = { + "disable_model_add_for_internal_users": True, + "require_auth_for_public_ai_hub": True, + } + mock_prisma.db.litellm_uisettings.find_unique = AsyncMock( + return_value=mock_db_record + ) + monkeypatch.setattr(proxy_server, "prisma_client", mock_prisma) + + store = SettingsStore("general_settings") + store.load_yaml( + { + "disable_model_add_for_internal_users": False, + "forward_client_headers_to_llm_api": True, + } + ) + store.apply_db_row( + "ui_settings", + {"disable_model_add_for_internal_users": True}, + ) + monkeypatch.setattr(proxy_server.proxy_config, "settings", store) + monkeypatch.setattr(proxy_server, "general_settings", store) + + response = client.get("/get/ui_settings") + + assert response.status_code == 200 + data = response.json() + assert data["values"]["disable_model_add_for_internal_users"] is False + assert data["values"]["forward_client_headers_to_llm_api"] is True + assert data["values"]["require_auth_for_public_ai_hub"] is True + assert data["source"]["disable_model_add_for_internal_users"] == "config" + assert data["source"]["forward_client_headers_to_llm_api"] == "config" + assert data["source"]["require_auth_for_public_ai_hub"] == "db" + def test_get_ui_settings_schema_description_preserved_with_extensions( self, mock_auth, monkeypatch ): @@ -3477,6 +3519,7 @@ class TestPtuCostAttributionUISetting: assert response.status_code == 200 assert response.json()["values"]["enable_ptu_cost_attribution"] is False + assert response.json()["source"]["enable_ptu_cost_attribution"] == "default" def test_reported_true_once_the_env_var_is_set(self, mock_auth, monkeypatch): from litellm.proxy.spend_tracking.ptu_feature_flag import PTU_COST_ATTRIBUTION_ENV_VAR @@ -3488,6 +3531,47 @@ class TestPtuCostAttributionUISetting: assert response.status_code == 200 assert response.json()["values"]["enable_ptu_cost_attribution"] is True + assert response.json()["source"]["enable_ptu_cost_attribution"] == "config" + + def test_reported_config_when_secret_manager_enables_the_flag( + self, mock_auth: None, monkeypatch: pytest.MonkeyPatch + ) -> None: + from litellm.proxy.spend_tracking.ptu_feature_flag import PTU_COST_ATTRIBUTION_ENV_VAR + + monkeypatch.delenv(PTU_COST_ATTRIBUTION_ENV_VAR, raising=False) + monkeypatch.setattr( + "litellm.proxy.ui_crud_endpoints.proxy_setting_endpoints.is_ptu_cost_attribution_enabled", + lambda: True, + ) + self._mock_prisma(monkeypatch) + + response = client.get("/get/ui_settings") + + assert response.status_code == 200 + assert response.json()["values"]["enable_ptu_cost_attribution"] is True + assert response.json()["source"]["enable_ptu_cost_attribution"] == "config" + + def test_reported_config_when_secret_manager_disables_the_flag( + self, mock_auth: None, monkeypatch: pytest.MonkeyPatch + ) -> None: + from litellm.proxy.spend_tracking.ptu_feature_flag import PTU_COST_ATTRIBUTION_ENV_VAR + + monkeypatch.delenv(PTU_COST_ATTRIBUTION_ENV_VAR, raising=False) + monkeypatch.setattr( + "litellm.proxy.ui_crud_endpoints.proxy_setting_endpoints.is_ptu_cost_attribution_enabled", + lambda: False, + ) + monkeypatch.setattr( + "litellm.proxy.ui_crud_endpoints.proxy_setting_endpoints.get_secret", + lambda *_args: False, + ) + self._mock_prisma(monkeypatch) + + response = client.get("/get/ui_settings") + + assert response.status_code == 200 + assert response.json()["values"]["enable_ptu_cost_attribution"] is False + assert response.json()["source"]["enable_ptu_cost_attribution"] == "config" def test_a_persisted_true_cannot_forge_the_derived_value(self, mock_auth, monkeypatch): """A row written before the allowlist existed must not be able to turn the feature on.""" diff --git a/tests/test_litellm/proxy/utils/helpers/test_model_access.py b/tests/test_litellm/proxy/utils/helpers/test_model_access.py index 5fb4392eec6..7f77f938323 100644 --- a/tests/test_litellm/proxy/utils/helpers/test_model_access.py +++ b/tests/test_litellm/proxy/utils/helpers/test_model_access.py @@ -110,8 +110,7 @@ def test_create_model_info_response_happy_path_no_metadata(): "owned_by": result["owned_by"], "created_is_int": isinstance(result["created"], int), "metadata_absent": "metadata" not in result, - "max_input_tokens_positive_int": isinstance(result["max_input_tokens"], int) - and result["max_input_tokens"] > 0, + "max_input_tokens_positive_int": isinstance(result["max_input_tokens"], int) and result["max_input_tokens"] > 0, "max_output_tokens_positive_int": isinstance(result["max_output_tokens"], int) and result["max_output_tokens"] > 0, } @@ -205,9 +204,7 @@ def test_validate_model_access_happy_path_single_model_in_list(): def test_validate_model_access_happy_path_batch_all_accessible(): summary = { - "result": validate_model_access( - "gpt-4o,claude-haiku", ["gpt-4o", "claude-haiku", "gemini"] - ), + "result": validate_model_access("gpt-4o,claude-haiku", ["gpt-4o", "claude-haiku", "gemini"]), "input": "gpt-4o,claude-haiku", "available": ["gpt-4o", "claude-haiku", "gemini"], } @@ -389,9 +386,7 @@ async def test_get_available_models_for_user_error_path_complete_list_raises( def _boom(**_kwargs): raise RuntimeError("downstream failure") - monkeypatch.setattr( - "litellm.proxy.auth.model_checks.get_complete_model_list", _boom - ) + monkeypatch.setattr("litellm.proxy.auth.model_checks.get_complete_model_list", _boom) user_api_key_dict = UserAPIKeyAuth( api_key="sk-test-key", user_id="user-1", @@ -481,6 +476,7 @@ async def test_get_available_models_for_user_without_access_groups_grants_nothin ) assert result == [] + @pytest.mark.asyncio async def test_get_available_models_for_user_resolves_key_access_group_models( monkeypatch, @@ -521,3 +517,78 @@ async def test_get_available_models_for_user_resolves_key_access_group_models( user_api_key_cache=MagicMock(), ) assert result == ["model-b"] + + +def _agent_ceiling(models: frozenset[str] | None): + from litellm.proxy.agent_endpoints.auth.agent_access_groups import AgentAccessGroupCeiling + + async def resolve(agent_id: str) -> AgentAccessGroupCeiling | None: + if models is None: + return None + return AgentAccessGroupCeiling( + access_group_ids=("ag-agent",), models=models, mcp_server_ids=frozenset(), agent_ids=frozenset() + ) + + return resolve + + +def _agent_key(models: list[str]) -> UserAPIKeyAuth: + return UserAPIKeyAuth(api_key="sk-agent-key", user_id="user-1", agent_id="agent-1", models=models) + + +@pytest.mark.asyncio +async def test_agent_key_listing_is_capped_to_its_access_groups(): + result = await get_available_models_for_user( + user_api_key_dict=_agent_key(["model-a", "model-b", "model-c"]), + llm_router=_router_with_models(["model-a", "model-b", "model-c"]), + general_settings={}, + user_model=None, + resolve_agent_ceiling=_agent_ceiling(frozenset({"model-b", "model-d"})), + ) + assert result == ["model-b"] + + +@pytest.mark.asyncio +async def test_agent_key_listing_is_empty_when_its_groups_grant_no_model(): + result = await get_available_models_for_user( + user_api_key_dict=_agent_key(["model-a"]), + llm_router=_router_with_models(["model-a"]), + general_settings={}, + user_model=None, + resolve_agent_ceiling=_agent_ceiling(frozenset()), + ) + assert result == [] + + +@pytest.mark.asyncio +async def test_agent_ceiling_expands_a_model_access_group_name_for_listing(): + router = _router_with_models(["model-a", "model-b"]) + router.get_model_access_groups.return_value = {"fast-models": ["model-b"]} + result = await get_available_models_for_user( + user_api_key_dict=_agent_key(["model-a", "model-b"]), + llm_router=router, + general_settings={}, + user_model=None, + resolve_agent_ceiling=_agent_ceiling(frozenset({"fast-models"})), + ) + assert result == ["model-b"] + + +@pytest.mark.asyncio +async def test_listing_is_unchanged_without_an_agent_or_without_attached_groups(): + router = _router_with_models(["model-a", "model-b"]) + plain_key = await get_available_models_for_user( + user_api_key_dict=UserAPIKeyAuth(api_key="sk-plain", user_id="user-1", models=["model-a", "model-b"]), + llm_router=router, + general_settings={}, + user_model=None, + resolve_agent_ceiling=_agent_ceiling(frozenset({"model-a"})), + ) + agent_without_groups = await get_available_models_for_user( + user_api_key_dict=_agent_key(["model-a", "model-b"]), + llm_router=router, + general_settings={}, + user_model=None, + resolve_agent_ceiling=_agent_ceiling(None), + ) + assert (plain_key, agent_without_groups) == (["model-a", "model-b"], ["model-a", "model-b"]) diff --git a/tests/test_litellm/responses/test_responses_api_bridge_flag.py b/tests/test_litellm/responses/test_responses_api_bridge_flag.py index 2f64cc8debc..16135106b41 100644 --- a/tests/test_litellm/responses/test_responses_api_bridge_flag.py +++ b/tests/test_litellm/responses/test_responses_api_bridge_flag.py @@ -262,6 +262,126 @@ class TestUseResponsesApiBridgeFlag: assert request_body["messages"] == [{"role": "user", "content": "Hello"}] assert response.output[0].content[0].text == "Answer" + def test_bridge_drops_client_metadata_even_when_allowed_openai_params_names_it( + self, respx_mock: respx.MockRouter + ): + upstream: Final = respx_mock.post("https://api.openai.com/v1/chat/completions").mock( + return_value=httpx.Response( + status_code=200, + json={ + "id": "chatcmpl-123", + "object": "chat.completion", + "created": 1677652288, + "model": "my-custom-model", + "choices": [ + {"index": 0, "message": {"role": "assistant", "content": "Answer"}, "finish_reason": "stop"} + ], + "usage": {"prompt_tokens": 9, "completion_tokens": 1, "total_tokens": 10}, + }, + ) + ) + + response: Final = litellm.responses( + model="openai/my-custom-model", + input="Hello", + use_chat_completions_api=True, + allowed_openai_params=["client_metadata"], + client_metadata={"turn_id": "turn-1", "thread_id": "thread-1"}, + api_key="fake-provider-api-key", + num_retries=0, + ) + + assert upstream.call_count == 1 + request_body: Final = json.loads(upstream.calls[0].request.read()) + assert "client_metadata" not in request_body + assert request_body["messages"] == [{"role": "user", "content": "Hello"}] + assert response.output[0].content[0].text == "Answer" + + def test_bridge_merges_instructions_and_developer_input_for_databricks(self, respx_mock: respx.MockRouter): + upstream: Final = respx_mock.post("https://example.databricks.test/serving-endpoints/chat/completions").mock( + return_value=httpx.Response( + status_code=200, + json={ + "id": "chatcmpl-123", + "object": "chat.completion", + "created": 1677652288, + "model": "my-custom-model", + "choices": [ + {"index": 0, "message": {"role": "assistant", "content": "Answer"}, "finish_reason": "stop"} + ], + "usage": {"prompt_tokens": 9, "completion_tokens": 1, "total_tokens": 10}, + }, + ) + ) + + response: Final = litellm.responses( + model="databricks/my-custom-model", + instructions="You are terse.", + input=[ + {"role": "developer", "content": [{"type": "input_text", "text": "Skills: none."}]}, + {"role": "user", "content": [{"type": "input_text", "text": "Hello"}]}, + ], + client_metadata={"turn_id": "turn-1", "thread_id": "thread-1"}, + chat_template_kwargs={"thinking": True}, + api_base="https://example.databricks.test/serving-endpoints", + api_key="fake-databricks-api-key", + num_retries=0, + ) + + assert upstream.call_count == 1 + request_body: Final = json.loads(upstream.calls[0].request.read()) + assert request_body["messages"] == [ + { + "role": "system", + "content": [{"type": "text", "text": "You are terse."}, {"type": "text", "text": "Skills: none."}], + }, + {"role": "user", "content": [{"type": "text", "text": "Hello"}]}, + ] + assert "client_metadata" not in request_body + assert request_body["chat_template_kwargs"] == {"thinking": True} + assert response.output[0].content[0].text == "Answer" + + def test_bridge_drops_client_metadata_for_provider_without_native_config(self, respx_mock: respx.MockRouter): + upstream: Final = respx_mock.post("https://example.databricks.test/serving-endpoints/chat/completions").mock( + return_value=httpx.Response( + status_code=200, + json={ + "id": "chatcmpl-123", + "object": "chat.completion", + "created": 1677652288, + "model": "my-custom-model", + "choices": [ + {"index": 0, "message": {"role": "assistant", "content": "Answer"}, "finish_reason": "stop"} + ], + "usage": {"prompt_tokens": 9, "completion_tokens": 1, "total_tokens": 10}, + }, + ) + ) + + response: Final = litellm.responses( + model="databricks/my-custom-model", + input="Hello", + client_metadata={ + "turn_id": "turn-1", + "thread_id": "thread-1", + "session_id": "session-1", + "root_turn_id": "turn-1", + "x-codex-installation-id": "install-1", + "x-codex-turn-metadata": '{"turn_id":"turn-1"}', + }, + chat_template_kwargs={"thinking": True}, + api_base="https://example.databricks.test/serving-endpoints", + api_key="fake-databricks-api-key", + num_retries=0, + ) + + assert upstream.call_count == 1 + request_body: Final = json.loads(upstream.calls[0].request.read()) + assert "client_metadata" not in request_body + assert request_body["chat_template_kwargs"] == {"thinking": True} + assert request_body["messages"] == [{"role": "user", "content": "Hello"}] + assert response.output[0].content[0].text == "Answer" + def test_bridge_keeps_deployment_credentials_while_dropping_unknown_params(self, respx_mock: respx.MockRouter): upstream: Final = respx_mock.post( "https://example-resource.openai.azure.com/openai/deployments/my-deployment/chat/completions", diff --git a/tests/test_litellm/router_strategy/test_complexity_router.py b/tests/test_litellm/router_strategy/test_complexity_router.py index 83f30dc52a4..baa15ac1568 100644 --- a/tests/test_litellm/router_strategy/test_complexity_router.py +++ b/tests/test_litellm/router_strategy/test_complexity_router.py @@ -13,6 +13,7 @@ import time from collections.abc import AsyncIterator, Mapping, Sequence from copy import deepcopy from functools import partial +from types import MappingProxyType from typing import Dict, Final, List, Literal from unittest.mock import AsyncMock, MagicMock, patch @@ -78,6 +79,7 @@ from litellm.router_strategy.complexity_router.jev_classifier import ( JevSystemOneResponse, JevUsage, ) +from litellm.router_strategy.complexity_router.llm_v2 import LLM_V2_PROMPT_VERSION from litellm.router_strategy.complexity_router.tier_predictor import ( TierGlobalStatistic, TrainedTierArtifact, @@ -90,6 +92,7 @@ from litellm.types.router import ( TaggedPreRoutingStrategy, ) from litellm.types.llms.openai import ResponsesAPIResponse +from litellm.types.management_endpoints.auto_router_endpoints import RequestComplexityRouterConfig from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler @@ -3238,6 +3241,41 @@ class TestCapabilityClassifier: ) assert response.model == "capable-model" assert response.routing_decision["cause"] == "capability_classifier_fallback" + assert "classifier_p_solve" not in response.routing_decision + assert "classifier_threshold" not in response.routing_decision + + @pytest.mark.asyncio + @pytest.mark.parametrize("bypass", ("literal_keyword_match", "session_affinity_pin", "housekeeping")) + async def test_bypasses_do_not_reuse_the_previous_capability_forecast( + self, + mock_router_instance: MagicMock, + bypass: Literal["literal_keyword_match", "session_affinity_pin", "housekeeping"], + ) -> None: + mock_router_instance.acompletion = AsyncMock(return_value=_llm_response(_capability_reply(p_solve=0.8))) + mock_router_instance.cache = DualCache() + router: Final = self._router( + mock_router_instance, + session_affinity=bypass == "session_affinity_pin", + keyword_tier_rules=[{"keywords": ["quick lookup"], "tier": "SIMPLE"}], + ) + original: Final = await router.async_pre_routing_hook( + model="capability-router", + request_kwargs={"metadata": {"session_id": "forecast-bypass"}}, + messages=[{"role": "user", "content": "Hello!"}], + ) + result: Final = await router.async_pre_routing_hook( + model="capability-router", + request_kwargs={"metadata": {"session_id": "forecast-bypass"}}, + messages=[{"role": "user", "content": TITLE_ASK if bypass == "housekeeping" else "quick lookup"}], + ) + + assert original is not None and original.routing_decision is not None + assert original.routing_decision["classifier_p_solve"] == 0.8 + assert result is not None and result.routing_decision is not None + assert result.routing_decision["cause"] == bypass + assert "classifier_p_solve" not in result.routing_decision + assert "classifier_threshold" not in result.routing_decision + mock_router_instance.acompletion.assert_awaited_once() CUSTOM_TIER_LABELS: Dict[str, str] = { @@ -6681,6 +6719,7 @@ class TestTierModelAffinity: litellm_router_instance=_windowed_router(_SMALL, _BIG), complexity_router_config={ "tiers": {"SIMPLE": ["small-model", "big-model"]}, + "enable_context_window_escalation": True, "adaptive": adaptive, "deployment_affinity": True, "session_affinity": False, @@ -13878,8 +13917,12 @@ _CJK_TURNS = [ ] -def _tier_config(**overrides) -> Dict: - return {"tiers": {"SIMPLE": "small-model", "COMPLEX": "big-model"}, **overrides} +def _tier_config(**overrides: object) -> dict[str, object]: + return { + "tiers": {"SIMPLE": "small-model", "COMPLEX": "big-model"}, + "enable_context_window_escalation": True, + **overrides, + } class TestContextWindowEscalation: @@ -13938,7 +13981,7 @@ class TestContextWindowEscalation: router = ComplexityRouter( model_name="test-router", litellm_router_instance=_windowed_router(_SMALL, ("mid-model", "openai/gpt-4o-mini", 200000), _BIG), - complexity_router_config={"tiers": {"SIMPLE": ["small-model", "mid-model"], "COMPLEX": "big-model"}}, + complexity_router_config=_tier_config(tiers={"SIMPLE": ["small-model", "mid-model"], "COMPLEX": "big-model"}), ) result = await router.async_pre_routing_hook(model="test-router", request_kwargs={}, messages=_OVERSIZED_TURNS) @@ -13975,7 +14018,7 @@ class TestContextWindowEscalation: }, ] ), - complexity_router_config={"tiers": {"SIMPLE": "mixed-pool", "COMPLEX": "big-model"}}, + complexity_router_config=_tier_config(tiers={"SIMPLE": "mixed-pool", "COMPLEX": "big-model"}), ) result = await router.async_pre_routing_hook(model="test-router", request_kwargs={}, messages=_OVERSIZED_TURNS) @@ -14026,7 +14069,7 @@ class TestContextWindowEscalation: router = ComplexityRouter( model_name="test-router", litellm_router_instance=_windowed_router(*deployments), - complexity_router_config={"tiers": tiers}, + complexity_router_config=_tier_config(tiers=tiers), ) result = await router.async_pre_routing_hook(model="test-router", request_kwargs={}, messages=_OVERSIZED_TURNS) @@ -14035,19 +14078,37 @@ class TestContextWindowEscalation: assert result.model == expected_model @pytest.mark.asyncio - async def test_the_disabled_gate_dispatches_on_complexity_alone(self): - """The escape hatch: enable_context_window_escalation false restores today's behavior.""" - router = ComplexityRouter( + @pytest.mark.parametrize("enabled", (None, False, True), ids=("omitted", "disabled", "enabled")) + @pytest.mark.parametrize("serialized", (False, True), ids=("config", "http-json")) + async def test_context_window_escalation_requires_opt_in(self, enabled: bool | None, serialized: bool) -> None: + setting: Final = ( + MappingProxyType({"enable_context_window_escalation": enabled}) + if enabled is not None + else MappingProxyType({}) + ) + raw_config: Final = RequestComplexityRouterConfig.model_validate( + MappingProxyType( + {"tiers": MappingProxyType({"SIMPLE": "small-model", "COMPLEX": "big-model"}), **setting} + ) + ) + config: Final = ( + RequestComplexityRouterConfig.model_validate_json(raw_config.model_dump_json()) + if serialized + else raw_config + ) + router: Final = ComplexityRouter( model_name="test-router", litellm_router_instance=_windowed_router(_SMALL, _BIG), - complexity_router_config=_tier_config(enable_context_window_escalation=False), + complexity_router_config=config.model_dump(exclude_unset=not serialized, exclude_none=True), ) - result = await router.async_pre_routing_hook(model="test-router", request_kwargs={}, messages=_OVERSIZED_TURNS) + result: Final = await router.async_pre_routing_hook( + model="test-router", request_kwargs={}, messages=_OVERSIZED_TURNS + ) assert result is not None - assert result.model == "small-model" - assert "context_escalated" not in result.routing_decision + assert result.model == ("big-model" if enabled else "small-model") + assert result.routing_decision.get("context_escalated", False) is (enabled is True) @pytest.mark.asyncio async def test_out_of_band_system_and_tools_count_against_the_window(self): @@ -14156,7 +14217,7 @@ class TestContextWindowEscalation: }, ] ), - complexity_router_config={"adaptive": True, "tiers": {"SIMPLE": ["small-model", "mid-model"]}}, + complexity_router_config=_tier_config(adaptive=True, tiers={"SIMPLE": ["small-model", "mid-model"]}), ) result = await router.async_pre_routing_hook(model="test-router", request_kwargs={}, messages=_OVERSIZED_TURNS) @@ -14186,7 +14247,7 @@ class TestContextWindowEscalation: }, ] ), - complexity_router_config={"tiers": {"SIMPLE": "cop-pool", "COMPLEX": "big-model"}}, + complexity_router_config=_tier_config(tiers={"SIMPLE": "cop-pool", "COMPLEX": "big-model"}), ) real_get_llm_provider = litellm.get_llm_provider copilot_resolutions: List = [] @@ -14219,7 +14280,7 @@ class TestContextWindowEscalation: "model_name": "smart-router", "litellm_params": { "model": "auto_router/complexity_router", - "complexity_router_config": {"tiers": {"SIMPLE": "small-model", "COMPLEX": "big-model"}}, + "complexity_router_config": _tier_config(), }, }, { @@ -14661,6 +14722,121 @@ class TestModalityRouting: @pytest.mark.usefixtures("local_model_cost_map") class TestHealthFallbackDispatch: + @pytest.mark.asyncio + @pytest.mark.parametrize("classifier", ("capability", "llm_v2")) + @pytest.mark.parametrize("calibrated", (False, True), ids=("raw", "calibrated")) + @pytest.mark.parametrize("rewrite", ("modality_escalation", "health_failover", "health_default_fallback")) + async def test_classifier_forecasts_survive_placement_rewrites( + self, + classifier: Literal["capability", "llm_v2"], + calibrated: bool, + rewrite: Literal["modality_escalation", "health_failover", "health_default_fallback"], + ) -> None: + calibration: Final = {"slope": 0.8, "intercept": 0.1} + classifier_config: Final = ( + { + "capability_classifier_config": { + "efficient_tier": "SIMPLE", + "capable_tier": "REASONING", + "base_threshold": 0.0, + "threshold_step": 0.1, + **({"calibration": {"version": "test-v1", **calibration}} if calibrated else {}), + } + } + if classifier == "capability" + else { + "llm_v2_config": { + "efficient_profile": "Small coding solver", + "capable_profile": "Large coding solver", + "harness": "Repository tools", + "max_quality_gap": 0.0, + **( + { + "calibration": { + "version": "test-v1", + "prompt_version": LLM_V2_PROMPT_VERSION, + "efficient": calibration, + "capable": calibration, + } + } + if calibrated + else {} + ), + } + } + ) + router: Final = self._router( + config={ + "classifier_type": classifier, + "classifier_llm_config": {"model": "fallback", "timeout_ms": 10000}, + "tiers": {"SIMPLE": "primary", "REASONING": "peer"}, + "tier_labels": {"SIMPLE": "Entry", "REASONING": "Advanced"}, + "modality_routing": True, + **classifier_config, + } + ) + verdict: Final = ( + _capability_reply(p_solve=0.0) + if classifier == "capability" + else json.dumps( + { + "crux": "Preserve existing behavior", + "demands": {"reasoning": "routine", "scope": "localized", "specification": "clear"}, + "verification": "relevant", + "forecasts": { + "efficient": {"likely_failure": "Miss an edge case", "p_solve": 0.0}, + "capable": {"likely_failure": "Miss an edge case", "p_solve": 0.0}, + }, + } + ) + ) + judge_response: Final = litellm.ModelResponse( + choices=[{"message": {"role": "assistant", "content": verdict}}], + usage={"prompt_tokens": 10, "completion_tokens": 1, "total_tokens": 11}, + ) + with respx.mock(assert_all_mocked=True) as upstream: + upstream.post(host="fallback.test").respond(json=judge_response.model_dump()) + original: Final = await router.async_pre_routing_hook( + model="health-router", request_kwargs={}, messages=[{"role": "user", "content": "Hello!"}] + ) + for deployment in router.model_list: + deployment["model_info"]["supports_vision"] = ( + rewrite != "modality_escalation" or deployment["model_name"] != "primary" + ) + if rewrite != "modality_escalation": + self._unavailable(router, "primary-id", "cooldown") + if rewrite == "health_default_fallback": + self._unavailable(router, "peer-id", "cooldown") + result: Final = await router.async_pre_routing_hook( + model="health-router", request_kwargs={}, messages=TestModalityRouting.IMAGE_MESSAGE + ) + + assert original is not None and original.routing_decision is not None + assert original.model == "primary" + assert result is not None and result.routing_decision is not None + decision: Final = result.routing_decision + assert decision["cause"] == rewrite + assert result.model == ("fallback" if rewrite == "health_default_fallback" else "peer") + expected: Final = { + field: value for field, value in original.routing_decision.items() if field.startswith("classifier_") + } + assert expected["classifier_p_solve" if classifier == "capability" else "classifier_efficient_p_solve"] == 0.0 + assert ("classifier_calibration_version" in expected) is calibrated + assert {field: value for field, value in decision.items() if field.startswith("classifier_")} == expected + if rewrite == "health_default_fallback": + assert "tier" not in decision and "tier_label" not in decision + else: + assert decision["tier"] == "REASONING" + assert decision["tier_label"] == "Advanced" + redacted: Final = Router._redact_prompt_text_if_needed( + request_kwargs={"metadata": {"headers": {"x-litellm-enable-message-redaction": True}}}, + routing_decision=decision, + ) + assert "classifier_crux" not in redacted and "signals" not in redacted + assert {field: value for field, value in redacted.items() if field.startswith("classifier_")} == { + field: value for field, value in expected.items() if field != "classifier_crux" + } + @pytest.mark.asyncio @pytest.mark.parametrize("peer", (True, False), ids=("peer_failover", "default_fallback")) async def test_health_rewrites_preserve_the_original_heuristic_v2_forecast(self, peer: bool) -> None: @@ -15139,7 +15315,12 @@ class TestHealthFallbackDispatch: ) -> None: from litellm.types.router import RouterRateLimitError - router: Final = self._router(config={"tiers": {"SIMPLE": "primary", "MEDIUM": "peer", "COMPLEX": "large"}}) + router: Final = self._router( + config={ + "tiers": {"SIMPLE": "primary", "MEDIUM": "peer", "COMPLEX": "large"}, + "enable_context_window_escalation": True, + } + ) router.add_deployment( Deployment( model_name="large", @@ -15216,7 +15397,13 @@ class TestHealthFallbackDispatch: @pytest.mark.asyncio @pytest.mark.parametrize("default_fits", [True, False]) async def test_modality_default_must_also_fit_context(self, default_fits: bool) -> None: - router: Final = self._router(config={"modality_routing": True, "tiers": {"SIMPLE": "primary"}}) + router: Final = self._router( + config={ + "modality_routing": True, + "tiers": {"SIMPLE": "primary"}, + "enable_context_window_escalation": True, + } + ) for deployment in router.model_list: deployment["model_info"]["supports_vision"] = deployment["model_name"] == "fallback" deployment["model_info"]["max_input_tokens"] = 10000 if default_fits else 10 diff --git a/tests/test_litellm/router_strategy/test_llm_v2.py b/tests/test_litellm/router_strategy/test_llm_v2.py index 27d31cbe640..6fb6df3265d 100644 --- a/tests/test_litellm/router_strategy/test_llm_v2.py +++ b/tests/test_litellm/router_strategy/test_llm_v2.py @@ -477,6 +477,10 @@ async def test_user_turn_mode_reuses_forecast_until_a_new_user_requirement() -> assert first.model == second.model == "efficient" assert first.routing_decision["cause"] == "llm_v2_classifier" assert first.routing_decision["classifier_cost"] == 0.001 + assert second is not None and second.routing_decision is not None + assert second.routing_decision["cause"] == "user_turn_continuation" + assert "classifier_efficient_p_solve" not in second.routing_decision + assert "classifier_capable_p_solve" not in second.routing_decision client.acompletion.assert_awaited_once() client.acompletion.return_value = _response(_verdict(0.3, 0.9).model_dump_json()) updated: Final = await router.async_pre_routing_hook( diff --git a/tests/test_litellm/router_strategy/test_lowest_tpm_rpm.py b/tests/test_litellm/router_strategy/test_lowest_tpm_rpm.py new file mode 100644 index 00000000000..7b13b196d5b --- /dev/null +++ b/tests/test_litellm/router_strategy/test_lowest_tpm_rpm.py @@ -0,0 +1,54 @@ +from datetime import datetime, timedelta +from typing import Final + +from litellm import Router +from litellm.types.router import DeploymentTypedDict, LiteLLMParamsTypedDict + +MODEL_GROUP: Final = "lowest-tpm-router" +HIGH_USAGE_DEPLOYMENT_ID: Final = "highest-usage" +LOW_USAGE_DEPLOYMENT_ID: Final = "lowest-usage" + + +def _deployment(deployment_id: str) -> DeploymentTypedDict: + params: LiteLLMParamsTypedDict = { + "model": "gpt-4o", + "api_key": "key", + "mock_response": f"from {deployment_id}", + } + return { + "model_name": MODEL_GROUP, + "litellm_params": params, + "model_info": {"id": deployment_id}, + } + + +def test_usage_based_routing_v1_selects_the_lowest_recorded_tpm() -> None: + router: Final = Router( + model_list=[ + _deployment(HIGH_USAGE_DEPLOYMENT_ID), + _deployment(LOW_USAGE_DEPLOYMENT_ID), + ], + routing_strategy="usage-based-routing", + num_retries=0, + ) + usage_by_deployment: Final = { + HIGH_USAGE_DEPLOYMENT_ID: 100, + LOW_USAGE_DEPLOYMENT_ID: 1, + } + now: Final = datetime.now() + cache_keys: Final = tuple( + f"{MODEL_GROUP}:tpm:{(now + timedelta(minutes=offset)).strftime('%H-%M')}" + for offset in range(60) + ) + + for cache_key in cache_keys: + router.cache.set_cache( + key=cache_key, value=usage_by_deployment, ttl=float("inf") + ) + + deployment: Final = router.get_available_deployment( + model=MODEL_GROUP, + messages=[{"role": "user", "content": "test"}], + ) + + assert deployment["model_info"]["id"] == LOW_USAGE_DEPLOYMENT_ID diff --git a/tests/test_litellm/rust_bridge/test_callbacks_legacy_python.py b/tests/test_litellm/rust_bridge/test_callbacks_legacy_python.py index 1f6a214398a..05f2d13a079 100644 --- a/tests/test_litellm/rust_bridge/test_callbacks_legacy_python.py +++ b/tests/test_litellm/rust_bridge/test_callbacks_legacy_python.py @@ -9,9 +9,10 @@ import pytest from pydantic import TypeAdapter import litellm +from litellm._internal_context import is_internal_call from litellm.litellm_core_utils.litellm_logging import Logging from litellm.rust_bridge import callbacks_legacy_python as legacy -from litellm.rust_bridge.callbacks_legacy_python import check_limits, setup +from litellm.rust_bridge.callbacks_legacy_python import check_limits, failure_handler, setup _OCR_KWARGS: Final = MappingProxyType( { @@ -81,6 +82,82 @@ def test_setup_builds_a_logger_when_none_is_supplied(call_type: str, kwargs: Map assert result.logger.litellm_call_id == result.kwargs["litellm_call_id"] +def _budget_reservation() -> dict: + return {"reserved_cost": 0.5, "entries": [], "finalized": False, "callback_bound": False} + + +def _kwargs_with_a_budget_reservation(reservation: dict) -> dict[str, object]: + return {**_OCR_KWARGS, "metadata": {"user_api_key_budget_reservation": reservation}} + + +def test_setup_claims_the_budget_reservation_for_an_async_call() -> None: + reservation: Final = _budget_reservation() + + setup("aocr", (), _kwargs_with_a_budget_reservation(reservation), datetime.datetime.now(), asynchronous=True) + + assert reservation["callback_bound"] is True + + +def test_setup_claims_the_budget_reservation_a_supplied_logger_already_saw() -> None: + reservation: Final = _budget_reservation() + supplied: Final = _supplied_logger() + supplied.update_environment_variables( + litellm_params={"metadata": {"user_api_key_budget_reservation": reservation}}, optional_params={} + ) + assert reservation["callback_bound"] is False + + setup("aocr", (), {**_OCR_KWARGS, "litellm_logging_obj": supplied}, datetime.datetime.now(), asynchronous=True) + + assert reservation["callback_bound"] is True + + +def test_setup_leaves_the_budget_reservation_alone_for_a_sync_call() -> None: + reservation: Final = _budget_reservation() + + setup("aocr", (), _kwargs_with_a_budget_reservation(reservation), datetime.datetime.now(), asynchronous=False) + + assert reservation["callback_bound"] is False + + +def test_setup_leaves_the_budget_reservation_alone_for_an_internal_call() -> None: + reservation: Final = _budget_reservation() + token: Final = is_internal_call.set(True) + try: + setup("aocr", (), _kwargs_with_a_budget_reservation(reservation), datetime.datetime.now(), asynchronous=True) + finally: + is_internal_call.reset(token) + + assert reservation["callback_bound"] is False + + +def test_failure_handler_hands_the_budget_reservation_back_for_an_async_call() -> None: + reservation: Final = _budget_reservation() + now: Final = datetime.datetime.now() + result: Final = setup("aocr", (), _kwargs_with_a_budget_reservation(reservation), now, asynchronous=True) + assert reservation["callback_bound"] is True + + pending: Final = failure_handler(result.logger, RuntimeError("upstream refused"), now, now, asynchronous=True) + + assert reservation["callback_bound"] is False + assert pending is not None + pending.close() + + +def test_failure_handler_of_an_internal_call_leaves_the_outer_budget_reservation_claim_in_place() -> None: + reservation: Final = _budget_reservation() + now: Final = datetime.datetime.now() + result: Final = setup("aocr", (), _kwargs_with_a_budget_reservation(reservation), now, asynchronous=True) + token: Final = is_internal_call.set(True) + try: + pending: Final = failure_handler(result.logger, RuntimeError("inner step failed"), now, now, asynchronous=True) + finally: + is_internal_call.reset(token) + + assert reservation["callback_bound"] is True + assert pending is not None + pending.close() + + CONTRACT_PATH: Final = ( Path(__file__).parents[3] / "litellm-rust/crates/callbacks-legacy-python/python_contract.json" ) diff --git a/tests/test_litellm/secret_managers/hashicorp_vault_parity.json b/tests/test_litellm/secret_managers/hashicorp_vault_parity.json new file mode 100644 index 00000000000..f1faefd48e8 --- /dev/null +++ b/tests/test_litellm/secret_managers/hashicorp_vault_parity.json @@ -0,0 +1,85 @@ +[ + { + "name": "defaults", + "env": { + "HCP_VAULT_TOKEN": "token" + }, + "secret_name": "OPENAI_API_KEY", + "expected_secret_url": "http://127.0.0.1:8200/v1/secret/data/OPENAI_API_KEY", + "expected_login_url": null, + "expected_login_namespace": null, + "expected_secret_namespace": null + }, + { + "name": "global_namespace", + "env": { + "HCP_VAULT_ADDR": "http://vault.test:8200", + "HCP_VAULT_TOKEN": "token", + "HCP_VAULT_NAMESPACE": "admin" + }, + "secret_name": "OPENAI_API_KEY", + "expected_secret_url": "http://vault.test:8200/v1/admin/secret/data/OPENAI_API_KEY", + "expected_login_url": null, + "expected_login_namespace": "admin", + "expected_secret_namespace": "admin" + }, + { + "name": "namespace_overrides", + "env": { + "HCP_VAULT_ADDR": "http://vault.test:8200", + "HCP_VAULT_TOKEN": "token", + "HCP_VAULT_NAMESPACE": "admin", + "HCP_VAULT_LOGIN_NAMESPACE": "root", + "HCP_VAULT_SECRET_NAMESPACE": "teams/team-a" + }, + "secret_name": "OPENAI_API_KEY", + "expected_secret_url": "http://vault.test:8200/v1/teams/team-a/secret/data/OPENAI_API_KEY", + "expected_login_url": null, + "expected_login_namespace": "root", + "expected_secret_namespace": "teams/team-a" + }, + { + "name": "custom_mount_and_prefix", + "env": { + "HCP_VAULT_ADDR": "http://vault.test:8200", + "HCP_VAULT_TOKEN": "token", + "HCP_VAULT_MOUNT_NAME": " /kv-prod/ ", + "HCP_VAULT_PATH_PREFIX": " /virtual-keys/ " + }, + "secret_name": "DB_PASSWORD", + "expected_secret_url": "http://vault.test:8200/v1/kv-prod/data/virtual-keys/DB_PASSWORD", + "expected_login_url": null, + "expected_login_namespace": null, + "expected_secret_namespace": null + }, + { + "name": "approle_custom_mount", + "env": { + "HCP_VAULT_ADDR": "http://vault.test:8200", + "HCP_VAULT_APPROLE_ROLE_ID": "role-id", + "HCP_VAULT_APPROLE_SECRET_ID": "secret-id", + "HCP_VAULT_APPROLE_MOUNT_PATH": "custom-approle", + "HCP_VAULT_NAMESPACE": "admin" + }, + "secret_name": "OPENAI_API_KEY", + "expected_secret_url": "http://vault.test:8200/v1/admin/secret/data/OPENAI_API_KEY", + "expected_login_url": "http://vault.test:8200/v1/auth/custom-approle/login", + "expected_login_namespace": "admin", + "expected_secret_namespace": "admin" + }, + { + "name": "tls_cert", + "env": { + "HCP_VAULT_ADDR": "http://vault.test:8200", + "HCP_VAULT_CLIENT_CERT": "/tmp/client.crt", + "HCP_VAULT_CLIENT_KEY": "/tmp/client.key", + "HCP_VAULT_CERT_ROLE": "vault-role", + "HCP_VAULT_NAMESPACE": "admin" + }, + "secret_name": "OPENAI_API_KEY", + "expected_secret_url": "http://vault.test:8200/v1/admin/secret/data/OPENAI_API_KEY", + "expected_login_url": "http://vault.test:8200/v1/auth/cert/login", + "expected_login_namespace": "admin", + "expected_secret_namespace": "admin" + } +] diff --git a/tests/test_litellm/secret_managers/test_hashicorp_secret_manager.py b/tests/test_litellm/secret_managers/test_hashicorp_secret_manager.py index 1676540e4ec..fc18cb8b8f7 100644 --- a/tests/test_litellm/secret_managers/test_hashicorp_secret_manager.py +++ b/tests/test_litellm/secret_managers/test_hashicorp_secret_manager.py @@ -1,4 +1,5 @@ import datetime +import json from collections.abc import Mapping from pathlib import Path from typing import Final @@ -18,6 +19,23 @@ LOGIN_RESPONSE: Final = {"auth": {"client_token": "hvs.login-token", "lease_dura SECRET_RESPONSE: Final = {"data": {"data": {"key": "sk-from-vault", "password": "pw-from-vault"}}} NAMESPACE_ENV_VARS: Final = ("HCP_VAULT_NAMESPACE", "HCP_VAULT_LOGIN_NAMESPACE", "HCP_VAULT_SECRET_NAMESPACE") +PARITY_ENV_VARS: Final = ( + "HCP_VAULT_ADDR", + "HCP_VAULT_TOKEN", + "HCP_VAULT_NAMESPACE", + "HCP_VAULT_LOGIN_NAMESPACE", + "HCP_VAULT_SECRET_NAMESPACE", + "HCP_VAULT_MOUNT_NAME", + "HCP_VAULT_PATH_PREFIX", + "HCP_VAULT_APPROLE_ROLE_ID", + "HCP_VAULT_APPROLE_SECRET_ID", + "HCP_VAULT_APPROLE_MOUNT_PATH", + "HCP_VAULT_CLIENT_CERT", + "HCP_VAULT_CLIENT_KEY", + "HCP_VAULT_CERT_ROLE", + "HCP_VAULT_REFRESH_INTERVAL", + "SECRET_MANAGER_REFRESH_INTERVAL", +) def _build_manager(monkeypatch: pytest.MonkeyPatch, env: Mapping[str, str]) -> HashicorpSecretManager: @@ -236,3 +254,35 @@ def test_tls_login_uses_login_namespace(monkeypatch: pytest.MonkeyPatch, tmp_pat assert manager._auth_via_tls_cert() == "hvs.login-token" assert login_route.calls.last.request.headers["X-Vault-Namespace"] == "root" + + +with Path(__file__).with_name("hashicorp_vault_parity.json").open() as parity_file: + PARITY_CASES: Final = json.load(parity_file) + + +@pytest.mark.parametrize("case", PARITY_CASES, ids=lambda case: case["name"]) +def test_configuration_matches_native_parity_fixture( + monkeypatch: pytest.MonkeyPatch, case: Mapping[str, object] +) -> None: + monkeypatch.setattr(litellm.proxy.proxy_server, "premium_user", True) + for name in PARITY_ENV_VARS: + monkeypatch.delenv(name, raising=False) + for name, value in case["env"].items(): + monkeypatch.setenv(name, value) + + manager: Final = HashicorpSecretManager() + env: Final = case["env"] + expected_login_url: Final = case["expected_login_url"] + if env.get("HCP_VAULT_APPROLE_ROLE_ID") and env.get("HCP_VAULT_APPROLE_SECRET_ID"): + login_url: str | None = ( + f"{manager.vault_addr}/v1/auth/{manager.approle_mount_path}/login" + ) + elif env.get("HCP_VAULT_CLIENT_CERT") and env.get("HCP_VAULT_CLIENT_KEY"): + login_url = f"{manager.vault_addr}/v1/auth/cert/login" + else: + login_url = None + + assert manager.get_url(case["secret_name"]) == case["expected_secret_url"] + assert manager.vault_login_namespace == case["expected_login_namespace"] + assert manager.vault_secret_namespace == case["expected_secret_namespace"] + assert login_url == expected_login_url diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py index 1e59f4d878e..cf61a6d9f65 100644 --- a/tests/test_litellm/test_utils.py +++ b/tests/test_litellm/test_utils.py @@ -4984,6 +4984,100 @@ async def test_wrapper_async_fires_post_call_failure_deployment_hook_on_internal assert isinstance(recorder.calls[0][1], litellm.AuthenticationError) +def _budget_reservation(callback_bound: bool = False) -> dict: + return {"reserved_cost": 0.5, "entries": [], "finalized": False, "callback_bound": callback_bound} + + +_BUDGET_RESERVATION_CALL_KWARGS: Final = {"model": "gpt-4o", "messages": [{"role": "user", "content": "hi"}]} +_BUDGET_RESERVATION_REFUSAL: Final = litellm.AuthenticationError(message="bad key", llm_provider="openai", model="gpt-4o") + + +@pytest.mark.asyncio +async def test_wrapper_async_claims_the_budget_reservation_for_the_cost_callback() -> None: + reservation = _budget_reservation() + + await litellm.acompletion( + **_BUDGET_RESERVATION_CALL_KWARGS, + mock_response="ok", + metadata={"user_api_key_budget_reservation": reservation}, + ) + + assert reservation["callback_bound"] is True + + +@pytest.mark.asyncio +async def test_wrapper_async_claims_the_budget_reservation_before_the_stream_is_consumed() -> None: + reservation = _budget_reservation() + + stream = await litellm.acompletion( + **_BUDGET_RESERVATION_CALL_KWARGS, + mock_response="ok", + stream=True, + metadata={"user_api_key_budget_reservation": reservation}, + ) + + assert reservation["callback_bound"] is True + async for _ in stream: + pass + + +@pytest.mark.asyncio +async def test_wrapper_async_claims_the_budget_reservation_a_supplied_logging_object_already_saw() -> None: + reservation = _budget_reservation() + logging_obj, kwargs = litellm.utils.function_setup( + original_function="acompletion", + rules_obj=litellm.utils.Rules(), + start_time=datetime.now(), + **_BUDGET_RESERVATION_CALL_KWARGS, + litellm_call_id="proxy-pre-call-setup", + metadata={"user_api_key_budget_reservation": reservation}, + ) + assert reservation["callback_bound"] is False + + await litellm.acompletion(**kwargs, litellm_logging_obj=logging_obj, mock_response="ok") + + assert reservation["callback_bound"] is True + + +@pytest.mark.asyncio +async def test_wrapper_async_hands_the_budget_reservation_back_when_the_call_fails() -> None: + reservation = _budget_reservation() + + with pytest.raises(litellm.AuthenticationError): + await litellm.acompletion( + **_BUDGET_RESERVATION_CALL_KWARGS, + mock_response=_BUDGET_RESERVATION_REFUSAL, + metadata={"user_api_key_budget_reservation": reservation}, + ) + + assert reservation["callback_bound"] is False + + +@pytest.mark.asyncio +async def test_wrapper_async_leaves_the_budget_reservation_alone_on_internal_calls() -> None: + claimed_by_the_outer_call = _budget_reservation(callback_bound=True) + never_claimed = _budget_reservation() + + token = is_internal_call.set(True) + try: + await litellm.acompletion( + **_BUDGET_RESERVATION_CALL_KWARGS, + mock_response="ok", + metadata={"user_api_key_budget_reservation": never_claimed}, + ) + with pytest.raises(litellm.AuthenticationError): + await litellm.acompletion( + **_BUDGET_RESERVATION_CALL_KWARGS, + mock_response=_BUDGET_RESERVATION_REFUSAL, + metadata={"user_api_key_budget_reservation": claimed_by_the_outer_call}, + ) + finally: + is_internal_call.reset(token) + + assert never_claimed["callback_bound"] is False + assert claimed_by_the_outer_call["callback_bound"] is True + + @pytest.mark.asyncio async def test_wrapper_async_does_not_fire_failure_hook_for_pre_call_budget_error( monkeypatch: pytest.MonkeyPatch, diff --git a/tests/test_litellm/types/test_types_utils.py b/tests/test_litellm/types/test_types_utils.py index 5f44ba1773e..0a8c9414a0d 100644 --- a/tests/test_litellm/types/test_types_utils.py +++ b/tests/test_litellm/types/test_types_utils.py @@ -1,9 +1,16 @@ +import json from typing import Final import pytest -from litellm.types.utils import HiddenParams, all_litellm_params, text_tokens_without_nested_reasoning +from litellm.types.utils import ( + HiddenParams, + ImageObject, + ImageResponse, + all_litellm_params, + text_tokens_without_nested_reasoning, +) def test_rust_is_a_known_litellm_param(): @@ -763,13 +770,70 @@ def test_delta_function_tool_call_unchanged_by_custom_support(): def test_image_response_keeps_background(): """https://github.com/BerriAI/litellm/issues/38649""" - from litellm.types.utils import ImageResponse - response = ImageResponse(created=1, data=[{"b64_json": "aGk="}], background="transparent", output_format="png") assert response.background == "transparent" assert response.model_dump()["background"] == "transparent" +def test_image_response_serialization_honors_dump_options(): + response: Final = ImageResponse( + data=[ + ImageObject( + url="https://example.com/image.png", + provider_specific_fields={"width": 1024, "height": 1536, "content_type": "image/png"}, + ) + ] + ) + expected: Final = [ + { + "url": "https://example.com/image.png", + "provider_specific_fields": {"width": 1024, "height": 1536, "content_type": "image/png"}, + } + ] + assert response.model_dump(exclude_none=True)["data"] == expected + assert json.loads(response.model_dump_json(exclude_none=True))["data"] == expected + assert response.model_dump()["data"][0]["provider_specific_fields"] == expected[0]["provider_specific_fields"] + assert "url" not in response.model_dump(exclude={"data": {0: {"url"}}})["data"][0] + assert response.model_dump(include={"data": {"__all__": {"url"}}})["data"] == [ + {"url": "https://example.com/image.png"} + ] + assert response.model_dump(include={"data": {0: True}})["data"] == [ + { + "b64_json": None, + "revised_prompt": None, + "url": "https://example.com/image.png", + "provider_specific_fields": {"width": 1024, "height": 1536, "content_type": "image/png"}, + } + ] + assert response.model_dump(exclude={"data": {0: True}})["data"] == [] + + two_image_response: Final = ImageResponse( + data=[ + ImageObject(url="https://example.com/image.png"), + ImageObject(url="https://example.com/second-image.png"), + ] + ) + assert two_image_response.model_dump(exclude={"data": {1}})["data"] == [ + { + "b64_json": None, + "revised_prompt": None, + "url": "https://example.com/image.png", + "provider_specific_fields": None, + } + ] + assert two_image_response.model_dump(exclude={"data": {-1}})["data"] == [ + { + "b64_json": None, + "revised_prompt": None, + "url": "https://example.com/image.png", + "provider_specific_fields": None, + } + ] + assert two_image_response.model_dump(include={"data": {-1: {"url"}}})["data"] == [ + {"url": "https://example.com/second-image.png"} + ] + + @pytest.mark.parametrize( ("completion_tokens", "text_tokens", "reasoning_tokens", "other_modality_tokens", "expected_text_tokens"), ( diff --git a/tests/test_litellm_rust/support/fake_gcs.py b/tests/test_litellm_rust/support/fake_gcs.py new file mode 100644 index 00000000000..67eb61798b9 --- /dev/null +++ b/tests/test_litellm_rust/support/fake_gcs.py @@ -0,0 +1,152 @@ +from __future__ import annotations + +import json +import threading +from collections.abc import Mapping +from dataclasses import dataclass +from functools import partial +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from socket import socket +from types import MappingProxyType +from typing import Final, cast +from urllib.parse import unquote, urlsplit + + +@dataclass(frozen=True, slots=True) +class RecordedRequest: + method: str + path: str + query: str + headers: Mapping[str, str] + body: bytes + + +class _FakeGcsHandler(BaseHTTPRequestHandler): + def __init__( + self, + request: socket | tuple[bytes, socket], + client_address: tuple[str, int], + server: ThreadingHTTPServer, + *, + fake: FakeGcs, + ) -> None: + self._fake: Final = fake + super().__init__(request, client_address, server) + + def _handle(self) -> None: + parsed: Final = urlsplit(self.path) + content_length: Final = int(self.headers.get("Content-Length", "0")) + body: Final = self.rfile.read(content_length) if content_length else b"" + headers: Final = MappingProxyType( + {name.title(): value for name, value in self.headers.items()} + ) + self._fake.record( + RecordedRequest( + method=self.command, + path=parsed.path, + query=parsed.query, + headers=headers, + body=body, + ) + ) + if self.headers.get("Authorization") != f"Bearer {self._fake.token}": + self._send_json(401, {"error": "unauthorized"}) + return + + upload_prefix: Final = "/upload/storage/v1/b/" + download_prefix: Final = "/storage/v1/b/" + if parsed.path.startswith(upload_prefix) and parsed.path.endswith("/o"): + self._upload(parsed.path[len(upload_prefix) : -2], parsed.query, body) + return + if parsed.path.startswith(download_prefix): + self._download(parsed.path[len(download_prefix) :], parsed.query) + return + self._send_json(404, {"error": "not found"}) + + def _upload(self, path: str, query: str, body: bytes) -> None: + values: Final = { + unquote(pair.partition("=")[0]): unquote(pair.partition("=")[2]) + for pair in query.split("&") + if pair + } + if not path or values.get("uploadType") != "media" or "name" not in values: + self._send_json(404, {"error": "not found"}) + return + self._fake.put_object(path, values["name"], body) + self._send_json(200, {"name": values["name"], "bucket": path}) + + def _download(self, path: str, query: str) -> None: + bucket, separator, encoded_name = path.partition("/o/") + if not separator or query != "alt=media": + self._send_json(404, {"error": "not found"}) + return + name: Final = unquote(encoded_name) + if name.endswith("/server-error") or name == "server-error": + self._send_json(500, {"error": "server error"}) + return + body: Final = self._fake.get_object(bucket, name) + if body is None: + self._send_json(404, {"error": "not found"}) + return + self._send(200, body, "application/octet-stream") + + def _send_json(self, status: int, value: object) -> None: + payload: Final = json.dumps(value).encode() + self._send(status, payload, "application/json") + + def _send(self, status: int, body: bytes, content_type: str) -> None: + self.send_response(status) + self.send_header("Content-Type", content_type) + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, format: str, *args: object) -> None: + pass + + do_GET = _handle + do_POST = _handle + + +class FakeGcs: + def __init__(self) -> None: + self._objects: dict[tuple[str, str], bytes] = {} # mutable-ok: fake object store + self._requests: list[RecordedRequest] = [] # mutable-ok: recorded request history + self._server = ThreadingHTTPServer( + ("127.0.0.1", 0), + partial(_FakeGcsHandler, fake=self), + ) + self._worker = threading.Thread(target=self._server.serve_forever, daemon=True) + self._worker.start() + self.token: Final = "test-token" + + @property + def url(self) -> str: + address: Final = cast(tuple[str, int], self._server.server_address) + host, port = address + return f"http://{host}:{port}" + + @property + def objects(self) -> Mapping[tuple[str, str], bytes]: + return MappingProxyType(self._objects) + + @property + def requests(self) -> tuple[RecordedRequest, ...]: + return tuple(self._requests) + + def put(self, bucket: str, name: str, body: bytes) -> None: + self.put_object(bucket, name, body) + + def close(self) -> None: + self._server.shutdown() + self._server.server_close() + self._worker.join(timeout=5) + + def record(self, request: RecordedRequest) -> None: + self._requests.append(request) + + def put_object(self, bucket: str, name: str, body: bytes) -> None: + self._objects[(bucket, name)] = body + + def get_object(self, bucket: str, name: str) -> bytes | None: + return self._objects.get((bucket, name)) diff --git a/tests/test_litellm_rust/support/s3_stub.py b/tests/test_litellm_rust/support/s3_stub.py new file mode 100644 index 00000000000..5a683fb78f3 --- /dev/null +++ b/tests/test_litellm_rust/support/s3_stub.py @@ -0,0 +1,112 @@ +"""In-process path-style S3 stub for native cache parity tests.""" + +import threading +from dataclasses import dataclass, field +from email.utils import parsedate_to_datetime +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from typing import Final +from urllib.parse import unquote, urlsplit + +_STORED_HEADERS: Final = ( + "cache-control", + "content-type", + "content-language", + "content-disposition", + "expires", +) + + +@dataclass +class S3Object: + body: bytes + headers: dict[str, str] = field(default_factory=dict) + + +class S3Stub: + """Minimal path-style S3 endpoint serving PUT and GET object operations.""" + + def __init__(self) -> None: + self._objects: dict[str, S3Object] = {} + stub: Final = self + + class Handler(BaseHTTPRequestHandler): + def _key(self) -> str: + parts: Final = urlsplit(self.path).path.lstrip("/").split("/", 1) + return unquote(parts[1]) if len(parts) == 2 else "" + + def _read_body(self) -> bytes: + transfer: Final = self.headers.get("transfer-encoding", "") + if "chunked" not in transfer: + return self.rfile.read(int(self.headers.get("content-length", 0))) + chunks: Final = bytearray() + while True: + size = int(self.rfile.readline().split(b";")[0].strip(), 16) + if size == 0: + while self.rfile.readline().strip(): + pass + return bytes(chunks) + chunks.extend(self.rfile.read(size)) + self.rfile.readline() + + def do_PUT(self) -> None: + body: Final = self._read_body() + headers: Final = {name: self.headers[name] for name in _STORED_HEADERS if name in self.headers} + stub._objects = {**stub._objects, self._key(): S3Object(body=body, headers=headers)} + self.send_response(200) + self.send_header("ETag", '"stub"') + self.send_header("Content-Length", "0") + self.end_headers() + + def do_HEAD(self) -> None: + self._object(send_body=False) + + def do_GET(self) -> None: + self._object(send_body=True) + + def _object(self, send_body: bool) -> None: + entry: Final = stub._objects.get(self._key()) + if entry is None: + self.send_response(404) + self.send_header("Content-Type", "application/xml") + body: Final = b'NoSuchKey' + self.send_header("Content-Length", str(len(body))) + self.end_headers() + if send_body: + self.wfile.write(body) + return + self.send_response(200) + for name, value in entry.headers.items(): + self.send_header(name, value) + self.send_header("ETag", '"stub"') + self.send_header("Content-Length", str(len(entry.body))) + self.end_headers() + if send_body: + self.wfile.write(entry.body) + + def log_message(self, format: str, *args: object) -> None: + pass + + self._server: Final = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + self._worker: Final = threading.Thread(target=self._server.serve_forever, daemon=True) + self._worker.start() + + @property + def url(self) -> str: + host, port = self._server.server_address[:2] + return f"http://{host}:{port}" + + @property + def objects(self) -> dict[str, S3Object]: + return self._objects + + def put_object(self, key: str, body: bytes, headers: dict[str, str] | None = None) -> None: + self._objects = {**self._objects, key: S3Object(body=body, headers=headers or {})} + + def expires(self, key: str) -> object: + header: Final = self._objects[key].headers.get("expires") + return parsedate_to_datetime(header) if header else None + + def close(self) -> None: + self._server.shutdown() + self._server.server_close() + self._worker.join(timeout=5) diff --git a/tests/test_litellm_rust/test_cache.py b/tests/test_litellm_rust/test_cache.py index a0f2396a25d..48ca6f5e165 100644 --- a/tests/test_litellm_rust/test_cache.py +++ b/tests/test_litellm_rust/test_cache.py @@ -1,17 +1,28 @@ import asyncio import contextvars import gc +import hashlib +import http.server import json +import math import os import threading import time import uuid import weakref -from collections.abc import Generator +from collections.abc import Callable, Generator +from contextlib import ExitStack +from datetime import datetime +from pathlib import Path from types import SimpleNamespace from typing import Final, Protocol, cast +from unittest.mock import Mock from urllib.parse import urlparse +from uuid import uuid4 +import boto3 +import botocore.config +import diskcache import fakeredis import pytest import redis @@ -20,23 +31,100 @@ from azure.storage.blob import ContainerClient import litellm from litellm.caching.azure_blob_cache import AzureBlobCache from litellm.caching.caching import Cache, disable_cache, enable_cache, update_cache +from litellm.caching.disk_cache import DiskCache +from litellm.caching.gcs_cache import GCSCache from litellm.caching.in_memory_cache import InMemoryCache from litellm.caching.redis_cluster_cache import RedisClusterCache +from litellm.caching.redis_semantic_cache import RedisSemanticCache +from litellm.caching.s3_cache import S3Cache from litellm.rust_bridge import _native from litellm.types.caching import LiteLLMCacheType +from litellm.types.llms.custom_llm import CustomLLMItem +from litellm.types.utils import EmbeddingResponse +from tests.test_litellm_rust.support.fake_gcs import FakeGcs from tests.test_litellm_rust.support.isolation import rebound +from tests.test_litellm_rust.support.s3_stub import S3Stub + +_CacheTestHandle: Final = _native._CacheTestHandle # pyright: ignore[reportPrivateUsage] # test-only handle has no public module name +_CacheTestResolver: Final = _native._CacheTestResolver # pyright: ignore[reportPrivateUsage] # test-only resolver has no public module name pytestmark: Final = pytest.mark.requires_rust_extension class CacheLookup(Protocol): def get_cache(self, **kwargs: object) -> object: ... + def flush_cache(self) -> object: ... def request(key: str = "key") -> dict[str, object]: return {"key": {"preset": key}} +def qdrant_request( + key: str, + messages: list[dict[str, object]], + **kwargs: object, +) -> dict[str, object]: + return {**request(key), "messages": messages, **kwargs} + + +def embedding_vector(text: str) -> list[float]: + raw: Final = hashlib.sha256(text.encode()).digest()[:8] + values: Final = [byte / 127.5 - 1 for byte in raw] + norm: Final = math.sqrt(sum(value * value for value in values)) + return [value / norm for value in values] + + +@pytest.fixture +def qdrant_url() -> str: + value: Final[str | None] = os.environ.get("QDRANT_URL") + if not value: + pytest.skip("QDRANT_URL is required for Qdrant semantic cache tests") + return value.rstrip("/") + + +@pytest.fixture +def fake_embedding_endpoint(monkeypatch: pytest.MonkeyPatch) -> Generator[str]: + class EmbeddingHandler(http.server.BaseHTTPRequestHandler): + def do_POST(self) -> None: + length: Final = int(self.headers["Content-Length"]) + body: Final = json.loads(self.rfile.read(length)) + text: Final = body["input"] + response: Final = { + "object": "list", + "data": [ + { + "object": "embedding", + "index": 0, + "embedding": embedding_vector(text), + } + ], + "model": body["model"], + "usage": {"prompt_tokens": 1, "total_tokens": 1}, + } + encoded: Final = json.dumps(response).encode() + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(encoded))) + self.end_headers() + self.wfile.write(encoded) + + def log_message(self, *_args: object) -> None: + return + + server: Final = http.server.ThreadingHTTPServer(("127.0.0.1", 0), EmbeddingHandler) + worker: Final = threading.Thread(target=server.serve_forever, daemon=True) + worker.start() + monkeypatch.setenv("OPENAI_API_BASE", f"http://127.0.0.1:{server.server_address[1]}") + monkeypatch.setenv("OPENAI_API_KEY", "sk-test") + try: + yield f"http://127.0.0.1:{server.server_address[1]}" + finally: + server.shutdown() + server.server_close() + worker.join(timeout=5) + + @pytest.fixture def redis_url() -> Generator[str]: server: Final = fakeredis.TcpFakeServer(("127.0.0.1", 0), server_type="redis") @@ -50,6 +138,15 @@ def redis_url() -> Generator[str]: worker.join(timeout=5) +@pytest.fixture +def fake_gcs() -> Generator[FakeGcs]: + server: Final = FakeGcs() + try: + yield server + finally: + server.close() + + @pytest.fixture def azure_blob_facade() -> Generator[Cache]: account_url: Final = os.environ.get("AZURE_BLOB_CACHE_ACCOUNT_URL") @@ -93,14 +190,14 @@ def test_existing_constructor_and_global_are_unchanged() -> None: assert type(facade.cache) is InMemoryCache assert "_native_cache_handle" not in vars(facade) with rebound(litellm, "cache", facade): - resolver: Final = _native._CacheTestResolver(litellm) + resolver: Final = _CacheTestResolver(litellm) assert resolver.resolve().kind == "python_callback" resolver.resolve().store(None, {"answer": 7}, callback_kwargs={"cache_key": "key"}) assert cast(CacheLookup, facade).get_cache(cache_key="key") == {"answer": 7} def test_existing_global_lifecycle_remains_the_resolver_source_of_truth() -> None: - resolver: Final = _native._CacheTestResolver(litellm) + resolver: Final = _CacheTestResolver(litellm) enable_cache(type=LiteLLMCacheType.LOCAL, ttl=30) enabled: Final = litellm.cache @@ -123,13 +220,13 @@ def test_existing_global_lifecycle_remains_the_resolver_source_of_truth() -> Non async def test_native_bindings_survive_replacement_and_capture_writes_before_dispatch() -> None: - namespace: Final = SimpleNamespace(cache=_native._CacheTestHandle.memory()) - resolver: Final = _native._CacheTestResolver(namespace) + namespace: Final = SimpleNamespace(cache=_CacheTestHandle.memory()) + resolver: Final = _CacheTestResolver(namespace) selected: Final = resolver.resolve() assert selected.kind == "native" selected.store(request(), {"answer": 1}) assert await selected.async_lookup(request()) == {"answer": 1} - with rebound(namespace, "cache", _native._CacheTestHandle.memory()): + with rebound(namespace, "cache", _CacheTestHandle.memory()): replacement: Final = resolver.resolve() await selected.async_store(request(), {"answer": 2}) assert replacement.lookup(request()) is None @@ -162,7 +259,7 @@ async def test_python_callback_preserves_identity_caller_task_context_and_errors raise failure namespace: Final = SimpleNamespace(cache=CustomCache()) - binding: Final = _native._CacheTestResolver(namespace).resolve() + binding: Final = _CacheTestResolver(namespace).resolve() assert binding.kind == "python_callback" assert await binding.async_lookup(None, callback_kwargs={"marker": sentinel}) is sentinel assert context.get() == "callback" @@ -183,7 +280,7 @@ async def test_callback_cancellation_stays_in_the_callers_task() -> None: finally: finished.set() - binding: Final = _native._CacheTestResolver(SimpleNamespace(cache=CustomCache())).resolve() + binding: Final = _CacheTestResolver(SimpleNamespace(cache=CustomCache())).resolve() async def lookup() -> object: return await binding.async_lookup(None, callback_kwargs={}) @@ -198,9 +295,9 @@ async def test_callback_cancellation_stays_in_the_callers_task() -> None: def test_registered_facade_uses_native_and_instance_overrides_fall_back() -> None: facade: Final = Cache(type=LiteLLMCacheType.LOCAL) - handle: Final = _native._CacheTestHandle.memory() + handle: Final = _CacheTestHandle.memory() handle._bind_facade(facade) - resolver: Final = _native._CacheTestResolver(SimpleNamespace(cache=facade)) + resolver: Final = _CacheTestResolver(SimpleNamespace(cache=facade)) native: Final = resolver.resolve() assert native.kind == "native" native.store(request(), {"source": "native"}) @@ -231,12 +328,12 @@ def test_facade_subclasses_backend_replacement_and_configuration_changes_are_not class CustomCache(Cache): pass - handle: Final = _native._CacheTestHandle.memory() + handle: Final = _CacheTestHandle.memory() with pytest.raises(TypeError): handle._bind_facade(CustomCache(type=LiteLLMCacheType.LOCAL)) facade: Final = Cache(type=LiteLLMCacheType.LOCAL) handle._bind_facade(facade) - resolver: Final = _native._CacheTestResolver(SimpleNamespace(cache=facade)) + resolver: Final = _CacheTestResolver(SimpleNamespace(cache=facade)) with rebound(facade, "cache", InMemoryCache()): assert resolver.resolve().kind == "python_callback" with rebound(facade, "ttl", 12): @@ -261,7 +358,7 @@ def test_resolver_and_callback_cycles_can_be_collected() -> None: def cyclic_reference() -> weakref.ReferenceType[CustomCache]: callback: Final = CustomCache() namespace: Final = SimpleNamespace(cache=callback) - binding: Final = _native._CacheTestResolver(namespace).resolve() + binding: Final = _CacheTestResolver(namespace).resolve() setattr(callback, "binding", binding) return weakref.ref(callback) @@ -272,8 +369,8 @@ def test_resolver_and_callback_cycles_can_be_collected() -> None: async def test_redis_reads_python_sync_and_async_entries_and_writes_without_hidden_prefix(redis_url: str) -> None: client: Final = redis.Redis.from_url(redis_url) - namespace: Final = SimpleNamespace(cache=_native._CacheTestHandle.redis(redis_url, namespace="team")) - binding: Final = _native._CacheTestResolver(namespace).resolve() + namespace: Final = SimpleNamespace(cache=_CacheTestHandle.redis(redis_url, namespace="team")) + binding: Final = _CacheTestResolver(namespace).resolve() response: Final = {"choices": [{"text": "cached"}], "usage": {"total_tokens": 3}, "flag": True, "empty": None} envelope: Final = {"timestamp": time.time(), "response": json.dumps(response)} client.set("team:sync", str(envelope)) @@ -295,33 +392,33 @@ async def test_redis_reads_python_sync_and_async_entries_and_writes_without_hidd def test_invalid_duration_and_request_shape_fail_before_storage() -> None: - binding: Final = _native._CacheTestResolver(SimpleNamespace(cache=_native._CacheTestHandle.memory())).resolve() + binding: Final = _CacheTestResolver(SimpleNamespace(cache=_CacheTestHandle.memory())).resolve() for seconds in (-1.0, float("nan"), float("inf")): with pytest.raises(ValueError, match="cache durations must be finite and nonnegative"): binding.store({**request(), "ttl_seconds": seconds}, {"answer": 1}) assert binding.lookup(request()) is None with pytest.raises(ValueError, match="cache durations must be finite and nonnegative"): - _native._CacheTestHandle.memory(ttl_seconds=-1) + _CacheTestHandle.memory(ttl_seconds=-1) async def test_memory_size_policy_is_applied_by_the_native_host() -> None: - handle: Final = _native._CacheTestHandle.memory(capacity=2, max_entry_bytes=128) - binding: Final = _native._CacheTestResolver(SimpleNamespace(cache=handle)).resolve() + handle: Final = _CacheTestHandle.memory(capacity=2, max_entry_bytes=128) + binding: Final = _CacheTestResolver(SimpleNamespace(cache=handle)).resolve() small: Final = {"answer": "ok"} binding.store(request("small"), small) assert await binding.async_lookup(request("small")) == small await binding.async_store(request("large"), {"answer": "x" * 256}) assert binding.lookup(request("large")) is None assert binding.lookup(request("small")) == small - disabled: Final = _native._CacheTestResolver( - SimpleNamespace(cache=_native._CacheTestHandle.memory(capacity=0)) + disabled: Final = _CacheTestResolver( + SimpleNamespace(cache=_CacheTestHandle.memory(capacity=0)) ).resolve() await disabled.async_store(request(), small) assert await disabled.async_lookup(request()) is None async def test_native_batch_lookup_and_store_report_partial_hits() -> None: - binding: Final = _native._CacheTestResolver(SimpleNamespace(cache=_native._CacheTestHandle.memory())).resolve() + binding: Final = _CacheTestResolver(SimpleNamespace(cache=_CacheTestHandle.memory())).resolve() requests: Final = [request("hit"), request("miss"), request("disabled")] requests[2]["controls"] = { "supported_call_type": True, @@ -359,7 +456,7 @@ async def test_python_batch_callbacks_use_the_builtin_cache_api() -> None: ) -> object: return result, kwargs - binding: Final = _native._CacheTestResolver( + binding: Final = _CacheTestResolver( SimpleNamespace(cache=CustomCache(type=LiteLLMCacheType.LOCAL)) ).resolve() assert binding.kind == "python_callback" @@ -389,7 +486,7 @@ async def test_unmodified_builtin_cache_callbacks_can_ping_and_flush() -> None: cache: Final = Cache(type=LiteLLMCacheType.LOCAL) cache.cache.set_cache("key", "value") - binding: Final = _native._CacheTestResolver(SimpleNamespace(cache=cache)).resolve() + binding: Final = _CacheTestResolver(SimpleNamespace(cache=cache)).resolve() assert binding.kind == "python_callback" setattr(cache.cache, "ping", ping) @@ -401,7 +498,7 @@ async def test_unmodified_builtin_cache_callbacks_can_ping_and_flush() -> None: def test_facade_registration_rejects_mismatched_capacity() -> None: facade: Final = Cache(type=LiteLLMCacheType.LOCAL) with pytest.raises(TypeError, match="capacities must match"): - _native._CacheTestHandle.memory(capacity=7)._bind_facade(facade) + _CacheTestHandle.memory(capacity=7)._bind_facade(facade) def test_azure_blob_facade_serves_natively_and_python_reads_the_same_blobs(azure_blob_facade: Cache) -> None: @@ -411,15 +508,20 @@ def test_azure_blob_facade_serves_natively_and_python_reads_the_same_blobs(azure assert handle.backend == "azure-blob" account_url: Final = backend.container_client.url.removesuffix(f"/{backend.container_client.container_name}") with pytest.raises(TypeError, match="containers must match"): - _native._CacheTestHandle.azure_blob(account_url, f"{backend.container_client.container_name}-other")._bind_facade( - azure_blob_facade - ) + _native._CacheTestHandle.azure_blob( + account_url, f"{backend.container_client.container_name}-other" + )._bind_facade(azure_blob_facade) handle._bind_facade(azure_blob_facade) resolver: Final = _native._CacheTestResolver(SimpleNamespace(cache=azure_blob_facade)) native: Final = resolver.resolve() assert native.kind == "native" - response: Final = {"choices": [{"text": "caf\u00e9 \u2603"}], "usage": {"total_tokens": 3}, "flag": True, "empty": None} + response: Final = { + "choices": [{"text": "caf\u00e9 \u2603"}], + "usage": {"total_tokens": 3}, + "flag": True, + "empty": None, + } native.store({**request("sync"), "ttl_seconds": 0.001}, response) native.store(request("sync"), {"choices": [{"text": "second"}]}) time.sleep(0.01) @@ -474,7 +576,9 @@ async def test_azure_blob_native_async_writes_overwrite_batch_and_flush_like_pyt await binding.async_store({**request("async"), "ttl_seconds": 0.001}, {"value": 2}) time.sleep(0.01) assert await binding.async_lookup(request("async")) == {"value": 2} - assert await backend.async_get_cache("async") == json.loads(backend.container_client.download_blob("async").readall()) + assert await backend.async_get_cache("async") == json.loads( + backend.container_client.download_blob("async").readall() + ) assert cast(CacheLookup, azure_blob_facade).get_cache(cache_key="async") == {"value": 2} await binding.async_store_batch([request("first"), request("second")], [{"value": 3}, {"value": 4}]) @@ -497,19 +601,19 @@ async def test_redis_facade_buffers_native_async_writes(redis_url: str) -> None: redis_flush_size=2, ) with pytest.raises(TypeError, match="default TTLs must match"): - _native._CacheTestHandle.redis(redis_url, ttl_seconds=61)._bind_facade(facade) + _CacheTestHandle.redis(redis_url, ttl_seconds=61)._bind_facade(facade) with pytest.raises(TypeError, match="namespaces must match"): - _native._CacheTestHandle.redis(redis_url, namespace="other")._bind_facade(facade) - _native._CacheTestHandle.redis(redis_url, ttl_seconds=60)._bind_facade(facade) - binding: Final = _native._CacheTestResolver(SimpleNamespace(cache=facade)).resolve() + _CacheTestHandle.redis(redis_url, namespace="other")._bind_facade(facade) + _CacheTestHandle.redis(redis_url, ttl_seconds=60)._bind_facade(facade) + binding: Final = _CacheTestResolver(SimpleNamespace(cache=facade)).resolve() client: Final = redis.Redis.from_url(redis_url) with rebound(facade.cache, "redis_kwargs", {**facade.cache.redis_kwargs, "ssl": True}): - assert _native._CacheTestResolver(SimpleNamespace(cache=facade)).resolve().kind == "python_callback" + assert _CacheTestResolver(SimpleNamespace(cache=facade)).resolve().kind == "python_callback" pool: Final = facade.cache.redis_client.connection_pool with rebound(pool, "connection_kwargs", {**pool.connection_kwargs, "db": 1}): - assert _native._CacheTestResolver(SimpleNamespace(cache=facade)).resolve().kind == "python_callback" + assert _CacheTestResolver(SimpleNamespace(cache=facade)).resolve().kind == "python_callback" await binding.async_store(request("first"), {"value": 1}) assert client.get("first") is None @@ -521,6 +625,508 @@ async def test_redis_facade_buffers_native_async_writes(redis_url: str) -> None: client.close() +async def test_disk_reads_python_entries_and_python_reads_native_entries(tmp_path: Path) -> None: + disk_cache: Final = DiskCache(disk_cache_dir=str(tmp_path)) + response: Final = {"choices": [{"text": "cached"}], "usage": {"total_tokens": 3}} + disk_cache.disk_cache.set( + "sync", + {"timestamp": time.time(), "response": json.dumps(response)}, + ) + disk_cache.disk_cache.set("async", json.dumps({"timestamp": time.time(), "response": response})) + disk_cache.disk_cache.set("raw", json.dumps(response)) + disk_cache.disk_cache.set("invalid", "not a cache entry") + disk_cache.disk_cache.set( + "large", + {"timestamp": time.time(), "response": {"text": "x" * 70_000}}, + ) + binding: Final = _native._CacheTestResolver( + SimpleNamespace(cache=_native._CacheTestHandle.disk(str(tmp_path))) + ).resolve() + + assert binding.lookup(request("sync")) == response + assert await binding.async_lookup(request("async")) == response + assert binding.lookup(request("raw")) == response + assert await binding.async_lookup(request("invalid")) is None + assert binding.lookup(request("large")) == {"text": "x" * 70_000} + + await binding.async_store({**request("native"), "ttl_seconds": 12.0}, response) + stored_response: Final = disk_cache.get_cache("native") + assert isinstance(stored_response, dict) + assert stored_response["response"] == response + stored, expire_time = disk_cache.disk_cache.get("native", expire_time=True) + assert stored is not None + assert time.time() < expire_time <= time.time() + 12.0 + await binding.async_store(request("no-ttl"), response) + _, no_expiry = disk_cache.disk_cache.get("no-ttl", expire_time=True) + assert no_expiry is None + + +async def test_disk_entries_survive_a_fresh_handle_and_expire_on_time(tmp_path: Path) -> None: + first: Final = _native._CacheTestResolver( + SimpleNamespace(cache=_native._CacheTestHandle.disk(str(tmp_path))) + ).resolve() + await first.async_store(request("persistent"), {"value": "persistent"}) + await first.async_store({**request("expiring"), "ttl_seconds": 0.3}, {"value": "expiring"}) + fresh: Final = _native._CacheTestResolver( + SimpleNamespace(cache=_native._CacheTestHandle.disk(str(tmp_path))) + ).resolve() + assert fresh.lookup(request("persistent")) == {"value": "persistent"} + assert fresh.lookup(request("expiring")) == {"value": "expiring"} + await asyncio.sleep(0.4) + assert fresh.lookup(request("expiring")) is None + assert fresh.lookup(request("persistent")) == {"value": "persistent"} + + +def test_disk_facade_registers_and_store_changes_fall_back(tmp_path: Path) -> None: + facade: Final = Cache(type=LiteLLMCacheType.DISK, disk_cache_dir=str(tmp_path)) + with pytest.raises(TypeError, match="directories must match"): + _native._CacheTestHandle.disk(str(tmp_path / "other"))._bind_facade(facade) + handle: Final = _native._CacheTestHandle.disk(str(tmp_path)) + handle._bind_facade(facade) + resolver: Final = _native._CacheTestResolver(SimpleNamespace(cache=facade)) + binding: Final = resolver.resolve() + assert binding.kind == "native" + binding.store(request("native"), {"value": "native"}) + assert facade.get_cache(cache_key="native") == {"value": "native"} + + with rebound(facade.cache, "disk_cache", diskcache.Cache(str(tmp_path))): + assert resolver.resolve().kind == "python_callback" + assert resolver.resolve().kind == "native" + + class CustomDiskCache(DiskCache): + pass + + with rebound(facade, "cache", CustomDiskCache(disk_cache_dir=str(tmp_path))): + assert resolver.resolve().kind == "python_callback" + + class CustomStore(diskcache.Cache): + pass + + custom_facade: Final = Cache(type=LiteLLMCacheType.DISK, disk_cache_dir=str(tmp_path)) + custom_facade.cache.disk_cache = CustomStore(str(tmp_path)) + with pytest.raises(TypeError, match="built-in diskcache store"): + _native._CacheTestHandle.disk(str(tmp_path))._bind_facade(custom_facade) + + +async def test_disk_native_batch_lookup_and_store_report_partial_hits(tmp_path: Path) -> None: + binding: Final = _native._CacheTestResolver( + SimpleNamespace(cache=_native._CacheTestHandle.disk(str(tmp_path))) + ).resolve() + requests: Final = [request("hit"), request("miss"), request("disabled")] + requests[2]["controls"] = { + "supported_call_type": True, + "configured": True, + "native_backend": True, + "default_on": True, + "caching": False, + "no_cache": False, + "no_store": False, + "use_cache": False, + } + await binding.async_store_batch(requests, [{"value": 1}, {"value": 2}, {"value": 3}]) + + partial: Final = await binding.async_lookup_batch(requests) + + assert partial == { + "values": [{"value": 1}, {"value": 2}, None], + "missing_indices": [2], + } + + +@pytest.fixture +def s3_stub() -> Generator[S3Stub]: + stub: Final = S3Stub() + try: + yield stub + finally: + stub.close() + + +def python_s3(url: str) -> S3Cache: + return S3Cache( + s3_bucket_name="cache-bucket", + s3_region_name="us-east-1", + s3_endpoint_url=url, + s3_aws_access_key_id="key", + s3_aws_secret_access_key="secret", + s3_path="team", + ) + + +async def test_s3_reads_python_entries_and_writes_with_python_metadata(s3_stub: S3Stub) -> None: + python_cache: Final = python_s3(s3_stub.url) + response: Final = {"choices": [{"text": "cached"}], "usage": {"total_tokens": 3}} + python_cache.set_cache("sync:key", {"timestamp": time.time(), "response": response}, ttl=90) + python_cache.set_cache("plain", {"timestamp": time.time(), "response": response}) + s3_stub.put_object("team/malformed", b"not a cache entry") + s3_stub.put_object( + "team/expired", + json.dumps({"timestamp": time.time(), "response": response}).encode(), + {"expires": "Thu, 01 Jan 1970 00:00:00 GMT"}, + ) + binding: Final = _native._CacheTestResolver( + SimpleNamespace( + cache=_native._CacheTestHandle.s3( + "cache-bucket", + region="us-east-1", + endpoint_url=s3_stub.url, + key_prefix="team/", + access_key_id="key", + secret_access_key="secret", + ) + ) + ).resolve() + + assert binding.lookup(request("sync:key")) == response + assert await binding.async_lookup(request("plain")) == response + assert binding.lookup(request("malformed")) is None + assert binding.lookup(request("expired")) is None + assert binding.lookup(request("absent")) is None + + binding.store({**request("native:key"), "ttl_seconds": 90.0}, response) + await binding.async_store(request("no_ttl"), response) + stored: Final = s3_stub.objects["team/native/key"] + assert stored.headers["content-type"] == "application/json" + assert stored.headers["content-language"] == "en" + assert stored.headers["content-disposition"] == 'inline; filename="team/native/key.json"' + assert stored.headers["cache-control"] == "immutable, max-age=90, s-maxage=90" + expires: Final = cast(datetime, s3_stub.expires("team/native/key")) + remaining: Final = (expires - datetime.now(expires.tzinfo)).total_seconds() + assert 60 < remaining <= 91 + no_ttl: Final = s3_stub.objects["team/no_ttl"] + assert no_ttl.headers["cache-control"] == "immutable, max-age=31536000, s-maxage=31536000" + assert "expires" not in no_ttl.headers + assert python_cache.get_cache("native:key")["response"] == response + + partial: Final = await binding.async_lookup_batch([request("native:key"), request("absent"), request("malformed")]) + assert partial == {"values": [response, None, None], "missing_indices": [1, 2]} + + +def test_s3_facade_binds_only_exact_configuration_and_falls_back_on_mutation(s3_stub: S3Stub) -> None: + facade: Final = Cache( + type=LiteLLMCacheType.S3, + s3_bucket_name="cache-bucket", + s3_region_name="us-east-1", + s3_endpoint_url=s3_stub.url, + s3_aws_access_key_id="key", + s3_aws_secret_access_key="secret", + s3_path="team", + ) + handle: Final = _native._CacheTestHandle.s3( + "cache-bucket", + region="us-east-1", + endpoint_url=s3_stub.url, + key_prefix="team/", + access_key_id="key", + secret_access_key="secret", + ) + with pytest.raises(TypeError, match="buckets must match"): + _native._CacheTestHandle.s3("other", region="us-east-1", endpoint_url=s3_stub.url)._bind_facade(facade) + with pytest.raises(TypeError, match="key prefixes must match"): + _native._CacheTestHandle.s3( + "cache-bucket", region="us-east-1", endpoint_url=s3_stub.url, key_prefix="other/" + )._bind_facade(facade) + handle._bind_facade(facade) + resolver: Final = _native._CacheTestResolver(SimpleNamespace(cache=facade)) + binding: Final = resolver.resolve() + assert binding.kind == "native" + + handler: Final = Mock() + facade.cache.s3_client.meta.events.register("before-call.s3.*", handler) + binding.store(request("native"), {"answer": 1}) + assert binding.lookup(request("native")) == {"answer": 1} + assert handler.call_count == 0 + assert "team/native" in s3_stub.objects + + with rebound(facade.cache, "bucket_name", "other"): + assert resolver.resolve().kind == "python_callback" + other_client: Final = boto3.client( + "s3", + region_name="us-east-1", + endpoint_url=s3_stub.url, + aws_access_key_id="key", + aws_secret_access_key="secret", + ) + with rebound(facade.cache, "s3_client", other_client): + assert resolver.resolve().kind == "python_callback" + + class CustomS3Cache(S3Cache): + pass + + subclassed: Final = Cache( + type=LiteLLMCacheType.S3, + s3_bucket_name="cache-bucket", + s3_region_name="us-east-1", + s3_endpoint_url=s3_stub.url, + s3_aws_access_key_id="key", + s3_aws_secret_access_key="secret", + s3_path="team", + ) + subclassed.cache = CustomS3Cache( + s3_bucket_name="cache-bucket", + s3_region_name="us-east-1", + s3_endpoint_url=s3_stub.url, + s3_aws_access_key_id="key", + s3_aws_secret_access_key="secret", + s3_path="team", + ) + with pytest.raises(TypeError): + handle._bind_facade(subclassed) + assert _native._CacheTestResolver(SimpleNamespace(cache=subclassed)).resolve().kind == "python_callback" + + +def test_s3_facade_rejects_configurations_that_require_python(s3_stub: S3Stub) -> None: + handle: Final = _native._CacheTestHandle.s3( + "cache-bucket", + region="us-east-1", + endpoint_url=s3_stub.url, + key_prefix="team/", + access_key_id="key", + secret_access_key="secret", + ) + unverified: Final = Cache( + type=LiteLLMCacheType.S3, + s3_bucket_name="cache-bucket", + s3_region_name="us-east-1", + s3_endpoint_url="https://s3.example.test", + s3_aws_access_key_id="key", + s3_aws_secret_access_key="secret", + s3_path="team", + s3_verify=False, + ) + with pytest.raises(TypeError, match="requires Python"): + handle._bind_facade(unverified) + proxied: Final = Cache( + type=LiteLLMCacheType.S3, + s3_bucket_name="cache-bucket", + s3_region_name="us-east-1", + s3_endpoint_url=s3_stub.url, + s3_aws_access_key_id="key", + s3_aws_secret_access_key="secret", + s3_path="team", + s3_config=botocore.config.Config(proxies={"https": "http://proxy.test"}), + ) + with pytest.raises(TypeError, match="requires Python"): + handle._bind_facade(proxied) + + +async def test_gcs_reads_python_entries_and_writes_python_compatible_objects( + fake_gcs: FakeGcs, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.delenv("GCS_PATH_SERVICE_ACCOUNT", raising=False) + monkeypatch.delenv("GCS_BUCKET_NAME", raising=False) + response: Final = {"choices": [{"text": "cached"}], "usage": {"total_tokens": 3}, "flag": True, "empty": None} + fake_gcs.put( + "bucket", + "cache/sync", + json.dumps({"timestamp": time.time(), "response": json.dumps(response)}).encode(), + ) + fake_gcs.put("bucket", "cache/async", json.dumps({"timestamp": time.time(), "response": response}).encode()) + fake_gcs.put("bucket", "cache/raw", json.dumps(response).encode()) + fake_gcs.put("bucket", "cache/invalid", b"not a cache entry") + binding: Final = _native._CacheTestResolver( + SimpleNamespace( + cache=_native._CacheTestHandle.gcs( + "bucket", + gcs_path="cache", + endpoint=fake_gcs.url, + token=fake_gcs.token, + ) + ) + ).resolve() + + assert binding.lookup(request("sync")) == response + assert await binding.async_lookup(request("async")) == response + assert binding.lookup(request("raw")) == response + assert await binding.async_lookup(request("invalid")) is None + assert binding.lookup(request("missing")) is None + + await binding.async_store({**request("native"), "ttl_seconds": 12.0}, response) + stored: Final = fake_gcs.objects[("bucket", "cache/native")] + stored_value: Final = cast(dict[str, object], json.loads(stored)) + assert stored_value["response"] == response + assert isinstance(stored_value["timestamp"], float) + upload: Final = next(item for item in fake_gcs.requests if item.method == "POST") + assert upload.path == "/upload/storage/v1/b/bucket/o" + assert upload.query == "uploadType=media&name=cache%2Fnative" + assert upload.headers["Authorization"] == f"Bearer {fake_gcs.token}" + assert upload.headers["Content-Type"] == "application/json" + upload_text: Final = f"{upload.path}?{upload.query}{upload.headers}" + assert "ttl" not in upload_text.lower() + assert "expiry" not in upload_text.lower() + download: Final = next(item for item in fake_gcs.requests if item.path.endswith("/cache%2Fsync")) + assert download.path == "/storage/v1/b/bucket/o/cache%2Fsync" + assert download.query == "alt=media" + + binding.store(request("sync2"), response) + assert binding.lookup(request("sync2")) == response + assert GCSCache(bucket_name="bucket", gcs_path="cache").key_prefix == "cache/" + assert GCSCache(bucket_name="bucket", gcs_path="cache/").key_prefix == "cache/" + assert GCSCache(bucket_name="bucket").key_prefix == "" + + +async def test_gcs_batch_lookup_preserves_order_and_treats_malformed_entries_as_misses(fake_gcs: FakeGcs) -> None: + fake_gcs.put("bucket", "cache/hit", json.dumps({"timestamp": time.time(), "response": {"value": 1}}).encode()) + fake_gcs.put("bucket", "cache/invalid", b"not a cache entry") + binding: Final = _native._CacheTestResolver( + SimpleNamespace( + cache=_native._CacheTestHandle.gcs( + "bucket", + gcs_path="cache", + endpoint=fake_gcs.url, + token=fake_gcs.token, + ) + ) + ).resolve() + requests: Final = [request("hit"), request("missing"), request("invalid")] + expected: Final = {"values": [{"value": 1}, None, None], "missing_indices": [1, 2]} + + assert await binding.async_lookup_batch(requests) == expected + assert binding.lookup_batch(requests) == expected + await binding.async_store_batch([request("first"), request("second")], [{"value": 1}, {"value": 2}]) + assert ("bucket", "cache/first") in fake_gcs.objects + assert ("bucket", "cache/second") in fake_gcs.objects + + +async def test_gcs_facade_binds_only_exact_matching_configuration( + fake_gcs: FakeGcs, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.delenv("GCS_PATH_SERVICE_ACCOUNT", raising=False) + monkeypatch.delenv("GCS_BUCKET_NAME", raising=False) + monkeypatch.setenv("GOOGLE_APPLICATION_CREDENTIALS", "/nonexistent") + facade: Final = Cache(type=LiteLLMCacheType.GCS, gcs_bucket_name="bucket", gcs_path="cache/") + assert type(facade.cache) is GCSCache + + mismatched_bucket: Final = _native._CacheTestHandle.gcs( + "other", + gcs_path="cache", + endpoint=fake_gcs.url, + token=fake_gcs.token, + ) + with pytest.raises(TypeError, match="buckets must match"): + mismatched_bucket._bind_facade(facade) + mismatched_prefix: Final = _native._CacheTestHandle.gcs( + "bucket", + gcs_path="x", + endpoint=fake_gcs.url, + token=fake_gcs.token, + ) + with pytest.raises(TypeError, match="key prefixes must match"): + mismatched_prefix._bind_facade(facade) + mismatched_credentials: Final = _native._CacheTestHandle.gcs( + "bucket", + gcs_path="cache", + path_service_account="sa.json", + endpoint=fake_gcs.url, + token=fake_gcs.token, + ) + with pytest.raises(TypeError, match="credentials must match"): + mismatched_credentials._bind_facade(facade) + with pytest.raises(TypeError, match="types must match"): + _native._CacheTestHandle.memory()._bind_facade(facade) + + matching: Final = _native._CacheTestHandle.gcs( + "bucket", + gcs_path="cache", + endpoint=fake_gcs.url, + token=fake_gcs.token, + ) + matching._bind_facade(facade) + resolver: Final = _native._CacheTestResolver(SimpleNamespace(cache=facade)) + binding: Final = resolver.resolve() + assert binding.kind == "native" + await binding.async_store(request("native"), {"value": "native"}) + assert await binding.async_lookup(request("native")) == {"value": "native"} + assert cast(CacheLookup, facade).get_cache(cache_key="native") is None + + with rebound(facade.cache, "bucket_name", "other"): + assert resolver.resolve().kind == "python_callback" + with rebound(facade.cache, "key_prefix", "x/"): + assert resolver.resolve().kind == "python_callback" + with rebound(facade.cache, "path_service_account", "sa.json"): + assert resolver.resolve().kind == "python_callback" + + def no_get_cache(*args: object, **kwargs: object) -> None: + return None + + with rebound(facade.cache, "get_cache", no_get_cache): + assert resolver.resolve().kind == "python_callback" + with rebound(facade, "ttl", 12): + assert resolver.resolve().kind == "python_callback" + + class CustomGcs(GCSCache): + pass + + with rebound(facade, "cache", CustomGcs(bucket_name="bucket", gcs_path="cache/")): + assert resolver.resolve().kind == "python_callback" + custom_facade: Final = Cache(type=LiteLLMCacheType.GCS, gcs_bucket_name="bucket", gcs_path="cache/") + with rebound(custom_facade, "cache", CustomGcs(bucket_name="bucket", gcs_path="cache/")): + with pytest.raises(TypeError, match="types must match"): + matching._bind_facade(custom_facade) + + missing_bucket: Final = Cache(type=LiteLLMCacheType.GCS) + with pytest.raises(TypeError, match="requires a configured bucket name"): + matching._bind_facade(missing_bucket) + + +async def test_gcs_flush_is_a_no_op_and_ping_is_not_implemented( + fake_gcs: FakeGcs, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.delenv("GCS_PATH_SERVICE_ACCOUNT", raising=False) + monkeypatch.delenv("GCS_BUCKET_NAME", raising=False) + binding: Final = _native._CacheTestResolver( + SimpleNamespace( + cache=_native._CacheTestHandle.gcs( + "bucket", + gcs_path="cache", + endpoint=fake_gcs.url, + token=fake_gcs.token, + ) + ) + ).resolve() + await binding.async_store(request("key"), {"value": "stored"}) + await binding.async_flush() + assert ("bucket", "cache/key") in fake_gcs.objects + assert await binding.async_lookup(request("key")) == {"value": "stored"} + with pytest.raises(NotImplementedError): + await binding.ping() + + facade: Final = Cache(type=LiteLLMCacheType.GCS, gcs_bucket_name="bucket", gcs_path="cache/") + with pytest.raises(AttributeError): + await facade.ping() + assert cast(CacheLookup, facade.cache).flush_cache() is None + + +async def test_gcs_unauthorized_and_server_errors_surface_as_runtime_errors(fake_gcs: FakeGcs) -> None: + wrong_token: Final = _native._CacheTestResolver( + SimpleNamespace( + cache=_native._CacheTestHandle.gcs( + "bucket", + gcs_path="cache", + endpoint=fake_gcs.url, + token="wrong-token", + ) + ) + ).resolve() + with pytest.raises(RuntimeError): + wrong_token.lookup(request("missing")) + assert not fake_gcs.objects + + binding: Final = _native._CacheTestResolver( + SimpleNamespace( + cache=_native._CacheTestHandle.gcs( + "bucket", + gcs_path="cache", + endpoint=fake_gcs.url, + token=fake_gcs.token, + ) + ) + ).resolve() + with pytest.raises(RuntimeError): + binding.lookup(request("server-error")) + assert binding.lookup(request("missing")) is None + + async def test_redis_cluster_facade_serves_multi_slot_batches_and_scoped_flush_natively( cluster_nodes: tuple[tuple[str, int], ...], ) -> None: @@ -574,9 +1180,776 @@ async def test_redis_cluster_facade_serves_multi_slot_batches_and_scoped_flush_n await binding.async_flush() - remaining: Final = tuple(sorted(key for node in client.get_primaries() for key in client.keys("parity:*", target_nodes=node))) + remaining: Final = tuple( + sorted(key for node in client.get_primaries() for key in client.keys("parity:*", target_nodes=node)) + ) assert remaining == (), remaining assert client.get("unscoped") == b"stays" client.delete("unscoped") client.close() facade.cache.redis_client.close() + + +PARAPHRASE_MARKER: Final = " (paraphrase)" +SEMANTIC_EMBEDDING_MODEL: Final = "semantic-test/deterministic" +SEMANTIC_INDEX_PREFIX: Final = "litellm_test_semantic_" +SEMANTIC_CONTEXT: Final = contextvars.ContextVar("semantic_test_context", default="unset") + + +def _normalized(vector: list[float]) -> list[float]: + norm: Final = math.sqrt(sum(component * component for component in vector)) + return [component / norm for component in vector] + + +def _base_embedding(prompt: str) -> list[float]: + digest: Final = hashlib.sha256(prompt.encode("utf-8")).digest() + return _normalized([float(digest[index] + 1) for index in range(8)]) + + +def _semantic_embedding(prompt: str) -> list[float]: + if PARAPHRASE_MARKER not in prompt: + return _base_embedding(prompt) + base: Final = _base_embedding(prompt.replace(PARAPHRASE_MARKER, "").strip()) + pivot: Final = min(range(8), key=lambda index: abs(base[index])) + direction: Final = _normalized( + [ + (1.0 - base[pivot] * base[pivot]) if index == pivot else -base[index] * base[pivot] + for index in range(8) + ] + ) + # Rotating an orthogonal unit direction by 0.329 produces ~0.05 cosine distance + return _normalized([base[index] + 0.329 * direction[index] for index in range(8)]) + + +class DeterministicEmbedding(litellm.CustomLLM): + def __init__(self) -> None: + self.calls: list[dict[str, object]] = [] + self.async_calls: list[dict[str, object]] = [] + self.entered = asyncio.Event() + self.gate: asyncio.Event | None = None + + def _respond( + self, + model: str, + input: object, + model_response: EmbeddingResponse, + ) -> EmbeddingResponse: + texts: Final = cast(list[object], input if isinstance(input, list) else [input]) + self.calls.append({"model": model, "input": texts}) + model_response.model = model + model_response.data = [ + {"object": "embedding", "index": index, "embedding": _semantic_embedding(str(text))} + for index, text in enumerate(texts) + ] + return model_response + + def embedding( + self, + model: str, + input: list[object], + model_response: EmbeddingResponse, + print_verbose: Callable[..., object], + logging_obj: object, + optional_params: dict[str, object], + api_key: object = None, + api_base: object = None, + timeout: object = None, + litellm_params: object = None, + ) -> EmbeddingResponse: + return self._respond(model, input, model_response) + + async def aembedding( + self, + model: str, + input: list[object], + model_response: EmbeddingResponse, + print_verbose: Callable[..., object], + logging_obj: object, + optional_params: dict[str, object], + api_key: object = None, + api_base: object = None, + timeout: object = None, + litellm_params: object = None, + ) -> EmbeddingResponse: + texts: Final = cast(list[object], input if isinstance(input, list) else [input]) + self.async_calls.append( + { + "model": model, + "input": texts, + "task": asyncio.current_task(), + "context": SEMANTIC_CONTEXT.get(), + } + ) + SEMANTIC_CONTEXT.set("written-in-aembedding") + self.entered.set() + if self.gate is not None: + await self.gate.wait() + return self._respond(model, input, model_response) + + +@pytest.fixture +def semantic_embedding() -> Generator[DeterministicEmbedding]: + handler: Final = DeterministicEmbedding() + with ExitStack() as stack: + stack.enter_context( + rebound( + litellm, + "custom_provider_map", + [ + *litellm.custom_provider_map, + cast( + CustomLLMItem, + {"provider": "semantic-test", "custom_handler": handler}, + ), + ], + ) + ) + stack.enter_context( + rebound( + litellm, + "_custom_providers", # pyright: ignore[reportPrivateUsage] # no public provider-registration hook + [*litellm._custom_providers, "semantic-test"], # pyright: ignore[reportPrivateUsage] # no public provider-registration hook + ) + ) + stack.enter_context( + rebound(litellm, "provider_list", [*litellm.provider_list, "semantic-test"]) + ) + yield handler + + +@pytest.fixture +def redis_stack() -> Generator[tuple[str, str]]: + url: Final = os.environ.get("LITELLM_REDIS_STACK_URL") + if url is None: + pytest.skip("LITELLM_REDIS_STACK_URL is not set") + index: Final = f"{SEMANTIC_INDEX_PREFIX}{uuid4().hex}" + yield url, index + client: Final = redis.Redis.from_url(url) + try: + client.execute_command("FT.DROPINDEX", index, "DD") # pyright: ignore[reportUnknownMemberType] # redis-py leaves execute_command partially unknown + except redis.RedisError: + pass + client.close() + + +def semantic_request(key: str, prompt: str, **extra: object) -> dict[str, object]: + return { + "key": {"preset": key}, + "messages": [{"role": "user", "content": prompt}], + **extra, + } + + +def semantic_messages(prompt: str) -> list[dict[str, object]]: + return [{"role": "user", "content": prompt}] + + +def semantic_entry_id(prompt: str, tag: str) -> str: + return hashlib.sha256(f"{prompt}litellm_cache_key{tag}".encode()).hexdigest() + + +def semantic_facade(url: str, index: str, *, similarity_threshold: float = 0.8) -> Cache: + facade: Final = Cache( + type=LiteLLMCacheType.REDIS_SEMANTIC, + redis_url=url, + similarity_threshold=similarity_threshold, + redis_semantic_cache_embedding_model=SEMANTIC_EMBEDDING_MODEL, + redis_semantic_cache_index_name=index, + ) + _CacheTestHandle.redis_semantic(facade.cache)._bind_facade(facade) + return facade + + +def test_redis_semantic_constructor_identity_and_provenance( + redis_stack: tuple[str, str], semantic_embedding: DeterministicEmbedding +) -> None: + url, index = redis_stack + facade: Final = semantic_facade(url, index) + backend: Final = cast(RedisSemanticCache, facade.cache) + assert backend.__class__.__module__ == "litellm.caching.redis_semantic_cache" + assert type(backend) is RedisSemanticCache + assert backend._redis_url == url # pyright: ignore[reportPrivateUsage] # provenance check needs the projected config + assert backend._index_name == index # pyright: ignore[reportPrivateUsage] # provenance check needs the projected config + assert backend.similarity_threshold == 0.8 + assert backend.embedding_model == SEMANTIC_EMBEDDING_MODEL + handle: Final = cast(object, getattr(facade, "_native_cache_handle")) + assert isinstance(handle, _CacheTestHandle) + assert handle.backend == "redis_semantic" + binding: Final = _CacheTestResolver(SimpleNamespace(cache=facade)).resolve() + assert binding.kind == "native" + + +def test_redis_semantic_native_and_python_sync_entries_share_one_layout( + redis_stack: tuple[str, str], semantic_embedding: DeterministicEmbedding +) -> None: + url, index = redis_stack + facade: Final = semantic_facade(url, index) + binding: Final = _CacheTestResolver(SimpleNamespace(cache=facade)).resolve() + client: Final = redis.Redis.from_url(url) + response: Final = {"choices": [{"text": "paris"}], "usage": {"total_tokens": 2}} + + binding.store(semantic_request("geo", "what is the capital of france"), response) + + native_hash_key: Final = f"{index}:{semantic_entry_id('what is the capital of france', 'geo')}" + stored: Final = client.hgetall(native_hash_key) + assert set(stored) == { + b"entry_id", + b"prompt", + b"response", + b"prompt_vector", + b"inserted_at", + b"updated_at", + b"litellm_cache_key", + }, stored + assert stored[b"entry_id"].decode() == native_hash_key.split(":", 1)[1] + assert stored[b"prompt"] == b"what is the capital of france" + assert stored[b"litellm_cache_key"] == b"geo" + assert len(stored[b"prompt_vector"]) == 32 + decoded: Final = cast(dict[str, object], json.loads(stored[b"response"])) + assert decoded["response"] == response + assert ( + cast(RedisSemanticCache, facade.cache).get_cache( # pyright: ignore[reportUnknownMemberType] # **kwargs stays unknown on the backend class + "geo", messages=semantic_messages("what is the capital of france") + ) + == decoded + ) + assert semantic_embedding.calls == [ + {"model": "deterministic", "input": ["what is the capital of france"]}, + {"model": "deterministic", "input": ["what is the capital of france"]}, + {"model": "deterministic", "input": ["dimension test"]}, + ] + + cast(RedisSemanticCache, facade.cache).set_cache( # pyright: ignore[reportUnknownMemberType] # **kwargs stays unknown on the backend class + "math", + json.dumps({"timestamp": 1700000000.0, "response": {"answer": 42}}), + messages=semantic_messages("what is 6 times 7"), + ) + python_hash_key: Final = f"{index}:{semantic_entry_id('what is 6 times 7', 'math')}" + assert json.loads(cast(bytes, client.hget(python_hash_key, "response"))) == { + "timestamp": 1700000000.0, + "response": {"answer": 42}, + } + assert binding.lookup(semantic_request("math", "what is 6 times 7")) == {"answer": 42} + client.close() + + +async def test_redis_semantic_async_paths_and_store_batch_share_one_layout( + redis_stack: tuple[str, str], semantic_embedding: DeterministicEmbedding +) -> None: + url, index = redis_stack + facade: Final = semantic_facade(url, index) + binding: Final = _CacheTestResolver(SimpleNamespace(cache=facade)).resolve() + client: Final = redis.Redis.from_url(url) + + await binding.async_store( + semantic_request("async", "name a primary color"), {"answer": "blue"} + ) + hash_key: Final = f"{index}:{semantic_entry_id('name a primary color', 'async')}" + decoded: Final = cast(dict[str, object], json.loads(cast(bytes, client.hget(hash_key, "response")))) + python_read: Final = await cast(RedisSemanticCache, facade.cache).async_get_cache( # pyright: ignore[reportUnknownMemberType] # **kwargs stays unknown on the backend class + "async", messages=semantic_messages("name a primary color") + ) + assert python_read == decoded + + await binding.async_store_batch( + [ + semantic_request("batch-one", "first batch prompt"), + semantic_request("batch-two", "second batch prompt"), + ], + [{"answer": 1}, {"answer": 2}], + ) + expected: Final = { + key: json.loads( + cast(bytes, client.hget(f"{index}:{semantic_entry_id(prompt, key)}", "response")) + ) + for key, prompt in ( + ("batch-one", "first batch prompt"), + ("batch-two", "second batch prompt"), + ) + } + for key, prompt in ( + ("batch-one", "first batch prompt"), + ("batch-two", "second batch prompt"), + ): + assert cast(RedisSemanticCache, facade.cache).get_cache( # pyright: ignore[reportUnknownMemberType] # **kwargs stays unknown on the backend class + key, messages=semantic_messages(prompt) + ) == expected[key], key + + cast(RedisSemanticCache, facade.cache).set_cache( # pyright: ignore[reportUnknownMemberType] # **kwargs stays unknown on the backend class + "async-python", + json.dumps({"timestamp": 1700000000.0, "response": {"answer": "python"}}), + messages=semantic_messages("python written prompt"), + ) + assert await binding.async_lookup( + semantic_request("async-python", "python written prompt") + ) == {"answer": "python"} + client.close() + + +async def test_native_semantic_async_embedding_runs_inline_in_the_callers_task( + redis_stack: tuple[str, str], semantic_embedding: DeterministicEmbedding +) -> None: + url, index = redis_stack + facade: Final = semantic_facade(url, index) + binding: Final = _CacheTestResolver(SimpleNamespace(cache=facade)).resolve() + assert binding.kind == "native" + caller: Final = asyncio.current_task() + SEMANTIC_CONTEXT.set("caller-sentinel") + response: Final = {"choices": [{"text": "paris"}]} + + await binding.async_store( + semantic_request("inline", "what is the capital of france"), response + ) + assert ( + await binding.async_lookup( + semantic_request("inline", f"what is the capital of france{PARAPHRASE_MARKER}") + ) + == response + ) + assert await binding.async_lookup(semantic_request("inline", "python written prompt")) is None + assert SEMANTIC_CONTEXT.get() == "written-in-aembedding" + assert semantic_embedding.async_calls == [ + { + "model": "deterministic", + "input": ["what is the capital of france"], + "task": caller, + "context": "caller-sentinel", + }, + { + "model": "deterministic", + "input": [f"what is the capital of france{PARAPHRASE_MARKER}"], + "task": caller, + "context": "written-in-aembedding", + }, + { + "model": "deterministic", + "input": ["python written prompt"], + "task": caller, + "context": "written-in-aembedding", + }, + ], semantic_embedding.async_calls + + +async def test_native_semantic_cancellation_during_embedding_skips_the_backend( + redis_stack: tuple[str, str], semantic_embedding: DeterministicEmbedding +) -> None: + url, index = redis_stack + facade: Final = semantic_facade(url, index) + binding: Final = _CacheTestResolver(SimpleNamespace(cache=facade)).resolve() + assert binding.kind == "native" + semantic_embedding.gate = asyncio.Event() + + async def lookup() -> object: + return await binding.async_lookup( + semantic_request("cancel", "cancelled prompt") + ) + + task: Final = asyncio.create_task(lookup()) + await semantic_embedding.entered.wait() + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + semantic_embedding.gate.set() + + assert len(semantic_embedding.async_calls) == 1 + assert ( + await cast(RedisSemanticCache, facade.cache).async_get_cache( # pyright: ignore[reportUnknownMemberType] # **kwargs stays unknown on the backend class + "cancel", messages=semantic_messages("cancelled prompt") + ) + is None + ) + + +def test_redis_semantic_similarity_tag_and_threshold_boundaries( + redis_stack: tuple[str, str], semantic_embedding: DeterministicEmbedding +) -> None: + url, index = redis_stack + facade: Final = semantic_facade(url, index) + binding: Final = _CacheTestResolver(SimpleNamespace(cache=facade)).resolve() + + binding.store(semantic_request("sim", "tell me a joke"), {"answer": "haha"}) + paraphrase: Final = f"tell me a joke{PARAPHRASE_MARKER}" + assert binding.lookup(semantic_request("sim", paraphrase)) == {"answer": "haha"} + assert binding.lookup(semantic_request("sim", "an unrelated question about spreadsheets")) is None + assert binding.lookup(semantic_request("other-key", "tell me a joke")) is None + + strict: Final = semantic_facade(url, index, similarity_threshold=0.99) + strict_binding: Final = _CacheTestResolver(SimpleNamespace(cache=strict)).resolve() + assert strict_binding.lookup(semantic_request("sim", paraphrase)) is None + assert strict_binding.lookup(semantic_request("sim", "tell me a joke")) == {"answer": "haha"} + + +def test_redis_semantic_ttl_is_written_only_when_requested( + redis_stack: tuple[str, str], semantic_embedding: DeterministicEmbedding +) -> None: + url, index = redis_stack + facade: Final = semantic_facade(url, index) + binding: Final = _CacheTestResolver(SimpleNamespace(cache=facade)).resolve() + client: Final = redis.Redis.from_url(url) + + binding.store( + {**semantic_request("ttl", "ttl prompt"), "ttl_seconds": 12.0}, {"answer": 1} + ) + expiring: Final = f"{index}:{semantic_entry_id('ttl prompt', 'ttl')}" + assert 0 < client.ttl(expiring) <= 12 + + binding.store(semantic_request("ttl-none", "untimed prompt"), {"answer": 2}) + persistent: Final = f"{index}:{semantic_entry_id('untimed prompt', 'ttl-none')}" + assert client.ttl(persistent) == -1 + + binding.store( + {**semantic_request("ttl-fraction", "fractional prompt"), "ttl_seconds": 1.5}, + {"answer": 3}, + ) + fractional: Final = f"{index}:{semantic_entry_id('fractional prompt', 'ttl-fraction')}" + assert client.ttl(fractional) == 2 + client.close() + + +def test_redis_semantic_malformed_response_is_a_miss_for_both_readers( + redis_stack: tuple[str, str], semantic_embedding: DeterministicEmbedding +) -> None: + url, index = redis_stack + facade: Final = semantic_facade(url, index) + binding: Final = _CacheTestResolver(SimpleNamespace(cache=facade)).resolve() + client: Final = redis.Redis.from_url(url) + + binding.store(semantic_request("bad", "corrupt me"), {"answer": 1}) + hash_key: Final = f"{index}:{semantic_entry_id('corrupt me', 'bad')}" + client.hset(hash_key, "response", b"{not json") + assert binding.lookup(semantic_request("bad", "corrupt me")) is None + assert ( + cast(RedisSemanticCache, facade.cache).get_cache( # pyright: ignore[reportUnknownMemberType] # **kwargs stays unknown on the backend class + "bad", messages=semantic_messages("corrupt me") + ) + is None + ) + client.close() + + +async def test_redis_semantic_unsupported_operations_raise_not_implemented( + redis_stack: tuple[str, str], semantic_embedding: DeterministicEmbedding +) -> None: + url, index = redis_stack + facade: Final = semantic_facade(url, index) + binding: Final = _CacheTestResolver(SimpleNamespace(cache=facade)).resolve() + + with pytest.raises(NotImplementedError): + binding.lookup_batch([semantic_request("batch", "prompt one")]) + with pytest.raises(NotImplementedError): + await binding.async_lookup_batch([semantic_request("batch", "prompt one")]) + with pytest.raises(NotImplementedError): + await binding.async_flush() + with pytest.raises(NotImplementedError): + await binding.ping() + + +def test_redis_semantic_requests_without_prompt_are_noops( + redis_stack: tuple[str, str], semantic_embedding: DeterministicEmbedding +) -> None: + url, index = redis_stack + facade: Final = semantic_facade(url, index) + binding: Final = _CacheTestResolver(SimpleNamespace(cache=facade)).resolve() + client: Final = redis.Redis.from_url(url) + + binding.store(request("plain"), {"answer": 1}) + assert binding.lookup(request("plain")) is None + assert semantic_embedding.calls == [] + assert client.keys(f"{index}:*") == [] + client.close() + + +def test_redis_semantic_scope_overrides_the_tag_and_isolates_entries( + redis_stack: tuple[str, str], semantic_embedding: DeterministicEmbedding +) -> None: + url, index = redis_stack + facade: Final = semantic_facade(url, index) + binding: Final = _CacheTestResolver(SimpleNamespace(cache=facade)).resolve() + client: Final = redis.Redis.from_url(url) + + scoped: Final = {**semantic_request("scoped", "scoped prompt"), "scope": "team-a"} + binding.store(scoped, {"answer": "kept"}) + hash_key: Final = f"{index}:{semantic_entry_id('scoped prompt', 'team-a')}" + assert client.hget(hash_key, "litellm_cache_key") == b"team-a" + assert binding.lookup(scoped) == {"answer": "kept"} + assert binding.lookup(semantic_request("scoped", "scoped prompt")) is None + assert binding.lookup({**scoped, "scope": "team-b"}) is None + client.close() + + +def test_redis_semantic_configuration_drift_falls_back_to_python( + redis_stack: tuple[str, str], + semantic_embedding: DeterministicEmbedding, + monkeypatch: pytest.MonkeyPatch, +) -> None: + url, index = redis_stack + facade: Final = semantic_facade(url, index) + resolver: Final = _CacheTestResolver(SimpleNamespace(cache=facade)) + assert resolver.resolve().kind == "native" + + with rebound(facade.cache, "similarity_threshold", 0.5): + assert resolver.resolve().kind == "python_callback" + with rebound(facade, "semantic_cache_scope", "end_user"): + assert resolver.resolve().kind == "python_callback" + with rebound(facade.cache, "embedding_model", "other-model"): + assert resolver.resolve().kind == "python_callback" + with rebound(facade.cache, "_index_name", "other-index"): + assert resolver.resolve().kind == "python_callback" + with rebound(facade.cache, "CACHE_KEY_FIELD_NAME", "other-field"): + assert resolver.resolve().kind == "python_callback" + + def patched_embedding(self: object, prompt: str, metadata: object = None) -> list[float]: + return _semantic_embedding(prompt) + + monkeypatch.setattr(RedisSemanticCache, "_get_embedding", patched_embedding) + assert resolver.resolve().kind == "python_callback" + + +def test_redis_semantic_handle_rejects_wrong_backends( + redis_stack: tuple[str, str], semantic_embedding: DeterministicEmbedding +) -> None: + url, index = redis_stack + + class CustomSemanticCache(RedisSemanticCache): + pass + + with pytest.raises(TypeError, match="built-in RedisSemanticCache"): + _CacheTestHandle.redis_semantic(object()) + with pytest.raises(TypeError, match="built-in RedisSemanticCache"): + _CacheTestHandle.redis_semantic( + CustomSemanticCache( + redis_url=url, + similarity_threshold=0.8, + embedding_model=SEMANTIC_EMBEDDING_MODEL, + index_name=f"{index}_subclass", + ) + ) + + facade: Final = semantic_facade(url, index) + with pytest.raises(TypeError, match="backend types must match"): + _CacheTestHandle.redis(url)._bind_facade(facade) + + subclassed_facade: Final = Cache( + type=LiteLLMCacheType.REDIS_SEMANTIC, + redis_url=url, + similarity_threshold=0.8, + redis_semantic_cache_embedding_model=SEMANTIC_EMBEDDING_MODEL, + redis_semantic_cache_index_name=index, + ) + subclassed_facade.cache = CustomSemanticCache( # pyright: ignore[reportAttributeAccessIssue] # facade backend slot is not declared + + redis_url=url, + similarity_threshold=0.8, + embedding_model=SEMANTIC_EMBEDDING_MODEL, + index_name=index, + ) + with pytest.raises(TypeError): + _CacheTestHandle.redis_semantic( + subclassed_facade.cache + )._bind_facade(subclassed_facade) + + replacement_facade: Final = Cache( + type=LiteLLMCacheType.REDIS_SEMANTIC, + redis_url=url, + similarity_threshold=0.8, + redis_semantic_cache_embedding_model=SEMANTIC_EMBEDDING_MODEL, + redis_semantic_cache_index_name=index, + ) + with pytest.raises(TypeError, match="must be the native embedder"): + _CacheTestHandle.redis_semantic(facade.cache)._bind_facade(replacement_facade) + + +def qdrant_facade(qdrant_url: str, collection_name: str) -> Cache: + return Cache( + type=LiteLLMCacheType.QDRANT_SEMANTIC, + qdrant_api_base=qdrant_url, + qdrant_collection_name=collection_name, + similarity_threshold=0.99, + qdrant_semantic_cache_embedding_model="text-embedding-3-small", + qdrant_semantic_cache_vector_size=8, + ) + + +def test_qdrant_semantic_facade_binds_native_and_shares_entries( + qdrant_url: str, fake_embedding_endpoint: str +) -> None: + del fake_embedding_endpoint + messages: Final = [{"role": "user", "content": "shared prompt"}] + collection: Final = f"cache_{uuid4().hex}" + facade: Final = qdrant_facade(qdrant_url, collection) + facade.cache.set_cache( + "python-key", + {"timestamp": time.time(), "response": json.dumps({"id": "py"})}, + messages=messages, + ) + handle: Final = _native._CacheTestHandle.qdrant_semantic( + qdrant_url, + collection_name=collection, + similarity_threshold=0.99, + vector_size=8, + ) + handle._bind_facade(facade) + binding: Final = _native._CacheTestResolver(SimpleNamespace(cache=facade)).resolve() + assert binding.kind == "native" + assert binding.lookup(qdrant_request("python-key", messages)) == {"id": "py"} + binding.store(qdrant_request("native-key", messages), {"id": "native"}) + python_value: Final = facade.cache.get_cache("native-key", messages=messages) + assert isinstance(python_value, dict) + assert python_value["response"] == {"id": "native"} + unrelated: Final = [{"role": "user", "content": "unrelated prompt"}] + assert binding.lookup(qdrant_request("native-key", unrelated)) is None + assert facade.cache.get_cache("native-key", messages=unrelated) is None + assert binding.lookup(qdrant_request("different-key", messages)) is None + assert facade.cache.get_cache("different-key", messages=messages) is None + + +async def test_qdrant_semantic_async_parity( + qdrant_url: str, fake_embedding_endpoint: str +) -> None: + del fake_embedding_endpoint + messages: Final = [{"role": "user", "content": "async prompt"}] + collection: Final = f"cache_{uuid4().hex}" + facade: Final = qdrant_facade(qdrant_url, collection) + handle: Final = _native._CacheTestHandle.qdrant_semantic( + qdrant_url, + collection_name=collection, + similarity_threshold=0.99, + vector_size=8, + ) + handle._bind_facade(facade) + binding: Final = _native._CacheTestResolver(SimpleNamespace(cache=facade)).resolve() + await facade.cache.async_set_cache( + "python-key", + {"timestamp": time.time(), "response": json.dumps({"id": "py"})}, + messages=messages, + ) + assert await binding.async_lookup(qdrant_request("python-key", messages)) == {"id": "py"} + await binding.async_store(qdrant_request("native-key", messages), {"id": "native"}) + python_value: Final = await facade.cache.async_get_cache("native-key", messages=messages) + assert isinstance(python_value, dict) + assert python_value["response"] == {"id": "native"} + + +async def test_qdrant_semantic_async_store_batch_shares_entries( + qdrant_url: str, fake_embedding_endpoint: str +) -> None: + del fake_embedding_endpoint + collection: Final = f"cache_{uuid4().hex}" + facade: Final = qdrant_facade(qdrant_url, collection) + handle: Final = _native._CacheTestHandle.qdrant_semantic( + qdrant_url, + collection_name=collection, + similarity_threshold=0.99, + vector_size=8, + ) + handle._bind_facade(facade) + binding: Final = _native._CacheTestResolver(SimpleNamespace(cache=facade)).resolve() + entries: Final = [ + qdrant_request("batch-one", [{"role": "user", "content": "first batch prompt"}]), + qdrant_request("batch-two", [{"role": "user", "content": "second batch prompt"}]), + ] + await binding.async_store_batch(entries, [{"id": "one"}, {"id": "two"}]) + + assert binding.lookup(entries[0]) == {"id": "one"} + assert binding.lookup(entries[1]) == {"id": "two"} + assert ( + (await facade.cache.async_get_cache("batch-one", messages=entries[0]["messages"]))["response"] + == {"id": "one"} + ) + assert ( + (await facade.cache.async_get_cache("batch-two", messages=entries[1]["messages"]))["response"] + == {"id": "two"} + ) + + +async def test_qdrant_semantic_malformed_entries_and_unsupported_operations( + qdrant_url: str, fake_embedding_endpoint: str +) -> None: + del fake_embedding_endpoint + messages: Final = [{"role": "user", "content": "malformed prompt"}] + collection: Final = f"cache_{uuid4().hex}" + facade: Final = qdrant_facade(qdrant_url, collection) + handle: Final = _native._CacheTestHandle.qdrant_semantic( + qdrant_url, + collection_name=collection, + similarity_threshold=0.99, + vector_size=8, + ) + handle._bind_facade(facade) + binding: Final = _native._CacheTestResolver(SimpleNamespace(cache=facade)).resolve() + key: Final = "malformed-key" + response: Final = { + "points": [ + { + "id": str(uuid4()), + "vector": embedding_vector("malformed prompt"), + "payload": { + "litellm_cache_key": key, + "text": "malformed prompt", + "response": "not json", + }, + } + ] + } + facade.cache.sync_client.put( + url=f"{qdrant_url}/collections/{collection}/points", + headers=facade.cache.headers, + json=response, + ) + assert binding.lookup(qdrant_request(key, messages)) is None + with pytest.raises(RuntimeError, match="operation is not supported"): + binding.lookup_batch([qdrant_request(key, messages)]) + with pytest.raises(RuntimeError, match="operation is not supported"): + await binding.async_flush() + with pytest.raises(RuntimeError, match="operation is not supported"): + await binding.ping() + + +def test_qdrant_semantic_ignores_request_expiry( + qdrant_url: str, fake_embedding_endpoint: str +) -> None: + del fake_embedding_endpoint + messages: Final = [{"role": "user", "content": "persistent prompt"}] + collection: Final = f"cache_{uuid4().hex}" + facade: Final = qdrant_facade(qdrant_url, collection) + handle: Final = _native._CacheTestHandle.qdrant_semantic( + qdrant_url, + collection_name=collection, + similarity_threshold=0.99, + vector_size=8, + ) + handle._bind_facade(facade) + binding: Final = _native._CacheTestResolver(SimpleNamespace(cache=facade)).resolve() + binding.store(qdrant_request("persistent-key", messages, ttl_seconds=1.0), {"id": "persistent"}) + time.sleep(1.2) + assert binding.lookup(qdrant_request("persistent-key", messages)) == {"id": "persistent"} + python_value: Final = facade.cache.get_cache("persistent-key", messages=messages) + assert isinstance(python_value, dict) + assert python_value["response"] == {"id": "persistent"} + + +def test_qdrant_semantic_mutation_and_projection_fallback( + qdrant_url: str, fake_embedding_endpoint: str +) -> None: + del fake_embedding_endpoint + collection: Final = f"cache_{uuid4().hex}" + facade: Final = qdrant_facade(qdrant_url, collection) + handle: Final = _native._CacheTestHandle.qdrant_semantic( + qdrant_url, + collection_name=collection, + similarity_threshold=0.99, + vector_size=8, + ) + handle._bind_facade(facade) + facade.cache.qdrant_api_key = "rotated" + assert _native._CacheTestResolver(SimpleNamespace(cache=facade)).resolve().kind == "python_callback" + facade.cache.similarity_threshold = 0.5 + assert _native._CacheTestResolver(SimpleNamespace(cache=facade)).resolve().kind == "python_callback" + unsupported: Final = qdrant_facade(qdrant_url, f"cache_{uuid4().hex}") + unsupported.cache.embedding_max_input_tokens = 100 + with pytest.raises(TypeError, match="requires Python"): + handle._bind_facade(unsupported) + unsupported.cache.embedding_max_input_tokens = None + unsupported.cache.qdrant_api_base = "http://127.0.0.1:7777" + with pytest.raises(TypeError, match="gRPC"): + handle._bind_facade(unsupported) diff --git a/tests/test_litellm_rust/test_valkey_semantic_cache_native.py b/tests/test_litellm_rust/test_valkey_semantic_cache_native.py new file mode 100644 index 00000000000..c87a9f86a80 --- /dev/null +++ b/tests/test_litellm_rust/test_valkey_semantic_cache_native.py @@ -0,0 +1,599 @@ +import asyncio +import contextvars +import hashlib +import os +import struct +import threading +import time +from collections.abc import Generator, Mapping +from types import SimpleNamespace +from typing import Final, cast +from uuid import uuid4 + +import pytest +import redis + +from litellm.caching.caching import Cache +from litellm.caching.valkey_semantic_cache import ValkeySemanticCache +from litellm.rust_bridge import _native +from litellm.types.caching import LiteLLMCacheType + +pytestmark: Final = pytest.mark.requires_rust_extension +embedding_context: Final = contextvars.ContextVar("embedding_context") + + +@pytest.fixture +def valkey_url() -> str: + url: Final = os.environ.get("LITELLM_TEST_VALKEY_URL") + if url is None: + pytest.skip("LITELLM_TEST_VALKEY_URL is not set") + return url + + +@pytest.fixture +def index_name(valkey_url: str) -> Generator[str]: + index: Final = f"litellm_test_{uuid4().hex}" + yield index + client: Final = redis.Redis.from_url(valkey_url) + try: + client.ft(index).dropindex(delete_documents=True) + except redis.ResponseError: + pass + finally: + client.close() + + +def _request(prompt: str = "semantic cache prompt") -> dict[str, object]: + return { + "key": {"preset": "key"}, + "messages": [{"role": "user", "content": prompt}], + } + + +def _field_request( + prompt: str, + metadata: Mapping[str, object], + *, + namespace: str | None = None, + litellm_metadata: Mapping[str, object] | None = None, + litellm_params: Mapping[str, object] | None = None, +) -> dict[str, object]: + request: Final = { + "key": { + "fields": [ + { + "name": "model", + "value": "gpt-4.1", + "api_parameter": True, + "internal_parameter": False, + }, + { + "name": "messages", + "value": prompt, + "api_parameter": True, + "internal_parameter": False, + }, + ], + "namespace": namespace, + }, + "messages": [{"role": "user", "content": prompt}], + "metadata": dict(metadata), + } + if litellm_metadata is not None: + request["litellm_metadata"] = dict(litellm_metadata) + if litellm_params is not None: + request["litellm_params"] = dict(litellm_params) + return request + + +def _facade( + url: str, + index_name: str, + embeddings: Mapping[str, list[float]], + *, + namespace: str | None = None, +) -> Cache: + facade: Final = Cache( + type=LiteLLMCacheType.VALKEY_SEMANTIC, + redis_url=url, + similarity_threshold=0.8, + valkey_semantic_cache_index_name=index_name, + namespace=namespace, + ) + vectors: Final = embeddings + + def embed(prompt: str, metadata: Mapping[str, object] | None = None) -> list[float]: + return vectors[prompt] + + async def async_embedding(prompt: str, metadata: dict[str, object] | None = None) -> list[float]: + return vectors[prompt] + + facade.cache._get_embedding = embed + facade.cache._get_async_embedding = async_embedding + return facade + + +def _backend( + url: str, + index_name: str, + embeddings: Mapping[str, list[float]] | None = None, +) -> ValkeySemanticCache: + vectors: Final = embeddings or {"semantic cache prompt": [1.0, 0.0]} + backend: Final = ValkeySemanticCache( + redis_url=url, + similarity_threshold=0.8, + index_name=index_name, + ) + + def embed(prompt: str, metadata: Mapping[str, object] | None = None) -> list[float]: + return vectors[prompt] + + async def async_embedding(prompt: str, metadata: dict[str, object] | None = None) -> list[float]: + return vectors[prompt] + + backend._get_embedding = embed + backend._get_async_embedding = async_embedding + return backend + + +def test_python_write_native_read( + valkey_url: str, + index_name: str, +) -> None: + backend: Final = _backend(valkey_url, index_name) + response: Final = {"answer": "python"} + backend.set_cache("key", response, messages=_request()["messages"]) + handle: Final = _native._CacheTestHandle.valkey_semantic( + valkey_url, + 0.8, + index_name, + backend, + ) + binding: Final = _native._CacheTestResolver(SimpleNamespace(cache=handle)).resolve() + assert binding.lookup(_request()) == response + + +def test_native_write_python_read( + valkey_url: str, + index_name: str, +) -> None: + backend: Final = _backend(valkey_url, index_name) + handle: Final = _native._CacheTestHandle.valkey_semantic( + valkey_url, + 0.8, + index_name, + backend, + ) + binding: Final = _native._CacheTestResolver(SimpleNamespace(cache=handle)).resolve() + response: Final = {"answer": "native"} + binding.store({**_request(), "ttl_seconds": 2.0}, response) + cached: Final = cast(Mapping[str, object], backend.get_cache("key", messages=_request()["messages"])) + assert cached["response"] == response + + +async def test_async_lookup_and_store( + valkey_url: str, + index_name: str, +) -> None: + backend: Final = _backend(valkey_url, index_name) + handle: Final = _native._CacheTestHandle.valkey_semantic( + valkey_url, + 0.8, + index_name, + backend, + ) + binding: Final = _native._CacheTestResolver(SimpleNamespace(cache=handle)).resolve() + request: Final = {**_request(), "ttl_seconds": 2.0} + await binding.async_store(request, {"answer": "async"}) + assert await binding.async_lookup(request) == {"answer": "async"} + + +async def test_disabled_cache_controls_skip_async_embedding( + valkey_url: str, + index_name: str, +) -> None: + backend: Final = _backend(valkey_url, index_name) + calls: Final = [] + + async def fail_embedding(prompt: str, metadata: dict[str, object] | None = None) -> list[float]: + calls.append(prompt) + raise AssertionError("embedding must not run") + + backend._get_async_embedding = fail_embedding + handle: Final = _native._CacheTestHandle.valkey_semantic( + valkey_url, + 0.8, + index_name, + backend, + ) + binding: Final = _native._CacheTestResolver(SimpleNamespace(cache=handle)).resolve() + controls: Final = { + "supported_call_type": True, + "configured": True, + "native_backend": True, + "default_on": True, + "caching": True, + "no_cache": False, + "no_store": False, + "use_cache": True, + } + no_read_request: Final = {**_request(), "controls": {**controls, "no_cache": True}} + assert await binding.async_lookup(no_read_request) is None + no_write_request: Final = {**_request(), "controls": {**controls, "no_store": True}} + await binding.async_store(no_write_request, {"answer": "blocked"}) + assert calls == [] + client: Final = redis.Redis.from_url(valkey_url) + assert list(client.scan_iter(f"{index_name}:*")) == [] + client.close() + + +async def test_async_embedding_runs_inline_in_caller_task( + valkey_url: str, + index_name: str, +) -> None: + backend: Final = _backend(valkey_url, index_name) + observed: dict[str, object] = {} + + async def async_embedding(prompt: str, metadata: dict[str, object] | None = None) -> list[float]: + observed["context"] = embedding_context.get("missing") + observed["task"] = asyncio.current_task() + observed["thread"] = threading.get_ident() + embedding_context.set("embedder") + return [1.0, 0.0] + + backend._get_async_embedding = async_embedding + handle: Final = _native._CacheTestHandle.valkey_semantic( + valkey_url, + 0.8, + index_name, + backend, + ) + binding: Final = _native._CacheTestResolver(SimpleNamespace(cache=handle)).resolve() + request: Final = {**_request(), "ttl_seconds": 2.0} + caller_task: Final = asyncio.current_task() + caller_thread: Final = threading.get_ident() + token: Final = embedding_context.set("caller") + try: + await binding.async_store(request, {"answer": "inline"}) + assert observed["context"] == "caller" + assert observed["task"] is caller_task + assert observed["thread"] == caller_thread + assert embedding_context.get() == "embedder" + assert await binding.async_lookup(request) == {"answer": "inline"} + finally: + embedding_context.reset(token) + + +def test_facade_activation_and_mutation_fallback( + valkey_url: str, + index_name: str, +) -> None: + facade: Final = Cache( + type=LiteLLMCacheType.VALKEY_SEMANTIC, + redis_url=valkey_url, + similarity_threshold=0.8, + valkey_semantic_cache_index_name=index_name, + ) + handle: Final = _native._CacheTestHandle.valkey_semantic( + valkey_url, + 0.8, + index_name, + facade.cache, + ) + handle._bind_facade(facade) + resolver: Final = _native._CacheTestResolver(SimpleNamespace(cache=facade)) + assert resolver.resolve().kind == "native" + facade.cache.similarity_threshold = 0.7 + assert resolver.resolve().kind == "python_callback" + + +def test_batch_lookup_is_unsupported( + valkey_url: str, + index_name: str, +) -> None: + backend: Final = _backend(valkey_url, index_name) + handle: Final = _native._CacheTestHandle.valkey_semantic( + valkey_url, + 0.8, + index_name, + backend, + ) + binding: Final = _native._CacheTestResolver(SimpleNamespace(cache=handle)).resolve() + with pytest.raises(NotImplementedError): + binding.lookup_batch([_request()]) + + +def test_ttl_expiry( + valkey_url: str, + index_name: str, +) -> None: + backend: Final = _backend(valkey_url, index_name) + handle: Final = _native._CacheTestHandle.valkey_semantic(valkey_url, 0.8, index_name, backend) + binding: Final = _native._CacheTestResolver(SimpleNamespace(cache=handle)).resolve() + binding.store({**_request(), "ttl_seconds": 1.0}, {"answer": "expires"}) + client: Final = redis.Redis.from_url(valkey_url) + documents: Final = list(client.scan_iter(f"{index_name}:*")) + assert len(documents) == 1 + assert client.ttl(documents[0]) > 0 + time.sleep(1.5) + assert binding.lookup(_request()) is None + + +def test_no_ttl_is_persistent_and_python_reads_native_value( + valkey_url: str, + index_name: str, +) -> None: + backend: Final = _backend(valkey_url, index_name) + handle: Final = _native._CacheTestHandle.valkey_semantic(valkey_url, 0.8, index_name, backend) + binding: Final = _native._CacheTestResolver(SimpleNamespace(cache=handle)).resolve() + response: Final = {"answer": "persistent"} + binding.store(_request(), response) + client: Final = redis.Redis.from_url(valkey_url) + documents: Final = list(client.scan_iter(f"{index_name}:*")) + assert len(documents) == 1 + assert client.ttl(documents[0]) == -1 + cached: Final = cast(Mapping[str, object], backend.get_cache("key", messages=_request()["messages"])) + assert cached["response"] == response + + +def test_below_threshold_misses_on_native_and_python( + valkey_url: str, + index_name: str, +) -> None: + backend: Final = _backend( + valkey_url, + index_name, + {"prompt A": [1.0, 0.0], "prompt B": [0.0, 1.0]}, + ) + handle: Final = _native._CacheTestHandle.valkey_semantic(valkey_url, 0.8, index_name, backend) + binding: Final = _native._CacheTestResolver(SimpleNamespace(cache=handle)).resolve() + binding.store(_request("prompt A"), {"answer": "A"}) + assert binding.lookup(_request("prompt B")) is None + assert backend.get_cache("key", messages=_request("prompt B")["messages"]) is None + + +def test_malformed_entry_is_a_miss_on_native_and_python( + valkey_url: str, + index_name: str, +) -> None: + backend: Final = _backend(valkey_url, index_name) + client: Final = redis.Redis.from_url(valkey_url) + scope: Final = hashlib.sha256(b"key").hexdigest() + document: Final = f"{index_name}:{scope}:{uuid4().hex}" + client.hset( + document, + mapping={ + "litellm_cache_key": scope, + "prompt": "semantic cache prompt", + "response": "not json", + "embedding": struct.pack("<2f", 1.0, 0.0), + }, + ) + handle: Final = _native._CacheTestHandle.valkey_semantic(valkey_url, 0.8, index_name, backend) + binding: Final = _native._CacheTestResolver(SimpleNamespace(cache=handle)).resolve() + assert binding.lookup(_request()) is None + assert backend.get_cache("key", messages=_request()["messages"]) is None + + +def test_mixed_content_parts_match_python_semantic_behavior( + valkey_url: str, + index_name: str, +) -> None: + backend: Final = _backend(valkey_url, index_name) + messages: Final = [{"role": "user", "content": ["raw", {"text": "hello"}]}] + backend.set_cache("key", {"answer": "mixed"}, messages=messages) + assert backend.get_cache("key", messages=messages) is None + + handle: Final = _native._CacheTestHandle.valkey_semantic(valkey_url, 0.8, index_name, backend) + binding: Final = _native._CacheTestResolver(SimpleNamespace(cache=handle)).resolve() + request: Final = {**_request(), "messages": messages} + binding.store(request, {"answer": "mixed"}) + assert binding.lookup(request) is None + client: Final = redis.Redis.from_url(valkey_url) + assert list(client.scan_iter(f"{index_name}:*")) == [] + client.close() + + +async def test_async_store_batch_and_lookup( + valkey_url: str, + index_name: str, +) -> None: + backend: Final = _backend( + valkey_url, + index_name, + {"prompt A": [1.0, 0.0], "prompt B": [0.0, 1.0]}, + ) + sync_calls: Final = [] + async_tasks: Final = [] + + def sync_embedding(prompt: str, metadata: Mapping[str, object] | None = None) -> list[float]: + sync_calls.append(prompt) + return {"prompt A": [1.0, 0.0], "prompt B": [0.0, 1.0]}[prompt] + + async def async_embedding( + prompt: str, + metadata: dict[str, object] | None = None, + ) -> list[float]: + async_tasks.append(asyncio.current_task()) + return {"prompt A": [1.0, 0.0], "prompt B": [0.0, 1.0]}[prompt] + + backend._get_embedding = sync_embedding + backend._get_async_embedding = async_embedding + handle: Final = _native._CacheTestHandle.valkey_semantic(valkey_url, 0.8, index_name, backend) + binding: Final = _native._CacheTestResolver(SimpleNamespace(cache=handle)).resolve() + requests: Final = [_request("prompt A"), _request("prompt B")] + responses: Final = [{"answer": "A"}, {"answer": "B"}] + caller_task: Final = asyncio.current_task() + await binding.async_store_batch(requests, responses) + assert sync_calls == [] + assert async_tasks + assert all(task is caller_task for task in async_tasks) + assert await binding.async_lookup(requests[0]) == responses[0] + assert await binding.async_lookup(requests[1]) == responses[1] + + +def test_subclass_backend_falls_back_to_python( + valkey_url: str, + index_name: str, +) -> None: + class Custom(ValkeySemanticCache): + pass + + facade: Final = Cache( + type=LiteLLMCacheType.VALKEY_SEMANTIC, + redis_url=valkey_url, + similarity_threshold=0.8, + valkey_semantic_cache_index_name=index_name, + ) + facade.cache = Custom(redis_url=valkey_url, similarity_threshold=0.8, index_name=index_name) + resolver: Final = _native._CacheTestResolver(SimpleNamespace(cache=facade)) + assert resolver.resolve().kind == "python_callback" + + +def test_field_key_matches_python_semantic_scope( + valkey_url: str, + index_name: str, +) -> None: + facade: Final = _facade(valkey_url, index_name, {"semantic cache prompt": [1.0, 0.0]}) + metadata: Final = {"user_api_key": "k1"} + expected: Final = facade.get_cache_key( + model="gpt-4.1", + messages=[{"role": "user", "content": "semantic cache prompt"}], + metadata=metadata, + ) + handle: Final = _native._CacheTestHandle.valkey_semantic( + valkey_url, + 0.8, + index_name, + facade.cache, + ) + binding: Final = _native._CacheTestResolver(SimpleNamespace(cache=handle)).resolve() + binding.store(_field_request("semantic cache prompt", metadata), {"answer": "scoped"}) + client: Final = redis.Redis.from_url(valkey_url) + documents: Final = list(client.scan_iter(f"{index_name}:*")) + assert len(documents) == 1 + document_parts: Final = documents[0].decode().split(":") + assert document_parts[1] == hashlib.sha256(expected.encode()).hexdigest() + client.close() + + +def test_field_key_reads_all_python_tenant_metadata_sources( + valkey_url: str, + index_name: str, +) -> None: + facade: Final = _facade(valkey_url, index_name, {"semantic cache prompt": [1.0, 0.0]}) + params_metadata: Final = {"user_api_key_team_id": "team-from-params"} + expected: Final = facade.get_cache_key( + model="gpt-4.1", + messages=[{"role": "user", "content": "semantic cache prompt"}], + metadata={}, + litellm_params={"metadata": params_metadata}, + ) + handle: Final = _native._CacheTestHandle.valkey_semantic( + valkey_url, + 0.8, + index_name, + facade.cache, + ) + binding: Final = _native._CacheTestResolver(SimpleNamespace(cache=handle)).resolve() + binding.store( + _field_request( + "semantic cache prompt", + {}, + litellm_params={"metadata": params_metadata}, + ), + {"answer": "params"}, + ) + client: Final = redis.Redis.from_url(valkey_url) + documents: Final = list(client.scan_iter(f"{index_name}:*")) + assert len(documents) == 1 + document_parts: Final = documents[0].decode().split(":") + assert document_parts[1] == hashlib.sha256(expected.encode()).hexdigest() + client.close() + + assert ( + binding.lookup( + _field_request( + "semantic cache prompt", + {}, + litellm_metadata={"user_api_key_team_id": "team-from-litellm"}, + ) + ) + is None + ) + + +def test_namespace_isolates_semantic_entries( + valkey_url: str, + index_name: str, +) -> None: + facade: Final = _facade( + valkey_url, + index_name, + {"semantic cache prompt": [1.0, 0.0]}, + namespace="team-a", + ) + handle: Final = _native._CacheTestHandle.valkey_semantic( + valkey_url, + 0.8, + index_name, + facade.cache, + ) + binding: Final = _native._CacheTestResolver(SimpleNamespace(cache=handle)).resolve() + team_a: Final = _field_request("semantic cache prompt", {}, namespace="team-a") + team_b: Final = _field_request("semantic cache prompt", {}, namespace="team-b") + binding.store(team_a, {"answer": "team-a"}) + assert binding.lookup(team_b) is None + assert binding.lookup(team_a) == {"answer": "team-a"} + cached: Final = cast( + Mapping[str, object], + facade.get_cache( + model="gpt-4.1", + messages=[{"role": "user", "content": "semantic cache prompt"}], + ), + ) + assert cached == {"answer": "team-a"} + + +def test_field_key_isolates_tenant_scope( + valkey_url: str, + index_name: str, +) -> None: + facade: Final = _facade(valkey_url, index_name, {"semantic cache prompt": [1.0, 0.0]}) + handle: Final = _native._CacheTestHandle.valkey_semantic( + valkey_url, + 0.8, + index_name, + facade.cache, + ) + binding: Final = _native._CacheTestResolver(SimpleNamespace(cache=handle)).resolve() + binding.store( + _field_request("semantic cache prompt", {"user_api_key": "k1"}), + {"answer": "tenant one"}, + ) + assert binding.lookup(_field_request("semantic cache prompt", {"user_api_key": "k2"})) is None + assert binding.lookup(_field_request("semantic cache prompt", {"user_api_key": "k1"})) == {"answer": "tenant one"} + + +def test_tls_valkey_facade_falls_back_to_python( + index_name: str, +) -> None: + facade: Final = Cache( + type=LiteLLMCacheType.VALKEY_SEMANTIC, + redis_url="rediss://127.0.0.1:6390/0", + similarity_threshold=0.8, + valkey_semantic_cache_index_name=index_name, + ) + resolver: Final = _native._CacheTestResolver(SimpleNamespace(cache=facade)) + assert resolver.resolve().kind == "python_callback" + + +async def test_ping_maps_unsupported_native_operation_to_not_implemented( + valkey_url: str, + index_name: str, +) -> None: + backend: Final = _backend(valkey_url, index_name) + handle: Final = _native._CacheTestHandle.valkey_semantic(valkey_url, 0.8, index_name, backend) + binding: Final = _native._CacheTestResolver(SimpleNamespace(cache=handle)).resolve() + with pytest.raises(NotImplementedError): + await binding.ping() diff --git a/tests/unit/llms/bedrock/files/test_bedrock_files_handler.py b/tests/unit/llms/bedrock/files/test_bedrock_files_handler.py index 639be272351..5c078affffc 100644 --- a/tests/unit/llms/bedrock/files/test_bedrock_files_handler.py +++ b/tests/unit/llms/bedrock/files/test_bedrock_files_handler.py @@ -270,3 +270,40 @@ async def test_afile_content_assumes_role_with_external_id(monkeypatch): assert s3_client_kwargs["aws_access_key_id"] == "ASIAFILESDOWNLOADROLE" assert s3_client_kwargs["aws_session_token"] == "assumed-session-token" assert response.content == b'{"custom_id": "req-1"}' + + +@pytest.mark.asyncio +async def test_afile_content_builds_the_s3_client_with_the_s3_pair_when_it_differs_from_the_aws_identity(): + import boto3 + + class FakeS3Body: + def read(self): + return b'{"custom_id": "req-1"}' + + class FakeS3Client: + def get_object(self, Bucket, Key): + return {"Body": FakeS3Body()} + + optional_params = { + "_litellm_internal_model_credentials": MappingProxyType({"s3_bucket_name": "safe-bucket"}), + "aws_region_name": "us-east-1", + "aws_access_key_id": "AKIABEDROCKONLY", + "aws_secret_access_key": "bedrock-only-secret", + "aws_session_token": "bedrock-only-token", + "s3_access_key_id": "AKIAS3ONLY", + "s3_secret_access_key": "s3-only-secret", + } + + with patch.object(boto3, "client", return_value=FakeS3Client()) as mock_boto3_client: + response = await BedrockFilesHandler().afile_content( + file_content_request={"file_id": "s3://safe-bucket/litellm-bedrock-files-model-id-abc.jsonl"}, + optional_params=optional_params, + timeout=10.0, + max_retries=None, + ) + + s3_client_kwargs = mock_boto3_client.call_args.kwargs + assert s3_client_kwargs["aws_access_key_id"] == "AKIAS3ONLY" + assert s3_client_kwargs["aws_secret_access_key"] == "s3-only-secret" + assert s3_client_kwargs["aws_session_token"] is None, "the aws_* session token belongs to the Bedrock identity" + assert response.content == b'{"custom_id": "req-1"}' diff --git a/tests/unit/llms/bedrock/files/test_bedrock_files_transformation.py b/tests/unit/llms/bedrock/files/test_bedrock_files_transformation.py index 2d2de77269b..d0921e68424 100644 --- a/tests/unit/llms/bedrock/files/test_bedrock_files_transformation.py +++ b/tests/unit/llms/bedrock/files/test_bedrock_files_transformation.py @@ -6,6 +6,7 @@ import json import os from collections.abc import Mapping from contextlib import AsyncExitStack, closing +from types import MappingProxyType from typing import Final from unittest.mock import MagicMock from urllib.parse import unquote, urlparse @@ -3789,3 +3790,82 @@ class TestBedrockFileListTransformation: assert denied.value.status_code == 403 assert "AccessDenied" in denied.value.message + + +_SPLIT_IDENTITY_PARAMS: Final = { + "aws_region_name": "us-east-1", + "aws_access_key_id": "AKIABEDROCKONLY", + "aws_secret_access_key": "bedrock-only-secret", + "s3_access_key_id": "AKIAS3ONLY", + "s3_secret_access_key": "s3-only-secret", + "s3_bucket_name": "safe-bucket", +} + + +def _authorization(headers: Mapping[str, str]) -> str: + return {key.lower(): value for key, value in headers.items()}["authorization"] + + +def test_sign_s3_request_uses_the_s3_pair_when_it_differs_from_the_aws_identity(): + from litellm.llms.bedrock.files.transformation import BedrockFilesConfig + + signed_headers, _signed_body = BedrockFilesConfig()._sign_s3_request( + content='{"custom_id": "req-1"}', + api_base="https://s3.us-east-1.amazonaws.com/safe-bucket/litellm-bedrock-files-model-id-abc.jsonl", + optional_params=dict(_SPLIT_IDENTITY_PARAMS), + ) + + assert _authorization(signed_headers).startswith("AWS4-HMAC-SHA256 Credential=AKIAS3ONLY/"), ( + "the S3 PutObject must be signed by s3_access_key_id, not the Bedrock aws_access_key_id" + ) + + +def test_sign_s3_request_with_the_s3_pair_ignores_ambient_aws_session_token_role_and_profile(monkeypatch): + from litellm.llms.bedrock.files.transformation import BedrockFilesConfig + + monkeypatch.setenv("AWS_SESSION_TOKEN", "pod-token") + monkeypatch.setenv("AWS_ROLE_NAME", "arn:aws:iam::123456789012:role/pod") + monkeypatch.setenv("AWS_PROFILE_NAME", "pod-profile") + signed_headers, _signed_body = BedrockFilesConfig()._sign_s3_request( + content='{"custom_id": "req-1"}', + api_base="https://s3.us-east-1.amazonaws.com/safe-bucket/litellm-bedrock-files-model-id-abc.jsonl", + optional_params=dict(_SPLIT_IDENTITY_PARAMS), + ) + + lowered: Final = {key.lower(): value for key, value in signed_headers.items()} + assert lowered["authorization"].startswith("AWS4-HMAC-SHA256 Credential=AKIAS3ONLY/") + assert "x-amz-security-token" not in lowered, "an ambient AWS_SESSION_TOKEN must not be mixed into the s3_* pair" + + +@pytest.mark.parametrize("method", ["GET", "DELETE"]) +def test_sign_s3_request_without_body_uses_the_s3_pair_when_it_differs_from_the_aws_identity(method): + from litellm.llms.bedrock.files.transformation import BedrockFilesConfig, _BedrockS3RequestParams + + signed_headers = BedrockFilesConfig()._sign_s3_request_without_body( + method=method, + api_base="https://s3.us-east-1.amazonaws.com/safe-bucket/litellm-bedrock-files-model-id-abc.jsonl", + aws_region_name="us-east-1", + request_params=_BedrockS3RequestParams.model_validate(_SPLIT_IDENTITY_PARAMS), + ) + + assert _authorization(signed_headers).startswith("AWS4-HMAC-SHA256 Credential=AKIAS3ONLY/"), ( + f"the S3 {method} must be signed by s3_access_key_id, not the Bedrock aws_access_key_id" + ) + + +def test_transform_file_content_request_signs_with_the_s3_pair_from_litellm_params(): + from litellm.llms.bedrock.files.transformation import S3_SIGNED_REQUEST_HEADERS_PARAM, BedrockFilesConfig + + litellm_params = { + **_SPLIT_IDENTITY_PARAMS, + "_litellm_internal_model_credentials": MappingProxyType({"s3_bucket_name": "safe-bucket"}), + } + BedrockFilesConfig().transform_file_content_request( + file_content_request={"file_id": "s3://safe-bucket/litellm-bedrock-files-model-id-abc.jsonl"}, + optional_params={}, + litellm_params=litellm_params, + ) + + assert _authorization(litellm_params[S3_SIGNED_REQUEST_HEADERS_PARAM]).startswith( + "AWS4-HMAC-SHA256 Credential=AKIAS3ONLY/" + ) diff --git a/tests/unit/models/test_models.py b/tests/unit/models/test_models.py index b8bf55f1b4a..ab456bb1624 100644 --- a/tests/unit/models/test_models.py +++ b/tests/unit/models/test_models.py @@ -324,6 +324,8 @@ class TestUser: assert user_no_models.has_model_access("any-model") def test_password_hash_excluded_from_serialization(self): + import json + from litellm.proxy._types import LiteLLM_UserTableWithKeyCount secret = "$2b$12$abcdefghijklmnopqrstuv" @@ -331,12 +333,12 @@ class TestUser: assert user.password == secret assert "password" not in user.model_dump() - assert "password" not in user.model_dump_json() + assert "password" not in json.loads(user.model_dump_json()) with_keys = LiteLLM_UserTableWithKeyCount(user_id="u1", user_email="a@b.c", password=secret, key_count=2) assert with_keys.password == secret assert "password" not in with_keys.model_dump() - assert "password" not in with_keys.model_dump_json() + assert "password" not in json.loads(with_keys.model_dump_json()) class TestVerificationToken: diff --git a/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/AgentFormKit.tsx b/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/AgentFormKit.tsx index 15b85001cdc..8e100d0c3ed 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/AgentFormKit.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/AgentFormKit.tsx @@ -90,6 +90,7 @@ export interface AgentFormValues { guardrails?: string[]; entitlement_models?: string[]; entitlement_agents?: string[]; + access_group_ids?: string[]; allowed_mcp_servers_and_groups?: McpServerSelection; mcp_tool_permissions?: Record; defaultInputModes?: string[]; @@ -121,6 +122,7 @@ export interface AgentRequestPayload { agent_card_params?: Record; litellm_params?: Record; object_permission?: Record; + access_group_ids?: string[]; } interface AgentFormFieldProps { diff --git a/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/add_agent_form.integration.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/add_agent_form.integration.test.tsx index ebc97891744..457ee656415 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/add_agent_form.integration.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/add_agent_form.integration.test.tsx @@ -21,6 +21,9 @@ vi.mock("./agent_card_discovery", () => ({ default: () =>
({ default: () =>
})); vi.mock("@/components/mcp_server_management/MCPToolPermissions", () => ({ default: () =>
})); vi.mock("@/components/guardrails/GuardrailSelector", () => ({ default: () =>
})); +vi.mock("@/app/(dashboard)/hooks/accessGroups/useAccessGroups", () => ({ + useAccessGroups: () => ({ data: [], isLoading: false, isError: false }), +})); vi.mock("@/components/common_components/team_dropdown", () => ({ default: () =>
})); const a2aInfo: AgentCreateInfo = { diff --git a/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/add_agent_form.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/add_agent_form.test.tsx index ccac244f019..fbf5cf8c1fb 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/add_agent_form.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/add_agent_form.test.tsx @@ -44,6 +44,14 @@ vi.mock("@/components/mcp_server_management/MCPToolPermissions", () => ({ default: () => null, })); +vi.mock("@/components/common_components/AccessGroupSelector", () => ({ + default: ({ onChange }: { onChange: (value: string[]) => void }) => ( + + ), +})); + vi.mock("@/components/common_components/team_dropdown", () => ({ default: () => null, })); @@ -141,5 +149,29 @@ describe("AddAgentForm logos", () => { await vi.waitFor(() => expect(networking.createAgentCall).toHaveBeenCalled()); const [, payload] = vi.mocked(networking.createAgentCall).mock.calls[0]; expect(payload.object_permission).toEqual({ mcp_toolsets: ["ts-1"] }); + expect(payload).not.toHaveProperty("access_group_ids"); + }); + + it("includes selected access groups in the create payload", async () => { + const user = userEvent.setup({ pointerEventsCheck: PointerEventsCheckLevel.Never }); + vi.mocked(networking.createAgentCall) + .mockReset() + .mockResolvedValue({ + agent_id: "agent-1", + agent_name: "Test Agent", + } as never); + vi.mocked(networking.keyListCall).mockResolvedValue({ keys: [] }); + + renderForm(); + await user.click(screen.getByRole("button", { name: "Next →" })); + await user.click(screen.getByTestId("select-access-group")); + await user.click(screen.getByRole("button", { name: "Next →" })); + await user.click(screen.getByRole("button", { name: "Next →" })); + await user.click(screen.getByText(/Skip for now/)); + await user.click(screen.getByRole("button", { name: "Create Agent →" })); + + await vi.waitFor(() => expect(networking.createAgentCall).toHaveBeenCalled()); + const [, payload] = vi.mocked(networking.createAgentCall).mock.calls[0]; + expect(payload.access_group_ids).toEqual(["ag-1", "ag-2"]); }); }); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/add_agent_form.tsx b/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/add_agent_form.tsx index e71fed40209..5bd6ea9b83a 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/add_agent_form.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/add_agent_form.tsx @@ -50,6 +50,7 @@ import { } from "./AgentFormKit"; import MCPServerSelector from "@/components/mcp_server_management/MCPServerSelector"; import MCPToolPermissions from "@/components/mcp_server_management/MCPToolPermissions"; +import AccessGroupSelector from "@/components/common_components/AccessGroupSelector"; import GuardrailSelector from "@/components/guardrails/GuardrailSelector"; import { Dialog, DialogContent, DialogHeader, DialogTitle } from "@/components/ui/dialog"; @@ -113,6 +114,7 @@ const SHARED_INITIAL_VALUES: AgentFormValues = { mcp_tool_permissions: {}, entitlement_models: [], entitlement_agents: [], + access_group_ids: [], guardrails: [], }; @@ -374,6 +376,9 @@ const AddAgentForm: React.FC = ({ visible, onClose, accessTok if (Object.keys(objectPermission).length > 0) { agentData.object_permission = objectPermission; } + if (values.access_group_ids?.length) { + agentData.access_group_ids = values.access_group_ids; + } // Wire trace-id flags and budget controls into agent litellm_params (before create call) if (requireTraceIdInbound || requireTraceIdOutbound) { @@ -494,6 +499,22 @@ const AddAgentForm: React.FC = ({ visible, onClose, accessTok )} + + {({ value, onChange }) => ( + + )} + + { return agentData; }; +export const parseAccessGroupIdsForForm = (agent: { access_group_ids?: string[] | null }) => ({ + access_group_ids: agent.access_group_ids ?? [], +}); + export const parseMcpPermissionsForForm = (agent: any) => ({ allowed_mcp_servers_and_groups: { servers: agent.object_permission?.mcp_servers ?? [], @@ -377,5 +381,6 @@ export const parseAgentForForm = (agent: any) => { // extra_headers: already an array of strings extra_headers: agent.extra_headers ?? [], ...parseMcpPermissionsForForm(agent), + ...parseAccessGroupIdsForForm(agent), }; }; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/agent_info.integration.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/agent_info.integration.test.tsx index 357f924cbe7..37e00766a75 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/agent_info.integration.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/agent_info.integration.test.tsx @@ -25,6 +25,10 @@ vi.mock("@/app/(dashboard)/hooks/keys/useKeys", () => ({ vi.mock("./agent_card_discovery", () => ({ default: () =>
})); +vi.mock("@/app/(dashboard)/hooks/accessGroups/useAccessGroups", () => ({ + useAccessGroups: () => ({ data: [], isLoading: false, isError: false }), +})); + const A2A_AGENT = { agent_id: "agent-1", agent_name: "my-agent", @@ -176,6 +180,7 @@ describe("AgentInfoView update payload", () => { session_tpm_limit: 333, session_rpm_limit: 444, object_permission: { mcp_servers: [], mcp_access_groups: [], mcp_toolsets: [], mcp_tool_permissions: {} }, + access_group_ids: [], }); }); @@ -217,6 +222,7 @@ describe("AgentInfoView update payload", () => { session_tpm_limit: 333, session_rpm_limit: 444, object_permission: { mcp_servers: [], mcp_access_groups: [], mcp_toolsets: [], mcp_tool_permissions: {} }, + access_group_ids: [], }); }); @@ -295,6 +301,7 @@ describe("AgentInfoView update payload", () => { model: "langgraph/asst_1", }, object_permission: { mcp_servers: [], mcp_access_groups: [], mcp_toolsets: [], mcp_tool_permissions: {} }, + access_group_ids: [], }); }); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/agent_info.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/agent_info.test.tsx index 29d97a20afe..7e6c7c0e05c 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/agent_info.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/agent_info.test.tsx @@ -28,6 +28,28 @@ vi.mock("@/app/(dashboard)/hooks/mcpServers/useMCPServers", () => ({ useMCPServers: () => ({ data: [{ server_id: "srv-1", server_name: "github" }] }), })); +vi.mock("@/app/(dashboard)/hooks/accessGroups/useAccessGroups", () => ({ + useAccessGroups: () => ({ + data: [{ access_group_id: "ag-1", access_group_name: "support-tools" }], + isLoading: false, + isError: false, + }), +})); + +vi.mock("@/components/common_components/AccessGroupSelector", () => ({ + default: ({ value, onChange }: { value?: string[]; onChange: (value: string[]) => void }) => ( +
+ {(value ?? []).join(",")} + + +
+ ), +})); + vi.mock("@/components/mcp_server_management/MCPServerSelector", () => ({ default: () =>
, })); @@ -76,6 +98,38 @@ describe("AgentInfoView settings", () => { expect(payload.tpm_limit).toBe(42); const clearedMcpGrants = { mcp_servers: [], mcp_access_groups: [], mcp_toolsets: [], mcp_tool_permissions: {} }; expect(payload.object_permission).toEqual(clearedMcpGrants); + expect(payload.access_group_ids).toEqual([]); + }); + + it("sends the newly attached access group in the update payload", async () => { + render(); + + fireEvent.click(await screen.findByRole("tab", { name: "Settings" })); + fireEvent.click(screen.getByRole("button", { name: "Edit Settings" })); + fireEvent.click(await screen.findByRole("button", { name: "Attach ag-1" })); + expect(screen.getByTestId("selected-access-groups")).toHaveTextContent("ag-1"); + + fireEvent.click(screen.getByRole("button", { name: /Save Changes/ })); + + await waitFor(() => expect(networking.patchAgentCall).toHaveBeenCalledTimes(1)); + const [, , payload] = vi.mocked(networking.patchAgentCall).mock.calls[0]; + expect(payload.access_group_ids).toEqual(["ag-1"]); + }); + + it("loads the attached access groups into the editor and sends an empty list once detached", async () => { + vi.mocked(networking.getAgentInfo).mockResolvedValue({ ...agent, access_group_ids: ["ag-1"] }); + render(); + + fireEvent.click(await screen.findByRole("tab", { name: "Settings" })); + fireEvent.click(screen.getByRole("button", { name: "Edit Settings" })); + expect(await screen.findByTestId("selected-access-groups")).toHaveTextContent("ag-1"); + + fireEvent.click(screen.getByRole("button", { name: "Detach all access groups" })); + fireEvent.click(screen.getByRole("button", { name: /Save Changes/ })); + + await waitFor(() => expect(networking.patchAgentCall).toHaveBeenCalledTimes(1)); + const [, , payload] = vi.mocked(networking.patchAgentCall).mock.calls[0]; + expect(payload.access_group_ids).toEqual([]); }); it("shows MCP grants with server names on the overview tab", async () => { @@ -88,4 +142,20 @@ describe("AgentInfoView settings", () => { expect(await screen.findByText("github (srv-1)")).toBeInTheDocument(); }); + + it("shows attached access groups with their names on the overview tab", async () => { + vi.mocked(networking.getAgentInfo).mockResolvedValue({ ...agent, access_group_ids: ["ag-1", "ag-unknown"] }); + + render(); + + expect(await screen.findByText("support-tools (ag-1)")).toBeInTheDocument(); + expect(screen.getByText("ag-unknown")).toBeInTheDocument(); + }); + + it("shows None when the agent has no access groups attached", async () => { + render(); + + expect(await screen.findByText("Access Groups")).toBeInTheDocument(); + expect(screen.getByText("None")).toBeInTheDocument(); + }); }); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/agent_info.tsx b/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/agent_info.tsx index 6cb99e9692f..adac456f232 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/agent_info.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/agent_info.tsx @@ -16,6 +16,8 @@ import { Agent } from "@/components/agents/types"; import { KeyResponse } from "@/components/key_team_helpers/key_list"; import { useKeys } from "@/app/(dashboard)/hooks/keys/useKeys"; import { useMCPServers } from "@/app/(dashboard)/hooks/mcpServers/useMCPServers"; +import { useAccessGroups } from "@/app/(dashboard)/hooks/accessGroups/useAccessGroups"; +import AccessGroupSelector from "@/components/common_components/AccessGroupSelector"; import KeyInfoView from "@/components/templates/key_info_view"; import MCPServerSelector from "@/components/mcp_server_management/MCPServerSelector"; import MCPToolPermissions from "@/components/mcp_server_management/MCPToolPermissions"; @@ -26,6 +28,7 @@ import { AGENT_FORM_CONFIG, buildAgentDataFromForm, buildMcpObjectPermission, + parseAccessGroupIdsForForm, parseAgentForForm, parseMcpPermissionsForForm, } from "./agent_config"; @@ -122,7 +125,11 @@ const AgentInfoView: React.FC = ({ agentId, onClose, accessT } else { const typeInfo = agentTypeMetadata.find((t) => t.agent_type === agentType); if (typeInfo) { - form.reset({ ...parseDynamicAgentForForm(data, typeInfo), ...parseMcpPermissionsForForm(data) }); + form.reset({ + ...parseDynamicAgentForForm(data, typeInfo), + ...parseMcpPermissionsForForm(data), + ...parseAccessGroupIdsForForm(data), + }); } else { form.reset(parseAgentForForm(data)); } @@ -142,7 +149,11 @@ const AgentInfoView: React.FC = ({ agentId, onClose, accessT if (agentType !== "a2a") { const typeInfo = agentTypeMetadata.find((t) => t.agent_type === agentType); if (typeInfo) { - form.reset({ ...parseDynamicAgentForForm(agent, typeInfo), ...parseMcpPermissionsForForm(agent) }); + form.reset({ + ...parseDynamicAgentForForm(agent, typeInfo), + ...parseMcpPermissionsForForm(agent), + ...parseAccessGroupIdsForForm(agent), + }); } } } @@ -153,12 +164,18 @@ const AgentInfoView: React.FC = ({ agentId, onClose, accessT const mcpSelection = useWatch({ control: form.control, name: "allowed_mcp_servers_and_groups" }); const mcpToolPermissions = useWatch({ control: form.control, name: "mcp_tool_permissions" }); const { data: mcpServers = [] } = useMCPServers(); + const { data: accessGroups = [] } = useAccessGroups(); const mcpServerLabel = (serverId: string) => { const server = mcpServers.find((s) => s.server_id === serverId); return server?.server_name ? `${server.server_name} (${serverId})` : serverId; }; + const accessGroupLabel = (accessGroupId: string) => { + const group = accessGroups.find((g) => g.access_group_id === accessGroupId); + return group ? `${group.access_group_name} (${accessGroupId})` : accessGroupId; + }; + const discoveryRequest = useMemo( () => buildDiscoveryRequest(detectedAgentType, watchedFormValues || {}, selectedAgentTypeInfo), [watchedFormValues, selectedAgentTypeInfo, detectedAgentType], @@ -221,6 +238,7 @@ const AgentInfoView: React.FC = ({ agentId, onClose, accessT await patchAgentCall(accessToken, agentId, { ...updateData, object_permission: buildMcpObjectPermission(values), + access_group_ids: values.access_group_ids ?? [], }); toast.success("Agent updated successfully"); setIsEditing(false); @@ -350,6 +368,17 @@ const AgentInfoView: React.FC = ({ agentId, onClose, accessT {agent.rpm_limit ?? "Unlimited"} {agent.session_tpm_limit ?? "Unlimited"} {agent.session_rpm_limit ?? "Unlimited"} + + {agent.access_group_ids?.length ? ( +
+ {agent.access_group_ids.map((accessGroupId) => ( +
{accessGroupLabel(accessGroupId)}
+ ))} +
+ ) : ( + "None" + )} + {formatDate(agent.created_at)} {formatDate(agent.updated_at)} @@ -489,6 +518,26 @@ const AgentInfoView: React.FC = ({ agentId, onClose, accessT {rateLimitField("session_rpm_limit", "Session RPM Limit")}
+ +

Access Groups

+ + + {({ value, onChange }) => ( + + )} + + +

MCP Servers

diff --git a/ui/litellm-dashboard/src/app/(dashboard)/change-password/ChangePasswordForm.integration.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/change-password/ChangePasswordForm.integration.test.tsx new file mode 100644 index 00000000000..c4cf8ebcf9d --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/change-password/ChangePasswordForm.integration.test.tsx @@ -0,0 +1,110 @@ +import { fireEvent, render, screen, waitFor } from "@testing-library/react"; +import { beforeEach, describe, expect, it, vi } from "vitest"; +import ChangePasswordForm from "./ChangePasswordForm"; + +const mockChangePasswordCall = vi.fn(); +const mockToastSuccess = vi.fn(); +const mockClearTokenCookies = vi.fn(); +let mockPasswordResetRequired = false; + +vi.mock("@/components/networking", () => ({ + changePasswordCall: (...args: unknown[]) => mockChangePasswordCall(...args), + getProxyBaseUrl: () => "", +})); + +vi.mock("@/app/(dashboard)/hooks/useAuthorized", () => ({ + default: () => ({ accessToken: "sk-session-token", passwordResetRequired: mockPasswordResetRequired }), +})); + +vi.mock("@/lib/toast", () => ({ + toast: { + success: (...args: unknown[]) => mockToastSuccess(...args), + fromError: vi.fn(), + }, +})); + +vi.mock("@/utils/cookieUtils", () => ({ + clearTokenCookies: (...args: unknown[]) => mockClearTokenCookies(...args), +})); + +const fillForm = (values: { current: string; next: string; confirm: string }) => { + fireEvent.change(screen.getByLabelText("Current Password"), { target: { value: values.current } }); + fireEvent.change(screen.getByLabelText("New Password"), { target: { value: values.next } }); + fireEvent.change(screen.getByLabelText("Confirm New Password"), { target: { value: values.confirm } }); +}; + +const submit = () => fireEvent.click(screen.getByRole("button", { name: "Change Password" })); + +describe("ChangePasswordForm", () => { + beforeEach(() => { + vi.clearAllMocks(); + mockPasswordResetRequired = false; + }); + + it("sends the current and new password to the change endpoint and resets on success", async () => { + mockChangePasswordCall.mockResolvedValue({ user_id: "user-123", message: "Password updated successfully." }); + render(); + + fillForm({ current: "OldP@ssw0rd-2026", next: "NewP@ssw0rd-2026", confirm: "NewP@ssw0rd-2026" }); + submit(); + + expect(await screen.findByLabelText("Current Password")).toHaveValue(""); + expect(mockChangePasswordCall).toHaveBeenCalledWith("sk-session-token", "OldP@ssw0rd-2026", "NewP@ssw0rd-2026"); + expect(mockToastSuccess).toHaveBeenCalled(); + }); + + it("blocks submission when the confirmation does not match", async () => { + render(); + + fillForm({ current: "OldP@ssw0rd-2026", next: "NewP@ssw0rd-2026", confirm: "Different-2026" }); + submit(); + + expect(await screen.findByText("New passwords do not match")).toBeInTheDocument(); + expect(mockChangePasswordCall).not.toHaveBeenCalled(); + }); + + it("shows the proxy's rejection message unwrapped", async () => { + mockChangePasswordCall.mockRejectedValue(new Error("{'error': 'Current password is incorrect.'}")); + render(); + + fillForm({ current: "wrong-password", next: "NewP@ssw0rd-2026", confirm: "NewP@ssw0rd-2026" }); + submit(); + + expect(await screen.findByText("Current password is incorrect.")).toBeInTheDocument(); + expect(mockToastSuccess).not.toHaveBeenCalled(); + }); + + describe("forced password reset", () => { + it("shows the forced-reset warning only when the session is flagged", () => { + mockPasswordResetRequired = true; + render(); + + expect(screen.getByText(/must be changed before you can use the dashboard/)).toBeInTheDocument(); + }); + + it("hides the forced-reset warning for a normal session", () => { + render(); + + expect(screen.queryByText(/must be changed before you can use the dashboard/)).not.toBeInTheDocument(); + }); + + it("signs the user out to re-login after a successful forced change", async () => { + mockPasswordResetRequired = true; + mockChangePasswordCall.mockResolvedValue({ user_id: "user-123", message: "Password updated successfully." }); + const replaceMock = vi.fn(); + const realLocation = window.location; + Object.defineProperty(window, "location", { configurable: true, value: { replace: replaceMock } }); + + try { + render(); + fillForm({ current: "OldP@ssw0rd-2026", next: "NewP@ssw0rd-2026", confirm: "NewP@ssw0rd-2026" }); + submit(); + + await waitFor(() => expect(replaceMock).toHaveBeenCalledWith("/ui/login/")); + expect(mockClearTokenCookies).toHaveBeenCalled(); + } finally { + Object.defineProperty(window, "location", { configurable: true, value: realLocation }); + } + }); + }); +}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/change-password/ChangePasswordForm.tsx b/ui/litellm-dashboard/src/app/(dashboard)/change-password/ChangePasswordForm.tsx new file mode 100644 index 00000000000..05a6bf3ae94 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/change-password/ChangePasswordForm.tsx @@ -0,0 +1,120 @@ +"use client"; + +import React, { useState } from "react"; +import { CircleAlert } from "lucide-react"; +import { z } from "zod/v4"; +import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; +import { Alert, AlertTitle } from "@/components/shared/Alert"; +import { PasswordInput } from "@/components/shared/PasswordInput"; +import { FormField } from "@/components/shared/form/FormField"; +import { Button } from "@/components/ui/button"; +import { Card, CardContent } from "@/components/ui/card"; +import { FieldGroup } from "@/components/ui/field"; +import { UiLoadingSpinner } from "@/components/ui/ui-loading-spinner"; +import { changePasswordCall, getProxyBaseUrl } from "@/components/networking"; +import { extractProxyErrorMessage } from "@/lib/http/client"; +import { useZodForm } from "@/lib/forms/useZodForm"; +import { toast } from "@/lib/toast"; +import { clearTokenCookies } from "@/utils/cookieUtils"; +import { getLoginUrl } from "@/utils/returnUrlUtils"; + +const changePasswordSchema = z + .object({ + currentPassword: z.string().min(1, "Current password is required"), + newPassword: z.string().min(1, "New password is required"), + confirmNewPassword: z.string().min(1, "Confirm your new password"), + }) + .refine((values) => values.newPassword === values.confirmNewPassword, { + message: "New passwords do not match", + path: ["confirmNewPassword"], + }); + +type ChangePasswordValues = z.infer; + +export function ChangePasswordForm() { + const { accessToken, passwordResetRequired } = useAuthorized(); + const form = useZodForm(changePasswordSchema, { + defaultValues: { currentPassword: "", newPassword: "", confirmNewPassword: "" }, + }); + const [isPending, setIsPending] = useState(false); + const [submitError, setSubmitError] = useState(null); + + const handleSubmit = async (values: ChangePasswordValues) => { + if (!accessToken) return; + setSubmitError(null); + setIsPending(true); + try { + await changePasswordCall(accessToken, values.currentPassword, values.newPassword); + if (passwordResetRequired) { + // The session key was minted restricted; only a fresh login lifts it. + toast.success("Password updated. Please log in with your new password."); + clearTokenCookies(); + window.location.replace(getLoginUrl(getProxyBaseUrl())); + return; + } + toast.success("Password updated"); + form.reset(); + } catch (error) { + setSubmitError(extractProxyErrorMessage(error)); + } finally { + setIsPending(false); + } + }; + + return ( +
+ + +

Change Password

+

+ Enter your current password and choose a new one. The new password must meet this proxy's password + policy. +

+ + {passwordResetRequired && ( + + + + Your password must be changed before you can use the dashboard: it was either found in a known data + breach or set by an administrator as a temporary password. After updating it, you will be signed out to + log in again. + + + )} + +
+ + + {({ ref, ...field }) => } + + + + {({ ref, ...field }) => } + + + + {({ ref, ...field }) => } + + + + {submitError && ( + + + {submitError} + + )} + +
+ +
+
+ + +
+ ); +} + +export default ChangePasswordForm; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/change-password/page.tsx b/ui/litellm-dashboard/src/app/(dashboard)/change-password/page.tsx new file mode 100644 index 00000000000..0a6ae926ceb --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/change-password/page.tsx @@ -0,0 +1,7 @@ +"use client"; + +import ChangePasswordForm from "./ChangePasswordForm"; + +export default function ChangePasswordPage() { + return ; +} diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/AutoRouterBenchmarksTab.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/AutoRouterBenchmarksTab.test.tsx index 5c7453c1394..a144630cdd0 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/AutoRouterBenchmarksTab.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/AutoRouterBenchmarksTab.test.tsx @@ -434,7 +434,7 @@ describe("AutoRouterBenchmarksTab", () => { mockHook({ data: response([group()]) }); const { dateValue, onDateChange } = renderTab(); - expect(vi.mocked(useAutoRouterBenchmarks)).toHaveBeenCalledWith("sk-test", dateValue, undefined); + expect(vi.mocked(useAutoRouterBenchmarks)).toHaveBeenCalledWith("sk-test", dateValue, undefined, undefined); expect(screen.getByText("Jul 6 – Aug 5 (UTC)")).toBeInTheDocument(); fireEvent.click(screen.getByTestId("date-picker")); @@ -460,7 +460,7 @@ describe("AutoRouterBenchmarksTab", () => { , ); - expect(vi.mocked(useAutoRouterBenchmarks)).toHaveBeenCalledWith("sk-test", dateValue, "key-hash-1"); + expect(vi.mocked(useAutoRouterBenchmarks)).toHaveBeenCalledWith("sk-test", dateValue, "key-hash-1", undefined); expect(screen.getByText("Total estimated savings")).toBeInTheDocument(); expect(screen.queryByRole("tab", { name: "Shadow Evals" })).not.toBeInTheDocument(); }); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/AutoRouterBenchmarksTab.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/AutoRouterBenchmarksTab.tsx index ce55b633b60..063598bd46e 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/AutoRouterBenchmarksTab.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/AutoRouterBenchmarksTab.tsx @@ -312,8 +312,7 @@ const BenchmarksBody: React.FC = ({ isPending, error, data, length. Total actual spend includes every turn; savings and baseline spend include only turns with a current estimate, including turns with zero savings. Savings are net of recorded LLM classification cost. Classification cost per 1K turns is averaged over all auto-router turns, including those that skip classification. The range - counts whole sessions that overlap it, so totals can differ slightly from the Overall tab, which buckets savings - by UTC day. + counts whole sessions that overlap it, so totals can differ from savings views that group usage by UTC day.

@@ -333,11 +332,17 @@ interface AutoRouterBenchmarksTabProps { accessToken: string | null; activity: Pick; apiKey?: string; + userId?: string; } -export const AutoRouterUsageView: React.FC = ({ accessToken, activity, apiKey }) => { +export const AutoRouterUsageView: React.FC = ({ + accessToken, + activity, + apiKey, + userId, +}) => { const { dateValue, onDateChange } = activity; - const { data, isPending, error } = useAutoRouterBenchmarks(accessToken, dateValue, apiKey); + const { data, isPending, error } = useAutoRouterBenchmarks(accessToken, dateValue, apiKey, userId); const [selectedKey, setSelectedKey] = useState(ALL_ROUTERS); const { data: autoRouters } = useAutoRouters(); @@ -372,6 +377,12 @@ export const AutoRouterUsageView: React.FC = ({ ac
+ {userId && ( +

+ Usage for this user across API keys and JWT-authenticated requests. Older sessions recorded without a user ID + are not included. +

+ )} +export const useAutoRouterBenchmarks = ( + accessToken: string | null, + range: DateRange, + apiKey?: string, + userId?: string, +) => $api.useQuery( "get", "/auto_router/benchmarks", - { params: { query: { ...benchmarksWindow(range, new Date()), api_key: apiKey } } }, + { params: { query: { ...benchmarksWindow(range, new Date()), api_key: apiKey, user_id: userId } } }, { enabled: Boolean(accessToken && range.from && range.to), retry: false }, ); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/useDailyActivityRange.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/useDailyActivityRange.test.tsx index 4059303d5a5..e501cf00b90 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/useDailyActivityRange.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/useDailyActivityRange.test.tsx @@ -15,6 +15,8 @@ vi.mock("@/app/(dashboard)/usage/_components/hooks/usePaginatedDailyActivity", ( isFetchingMore: false, progress: { currentPage: 4, totalPages: 9 }, cancelled: false, + failed: false, + coversRange: true, cancel: mockCancel, }; }, diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/useDailyActivityRange.ts b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/useDailyActivityRange.ts index 92dd24b8d6d..4eb9f257d30 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/useDailyActivityRange.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/useDailyActivityRange.ts @@ -67,14 +67,16 @@ export const useScopedDailyActivityRange = ( args: [accessToken, startTime, endTime, userId, true, apiKey], enabled: !!accessToken && !!startTime && !!endTime, }; - const { data, loading, isFetchingMore, progress, cancelled, failed, cancel } = + const { data, loading, isFetchingMore, progress, cancelled, failed, coversRange, cancel } = usePaginatedDailyActivity(activityQueryOptions); + const readUnavailable = failed || cancelled; + const waitingForRange = activityQueryOptions.enabled && !coversRange && !readUnavailable; return { dateValue, onDateChange, results: data.results as DailyData[], - loading, + loading: loading || waitingForRange, isFetchingMore, progress, cancelled, diff --git a/ui/litellm-dashboard/src/app/(dashboard)/hooks/useAuthorized.ts b/ui/litellm-dashboard/src/app/(dashboard)/hooks/useAuthorized.ts index 40d1ec09d1f..581ee8b2580 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/hooks/useAuthorized.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/hooks/useAuthorized.ts @@ -50,7 +50,9 @@ const useAuthorized = () => { isViewOnly: isViewOnlySessionRole(decoded?.user_role), premiumUser: decoded?.premium_user ?? null, disabledPersonalKeyCreation: decoded?.disabled_non_admin_personal_key_creation ?? null, + loginMethod: decoded?.login_method ?? null, showSSOBanner: decoded?.login_method === "username_password", + passwordResetRequired: decoded?.password_reset_required === true, }; }; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/layout.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/layout.test.tsx index d7e1bb82564..d854befa197 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/layout.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/layout.test.tsx @@ -1,4 +1,4 @@ -import { describe, it, expect, vi, beforeEach } from "vitest"; +import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; import { render, screen, waitFor } from "@testing-library/react"; import { AuthProvider } from "@/contexts/AuthContext"; import Layout from "./layout"; @@ -121,4 +121,60 @@ describe("(dashboard) Layout", () => { expect(screen.queryByTestId("dashboard-header")).not.toBeInTheDocument(); expect(screen.queryByTestId("sidebar")).not.toBeInTheDocument(); }); + + describe("forced password reset routing", () => { + const sessionCookie = (claims: Record) => { + const encode = (part: Record) => + btoa(JSON.stringify(part)).replace(/\+/g, "-").replace(/\//g, "_").replace(/=+$/, ""); + const exp = Math.floor(Date.now() / 1000) + 3600; + return `${encode({ alg: "HS256", typ: "JWT" })}.${encode({ ...claims, exp })}.sig`; + }; + + afterEach(() => { + document.cookie = "token=; Max-Age=0; Path=/"; + }); + + it("routes a session flagged password_reset_required to the change-password page", async () => { + const flaggedClaims = { + user_id: "flagged-user", + key: "sk-session", + login_method: "username_password", + password_reset_required: true, + }; + document.cookie = `token=${sessionCookie(flaggedClaims)}; Path=/`; + + render( + + +
+ + , + ); + + pendingUiConfig.resolve(); + + await waitFor(() => expect(replaceMock).toHaveBeenCalledWith(expect.stringContaining("/change-password"))); + }); + + it("does not reroute an unflagged session", async () => { + document.cookie = `token=${sessionCookie({ + user_id: "normal-user", + key: "sk-session", + login_method: "username_password", + })}; Path=/`; + + render( + + +
+ + , + ); + + pendingUiConfig.resolve(); + + expect(await screen.findByTestId("page-content")).toBeInTheDocument(); + expect(replaceMock).not.toHaveBeenCalled(); + }); + }); }); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/layout.tsx b/ui/litellm-dashboard/src/app/(dashboard)/layout.tsx index 6d326f5280e..2e903c7b150 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/layout.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/layout.tsx @@ -7,7 +7,7 @@ import LoadingScreen from "@/components/common_components/LoadingScreen"; import { ThemeProvider } from "@/contexts/ThemeContext"; import { useAuth } from "@/contexts/AuthContext"; import SidebarProvider from "@/app/(dashboard)/components/SidebarProvider"; -import { useRouter, useSearchParams } from "next/navigation"; +import { usePathname, useRouter, useSearchParams } from "next/navigation"; import { DebugWarningBanner } from "@/components/DebugWarningBanner"; import { NoRedisWarningBanner } from "@/components/NoRedisWarningBanner"; import { EnvCredentialLoginWarningBanner } from "@/components/EnvCredentialLoginWarningBanner"; @@ -149,7 +149,8 @@ function DashboardShell({ children }: { children: React.ReactNode }) { function LayoutContent({ children }: { children: React.ReactNode }) { const router = useRouter(); const searchParams = useSearchParams(); - const { accessToken, authLoading } = useAuth(); + const pathname = usePathname(); + const { accessToken, authLoading, passwordResetRequired } = useAuth(); const isInvitationFlow = Boolean(searchParams.get("invitation_id")); // Legacy invitation links point at /ui/?invitation_id=; the onboarding form now lives at its own @@ -160,6 +161,14 @@ function LayoutContent({ children }: { children: React.ReactNode }) { } }, [authLoading, isInvitationFlow, router, searchParams]); + // A session flagged for a forced password reset can only reach the change-password + // endpoint server-side; keep the UI on the matching page. + useEffect(() => { + if (!authLoading && passwordResetRequired && !pathname?.endsWith("/change-password")) { + router.replace(uiHref("change-password")); + } + }, [authLoading, passwordResetRequired, pathname, router]); + if (authLoading || isInvitationFlow) { return ; } diff --git a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/mcp_servers.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/mcp_servers.test.tsx index 091b2f1403f..1217d878489 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/mcp_servers.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/mcp_servers.test.tsx @@ -3,7 +3,8 @@ import { render, waitFor, screen, act, within } from "@testing-library/react"; import userEvent from "@testing-library/user-event"; import { describe, it, expect, vi, beforeEach } from "vitest"; import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; -import MCPServers from "./mcp_servers"; +import MCPServers, { compareServers, type SortKey } from "./mcp_servers"; +import type { MCPServer } from "@/components/mcp_tools/types"; import * as networking from "@/components/networking"; // Mock the networking module @@ -31,6 +32,98 @@ const createQueryClient = () => }, }); +describe("compareServers", () => { + const server = (server_id: string, name: string, created_at = ""): MCPServer => ({ + server_id, + server_name: name, + created_at, + updated_at: created_at, + created_by: "user", + updated_by: "user", + }); + + const shuffled = [server("c", "github"), server("a", "slack"), server("b", "Jira")]; + + it("orders servers without timestamps by name so config.yaml servers render in a stable order", () => { + const byCreated = [...shuffled].sort((a, b) => compareServers(a, b, "created_desc")).map((s) => s.server_id); + const byUpdated = [...shuffled].sort((a, b) => compareServers(a, b, "updated_desc")).map((s) => s.server_id); + const byHealth = [...shuffled].sort((a, b) => compareServers(a, b, "health")).map((s) => s.server_id); + + expect(byCreated).toEqual(["c", "b", "a"]); + expect(byUpdated).toEqual(["c", "b", "a"]); + expect(byHealth).toEqual(["c", "b", "a"]); + }); + + it("keeps newest-first when timestamps differ", () => { + const newest = server("new", "zzz", "2026-02-01T00:00:00Z"); + const oldest = server("old", "aaa", "2026-01-01T00:00:00Z"); + expect([oldest, newest].sort((a, b) => compareServers(a, b, "created_desc")).map((s) => s.server_id)).toEqual([ + "new", + "old", + ]); + }); + + it.each(["created_desc", "updated_desc", "name_asc", "health"])( + "breaks equal timestamps and names by ID for %s regardless of input order", + (sort) => { + const servers = [ + server("b", "GitHub", "2026-01-01T00:00:00Z"), + server("c", "Slack", "2026-01-01T00:00:00Z"), + server("a", "github", "2026-01-01T00:00:00Z"), + ]; + for (const input of [servers, [...servers].reverse()]) { + expect([...input].sort((a, b) => compareServers(a, b, sort)).map((s) => s.server_id)).toEqual(["a", "b", "c"]); + } + }, + ); + + it("uses the display name before alias, then falls back to alias and ID", () => { + const servers: MCPServer[] = [ + { ...server("s-slack", "Slack"), alias: "aaa" }, + { ...server("s-github", ""), server_name: null, alias: "GitHub" }, + { ...server("confluence", ""), alias: "" }, + ]; + for (const input of [servers, [...servers].reverse()]) { + expect([...input].sort((a, b) => compareServers(a, b, "name_asc")).map((s) => s.server_id)).toEqual([ + "confluence", + "s-github", + "s-slack", + ]); + } + }); + + it.each(["created_desc", "updated_desc", "health"])( + "keeps timestamped servers before missing timestamps for %s", + (sort) => { + const servers = [ + server("config", "aaa"), + server("older", "bbb", "2026-01-01T00:00:00Z"), + server("newer", "zzz", "2026-02-01T00:00:00Z"), + ]; + for (const input of [servers, [...servers].reverse()]) { + expect([...input].sort((a, b) => compareServers(a, b, sort)).map((s) => s.server_id)).toEqual([ + "newer", + "older", + "config", + ]); + } + }, + ); + + it("sorts health before recency and display name", () => { + const servers: MCPServer[] = [ + { ...server("healthy", "aaa", "2026-03-01T00:00:00Z"), status: "healthy" }, + { ...server("unknown", "bbb", "2026-02-01T00:00:00Z"), status: "unknown" }, + { ...server("unhealthy", "zzz", "2026-01-01T00:00:00Z"), status: "unhealthy" }, + ]; + expect(servers.sort((a, b) => compareServers(a, b, "health")).map((s) => s.server_id)).toEqual([ + "unhealthy", + "unknown", + "healthy", + ]); + }); +}); + describe("MCPServers", () => { const defaultProps = { accessToken: "123", @@ -74,6 +167,134 @@ describe("MCPServers", () => { const myConnections = await screen.findByRole("link", { name: "My Connections" }); expect(myConnections).toBeVisible(); expect(myConnections).toHaveAttribute("href", "/ui/connect"); + for (const name of ["Semantic Filter", "Tool Search", "Network Settings", "Submitted MCPs"]) { + const tab = screen.queryByRole("tab", { name }); + if (userRole === "Admin") { + expect(tab).toBeVisible(); + } else { + expect(tab).not.toBeInTheDocument(); + } + } + expect( + screen.getByRole("button", { + name: userRole === "Admin" ? "+ Add New MCP Server" : "+ Submit MCP Server", + }), + ).toBeVisible(); + }); + + it.each(["cancel", "success", "failure", "unnamed"])("preserves delete confirmation on %s", async (outcome) => { + const server: MCPServer = { + created_at: "", + updated_at: "", + server_id: "delete-server", + server_name: outcome === "unnamed" ? null : "Delete fixture", + alias: "delete-alias", + url: outcome === "unnamed" ? null : "https://example.com/mcp", + created_by: "user", + updated_by: "user", + }; + vi.mocked(networking.fetchMCPServers).mockResolvedValue([server]); + vi.mocked(networking.fetchMCPServerHealth).mockResolvedValue([]); + let finishDelete: () => void = () => {}; + vi.mocked(networking.deleteMCPServer).mockImplementation( + () => + new Promise((resolve, reject) => { + finishDelete = () => (outcome === "failure" ? reject(new Error("Delete failed")) : resolve(undefined)); + }), + ); + render( + + + , + ); + await userEvent.click(await screen.findByRole("button", { name: "Server actions" })); + await userEvent.click(await screen.findByRole("menuitem", { name: "Delete" })); + const dialog = await screen.findByRole("alertdialog", { name: "Delete MCP Server?" }); + expect(within(dialog).getByText("delete-server")).toBeVisible(); + if (outcome === "unnamed") { + expect(within(dialog).queryByText("Name")).not.toBeInTheDocument(); + expect(within(dialog).queryByText("URL")).not.toBeInTheDocument(); + } else { + expect(within(dialog).getByText("Delete fixture")).toBeVisible(); + expect(within(dialog).getByText("https://example.com/mcp")).toBeVisible(); + } + if (outcome === "cancel") { + await userEvent.click(within(dialog).getByRole("button", { name: "Cancel" })); + expect(networking.deleteMCPServer).not.toHaveBeenCalled(); + } else { + await userEvent.click(within(dialog).getByRole("button", { name: "Delete" })); + expect(within(dialog).getByRole("button", { name: "Deleting..." })).toBeDisabled(); + expect(within(dialog).getByRole("button", { name: "Cancel" })).toBeDisabled(); + expect(networking.deleteMCPServer).toHaveBeenCalledWith("123", "delete-server"); + await act(async () => finishDelete()); + } + await waitFor(() => expect(screen.queryByRole("alertdialog")).not.toBeInTheDocument()); + }); + + it("filters servers by access group", async () => { + const server = { created_by: "user", updated_by: "user" }; + vi.mocked(networking.fetchMCPServers).mockResolvedValue([ + { + ...server, + server_id: "string-group", + server_name: "String group", + alias: "string-alias", + mcp_access_groups: ["shared"], + }, + { + ...server, + server_id: "legacy-group", + server_name: "Legacy group", + alias: "legacy-alias", + mcp_access_groups: ["shared"], + }, + { + ...server, + server_id: "other-group", + server_name: "Other group", + alias: "other-alias", + mcp_access_groups: ["different"], + }, + ]); + vi.mocked(networking.fetchMCPServerHealth).mockResolvedValue([]); + render( + + + , + ); + await screen.findByText("String group"); + await userEvent.click(screen.getByRole("combobox", { name: "Access Group" })); + await userEvent.click(await screen.findByRole("option", { name: "shared" })); + expect(screen.getByText("String group")).toBeVisible(); + expect(screen.getByText("Legacy group")).toBeVisible(); + expect(screen.queryByText("Other group")).not.toBeInTheDocument(); + }); + + it.each(["server_name", "alias", "url", "server_id"] as const)("searches by %s case-insensitively", async (field) => { + const server: MCPServer = { + created_at: "", + updated_at: "", + server_id: "search-server", + server_name: "Search fixture", + created_by: "user", + updated_by: "user", + [field]: "Needle", + }; + vi.mocked(networking.fetchMCPServers).mockResolvedValue([server]); + vi.mocked(networking.fetchMCPServerHealth).mockResolvedValue([]); + render( + + + , + ); + await screen.findByTestId("mcp-servers-grid"); + const search = screen.getByPlaceholderText("Search by name, alias, URL, or ID"); + await userEvent.type(search, " NEEDLE "); + expect(screen.getByTestId("mcp-servers-grid")).toBeVisible(); + await userEvent.clear(search); + await userEvent.type(search, "no-match"); + expect(screen.queryByTestId("mcp-servers-grid")).not.toBeInTheDocument(); + expect(screen.getByText("No servers match the current filters or search.")).toBeVisible(); }); it("should render mocked MCP servers data in the table", async () => { @@ -316,9 +537,7 @@ describe("MCPServers", () => { expect(screen.getByText("Team B Server")).toBeInTheDocument(); expect(screen.getByText("Team A Server 2")).toBeInTheDocument(); - // Find the team select by its "Team" label, then the combobox it labels - const teamLabel = screen.getByText("Team"); - const teamSelect = within(teamLabel.parentElement!).getByRole("combobox"); + const teamSelect = screen.getByRole("combobox", { name: "Team" }); await userEvent.click(teamSelect); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/mcp_servers.tsx b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/mcp_servers.tsx index 2df304eff96..818b5150650 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/mcp_servers.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/mcp_servers.tsx @@ -49,7 +49,7 @@ import { cn } from "@/lib/cva.config"; import UserEnvVarsModal from "./UserEnvVarsModal"; import { listMCPUserEnvVarStatus } from "@/components/networking"; -type SortKey = "created_desc" | "updated_desc" | "name_asc" | "health"; +export type SortKey = "created_desc" | "updated_desc" | "name_asc" | "health"; const SORT_OPTIONS: { value: SortKey; label: string }[] = [ { value: "created_desc", label: "Recently created" }, @@ -64,32 +64,33 @@ const HEALTH_RANK: Record = { healthy: 2, }; -const compareServers = (a: MCPServer, b: MCPServer, sort: SortKey): number => { +const compareByName = (a: MCPServer, b: MCPServer): number => { + const nameA = (a.server_name || a.alias || a.server_id).toLowerCase(); + const nameB = (b.server_name || b.alias || b.server_id).toLowerCase(); + return nameA.localeCompare(nameB) || a.server_id.localeCompare(b.server_id); +}; + +const compareByTimestampDesc = (a: string | null | undefined, b: string | null | undefined): number => { + const ta = a ? new Date(a).getTime() : 0; + const tb = b ? new Date(b).getTime() : 0; + return tb - ta; +}; + +export const compareServers = (a: MCPServer, b: MCPServer, sort: SortKey): number => { switch (sort) { - case "name_asc": { - const nameA = (a.server_name || a.alias || a.server_id).toLowerCase(); - const nameB = (b.server_name || b.alias || b.server_id).toLowerCase(); - return nameA.localeCompare(nameB); - } - case "updated_desc": { - const ta = a.updated_at ? new Date(a.updated_at).getTime() : 0; - const tb = b.updated_at ? new Date(b.updated_at).getTime() : 0; - return tb - ta; - } + case "name_asc": + return compareByName(a, b); + case "updated_desc": + return compareByTimestampDesc(a.updated_at, b.updated_at) || compareByName(a, b); case "health": { const ra = HEALTH_RANK[a.status ?? "unknown"] ?? 1; const rb = HEALTH_RANK[b.status ?? "unknown"] ?? 1; if (ra !== rb) return ra - rb; - const ta = a.created_at ? new Date(a.created_at).getTime() : 0; - const tb = b.created_at ? new Date(b.created_at).getTime() : 0; - return tb - ta; + return compareByTimestampDesc(a.created_at, b.created_at) || compareByName(a, b); } case "created_desc": - default: { - const ta = a.created_at ? new Date(a.created_at).getTime() : 0; - const tb = b.created_at ? new Date(b.created_at).getTime() : 0; - return tb - ta; - } + default: + return compareByTimestampDesc(a.created_at, b.created_at) || compareByName(a, b); } }; @@ -112,6 +113,62 @@ const readToolsOAuthServerId = (): string | null => { } }; +function DeleteServerDialog({ + open, + onOpenChange, + server, + isDeleting, + onConfirm, +}: { + open: boolean; + onOpenChange: (open: boolean) => void; + server: MCPServer | undefined; + isDeleting: boolean; + onConfirm: () => Promise; +}) { + return ( + + + + Delete MCP Server? + +
+

+ This action is permanent and cannot be undone. All associated configurations will be removed. +

+ + {server && ( +
+ {server.server_name && ( +
+
Name
+
{server.server_name}
+
+ )} +
+
ID
+
{server.server_id}
+
+ {server.url && ( +
+
URL
+
{server.url}
+
+ )} +
+ )} +
+ + Cancel + + + + + ); +} + const MCPServers: React.FC = ({ accessToken, userRole, userID, isViewOnly = false }) => { const { data: mcpServers, isLoading: isLoadingServers, refetch } = useMCPServers(); @@ -298,16 +355,12 @@ const MCPServers: React.FC = ({ accessToken, userRole, userID, i } if (group !== "all") { filtered = filtered.filter((server) => - server.mcp_access_groups?.some((g: any) => (typeof g === "string" ? g === group : g && g.name === group)), + server.mcp_access_groups?.some((g: string | { name?: string } | null) => + typeof g === "string" ? g === group : g?.name === group, + ), ); } - const sorted = [...filtered].sort((a, b) => { - if (!a.created_at && !b.created_at) return 0; - if (!a.created_at) return 1; - if (!b.created_at) return -1; - return new Date(b.created_at).getTime() - new Date(a.created_at).getTime(); - }); - setFilteredServers(sorted); + setFilteredServers(filtered); }, [serversWithHealth], ); @@ -338,7 +391,7 @@ const MCPServers: React.FC = ({ accessToken, userRole, userID, i const alias = (s.alias || "").toLowerCase(); const url = (s.url || "").toLowerCase(); const id = s.server_id.toLowerCase(); - return name.includes(q) || alias.includes(q) || url.includes(q) || id.includes(q); + return [name, alias, url, id].some((value) => value.includes(q)); }) : filteredServers; return [...matches].sort((a, b) => compareServers(a, b, sortKey)); @@ -381,9 +434,7 @@ const MCPServers: React.FC = ({ accessToken, userRole, userID, i }; // Find the server to delete from the servers list - const serverToDelete = serverIdToDelete - ? (mcpServers || []).find((server) => server.server_id === serverIdToDelete) - : null; + const serverToDelete = mcpServers?.find((server) => server.server_id === serverIdToDelete); const handleCreateSuccess = (newMcpServer: MCPServer) => { setFilteredServers((prev) => [...prev, newMcpServer]); @@ -425,45 +476,13 @@ const MCPServers: React.FC = ({ accessToken, userRole, userID, i return (
- !open && cancelDelete()}> - - - Delete MCP Server? - -
-

- This action is permanent and cannot be undone. All associated configurations will be removed. -

- - {serverToDelete && ( -
- {serverToDelete.server_name && ( -
-
Name
-
{serverToDelete.server_name}
-
- )} -
-
ID
-
{serverToDelete.server_id}
-
- {serverToDelete.url && ( -
-
URL
-
{serverToDelete.url}
-
- )} -
- )} -
- - Cancel - - - - + !open && cancelDelete()} + server={serverToDelete} + isDeleting={isDeletingServer} + onConfirm={confirmDelete} + /> = ({ accessToken, userRole, userID, i My Connections - {isAdminRole(userRole) && ( + {isAdminRole(userRole) ? ( <> - )} - {!isAdminRole(userRole) && ( + ) : ( + )} + )} +
When a prompt provably cannot fit the decided tier's context windows, route it to the lowest tier whose - window holds it instead of letting the provider reject it. Off means requests dispatch on complexity alone. + window holds it instead of letting the provider reject it. Disabled by default. Off means requests dispatch on + complexity alone. {enabled && (
diff --git a/ui/litellm-dashboard/src/components/add_model/HeuristicKeywordOverrides.tsx b/ui/litellm-dashboard/src/components/add_model/HeuristicKeywordOverrides.tsx new file mode 100644 index 00000000000..185f187bbfc --- /dev/null +++ b/ui/litellm-dashboard/src/components/add_model/HeuristicKeywordOverrides.tsx @@ -0,0 +1,42 @@ +import React from "react"; +import { MultiSelect } from "@/components/shared/MultiSelect"; +import type { ComplexityRouterConfigValue } from "./ComplexityRouterConfig"; + +const fields = [ + ["code_keywords", "Code keywords"], + ["reasoning_keywords", "Reasoning keywords"], + ["technical_keywords", "Technical keywords"], + ["simple_keywords", "Simple keywords"], +] as const; + +const HeuristicKeywordOverrides: React.FC<{ + value: ComplexityRouterConfigValue; + onChange: (value: ComplexityRouterConfigValue) => void; +}> = ({ value, onChange }) => ( +
+

+ Each list replaces the built-in keyword list of the same name for the heuristic scorer. Leave a list empty to keep + the built-in one. To add technical terms without replacing the list, use custom technical keywords under + Classification Method. +

+ {fields.map(([key, label]) => { + const keywords = value[key] ?? []; + return ( +
+ {label} + ({ label: keyword, value: keyword }))} + value={keywords} + onValueChange={(next) => onChange({ ...value, [key]: next.length > 0 ? next : undefined })} + placeholder={`Add ${label.toLowerCase()}`} + emptyText="Type to add a keyword" + allowCustomValues + className="w-full" + /> +
+ ); + })} +
+); + +export default HeuristicKeywordOverrides; diff --git a/ui/litellm-dashboard/src/components/add_model/HousekeepingRoutingControls.tsx b/ui/litellm-dashboard/src/components/add_model/HousekeepingRoutingControls.tsx new file mode 100644 index 00000000000..2b6caba6291 --- /dev/null +++ b/ui/litellm-dashboard/src/components/add_model/HousekeepingRoutingControls.tsx @@ -0,0 +1,44 @@ +import React from "react"; +import { MultiSelect } from "@/components/shared/MultiSelect"; +import { Switch } from "@/components/ui/switch"; +import type { ComplexityRouterConfigValue } from "./ComplexityRouterConfig"; + +const HousekeepingRoutingControls: React.FC<{ + value: ComplexityRouterConfigValue; + onChange: (value: ComplexityRouterConfigValue) => void; +}> = ({ value, onChange }) => { + const enabled = value.route_housekeeping_to_cheapest_tier ?? true; + const patterns = value.housekeeping_patterns ?? []; + return ( + <> +
+ onChange({ ...value, route_housekeeping_to_cheapest_tier: next })} + aria-label="Route housekeeping calls to the cheapest tier" + /> + Route housekeeping calls to the cheapest tier +
+ + Conversation-title style calls skip the classifier and go to the cheapest tier. + + Additional housekeeping sentinels + ({ label: pattern, value: pattern }))} + value={patterns} + onValueChange={(next) => onChange({ ...value, housekeeping_patterns: next.length > 0 ? next : undefined })} + placeholder="e.g., conversation title" + emptyText="Type to add a sentinel" + allowCustomValues + disabled={!enabled} + className="w-full" + /> + + Case-sensitive literal strings added to the built-in conversation-title sentinels. + {!enabled && " Turn housekeeping routing on for these to take effect."} + + + ); +}; + +export default HousekeepingRoutingControls; diff --git a/ui/litellm-dashboard/src/components/add_model/PlanModeOverrideControls.tsx b/ui/litellm-dashboard/src/components/add_model/PlanModeOverrideControls.tsx index 83fd0c42d16..e195fb62222 100644 --- a/ui/litellm-dashboard/src/components/add_model/PlanModeOverrideControls.tsx +++ b/ui/litellm-dashboard/src/components/add_model/PlanModeOverrideControls.tsx @@ -1,5 +1,6 @@ import React from "react"; import { Switch } from "@/components/ui/switch"; +import { MultiSelect } from "@/components/shared/MultiSelect"; import TierRowSelect from "./TierRowSelect"; import type { ComplexityRouterConfigValue } from "./ComplexityRouterConfig"; @@ -38,6 +39,23 @@ const PlanModeOverrideControls: React.FC<{ />
)} +
+ Additional plan-mode sentinels + ({ label: pattern, value: pattern }))} + value={value.plan_mode_patterns ?? []} + onValueChange={(patterns) => + onChange({ ...value, plan_mode_patterns: patterns.length > 0 ? patterns : undefined }) + } + placeholder="e.g., enter plan mode" + emptyText="Type to add a sentinel" + allowCustomValues + className="w-full" + /> + + Case-sensitive literal strings added to the built-in Claude Code and Copilot plan-mode markers. + +
); diff --git a/ui/litellm-dashboard/src/components/add_model/ReminderMarkers.tsx b/ui/litellm-dashboard/src/components/add_model/ReminderMarkers.tsx new file mode 100644 index 00000000000..c7f9ee48e0b --- /dev/null +++ b/ui/litellm-dashboard/src/components/add_model/ReminderMarkers.tsx @@ -0,0 +1,82 @@ +import React from "react"; +import { Plus, Trash2 } from "lucide-react"; +import { Button } from "@/components/ui/button"; +import { Input } from "@/components/ui/input"; +import type { ComplexityRouterConfigValue } from "./ComplexityRouterConfig"; +import { getReminderMarkersError, type ReminderMarkerPair } from "./build_complexity_router_config"; + +const ReminderMarkers: React.FC<{ + value: ComplexityRouterConfigValue; + onChange: (value: ComplexityRouterConfigValue) => void; + showValidationErrors?: boolean; +}> = ({ value, onChange, showValidationErrors = false }) => { + const markers = value.reminder_markers ?? []; + const update = (index: number, patch: Partial) => + onChange({ + ...value, + reminder_markers: markers.map((marker, markerIndex) => + markerIndex === index ? { ...marker, ...patch } : marker, + ), + }); + const remove = (index: number) => { + const next = markers.filter((_, markerIndex) => markerIndex !== index); + onChange({ ...value, reminder_markers: next.length > 0 ? next : undefined }); + }; + const error = getReminderMarkersError(value.reminder_markers); + return ( +
+

+ Delimiter pairs that wrap harness-injected reminder blocks, which are stripped before classification. Setting + any pair replaces the built-in pairs, so list every pair your harness emits. Matching is case-insensitive and + values are saved lowercased. +

+
+ {markers.map((marker, index) => ( +
+
+ + update(index, { open: event.target.value })} + /> +
+
+ + update(index, { close: event.target.value })} + /> +
+ +
+ ))} +
+ + {showValidationErrors && error &&

{error}

} +
+ ); +}; + +export default ReminderMarkers; diff --git a/ui/litellm-dashboard/src/components/add_model/ResponseFormatControls.tsx b/ui/litellm-dashboard/src/components/add_model/ResponseFormatControls.tsx index 68dd880a684..6e7b1489a4c 100644 --- a/ui/litellm-dashboard/src/components/add_model/ResponseFormatControls.tsx +++ b/ui/litellm-dashboard/src/components/add_model/ResponseFormatControls.tsx @@ -18,6 +18,18 @@ const ResponseFormatControls: React.FC<{ Return the resolved underlying model name in responses instead of the autorouter alias. +
+ onChange({ ...value, max_tokens_from_tier_model: enabled })} + aria-label="Cap max_tokens at the tier model's output ceiling" + /> + Cap max_tokens at the tier model's output ceiling +
+ + Replace the caller's max_tokens with the routed tier model's output ceiling so one client value fits + every tier. Off forwards the caller's value unchanged. + ); diff --git a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.test.tsx b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.test.tsx index fdff7ad001c..242893e2ae8 100644 --- a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.test.tsx +++ b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.test.tsx @@ -744,7 +744,7 @@ describe("AddAutoRouterTab", () => { ); }); - it("carries a context-window escalation opt-out through to the create payload", async () => { + it("starts context-window escalation disabled and carries an explicit opt-in to the create payload", async () => { const user = userEvent.setup(); vi.mocked(getMissingTiersError).mockReturnValue(null); @@ -754,14 +754,15 @@ describe("AddAutoRouterTab", () => { expandDetailedConfiguration(); await user.click(screen.getByText("Advanced: Context Window Escalation")); const toggle = await screen.findByRole("switch", { name: "Escalate oversized prompts to a tier that fits" }); - expect(toggle).toBeChecked(); + expect(toggle).not.toBeChecked(); + expect(screen.queryByLabelText("Window fit buffer")).not.toBeInTheDocument(); await user.click(toggle); await user.click(screen.getByRole("button", { name: /add auto router/i })); await waitFor(() => expect(handleAddAutoRouterSubmit).toHaveBeenCalled()); expect(vi.mocked(handleAddAutoRouterSubmit).mock.calls.at(-1)?.[0].complexity_router_config).toMatchObject({ - enable_context_window_escalation: false, + enable_context_window_escalation: true, }); }); @@ -774,6 +775,7 @@ describe("AddAutoRouterTab", () => { await user.type(screen.getByPlaceholderText(/smart_router/i), "ctx-buffer-router"); expandDetailedConfiguration(); await user.click(screen.getByText("Advanced: Context Window Escalation")); + await user.click(screen.getByRole("switch", { name: "Escalate oversized prompts to a tier that fits" })); const buffer = await screen.findByLabelText("Window fit buffer"); fireEvent.change(buffer, { target: { value: "1.5" } }); fireEvent.blur(buffer, { target: { value: "1.5" } }); @@ -783,7 +785,7 @@ describe("AddAutoRouterTab", () => { await waitFor(() => expect(handleAddAutoRouterSubmit).toHaveBeenCalled()); const config = vi.mocked(handleAddAutoRouterSubmit).mock.calls.at(-1)?.[0].complexity_router_config; expect(config).toMatchObject({ context_window_escalation_buffer: 1 }); - expect(config).not.toHaveProperty("enable_context_window_escalation"); + expect(config).toHaveProperty("enable_context_window_escalation", true); }); it("clearing the buffer removes it from the payload so the router tracks the backend default", async () => { @@ -795,6 +797,7 @@ describe("AddAutoRouterTab", () => { await user.type(screen.getByPlaceholderText(/smart_router/i), "ctx-clear-router"); expandDetailedConfiguration(); await user.click(screen.getByText("Advanced: Context Window Escalation")); + await user.click(screen.getByRole("switch", { name: "Escalate oversized prompts to a tier that fits" })); const buffer = await screen.findByLabelText("Window fit buffer"); fireEvent.change(buffer, { target: { value: "0.8" } }); fireEvent.blur(buffer, { target: { value: "0.8" } }); diff --git a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx index 84e44fee9c3..9be49edf08d 100644 --- a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx +++ b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx @@ -28,10 +28,6 @@ import ComplexityRouterConfig, { effectiveClassifierType, usesLlmClassifier, heuristicScoringRole, - DEFAULT_ADAPTIVE_WEIGHTS, - DEFAULT_SESSION_AFFINITY, - DEFAULT_DEPLOYMENT_AFFINITY, - DEFAULT_TIER_DISTANCE_PENALTY, } from "./ComplexityRouterConfig"; import { KeywordTierRule } from "./KeywordTierRules"; import { customDimensionsError } from "./custom_dimensions"; @@ -48,6 +44,8 @@ import { getKeywordTierRulesError, getClassifierModelError, getHeuristicV2SuccessThresholdError, + getReminderMarkersError, + getClassifierPluginTimeoutError, getClassifierReasoningEffortError, getMissingTiersError, getPlanModeTierError, @@ -55,6 +53,7 @@ import { getTierLabelsError, dryRunRejection, } from "./build_complexity_router_config"; +import { builderParamsFromValue } from "./complexity_router_builder_params"; import { activeTierName, activeTierRows, getCustomTierRowsError, resolveComplexityDefaultModel } from "./tier_rows"; import { tierRowLabel } from "./complexity_router_tiers"; import { buildAutoRouterTestTargets, AutoRouterTestTarget } from "./build_auto_router_test_targets"; @@ -152,6 +151,8 @@ export const getSubmitBlockedReason = ( getKeywordTierRulesError(keywordTierRules, activeTierRows(config)) ?? getClassifierModelError(config) ?? getHeuristicV2SuccessThresholdError(config.heuristic_v2_success_threshold) ?? + getReminderMarkersError(config.reminder_markers) ?? + getClassifierPluginTimeoutError(config.classifier_type, config.classifier_plugin_timeout_ms) ?? (heuristicScoringRole(config) === "decides" ? customDimensionsError(config.custom_dimensions) : null) ?? getClassifierReasoningEffortError(config, modelInfo) ?? getReferencedModelsError(referencedModelsParams, availability) @@ -399,56 +400,25 @@ const AddAutoRouterTab: React.FC = ({ ); const complexityRouterConfigParams: BuildComplexityRouterConfigParams = { - tiers: complexityRouterConfig.tiers, - enableNonReasoningTier: complexityRouterConfig.enable_non_reasoning_tier, - customTierSet: complexityRouterConfig.custom_tier_set, - defaultModel: complexityRouterConfig.default_model, - planModeMinTier: complexityRouterConfig.plan_mode_min_tier, - classificationPrompt: complexityRouterConfig.classification_prompt, - classificationExamples: complexityRouterConfig.classification_examples, - heuristicFirstMaxTier: complexityRouterConfig.heuristic_first_max_tier, - hybridBoundaryMargin: complexityRouterConfig.hybrid_boundary_margin, - classificationMode: complexityRouterConfig.classification_mode, - tierLabels: complexityRouterConfig.tier_labels, - classifierType: complexityRouterConfig.classifier_type, - jevClassifierConfig: complexityRouterConfig.jev_classifier_config, - heuristicV2SuccessThreshold: complexityRouterConfig.heuristic_v2_success_threshold, - capabilityClassifierConfig: complexityRouterConfig.capability_classifier_config, - llmV2Config: complexityRouterConfig.llm_v2_config, - classifierLlmConfig: complexityRouterConfig.classifier_llm_config, - classifierContextWindowSize: complexityRouterConfig.classifier_context_window_size, - classifierContextBudgetChars: complexityRouterConfig.classifier_context_budget_chars, - classifierContextPerTurnChars: complexityRouterConfig.classifier_context_per_turn_chars, - classifierContextIncludeAssistantTurns: complexityRouterConfig.classifier_context_include_assistant_turns, - classifierFallback: complexityRouterConfig.classifier_fallback, - sessionAffinity: complexityRouterConfig.session_affinity ?? DEFAULT_SESSION_AFFINITY, - modalityRouting: complexityRouterConfig.modality_routing ?? false, - modalityPinOverride: complexityRouterConfig.modality_pin_override ?? false, - deploymentAffinity: complexityRouterConfig.deployment_affinity ?? DEFAULT_DEPLOYMENT_AFFINITY, + ...builderParamsFromValue(complexityRouterConfig), customTechnicalKeywords, keywordTierRules, semanticMatchingEnabled, embeddingModel, matchThreshold, escalationKeywords, - stallEscalationEnabled: complexityRouterConfig.stall_escalation_enabled, - stallEscalationWindow: complexityRouterConfig.stall_escalation_window, - stallEscalationRepeatThreshold: complexityRouterConfig.stall_escalation_repeat_threshold, - adaptive: complexityRouterConfig.adaptive ?? false, - adaptiveWeights: complexityRouterConfig.adaptive_weights ?? DEFAULT_ADAPTIVE_WEIGHTS, - tierDistancePenalty: complexityRouterConfig.tier_distance_penalty ?? DEFAULT_TIER_DISTANCE_PENALTY, - adaptiveEligible: complexityRouterConfig.adaptive_eligible ?? "all", - returnRawModelName: complexityRouterConfig.return_raw_model_name ?? false, - tierModelParams: complexityRouterConfig.tier_model_params, - tierBoundaries: complexityRouterConfig.tier_boundaries, - tokenThresholds: complexityRouterConfig.token_thresholds, - dimensionWeights: complexityRouterConfig.dimension_weights, - customDimensions: complexityRouterConfig.custom_dimensions, - reasoningOverrideMinScore: complexityRouterConfig.reasoning_override_min_score, - enableContextWindowEscalation: complexityRouterConfig.enable_context_window_escalation, - contextWindowEscalationBuffer: complexityRouterConfig.context_window_escalation_buffer, - sessionAffinityTtlSeconds: complexityRouterConfig.session_affinity_ttl_seconds, }; + const jevRequestParams = + effectiveClassifierType(complexityRouterConfig) === "jev" + ? { + prompt: JEV_CONNECTION_TEST_PROMPT, + config: buildComplexityRouterConfig(complexityRouterConfigParams), + defaultModel: resolveComplexityDefaultModel(complexityRouterConfig, complexityRouterConfig.default_model), + routerName: watchedName, + teamId: requiresTeamScope ? watchedTeamId ?? undefined : undefined, + } + : undefined; + const jevRequest = jevRequestParams ? buildAutoRouterRoutingTestRequest(jevRequestParams) : undefined; const submitRecommendedRouter = async (name: string) => { // The one answer the submit button reads, so a disabled button and a refused submit cannot @@ -857,20 +827,7 @@ const AddAutoRouterTab: React.FC = ({ testId={connectionTestId} accessToken={accessToken} targets={testTargets} - jevRequest={ - effectiveClassifierType(complexityRouterConfig) === "jev" - ? buildAutoRouterRoutingTestRequest({ - prompt: JEV_CONNECTION_TEST_PROMPT, - config: buildComplexityRouterConfig(complexityRouterConfigParams), - defaultModel: resolveComplexityDefaultModel( - complexityRouterConfig, - complexityRouterConfig.default_model, - ), - routerName: watchedName, - teamId: requiresTeamScope ? watchedTeamId ?? undefined : undefined, - }) - : undefined - } + jevRequest={jevRequest} onTestComplete={() => setIsTestingConnection(false)} /> diff --git a/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.test.ts b/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.test.ts index 054aba7a6aa..6d458d61797 100644 --- a/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.test.ts +++ b/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.test.ts @@ -6,6 +6,8 @@ import { getKeywordTierRulesError, getClassifierModelError, getHeuristicV2SuccessThresholdError, + getReminderMarkersError, + getClassifierPluginTimeoutError, getClassifierReasoningEffortError, getMissingTiersError, hydrateCustomTierSet, @@ -166,7 +168,7 @@ describe("buildComplexityRouterConfig", () => { }); it.each(["capability", "llm_v2", "heuristic"] as const)( - "disables the removed overrides only for forecast creates: %s", + "preserves explicit context-window opt-in beside forecast restrictions: %s", (classifierType) => { const forecast = classifierType !== "heuristic"; const params = { @@ -178,14 +180,10 @@ describe("buildComplexityRouterConfig", () => { }; const config = buildComplexityRouterConfig(params); expect(config.adaptive).toBe(!forecast); - expect(config.enable_context_window_escalation).toBe(!forecast); + expect(config.enable_context_window_escalation).toBe(true); + expect(config.context_window_escalation_buffer).toBe(0.9); expect(config.escalation_keywords).toEqual(forecast ? [] : ["LITELLM ESCALATE"]); - for (const key of [ - "adaptive_weights", - "adaptive_eligible", - "tier_distance_penalty", - "context_window_escalation_buffer", - ]) { + for (const key of ["adaptive_weights", "adaptive_eligible", "tier_distance_penalty"]) { expect(Object.hasOwn(config, key)).toBe(!forecast); } if (forecast) { @@ -224,13 +222,14 @@ describe("buildComplexityRouterConfig", () => { expect(config).toEqual(expected); }); - it("carries an explicit context-window escalation opt-out and buffer, false included", () => { + it.each([undefined, false, true])("preserves the context-window escalation setting: %s", (enabled) => { const config = buildComplexityRouterConfig({ ...baseParams, - enableContextWindowEscalation: false, + enableContextWindowEscalation: enabled, contextWindowEscalationBuffer: 0.9, }); - expect(config.enable_context_window_escalation).toBe(false); + expect(config.enable_context_window_escalation).toBe(enabled); + expect(Object.hasOwn(config, "enable_context_window_escalation")).toBe(enabled !== undefined); expect(config.context_window_escalation_buffer).toBe(0.9); }); @@ -1482,3 +1481,75 @@ describe("classifier vision wire payload", () => { expect(payload.classifier_llm_config).not.toHaveProperty("vision"); }); }); + +describe("advanced complexity router fields", () => { + it("normalizes lists, reminder markers, and explicit false values", () => { + const payload = buildComplexityRouterConfig({ + ...baseParams, + codeKeywords: [" async ", " "], + reasoningKeywords: ["prove"], + technicalKeywords: ["api"], + simpleKeywords: ["hello"], + planModePatterns: [" plan "], + routeHousekeepingToCheapestTier: false, + housekeepingPatterns: [" title "], + reminderMarkers: [{ open: " ", close: " " }], + maxTokensFromTierModel: false, + classifierType: "custom", + classifierPluginTimeoutMs: 3000, + }); + expect(payload).toMatchObject({ + code_keywords: ["async"], + reasoning_keywords: ["prove"], + technical_keywords: ["api"], + simple_keywords: ["hello"], + plan_mode_patterns: ["plan"], + route_housekeeping_to_cheapest_tier: false, + housekeeping_patterns: ["title"], + reminder_markers: [{ open: "", close: "" }], + max_tokens_from_tier_model: false, + classifier_plugin_timeout_ms: 3000, + }); + }); + + it("omits defaults, empty lists, and timeout values for non-custom classifiers", () => { + const payload = buildComplexityRouterConfig({ + ...baseParams, + codeKeywords: [" ", ""], + reminderMarkers: [], + routeHousekeepingToCheapestTier: true, + maxTokensFromTierModel: true, + classifierPluginTimeoutMs: 3000, + }); + expect(payload).not.toHaveProperty("code_keywords"); + expect(payload).not.toHaveProperty("reminder_markers"); + expect(payload).not.toHaveProperty("route_housekeeping_to_cheapest_tier"); + expect(payload).not.toHaveProperty("max_tokens_from_tier_model"); + expect(payload).not.toHaveProperty("classifier_plugin_timeout_ms"); + }); + + it.each([ + "code_keywords", + "reasoning_keywords", + "technical_keywords", + "simple_keywords", + "plan_mode_patterns", + "route_housekeeping_to_cheapest_tier", + "housekeeping_patterns", + "reminder_markers", + "max_tokens_from_tier_model", + "classifier_plugin_timeout_ms", + ])("omits unset advanced field %s", (key) => { + const payload = buildComplexityRouterConfig(baseParams); + expect(payload).not.toHaveProperty(key); + }); + + it("validates marker pairs and custom classifier timeout", () => { + expect(getReminderMarkersError([{ open: " ", close: " " }])).toContain("different"); + expect(getReminderMarkersError([{ open: "", close: "" }])).toContain("needs both"); + expect(getReminderMarkersError([{ open: "", close: "" }])).toBeNull(); + expect(getClassifierPluginTimeoutError("custom", 0)).toContain("whole number"); + expect(getClassifierPluginTimeoutError("custom", 3000)).toBeNull(); + expect(getClassifierPluginTimeoutError("heuristic", 0)).toBeNull(); + }); +}); diff --git a/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts b/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts index 15ae2b4c0b0..9b4f1980f62 100644 --- a/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts +++ b/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts @@ -54,6 +54,10 @@ import { export type ClassifierVisionConfig = { enabled?: boolean; max_images?: number }; export type ClassifierLLMConfigWire = ClassifierLLMConfig & { vision?: ClassifierVisionConfig }; +export interface ReminderMarkerPair { + open: string; + close: string; +} /** * Drop an empty system_prompt so the payload carries an override only when there is one. The @@ -181,6 +185,16 @@ export interface StoredComplexityRouterConfig { stall_escalation_enabled?: unknown; stall_escalation_window?: unknown; stall_escalation_repeat_threshold?: unknown; + code_keywords?: unknown; + reasoning_keywords?: unknown; + technical_keywords?: unknown; + simple_keywords?: unknown; + plan_mode_patterns?: unknown; + route_housekeeping_to_cheapest_tier?: unknown; + housekeeping_patterns?: unknown; + reminder_markers?: unknown; + max_tokens_from_tier_model?: unknown; + classifier_plugin_timeout_ms?: unknown; } export interface BuildComplexityRouterConfigParams { @@ -233,6 +247,16 @@ export interface BuildComplexityRouterConfigParams { enableContextWindowEscalation?: boolean; contextWindowEscalationBuffer?: number; sessionAffinityTtlSeconds?: number; + codeKeywords?: string[]; + reasoningKeywords?: string[]; + technicalKeywords?: string[]; + simpleKeywords?: string[]; + planModePatterns?: string[]; + routeHousekeepingToCheapestTier?: boolean; + housekeepingPatterns?: string[]; + reminderMarkers?: ReminderMarkerPair[]; + maxTokensFromTierModel?: boolean; + classifierPluginTimeoutMs?: number; } /** @@ -302,6 +326,16 @@ export interface ComplexityRouterConfigPayload { enable_context_window_escalation?: boolean; context_window_escalation_buffer?: number; tier_model_configs?: Record; + code_keywords?: string[]; + reasoning_keywords?: string[]; + technical_keywords?: string[]; + simple_keywords?: string[]; + plan_mode_patterns?: string[]; + route_housekeeping_to_cheapest_tier?: boolean; + housekeeping_patterns?: string[]; + reminder_markers?: ReminderMarkerPair[]; + max_tokens_from_tier_model?: boolean; + classifier_plugin_timeout_ms?: number; } export const serializeTierLabels = (tierLabels: ComplexityTierLabels | undefined): ComplexityTierLabels | undefined => { @@ -376,6 +410,26 @@ export const getHeuristicV2SuccessThresholdError = (threshold: number | undefine return validProbability ? null : "Success threshold must be a number between 0 and 1"; }; +export const getReminderMarkersError = (pairs: ReminderMarkerPair[] | undefined): string | null => { + for (const [index, pair] of (pairs ?? []).entries()) { + const open = pair.open.trim().toLowerCase(); + const close = pair.close.trim().toLowerCase(); + if (!open || !close) return `Reminder marker pair ${index + 1} needs both an opening and a closing delimiter`; + if (open === close) return `Reminder marker pair ${index + 1} must use different opening and closing delimiters`; + } + return null; +}; + +export const getClassifierPluginTimeoutError = ( + classifierType: ClassifierType, + timeoutMs: number | undefined, +): string | null => { + if (classifierType !== "custom" || timeoutMs === undefined) return null; + return Number.isInteger(timeoutMs) && timeoutMs > 0 + ? null + : "Classifier plugin timeout must be a whole number of milliseconds greater than 0"; +}; + export const getClassifierModelError = ( config: Pick< ComplexityRouterConfigValue, @@ -640,6 +694,16 @@ export const buildComplexityRouterConfig = ({ enableContextWindowEscalation, contextWindowEscalationBuffer, sessionAffinityTtlSeconds, + codeKeywords, + reasoningKeywords, + technicalKeywords, + simpleKeywords, + planModePatterns, + routeHousekeepingToCheapestTier, + housekeepingPatterns, + reminderMarkers, + maxTokensFromTierModel, + classifierPluginTimeoutMs, }: BuildComplexityRouterConfigParams): ComplexityRouterConfigPayload => { const serializedTierModelConfigs = customTierSet ? serializeTierModelConfigs( @@ -672,6 +736,29 @@ export const buildComplexityRouterConfig = ({ }; const effectiveType = effectiveClassifierType({ custom_tier_set: customTierSet, classifier_type: classifierType }); const forecast = isForecastClassifier(effectiveType); + const preserveContextWindowBuffer = !forecast || enableContextWindowEscalation === true; + const cleanList = (items: string[] | undefined): string[] | undefined => { + const cleaned = (items ?? []).map((item) => item.trim()).filter(Boolean); + return cleaned.length > 0 ? cleaned : undefined; + }; + const cleanedReminderMarkers = reminderMarkers?.map(({ open, close }) => ({ + open: open.trim().toLowerCase(), + close: close.trim().toLowerCase(), + })); + const cleanedListValues = { + code_keywords: cleanList(codeKeywords), + reasoning_keywords: cleanList(reasoningKeywords), + technical_keywords: cleanList(technicalKeywords), + simple_keywords: cleanList(simpleKeywords), + plan_mode_patterns: cleanList(planModePatterns), + housekeeping_patterns: cleanList(housekeepingPatterns), + }; + const cleanedLists = Object.fromEntries(Object.entries(cleanedListValues).filter(([, list]) => list !== undefined)); + const hasValidCustomClassifierTimeout = + classifierType === "custom" && + classifierPluginTimeoutMs !== undefined && + Number.isInteger(classifierPluginTimeoutMs) && + classifierPluginTimeoutMs > 0; const supportsOpeningPrompt = !customTierSet && !forecast && usesLlmClassifier(effectiveType); const payload: ComplexityRouterConfigPayload = { @@ -729,17 +816,21 @@ export const buildComplexityRouterConfig = ({ adaptive_eligible: adaptiveEligible, }), ...(returnRawModelName && { return_raw_model_name: true }), - // Omission enables the backend default, so hidden forecast controls need an explicit opt-out. ...((forecast || enableContextWindowEscalation !== undefined) && { - enable_context_window_escalation: forecast ? false : enableContextWindowEscalation, + enable_context_window_escalation: enableContextWindowEscalation ?? false, }), - ...(!forecast && + ...(preserveContextWindowBuffer && contextWindowEscalationBuffer !== undefined && { context_window_escalation_buffer: contextWindowEscalationBuffer, }), ...(sessionAffinityTtlSeconds !== undefined && { session_affinity_ttl_seconds: sessionAffinityTtlSeconds, }), + ...cleanedLists, + ...(routeHousekeepingToCheapestTier === false && { route_housekeeping_to_cheapest_tier: false }), + ...(cleanedReminderMarkers && cleanedReminderMarkers.length > 0 && { reminder_markers: cleanedReminderMarkers }), + ...(maxTokensFromTierModel === false && { max_tokens_from_tier_model: false }), + ...(hasValidCustomClassifierTimeout && { classifier_plugin_timeout_ms: classifierPluginTimeoutMs }), ...scorerKnobs, }; if (!customTierSet) return payload; diff --git a/ui/litellm-dashboard/src/components/add_model/classifier_types.ts b/ui/litellm-dashboard/src/components/add_model/classifier_types.ts index ec88166ed2e..aa9d5619052 100644 --- a/ui/litellm-dashboard/src/components/add_model/classifier_types.ts +++ b/ui/litellm-dashboard/src/components/add_model/classifier_types.ts @@ -6,7 +6,8 @@ export type ClassifierType = | "heuristic_first" | "hybrid" | "capability" - | "llm_v2"; + | "llm_v2" + | "custom"; export const usesLlmClassifier = (classifierType: ClassifierType): boolean => (["llm", "heuristic_first", "hybrid", "capability", "llm_v2"] as const).some((type) => type === classifierType); diff --git a/ui/litellm-dashboard/src/components/add_model/complexity_router_builder_params.ts b/ui/litellm-dashboard/src/components/add_model/complexity_router_builder_params.ts new file mode 100644 index 00000000000..124a85ce9a3 --- /dev/null +++ b/ui/litellm-dashboard/src/components/add_model/complexity_router_builder_params.ts @@ -0,0 +1,74 @@ +import type { BuildComplexityRouterConfigParams } from "./build_complexity_router_config"; +import type { ComplexityRouterConfigValue } from "./ComplexityRouterConfig"; +import { + DEFAULT_ADAPTIVE_WEIGHTS, + DEFAULT_DEPLOYMENT_AFFINITY, + DEFAULT_SESSION_AFFINITY, + DEFAULT_TIER_DISTANCE_PENALTY, +} from "./ComplexityRouterConfig"; + +export const builderParamsFromValue = ( + value: ComplexityRouterConfigValue, +): Omit< + BuildComplexityRouterConfigParams, + | "customTechnicalKeywords" + | "keywordTierRules" + | "semanticMatchingEnabled" + | "embeddingModel" + | "matchThreshold" + | "escalationKeywords" +> => ({ + tiers: value.tiers, + enableNonReasoningTier: value.enable_non_reasoning_tier, + customTierSet: value.custom_tier_set, + defaultModel: value.default_model, + planModeMinTier: value.plan_mode_min_tier, + classificationPrompt: value.classification_prompt, + classificationExamples: value.classification_examples, + heuristicFirstMaxTier: value.heuristic_first_max_tier, + hybridBoundaryMargin: value.hybrid_boundary_margin, + classificationMode: value.classification_mode, + tierLabels: value.tier_labels, + classifierType: value.classifier_type, + jevClassifierConfig: value.jev_classifier_config, + heuristicV2SuccessThreshold: value.heuristic_v2_success_threshold, + capabilityClassifierConfig: value.capability_classifier_config, + llmV2Config: value.llm_v2_config, + classifierLlmConfig: value.classifier_llm_config, + classifierContextWindowSize: value.classifier_context_window_size, + classifierContextBudgetChars: value.classifier_context_budget_chars, + classifierContextPerTurnChars: value.classifier_context_per_turn_chars, + classifierContextIncludeAssistantTurns: value.classifier_context_include_assistant_turns, + classifierFallback: value.classifier_fallback, + sessionAffinity: value.session_affinity ?? DEFAULT_SESSION_AFFINITY, + sessionAffinityTtlSeconds: value.session_affinity_ttl_seconds, + modalityRouting: value.modality_routing ?? false, + modalityPinOverride: value.modality_pin_override ?? false, + deploymentAffinity: value.deployment_affinity ?? DEFAULT_DEPLOYMENT_AFFINITY, + adaptive: value.adaptive ?? false, + adaptiveWeights: value.adaptive_weights ?? DEFAULT_ADAPTIVE_WEIGHTS, + tierDistancePenalty: value.tier_distance_penalty ?? DEFAULT_TIER_DISTANCE_PENALTY, + adaptiveEligible: value.adaptive_eligible ?? "all", + returnRawModelName: value.return_raw_model_name ?? false, + tierBoundaries: value.tier_boundaries, + tokenThresholds: value.token_thresholds, + dimensionWeights: value.dimension_weights, + customDimensions: value.custom_dimensions, + reasoningOverrideMinScore: value.reasoning_override_min_score, + tierModelParams: value.tier_model_params, + enableContextWindowEscalation: value.enable_context_window_escalation, + contextWindowEscalationBuffer: value.context_window_escalation_buffer, + stallEscalationEnabled: value.stall_escalation_enabled, + stallEscalationWindow: value.stall_escalation_window, + stallEscalationRepeatThreshold: value.stall_escalation_repeat_threshold, + codeKeywords: value.code_keywords, + reasoningKeywords: value.reasoning_keywords, + technicalKeywords: value.technical_keywords, + simpleKeywords: value.simple_keywords, + planModePatterns: value.plan_mode_patterns, + routeHousekeepingToCheapestTier: value.route_housekeeping_to_cheapest_tier, + housekeepingPatterns: value.housekeeping_patterns, + reminderMarkers: value.reminder_markers, + maxTokensFromTierModel: value.max_tokens_from_tier_model, + classifierPluginTimeoutMs: value.classifier_plugin_timeout_ms, +}); diff --git a/ui/litellm-dashboard/src/components/add_model/handle_add_model_submit.test.tsx b/ui/litellm-dashboard/src/components/add_model/handle_add_model_submit.test.tsx index 9d792480c9f..923cf2aa0e3 100644 --- a/ui/litellm-dashboard/src/components/add_model/handle_add_model_submit.test.tsx +++ b/ui/litellm-dashboard/src/components/add_model/handle_add_model_submit.test.tsx @@ -101,4 +101,23 @@ describe("prepareModelAddRequest", () => { expect(deployment.litellmParamsObj.litellm_credential_name).toBe("from-json"); expect(deployment.litellmParamsObj.timeout).toBe(5); }); + + it.each([ + ["OpenAI", "openai/*"], + ["Azure_AI_Studio", "azure_ai/*"], + ["Petals", "petals/*"], + ])("composes wildcard names for the all-model selection", async (custom_llm_provider, wildcardModel) => { + const formValues = { + model_mappings: [], + model: "all-wildcard", + custom_llm_provider, + }; + + const deployments = await prepareModelAddRequest({ ...formValues }, "token", null); + + expect(deployments).toHaveLength(1); + const [deployment] = deployments!; + expect(deployment.modelName).toBe(wildcardModel); + expect(deployment.litellmParamsObj.model).toBe(wildcardModel); + }); }); diff --git a/ui/litellm-dashboard/src/components/agents/types.ts b/ui/litellm-dashboard/src/components/agents/types.ts index 24ff0c0e12c..6adb3fa9dda 100644 --- a/ui/litellm-dashboard/src/components/agents/types.ts +++ b/ui/litellm-dashboard/src/components/agents/types.ts @@ -21,6 +21,7 @@ export interface Agent { [key: string]: any; }; object_permission?: AgentObjectPermission; + access_group_ids?: string[] | null; keys?: AgentAttachedKey[] | null; spend?: number; tpm_limit?: number | null; diff --git a/ui/litellm-dashboard/src/components/edit_auto_router/build_updated_complexity_router_config.test.ts b/ui/litellm-dashboard/src/components/edit_auto_router/build_updated_complexity_router_config.test.ts index 2450f7bce27..162414b7814 100644 --- a/ui/litellm-dashboard/src/components/edit_auto_router/build_updated_complexity_router_config.test.ts +++ b/ui/litellm-dashboard/src/components/edit_auto_router/build_updated_complexity_router_config.test.ts @@ -189,14 +189,10 @@ describe("buildUpdatedComplexityRouterConfig keyword matching", () => { const saved = buildUpdatedComplexityRouterConfig(stored, value, undefined, keywordState); const forecast = classifier_type !== "heuristic"; expect(saved.adaptive).toBe(!forecast); - expect(saved.enable_context_window_escalation).toBe(!forecast); + expect(saved.enable_context_window_escalation).toBe(true); + expect(saved.context_window_escalation_buffer).toBe(0.9); expect(saved.escalation_keywords).toEqual(forecast ? [] : stored.escalation_keywords); - for (const key of [ - "adaptive_weights", - "adaptive_eligible", - "tier_distance_penalty", - "context_window_escalation_buffer", - ]) { + for (const key of ["adaptive_weights", "adaptive_eligible", "tier_distance_penalty"]) { expect(Object.hasOwn(saved, key)).toBe(!forecast); } expect(saved.keyword_tier_rules).toEqual(STORED.keyword_tier_rules); @@ -208,6 +204,19 @@ describe("buildUpdatedComplexityRouterConfig keyword matching", () => { }, ); + it.each([undefined, false, true])("preserves stored context-window escalation on save: %s", (enabled) => { + const stored = { + ...STORED, + ...(enabled !== undefined && { enable_context_window_escalation: enabled }), + }; + const value = hydrateComplexityRouterConfig(stored, undefined); + const saved = buildUpdatedComplexityRouterConfig(stored, value, undefined, hydratedState); + const serialized: typeof saved = JSON.parse(JSON.stringify(saved)); + expect(value.enable_context_window_escalation).toBe(enabled); + expect(serialized.enable_context_window_escalation).toBe(enabled); + expect(Object.hasOwn(serialized, "enable_context_window_escalation")).toBe(enabled !== undefined); + }); + it("round-trips an untouched edit without changing any keyword-matching value", () => { // Opening the modal hydrates state from STORED; saving with nothing changed must be a // no-op. These keys are now MANAGED, so a hydration bug silently wipes them. @@ -854,6 +863,16 @@ describe("managed keys survive an untouched open-and-save", () => { reasoning_override_min_score: 0.3, enable_context_window_escalation: false, context_window_escalation_buffer: 0.9, + code_keywords: ["async", "await"], + reasoning_keywords: ["prove"], + technical_keywords: ["api"], + simple_keywords: ["hello"], + plan_mode_patterns: ["plan now"], + route_housekeeping_to_cheapest_tier: false, + housekeeping_patterns: ["conversation title"], + reminder_markers: [{ open: "", close: "" }], + max_tokens_from_tier_model: false, + classifier_plugin_timeout_ms: 3000, }; // tier_definitions and fallback_tier cannot sit beside heuristic_first, which this fixture uses, @@ -864,6 +883,7 @@ describe("managed keys survive an untouched open-and-save", () => { "fallback_tier", "hybrid_boundary_margin", "jev_classifier_config", + "classifier_plugin_timeout_ms", ]); // The stall keys are rejected beside the session pinning and user-turn classification this @@ -894,6 +914,12 @@ describe("managed keys survive an untouched open-and-save", () => { expect(dropped).toEqual([]); }); + it("keeps the custom classifier plugin timeout through an untouched save", () => { + const stored = { ...STORED_ALL_MANAGED, classifier_type: "custom", classifier_plugin_timeout_ms: 3000 }; + const hydrated = hydrateComplexityRouterConfig(stored, undefined); + expect(buildUpdatedComplexityRouterConfig(stored, hydrated).classifier_plugin_timeout_ms).toBe(3000); + }); + it("carries an enabled non-reasoning tier and its models through their own round trip", () => { // `tiers` is rewritten wholesale on save, so this is the regression that matters: opening an // enabled router and saving an unrelated edit must not delete the tier or its pool. diff --git a/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.integration.test.tsx b/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.integration.test.tsx index 34db61483cf..aa6cf92ceb9 100644 --- a/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.integration.test.tsx +++ b/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.integration.test.tsx @@ -347,6 +347,77 @@ describe("EditAutoRouterModal keyword matching", () => { }); }); +describe("EditAutoRouterModal advanced field round trips", () => { + const storedAdvancedConfig = { + ...STORED_CONFIG, + route_housekeeping_to_cheapest_tier: false, + housekeeping_patterns: ["conversation title"], + reminder_markers: [{ open: "", close: "" }], + max_tokens_from_tier_model: false, + }; + + const renderAdvancedModal = (props: Partial> = {}) => + renderModal({ + modelData: { + ...MODEL_DATA, + litellm_params: { ...MODEL_DATA.litellm_params, complexity_router_config: storedAdvancedConfig }, + }, + ...props, + }); + + beforeEach(() => { + modelPatchUpdateCall.mockClear(); + }); + + it("hydrates housekeeping and reminder fields, then omits the default max-token value after editing", async () => { + const user = userEvent.setup(); + renderAdvancedModal(); + + await user.click(await screen.findByText("Advanced: Housekeeping Routing")); + expect(screen.getByRole("switch", { name: "Route housekeeping calls to the cheapest tier" })).not.toBeChecked(); + expect(screen.getByRole("combobox", { name: "e.g., conversation title" })).toHaveValue(""); + + await user.click(screen.getByText("Advanced: Reminder Markers")); + expect(screen.getByLabelText("Opening delimiter")).toHaveValue(""); + expect(screen.getByLabelText("Closing delimiter")).toHaveValue(""); + + await user.click(screen.getByText("Advanced: Response Format")); + const maxTokensSwitch = screen.getByRole("switch", { name: "Cap max_tokens at the tier model's output ceiling" }); + await user.click(maxTokensSwitch); + await user.click(screen.getByRole("button", { name: /save changes/i })); + await waitFor(() => expect(modelPatchUpdateCall).toHaveBeenCalledOnce()); + + expect(savedConfig()).not.toHaveProperty("max_tokens_from_tier_model"); + expect(savedConfig()).toMatchObject({ + route_housekeeping_to_cheapest_tier: false, + housekeeping_patterns: ["conversation title"], + reminder_markers: [{ open: "", close: "" }], + }); + }); + + it("does not PATCH when the edit is cancelled", async () => { + const user = userEvent.setup(); + const onCancel = vi.fn(); + renderAdvancedModal({ onCancel }); + await user.click(screen.getByRole("button", { name: /cancel/i })); + expect(onCancel).toHaveBeenCalledOnce(); + expect(modelPatchUpdateCall).not.toHaveBeenCalled(); + }); + + it("preserves all stored advanced fields through an untouched save", async () => { + const user = userEvent.setup(); + renderAdvancedModal(); + await user.click(screen.getByRole("button", { name: /save changes/i })); + await waitFor(() => expect(modelPatchUpdateCall).toHaveBeenCalledOnce()); + expect(savedConfig()).toMatchObject({ + route_housekeeping_to_cheapest_tier: false, + housekeeping_patterns: ["conversation title"], + reminder_markers: [{ open: "", close: "" }], + max_tokens_from_tier_model: false, + }); + }); +}); + describe("EditAutoRouterModal classifier context window", () => { beforeEach(() => { modelPatchUpdateCall.mockClear(); diff --git a/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.tsx b/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.tsx index c88bbb101f7..0dafa9b330a 100644 --- a/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.tsx +++ b/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.tsx @@ -1,14 +1,7 @@ import AutoRouterClassifierTabs from "../add_model/AutoRouterClassifierTabs"; import { usesClassifierContext } from "../add_model/classifier_types"; -import { defaultJevClassifierConfig, jevClassifierConfigSchema } from "../add_model/jev_classifier_config"; -import type { StoredComplexityRouterConfig } from "../add_model/build_complexity_router_config"; export type { StoredComplexityRouterConfig } from "../add_model/build_complexity_router_config"; -import { - getForecastConfigError, - isForecastClassifier, - capabilitySettingsSchema, - fuseSettingsSchema, -} from "../add_model/forecast_classifier_config"; +import { getForecastConfigError, isForecastClassifier } from "../add_model/forecast_classifier_config"; import React, { useEffect, useMemo, useState } from "react"; import { complexityRouterSchema, @@ -30,13 +23,10 @@ import ModelChoiceCombobox, { type ModelChoice } from "../add_model/ModelChoiceC import { modelAvailableCall, modelPatchUpdateCall, validateAutoRouterConfig } from "../networking"; import { fetchAutoRouterModels, fetchAvailableModels, ModelGroup } from "@/components/llm_calls/fetch_models"; import RouterConfigBuilder, { type RouterConfig, serializeRouterConfig } from "../add_model/RouterConfigBuilder"; -import { hydrateTierModelParams } from "../add_model/complexity_router_tiers"; import { - type ActiveTierSet, CUSTOM_TIER_OMITTED_KEYS, activeTierRows, getCustomTierRowsError, - tierParamsByRowId, resolveComplexityDefaultModel, } from "../add_model/tier_rows"; import { isComplexityRouter } from "../add_model/auto_router_strategies"; @@ -45,16 +35,14 @@ import { buildComplexityRouterConfig, getClassifierModelError, getHeuristicV2SuccessThresholdError, + getReminderMarkersError, + getClassifierPluginTimeoutError, getClassifierReasoningEffortError, getKeywordTierRulesError, getMissingTiersError, getSemanticConfigError, getPlanModeTierError, getTierLabelsError, - hydrateBuiltInTiers, - hydrateCustomTierSet, - hydratePlanModeMinTier, - hydrateTierLabels, dryRunRejection, } from "../add_model/build_complexity_router_config"; import { KeywordTierRule } from "../add_model/KeywordTierRules"; @@ -66,22 +54,14 @@ import { hydrateAutoRouterCompression, } from "../add_model/buildAutoRouterCompression"; import { hydrateKeywordTierRules } from "../add_model/complexity_router_keywords"; -import { customDimensionsError, hydrateCustomDimensions } from "../add_model/custom_dimensions"; -import { - hydrateDimensionWeights, - hydrateReasoningOverrideMinScore, - hydrateTierBoundaries, - hydrateTokenThresholds, -} from "../add_model/heuristic_scoring_knobs"; +import { customDimensionsError } from "../add_model/custom_dimensions"; import ComplexityRouterConfig, { ComplexityRouterConfigValue, effectiveClassifierType, heuristicScoringRole, - DEFAULT_ADAPTIVE_WEIGHTS, - DEFAULT_SESSION_AFFINITY, - DEFAULT_DEPLOYMENT_AFFINITY, - DEFAULT_TIER_DISTANCE_PENALTY, } from "../add_model/ComplexityRouterConfig"; +import { builderParamsFromValue } from "../add_model/complexity_router_builder_params"; +import { hydrateComplexityRouterConfig, hydratePinnedDefaultModel } from "./hydrate_complexity_router_config"; import { Dialog, DialogContent, @@ -104,124 +84,7 @@ interface EditAutoRouterModalProps { // Keys this modal rewrites from its own form state on save. Anything absent from this set is // carried through untouched from the stored config, so a key only belongs here once the modal // actually renders a control that can set it. - -/** - * The stored complexity_router_config as form state. Every key in MANAGED_COMPLEXITY_ROUTER_KEYS is - * rewritten from this state on save, so a key missing here is silently dropped from the saved config. - */ -export const hydrateComplexityRouterConfig = ( - parsedConfig: StoredComplexityRouterConfig, - complexityRouterDefaultModel: string | null | undefined, -): ComplexityRouterConfigValue => { - const builtIn = hydrateBuiltInTiers(parsedConfig.tiers, parsedConfig.enable_non_reasoning_tier); - const { tiers: hydratedTiers, enable_non_reasoning_tier } = builtIn; - const custom_tier_set = hydrateCustomTierSet(parsedConfig); - const activeTiers = { ...builtIn, custom_tier_set }; - - return { - tiers: hydratedTiers, - enable_non_reasoning_tier, - custom_tier_set, - tier_model_params: tierParamsByRowId( - hydrateTierModelParams(parsedConfig.tiers, parsedConfig.tier_model_configs), - activeTierRows(activeTiers), - ), - default_model: hydratePinnedDefaultModel(parsedConfig.default_model, complexityRouterDefaultModel, activeTiers), - plan_mode_min_tier: hydratePlanModeMinTier(parsedConfig.plan_mode_min_tier, custom_tier_set), - tier_labels: hydrateTierLabels(parsedConfig.tier_labels), - classifier_type: parsedConfig.classifier_type || "heuristic", - heuristic_v2_success_threshold: - typeof parsedConfig.heuristic_v2_success_threshold === "number" - ? parsedConfig.heuristic_v2_success_threshold - : undefined, - capability_classifier_config: capabilitySettingsSchema.safeParse(parsedConfig.capability_classifier_config).data, - llm_v2_config: fuseSettingsSchema.safeParse(parsedConfig.llm_v2_config).data, - classifier_llm_config: parsedConfig.classifier_type === "jev" ? undefined : parsedConfig.classifier_llm_config, - jev_classifier_config: - parsedConfig.classifier_type === "jev" - ? jevClassifierConfigSchema.safeParse(parsedConfig.jev_classifier_config ?? {}).data ?? - defaultJevClassifierConfig() - : undefined, - classifier_context_window_size: - typeof parsedConfig.classifier_context_window_size === "number" - ? parsedConfig.classifier_context_window_size - : undefined, - classifier_context_budget_chars: - typeof parsedConfig.classifier_context_budget_chars === "number" - ? parsedConfig.classifier_context_budget_chars - : undefined, - classifier_context_per_turn_chars: - typeof parsedConfig.classifier_context_per_turn_chars === "number" - ? parsedConfig.classifier_context_per_turn_chars - : undefined, - classifier_context_include_assistant_turns: - typeof parsedConfig.classifier_context_include_assistant_turns === "boolean" - ? parsedConfig.classifier_context_include_assistant_turns - : undefined, - classifier_fallback: - parsedConfig.classifier_fallback === "default_model" || parsedConfig.classifier_fallback === "heuristic" - ? parsedConfig.classifier_fallback - : undefined, - classification_prompt: - typeof parsedConfig.classification_prompt === "string" && parsedConfig.classification_prompt.trim() !== "" - ? parsedConfig.classification_prompt - : undefined, - classification_examples: - typeof parsedConfig.classification_examples === "string" && parsedConfig.classification_examples.trim() !== "" - ? parsedConfig.classification_examples - : undefined, - heuristic_first_max_tier: - typeof parsedConfig.heuristic_first_max_tier === "string" && parsedConfig.heuristic_first_max_tier.trim() !== "" - ? parsedConfig.heuristic_first_max_tier - : undefined, - hybrid_boundary_margin: - typeof parsedConfig.hybrid_boundary_margin === "number" ? parsedConfig.hybrid_boundary_margin : undefined, - classification_mode: - parsedConfig.classification_mode === "user_turn" || parsedConfig.classification_mode === "every_request" - ? parsedConfig.classification_mode - : undefined, - tier_boundaries: hydrateTierBoundaries(parsedConfig.tier_boundaries), - token_thresholds: hydrateTokenThresholds(parsedConfig.token_thresholds), - dimension_weights: hydrateDimensionWeights(parsedConfig.dimension_weights), - custom_dimensions: hydrateCustomDimensions(parsedConfig.custom_dimensions), - reasoning_override_min_score: hydrateReasoningOverrideMinScore(parsedConfig.reasoning_override_min_score), - session_affinity: - typeof parsedConfig.session_affinity === "boolean" ? parsedConfig.session_affinity : DEFAULT_SESSION_AFFINITY, - session_affinity_ttl_seconds: - typeof parsedConfig.session_affinity_ttl_seconds === "number" && - Number.isFinite(parsedConfig.session_affinity_ttl_seconds) - ? parsedConfig.session_affinity_ttl_seconds - : undefined, - modality_routing: typeof parsedConfig.modality_routing === "boolean" ? parsedConfig.modality_routing : false, - modality_pin_override: - typeof parsedConfig.modality_pin_override === "boolean" ? parsedConfig.modality_pin_override : false, - deployment_affinity: - typeof parsedConfig.deployment_affinity === "boolean" - ? parsedConfig.deployment_affinity - : DEFAULT_DEPLOYMENT_AFFINITY, - adaptive: parsedConfig.adaptive || false, - adaptive_weights: parsedConfig.adaptive_weights, - tier_distance_penalty: parsedConfig.tier_distance_penalty, - adaptive_eligible: parsedConfig.adaptive_eligible || "all", - return_raw_model_name: parsedConfig.return_raw_model_name || false, - enable_context_window_escalation: - typeof parsedConfig.enable_context_window_escalation === "boolean" - ? parsedConfig.enable_context_window_escalation - : undefined, - context_window_escalation_buffer: - typeof parsedConfig.context_window_escalation_buffer === "number" - ? parsedConfig.context_window_escalation_buffer - : undefined, - stall_escalation_enabled: parsedConfig.stall_escalation_enabled === true || undefined, - stall_escalation_window: - typeof parsedConfig.stall_escalation_window === "number" ? parsedConfig.stall_escalation_window : undefined, - stall_escalation_repeat_threshold: - typeof parsedConfig.stall_escalation_repeat_threshold === "number" - ? parsedConfig.stall_escalation_repeat_threshold - : undefined, - }; -}; - +export { hydrateComplexityRouterConfig, hydratePinnedDefaultModel }; export const MANAGED_COMPLEXITY_ROUTER_KEYS = new Set([ "tiers", "enable_non_reasoning_tier", @@ -266,6 +129,16 @@ export const MANAGED_COMPLEXITY_ROUTER_KEYS = new Set([ "stall_escalation_enabled", "stall_escalation_window", "stall_escalation_repeat_threshold", + "code_keywords", + "reasoning_keywords", + "technical_keywords", + "simple_keywords", + "plan_mode_patterns", + "route_housekeeping_to_cheapest_tier", + "housekeeping_patterns", + "reminder_markers", + "max_tokens_from_tier_model", + "classifier_plugin_timeout_ms", ]); // Managed only when the caller passes the corresponding state. A caller that does not render @@ -285,24 +158,6 @@ const toRecord = (value: unknown): Record => { : {}; }; -// A pin lives in two places: complexity_router_config.default_model (this UI's own marker, added -// by PR #36615) and litellm_params.complexity_router_default_model (what the backend reads). Only -// the marker proves an operator picked it, because before #36615 every save wrote a tier-derived -// value into litellm_params. So with no marker, a litellm_params value counts as a pin only when -// it diverges from what the tiers alone derive; a match stays unpinned and keeps tracking tiers. -export const hydratePinnedDefaultModel = ( - storedConfigDefaultModel: unknown, - litellmParamsDefaultModel: string | null | undefined, - activeTiers: ActiveTierSet, -): string | undefined => { - if (typeof storedConfigDefaultModel === "string" && storedConfigDefaultModel.trim()) { - return storedConfigDefaultModel; - } - const tierDerived = resolveComplexityDefaultModel(activeTiers); - const externalOverride = litellmParamsDefaultModel?.trim(); - return externalOverride && externalOverride !== tierDerived ? externalOverride : undefined; -}; - export interface KeywordMatchingState { keywordTierRules: KeywordTierRule[]; escalationKeywords: string[]; @@ -338,55 +193,13 @@ export const buildUpdatedComplexityRouterConfig = ( ); const builderParams: BuildComplexityRouterConfigParams = { - tiers: value.tiers, - enableNonReasoningTier: value.enable_non_reasoning_tier, - customTierSet: value.custom_tier_set, - defaultModel: value.default_model, - planModeMinTier: value.plan_mode_min_tier, - classificationPrompt: value.classification_prompt, - classificationExamples: value.classification_examples, - heuristicFirstMaxTier: value.heuristic_first_max_tier, - hybridBoundaryMargin: value.hybrid_boundary_margin, - classificationMode: value.classification_mode, - tierLabels: value.tier_labels, - classifierType: value.classifier_type, - jevClassifierConfig: value.jev_classifier_config, - heuristicV2SuccessThreshold: value.heuristic_v2_success_threshold, - capabilityClassifierConfig: value.capability_classifier_config, - llmV2Config: value.llm_v2_config, - classifierLlmConfig: value.classifier_llm_config, - classifierContextWindowSize: value.classifier_context_window_size, - classifierContextBudgetChars: value.classifier_context_budget_chars, - classifierContextPerTurnChars: value.classifier_context_per_turn_chars, - classifierContextIncludeAssistantTurns: value.classifier_context_include_assistant_turns, - classifierFallback: value.classifier_fallback, - sessionAffinity: value.session_affinity ?? DEFAULT_SESSION_AFFINITY, - sessionAffinityTtlSeconds: value.session_affinity_ttl_seconds, - modalityRouting: value.modality_routing ?? false, - modalityPinOverride: value.modality_pin_override ?? false, - deploymentAffinity: value.deployment_affinity ?? DEFAULT_DEPLOYMENT_AFFINITY, + ...builderParamsFromValue(value), customTechnicalKeywords: customTechnicalKeywords ?? [], keywordTierRules: keywordMatching?.keywordTierRules ?? [], semanticMatchingEnabled: keywordMatching?.semanticMatchingEnabled ?? false, embeddingModel: keywordMatching?.embeddingModel, matchThreshold: keywordMatching?.matchThreshold ?? DEFAULT_MATCH_THRESHOLD, escalationKeywords: keywordMatching?.escalationKeywords ?? [], - adaptive: value.adaptive ?? false, - adaptiveWeights: value.adaptive_weights ?? DEFAULT_ADAPTIVE_WEIGHTS, - tierDistancePenalty: value.tier_distance_penalty ?? DEFAULT_TIER_DISTANCE_PENALTY, - adaptiveEligible: value.adaptive_eligible ?? "all", - returnRawModelName: value.return_raw_model_name ?? false, - tierBoundaries: value.tier_boundaries, - tokenThresholds: value.token_thresholds, - dimensionWeights: value.dimension_weights, - customDimensions: value.custom_dimensions, - reasoningOverrideMinScore: value.reasoning_override_min_score, - tierModelParams: value.tier_model_params, - enableContextWindowEscalation: value.enable_context_window_escalation, - contextWindowEscalationBuffer: value.context_window_escalation_buffer, - stallEscalationEnabled: value.stall_escalation_enabled, - stallEscalationWindow: value.stall_escalation_window, - stallEscalationRepeatThreshold: value.stall_escalation_repeat_threshold, }; const built = buildComplexityRouterConfig(builderParams); @@ -557,12 +370,13 @@ const EditAutoRouterModal: React.FC = ({ setRouterConfig(parsedConfig); // Set form values - form.reset({ + const routerFormValues = { auto_router_name: modelData.model_name, auto_router_default_model: modelData.litellm_params?.auto_router_default_model || null, auto_router_embedding_model: modelData.litellm_params?.auto_router_embedding_model || null, model_access_group: modelData.model_info?.access_groups || [], - }); + }; + form.reset(routerFormValues); } catch (error) { console.error("Error parsing auto router config:", error); toast.fromError("Error loading auto router configuration"); @@ -585,6 +399,11 @@ const EditAutoRouterModal: React.FC = ({ const classifierError = getClassifierModelError(complexityRouterConfig) ?? getHeuristicV2SuccessThresholdError(complexityRouterConfig.heuristic_v2_success_threshold) ?? + getReminderMarkersError(complexityRouterConfig.reminder_markers) ?? + getClassifierPluginTimeoutError( + complexityRouterConfig.classifier_type, + complexityRouterConfig.classifier_plugin_timeout_ms, + ) ?? getForecastConfigError(complexityRouterConfig) ?? (heuristicScoringRole(complexityRouterConfig) === "decides" ? customDimensionsError(complexityRouterConfig.custom_dimensions) @@ -635,11 +454,18 @@ const EditAutoRouterModal: React.FC = ({ // Dual write: complexity_router_config.default_model (the pin marker hydratePinnedDefaultModel // reads back) and complexity_router_default_model (what the backend routes on) must always be // written together from the same value. Same pairing in add_auto_router_tab.tsx. + const keywordMatching = { + keywordTierRules, + escalationKeywords, + semanticMatchingEnabled, + embeddingModel, + matchThreshold, + }; const updatedConfig = buildUpdatedComplexityRouterConfig( modelData.litellm_params?.complexity_router_config, complexityRouterConfig, customTechnicalKeywords, - { keywordTierRules, escalationKeywords, semanticMatchingEnabled, embeddingModel, matchThreshold }, + keywordMatching, ); const serverVerdict = await validateAutoRouterConfig(accessToken, updatedConfig, modelData?.model_info?.team_id); const dryRunError = dryRunRejection(serverVerdict); @@ -676,12 +502,13 @@ const EditAutoRouterModal: React.FC = ({ ); toast.success("Auto router configuration updated successfully"); - onSuccess({ + const updatedModelData = { ...modelData, model_name: values.auto_router_name, litellm_params: updatedLitellmParams, model_info: updatedModelInfo, - }); + }; + onSuccess(updatedModelData); onCancel(); return; } diff --git a/ui/litellm-dashboard/src/components/edit_auto_router/hydrate_complexity_router_config.ts b/ui/litellm-dashboard/src/components/edit_auto_router/hydrate_complexity_router_config.ts new file mode 100644 index 00000000000..6dbd2b19b52 --- /dev/null +++ b/ui/litellm-dashboard/src/components/edit_auto_router/hydrate_complexity_router_config.ts @@ -0,0 +1,186 @@ +import { defaultJevClassifierConfig, jevClassifierConfigSchema } from "../add_model/jev_classifier_config"; +import { capabilitySettingsSchema, fuseSettingsSchema } from "../add_model/forecast_classifier_config"; +import type { StoredComplexityRouterConfig } from "../add_model/build_complexity_router_config"; +import { + hydrateBuiltInTiers, + hydrateCustomTierSet, + hydratePlanModeMinTier, + hydrateTierLabels, +} from "../add_model/build_complexity_router_config"; +import { hydrateTierModelParams } from "../add_model/complexity_router_tiers"; +import { hydrateCustomDimensions } from "../add_model/custom_dimensions"; +import { + hydrateDimensionWeights, + hydrateReasoningOverrideMinScore, + hydrateTierBoundaries, + hydrateTokenThresholds, +} from "../add_model/heuristic_scoring_knobs"; +import type { ComplexityRouterConfigValue } from "../add_model/ComplexityRouterConfig"; +import { DEFAULT_DEPLOYMENT_AFFINITY, DEFAULT_SESSION_AFFINITY } from "../add_model/ComplexityRouterConfig"; +import { + type ActiveTierSet, + activeTierRows, + tierParamsByRowId, + resolveComplexityDefaultModel, +} from "../add_model/tier_rows"; + +const isReminderMarkerPair = (input: unknown): input is { open: string; close: string } => { + if (typeof input !== "object" || input === null) { + return false; + } + if (!("open" in input) || !("close" in input)) { + return false; + } + return typeof input.open === "string" && typeof input.close === "string"; +}; + +const stringList = (input: unknown): string[] | undefined => + Array.isArray(input) ? input.filter((item): item is string => typeof item === "string") : undefined; + +export const hydratePinnedDefaultModel = ( + storedConfigDefaultModel: unknown, + litellmParamsDefaultModel: string | null | undefined, + activeTiers: ActiveTierSet, +): string | undefined => { + if (typeof storedConfigDefaultModel === "string" && storedConfigDefaultModel.trim()) { + return storedConfigDefaultModel; + } + const tierDerived = resolveComplexityDefaultModel(activeTiers); + const externalOverride = litellmParamsDefaultModel?.trim(); + return externalOverride && externalOverride !== tierDerived ? externalOverride : undefined; +}; + +export const hydrateComplexityRouterConfig = ( + parsedConfig: StoredComplexityRouterConfig, + complexityRouterDefaultModel: string | null | undefined, +): ComplexityRouterConfigValue => { + const builtIn = hydrateBuiltInTiers(parsedConfig.tiers, parsedConfig.enable_non_reasoning_tier); + const { tiers: hydratedTiers, enable_non_reasoning_tier } = builtIn; + const custom_tier_set = hydrateCustomTierSet(parsedConfig); + const activeTiers = { ...builtIn, custom_tier_set }; + + return { + tiers: hydratedTiers, + enable_non_reasoning_tier, + custom_tier_set, + tier_model_params: tierParamsByRowId( + hydrateTierModelParams(parsedConfig.tiers, parsedConfig.tier_model_configs), + activeTierRows(activeTiers), + ), + default_model: hydratePinnedDefaultModel(parsedConfig.default_model, complexityRouterDefaultModel, activeTiers), + plan_mode_min_tier: hydratePlanModeMinTier(parsedConfig.plan_mode_min_tier, custom_tier_set), + tier_labels: hydrateTierLabels(parsedConfig.tier_labels), + classifier_type: parsedConfig.classifier_type || "heuristic", + heuristic_v2_success_threshold: + typeof parsedConfig.heuristic_v2_success_threshold === "number" + ? parsedConfig.heuristic_v2_success_threshold + : undefined, + capability_classifier_config: capabilitySettingsSchema.safeParse(parsedConfig.capability_classifier_config).data, + llm_v2_config: fuseSettingsSchema.safeParse(parsedConfig.llm_v2_config).data, + classifier_llm_config: parsedConfig.classifier_type === "jev" ? undefined : parsedConfig.classifier_llm_config, + jev_classifier_config: + parsedConfig.classifier_type === "jev" + ? jevClassifierConfigSchema.safeParse(parsedConfig.jev_classifier_config ?? {}).data ?? + defaultJevClassifierConfig() + : undefined, + classifier_context_window_size: + typeof parsedConfig.classifier_context_window_size === "number" + ? parsedConfig.classifier_context_window_size + : undefined, + classifier_context_budget_chars: + typeof parsedConfig.classifier_context_budget_chars === "number" + ? parsedConfig.classifier_context_budget_chars + : undefined, + classifier_context_per_turn_chars: + typeof parsedConfig.classifier_context_per_turn_chars === "number" + ? parsedConfig.classifier_context_per_turn_chars + : undefined, + classifier_context_include_assistant_turns: + typeof parsedConfig.classifier_context_include_assistant_turns === "boolean" + ? parsedConfig.classifier_context_include_assistant_turns + : undefined, + classifier_fallback: + parsedConfig.classifier_fallback === "default_model" || parsedConfig.classifier_fallback === "heuristic" + ? parsedConfig.classifier_fallback + : undefined, + classification_prompt: + typeof parsedConfig.classification_prompt === "string" && parsedConfig.classification_prompt.trim() !== "" + ? parsedConfig.classification_prompt + : undefined, + classification_examples: + typeof parsedConfig.classification_examples === "string" && parsedConfig.classification_examples.trim() !== "" + ? parsedConfig.classification_examples + : undefined, + heuristic_first_max_tier: + typeof parsedConfig.heuristic_first_max_tier === "string" && parsedConfig.heuristic_first_max_tier.trim() !== "" + ? parsedConfig.heuristic_first_max_tier + : undefined, + hybrid_boundary_margin: + typeof parsedConfig.hybrid_boundary_margin === "number" ? parsedConfig.hybrid_boundary_margin : undefined, + classification_mode: + parsedConfig.classification_mode === "user_turn" || parsedConfig.classification_mode === "every_request" + ? parsedConfig.classification_mode + : undefined, + tier_boundaries: hydrateTierBoundaries(parsedConfig.tier_boundaries), + token_thresholds: hydrateTokenThresholds(parsedConfig.token_thresholds), + dimension_weights: hydrateDimensionWeights(parsedConfig.dimension_weights), + custom_dimensions: hydrateCustomDimensions(parsedConfig.custom_dimensions), + reasoning_override_min_score: hydrateReasoningOverrideMinScore(parsedConfig.reasoning_override_min_score), + session_affinity: + typeof parsedConfig.session_affinity === "boolean" ? parsedConfig.session_affinity : DEFAULT_SESSION_AFFINITY, + session_affinity_ttl_seconds: + typeof parsedConfig.session_affinity_ttl_seconds === "number" && + Number.isFinite(parsedConfig.session_affinity_ttl_seconds) + ? parsedConfig.session_affinity_ttl_seconds + : undefined, + modality_routing: typeof parsedConfig.modality_routing === "boolean" ? parsedConfig.modality_routing : false, + modality_pin_override: + typeof parsedConfig.modality_pin_override === "boolean" ? parsedConfig.modality_pin_override : false, + deployment_affinity: + typeof parsedConfig.deployment_affinity === "boolean" + ? parsedConfig.deployment_affinity + : DEFAULT_DEPLOYMENT_AFFINITY, + adaptive: parsedConfig.adaptive || false, + adaptive_weights: parsedConfig.adaptive_weights, + tier_distance_penalty: parsedConfig.tier_distance_penalty, + adaptive_eligible: parsedConfig.adaptive_eligible || "all", + return_raw_model_name: parsedConfig.return_raw_model_name || false, + enable_context_window_escalation: + typeof parsedConfig.enable_context_window_escalation === "boolean" + ? parsedConfig.enable_context_window_escalation + : undefined, + context_window_escalation_buffer: + typeof parsedConfig.context_window_escalation_buffer === "number" + ? parsedConfig.context_window_escalation_buffer + : undefined, + stall_escalation_enabled: parsedConfig.stall_escalation_enabled === true || undefined, + stall_escalation_window: + typeof parsedConfig.stall_escalation_window === "number" ? parsedConfig.stall_escalation_window : undefined, + stall_escalation_repeat_threshold: + typeof parsedConfig.stall_escalation_repeat_threshold === "number" + ? parsedConfig.stall_escalation_repeat_threshold + : undefined, + code_keywords: stringList(parsedConfig.code_keywords), + reasoning_keywords: stringList(parsedConfig.reasoning_keywords), + technical_keywords: stringList(parsedConfig.technical_keywords), + simple_keywords: stringList(parsedConfig.simple_keywords), + plan_mode_patterns: stringList(parsedConfig.plan_mode_patterns), + route_housekeeping_to_cheapest_tier: + typeof parsedConfig.route_housekeeping_to_cheapest_tier === "boolean" + ? parsedConfig.route_housekeeping_to_cheapest_tier + : undefined, + housekeeping_patterns: stringList(parsedConfig.housekeeping_patterns), + reminder_markers: Array.isArray(parsedConfig.reminder_markers) + ? parsedConfig.reminder_markers.filter(isReminderMarkerPair) + : undefined, + max_tokens_from_tier_model: + typeof parsedConfig.max_tokens_from_tier_model === "boolean" + ? parsedConfig.max_tokens_from_tier_model + : undefined, + classifier_plugin_timeout_ms: + typeof parsedConfig.classifier_plugin_timeout_ms === "number" && + Number.isFinite(parsedConfig.classifier_plugin_timeout_ms) + ? parsedConfig.classifier_plugin_timeout_ms + : undefined, + }; +}; diff --git a/ui/litellm-dashboard/src/components/leftnav.test.tsx b/ui/litellm-dashboard/src/components/leftnav.test.tsx index 6eb0218c41d..5fa6e9728cf 100644 --- a/ui/litellm-dashboard/src/components/leftnav.test.tsx +++ b/ui/litellm-dashboard/src/components/leftnav.test.tsx @@ -21,6 +21,7 @@ const navState = vi.hoisted(() => ({ pathname: "/ui/api-keys" })); vi.mock("next/navigation", () => ({ usePathname: () => navState.pathname, + useRouter: () => ({ push: vi.fn() }), })); const { mockUseAuthorized, mockUseOrganizations } = vi.hoisted(() => { diff --git a/ui/litellm-dashboard/src/components/networking.tsx b/ui/litellm-dashboard/src/components/networking.tsx index f815a2cb557..ab750ac3c62 100644 --- a/ui/litellm-dashboard/src/components/networking.tsx +++ b/ui/litellm-dashboard/src/components/networking.tsx @@ -1599,6 +1599,20 @@ export const claimOnboardingToken = async ( } }; +export const changePasswordCall = async ( + accessToken: string, + currentPassword: string, + newPassword: string, +): Promise<{ user_id: string; message: string }> => { + return await apiClient.post(`/user/password/change`, { + accessToken, + body: { + current_password: currentPassword, + new_password: newPassword, + }, + }); +}; + export const regenerateKeyCall = async (accessToken: string, keyToRegenerate: string, formData: any) => { try { const url = proxyBaseUrl @@ -6253,6 +6267,7 @@ export const patchAgentCall = async ( rpm_limit?: number | null; session_tpm_limit?: number | null; session_rpm_limit?: number | null; + access_group_ids?: string[]; }, ) => { try { diff --git a/ui/litellm-dashboard/src/components/shared/ScopedSavingsTab.tsx b/ui/litellm-dashboard/src/components/shared/ScopedSavingsTab.tsx new file mode 100644 index 00000000000..7a0455f9d7e --- /dev/null +++ b/ui/litellm-dashboard/src/components/shared/ScopedSavingsTab.tsx @@ -0,0 +1,133 @@ +"use client"; + +import { useMemo, useState } from "react"; + +import { AreaChart, BarChart, CustomLegend } from "@/components/shared/charts"; +import AdvancedDatePicker from "@/components/shared/advanced_date_picker"; +import SavingsTiles from "@/components/shared/SavingsTiles"; +import { Card, CardAction, CardContent, CardDescription, CardHeader, CardTitle } from "@/components/ui/card"; +import { Tabs, TabsList, TabsTrigger } from "@/components/ui/tabs"; +import { + formatRangeLabel, + localIsoDay, + MAX_POINTS_WITH_DOTS, + SAVINGS_COLORS, + SAVINGS_SERIES, + SavingsAccumulation, + SavingsPoint, + savingsSeriesOf, + shortDate, + toCumulative, + usd, + withStartAnchor, +} from "@/app/(dashboard)/cost-optimization/_components/costOptimizationUtils"; +import { + useScopedDailyActivityRange, + type ActivityDateRange, + type DailyActivityScope, +} from "@/app/(dashboard)/cost-optimization/_components/useDailyActivityRange"; + +interface ScopedSavingsTabProps { + accessToken: string | null; + scope: DailyActivityScope; + activity: ActivityDateRange; + entityType: "key" | "user"; + scopeNote?: string; +} + +const ScopedSavingsTab = ({ accessToken, scope, activity, entityType, scopeNote }: ScopedSavingsTabProps) => { + const { dateValue, onDateChange, results, loading, isFetchingMore, failed, cancelled } = useScopedDailyActivityRange( + accessToken, + scope, + activity, + ); + const startTime = dateValue.from; + const endTime = dateValue.to; + + const [accumulation, setAccumulation] = useState("cumulative"); + + const perInterval = useMemo(() => savingsSeriesOf(results), [results]); + + const overTime = useMemo(() => { + if (accumulation !== "cumulative") return perInterval; + const startLabel = startTime ? shortDate(localIsoDay(startTime)) : ""; + return withStartAnchor(toCumulative(perInterval), startLabel); + }, [accumulation, perInterval, startTime]); + + const intervalLabel = "Per day"; + const rangeLabel = formatRangeLabel(startTime, endTime); + const savingsSubtitle = [ + accumulation === "cumulative" ? "Running total saved" : `Saved ${intervalLabel.toLowerCase()}`, + rangeLabel && `${rangeLabel} (UTC)`, + ] + .filter(Boolean) + .join(" · "); + + const isLoading = loading || isFetchingMore; + const unavailable = failed || cancelled; + const showResults = !isLoading && !unavailable; + const hasRows = results.length > 0; + const showEmpty = !unavailable && (isLoading || !hasRows); + const showChart = showResults && hasRows; + const chartProps = { + data: overTime, + index: "date", + categories: SAVINGS_SERIES, + colors: SAVINGS_COLORS, + valueFormatter: usd, + showLegend: false, + }; + + return ( +
+
+ Spend is bucketed by UTC day + +
+ + {scopeNote && ( +

+ {scopeNote} +

+ )} + + {unavailable && ( +

+ Savings are unavailable for this range. Try another date range or reopen this tab. +

+ )} + {showResults && } + + + + Savings + {savingsSubtitle} + + + setAccumulation(value as SavingsAccumulation)}> + + Cumulative + {intervalLabel} + + + + + + {showEmpty && ( +

+ {isLoading ? "Loading savings..." : `No usage recorded for this ${entityType} in this range.`} +

+ )} + {showChart && + (accumulation === "cumulative" ? ( + + ) : ( + + ))} + + +
+ ); +}; + +export default ScopedSavingsTab; diff --git a/ui/litellm-dashboard/src/components/shared/SearchSelect.integration.test.tsx b/ui/litellm-dashboard/src/components/shared/SearchSelect.integration.test.tsx index c981010dff9..9f320049b09 100644 --- a/ui/litellm-dashboard/src/components/shared/SearchSelect.integration.test.tsx +++ b/ui/litellm-dashboard/src/components/shared/SearchSelect.integration.test.tsx @@ -111,4 +111,36 @@ describe("SearchSelect", () => { expect(screen.queryByText("Growth")).not.toBeInTheDocument(); expect(onValueChange).not.toHaveBeenCalled(); }); + + it("supports keyboard select, clear, and reselect", async () => { + const onValueChange = vi.fn(); + const user = userEvent.setup(); + function Controlled() { + const [value, setValue] = useState(null); + return ( + { + setValue(next); + onValueChange(next); + }} + /> + ); + } + + render(); + const input = screen.getByRole("combobox"); + await user.tab(); + await user.keyboard("{Enter}"); + await user.keyboard("{ArrowDown}{Enter}"); + expect(onValueChange).toHaveBeenLastCalledWith("team-1"); + const clear = screen.getByRole("button", { name: "Clear" }); + clear.focus(); + await user.keyboard("{Enter}"); + expect(onValueChange).toHaveBeenLastCalledWith(null); + input.focus(); + await user.keyboard("{Enter}{ArrowDown}{Enter}"); + expect(onValueChange).toHaveBeenLastCalledWith("team-1"); + }); }); diff --git a/ui/litellm-dashboard/src/components/templates/KeySavingsTab.tsx b/ui/litellm-dashboard/src/components/templates/KeySavingsTab.tsx index c33529eb042..d1395e0be51 100644 --- a/ui/litellm-dashboard/src/components/templates/KeySavingsTab.tsx +++ b/ui/litellm-dashboard/src/components/templates/KeySavingsTab.tsx @@ -1,132 +1,29 @@ "use client"; -import React, { useMemo, useState } from "react"; - -import { AreaChart, BarChart, CustomLegend } from "@/components/shared/charts"; -import AdvancedDatePicker from "@/components/shared/advanced_date_picker"; -import SavingsTiles from "@/components/shared/SavingsTiles"; -import { Card, CardAction, CardContent, CardDescription, CardHeader, CardTitle } from "@/components/ui/card"; -import { Tabs, TabsList, TabsTrigger } from "@/components/ui/tabs"; +import ScopedSavingsTab from "@/components/shared/ScopedSavingsTab"; import { hasProxyWideSpendView, spendScopeUserId } from "@/utils/roles"; -import { - formatRangeLabel, - localIsoDay, - MAX_POINTS_WITH_DOTS, - SAVINGS_COLORS, - SAVINGS_SERIES, - SavingsAccumulation, - SavingsPoint, - savingsSeriesOf, - shortDate, - toCumulative, - usd, - withStartAnchor, -} from "@/app/(dashboard)/cost-optimization/_components/costOptimizationUtils"; -import { - useScopedDailyActivityRange, - type ActivityDateRange, -} from "@/app/(dashboard)/cost-optimization/_components/useDailyActivityRange"; +import type { ActivityDateRange } from "@/app/(dashboard)/cost-optimization/_components/useDailyActivityRange"; interface KeySavingsTabProps { accessToken: string | null; - /** The key's token hash — what spend rows are keyed by, not the one-time plaintext secret. */ keyToken: string; userId: string | null; userRole: string; activity: ActivityDateRange; } -const KeySavingsTab: React.FC = ({ accessToken, keyToken, userId, userRole, activity }) => { - // Proxy admins read the whole key. For anyone else the endpoint applies the caller's own user_id - // alongside the key filter, so the figures cover only that viewer's requests on this key -- said - // plainly in the scope note below rather than left to be misread as the key's total. - const readsWholeKey = hasProxyWideSpendView(userRole); - const { dateValue, onDateChange, results, loading, isFetchingMore } = useScopedDailyActivityRange( - accessToken, - { userId: spendScopeUserId(userRole, userId), apiKey: keyToken }, - activity, - ); - const startTime = dateValue.from ?? null; - const endTime = dateValue.to ?? null; - - const [accumulation, setAccumulation] = useState("cumulative"); - - const perInterval = useMemo(() => savingsSeriesOf(results), [results]); - - const overTime = useMemo(() => { - if (accumulation !== "cumulative") return perInterval; - const startLabel = startTime ? shortDate(localIsoDay(startTime)) : ""; - return withStartAnchor(toCumulative(perInterval), startLabel); - }, [accumulation, perInterval, startTime]); - - const intervalLabel = "Per day"; - const rangeLabel = formatRangeLabel(startTime ?? undefined, endTime ?? undefined); - const savingsSubtitle = [ - accumulation === "cumulative" ? "Running total saved" : `Saved ${intervalLabel.toLowerCase()}`, - rangeLabel && `${rangeLabel} (UTC)`, - ] - .filter(Boolean) - .join(" · "); - - const isLoading = loading || isFetchingMore; - const hasRows = results.length > 0; - const chartProps = { - data: overTime, - index: "date", - categories: SAVINGS_SERIES, - colors: SAVINGS_COLORS, - valueFormatter: usd, - showLegend: false, - }; - - return ( -
-
- Spend is bucketed by UTC day - -
- - {!readsWholeKey && ( -

- Showing your own requests on this key. A key shared across a team will have spend from other members that is - not counted here. -

- )} - - - - - - Savings - {savingsSubtitle} - - - setAccumulation(value as SavingsAccumulation)}> - - Cumulative - {intervalLabel} - - - - - - {/* Distinguishes "still fetching" from "this key genuinely had no traffic": an empty - chart alone reads as a broken panel, and a $0.00 tile reads as a real zero. */} - {!hasRows && ( -

- {isLoading ? "Loading savings..." : "No usage recorded for this key in this range."} -

- )} - {hasRows && accumulation === "cumulative" && ( - - )} - {/* Not stacked: auto-router can go negative on a cold-cache write, and stacking would - draw that below the axis while the rest of the bar still read as the total. */} - {hasRows && accumulation !== "cumulative" && } - - -
- ); -}; +const KeySavingsTab = ({ accessToken, keyToken, userId, userRole, activity }: KeySavingsTabProps) => ( + +); export default KeySavingsTab; diff --git a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/RoutingDecisionCard.test.tsx b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/RoutingDecisionCard.test.tsx index 17770c54482..5492ceffefc 100644 --- a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/RoutingDecisionCard.test.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/RoutingDecisionCard.test.tsx @@ -113,6 +113,148 @@ describe("RoutingDecisionCard", () => { }, ); + it.each(["capability_classifier", "modality_escalation"])( + "shows the recorded Capability forecast for %s", + (cause) => { + render( + , + ); + + expect(screen.getByText("Capability estimates")).toBeInTheDocument(); + expect(screen.getByText("Efficient model solve chance")).toBeInTheDocument(); + expect(screen.getAllByText(/^\d+\.\d%$/).map((value) => value.textContent)).toEqual(["0.0%", "86.4%", "82.0%"]); + expect(screen.getByText("Raw")).toBeInTheDocument(); + expect(screen.getByText("Calibrated")).toBeInTheDocument(); + expect(screen.getByText("Threshold")).toBeInTheDocument(); + expect(screen.getByText("uncertain")).toBeInTheDocument(); + expect(screen.getByText("UNC-2")).toBeInTheDocument(); + expect(screen.getByText("calibration-1")).toBeInTheDocument(); + expect(screen.getByText("Deep")).toBeInTheDocument(); + expect(screen.queryByText("FUSE v2 estimates")).not.toBeInTheDocument(); + }, + ); + + it("omits absent Capability fields while preserving a recorded zero threshold", () => { + render( + , + ); + + expect(screen.getAllByText(/^\d+\.\d%$/).map((value) => value.textContent)).toEqual(["25.0%", "0.0%"]); + for (const label of ["Calibrated", "Calibration", "Boundary", "Rule"]) { + expect(screen.queryByText(label)).not.toBeInTheDocument(); + } + }); + + it.each(["llm_v2_classifier", "default_fallback"])( + "shows the original calibrated FUSE v2 forecast for %s", + (cause) => { + render( + , + ); + + expect(screen.getByText("FUSE v2 estimates")).toBeInTheDocument(); + expect(screen.getAllByText(/^\d+\.\d%$/).map((value) => value.textContent)).toEqual([ + "25.0%", + "91.0%", + "75.0%", + "80.0%", + ]); + expect(screen.getByText("Efficient (raw)")).toBeInTheDocument(); + expect(screen.getByText("Capable (raw)")).toBeInTheDocument(); + expect(screen.getByText("Efficient (calibrated)")).toBeInTheDocument(); + expect(screen.getByText("Capable (calibrated)")).toBeInTheDocument(); + expect(screen.getAllByText(/percentage points$/).map((value) => value.textContent)).toEqual([ + "5.0 percentage points", + "10.0 percentage points", + ]); + expect(screen.getByText("Applied gap")).toBeInTheDocument(); + expect(screen.getByText("Allowed gap")).toBeInTheDocument(); + expect(screen.getByText("calibration-2")).toBeInTheDocument(); + expect(screen.getByText("llm-v2:verification=tests")).toBeInTheDocument(); + expect(screen.getByText("fallback-model")).toBeInTheDocument(); + expect(screen.queryByText("Capability estimates")).not.toBeInTheDocument(); + }, + ); + + it("uses raw FUSE v2 probabilities without calibration and preserves negative and zero gaps", () => { + render( + , + ); + + expect(screen.getAllByText(/^\d+\.\d%$/).map((value) => value.textContent)).toEqual(["50.0%", "0.0%"]); + expect(screen.getAllByText(/percentage points$/).map((value) => value.textContent)).toEqual([ + "-50.0 percentage points", + "0.0 percentage points", + ]); + expect(screen.queryByText(/calibrated|Calibration/)).not.toBeInTheDocument(); + }); + + it.each([ + { classifier_efficient_p_solve: 0, classifier_max_quality_gap: 0.2 }, + { + classifier_efficient_p_solve: 0.4, + classifier_capable_p_solve: 0.9, + classifier_calibrated_efficient_p_solve: 0, + classifier_calibration_version: "partial-calibration", + classifier_max_quality_gap: 0.2, + }, + ])("shows partial FUSE v2 estimates without inventing an applied gap: %j", (fields) => { + render(); + + expect(screen.getByText("FUSE v2 estimates")).toBeInTheDocument(); + expect(screen.getByText("0.0%")).toBeInTheDocument(); + expect(screen.getByText("20.0 percentage points")).toBeInTheDocument(); + expect(screen.queryByText("Applied gap")).not.toBeInTheDocument(); + expect(screen.queryByText("Capable (calibrated)")).not.toBeInTheDocument(); + }); + + it.each([ + ["capability_classifier", "Capability"], + ["llm_v2_classifier", "FUSE v2"], + ["capability_classifier_fallback", "Capable tier, Capability classifier failed"], + ["llm_v2_fallback", "Capable tier, FUSE v2 classifier failed"], + ["session_affinity_pin", "Pinned to session"], + ])("labels %s without inventing a missing forecast", (cause, label) => { + render(); + + expect(screen.getByText(label)).toBeInTheDocument(); + expect(screen.getByText("MEDIUM")).toBeInTheDocument(); + expect(screen.queryByText(/estimates/)).not.toBeInTheDocument(); + }); + it("uses the persisted boundary snapshot, not today's defaults", () => { // Same score, boundaries the operator had configured lower: it lands in a // different band, and the card must say so. diff --git a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/RoutingDecisionCard.tsx b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/RoutingDecisionCard.tsx index c497b2a94a7..6a8cedd3746 100644 --- a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/RoutingDecisionCard.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/RoutingDecisionCard.tsx @@ -24,6 +24,17 @@ export interface RoutingDecision { matched_keyword?: string; escalation_keyword?: string; classifier_model?: string; + classifier_p_solve?: number; + classifier_calibrated_p_solve?: number; + classifier_threshold?: number; + classifier_capability_boundary?: string; + classifier_primary_rule?: string; + classifier_calibration_version?: string; + classifier_efficient_p_solve?: number; + classifier_capable_p_solve?: number; + classifier_calibrated_efficient_p_solve?: number; + classifier_calibrated_capable_p_solve?: number; + classifier_max_quality_gap?: number; classifier_confidence?: number; classifier_probabilities?: Record; classifier_cost?: number; @@ -94,6 +105,10 @@ function describeReasoningOverride(tierLabel: string | undefined, floor: number const CONSTANT_CAUSE_LABELS: Record = { heuristic_scorer: "Heuristic scorer", heuristic_v2: "Heuristic v2", + capability_classifier: "Capability", + capability_classifier_fallback: "Capable tier, Capability classifier failed", + llm_v2_classifier: "FUSE v2", + llm_v2_fallback: "Capable tier, FUSE v2 classifier failed", heuristic_first_short_circuit: "Heuristic scorer, classifier skipped", hybrid_short_circuit: "Heuristic scorer, score clear of every boundary", classifier_plugin: "Custom classifier plugin", @@ -161,6 +176,71 @@ function Row({ label, children }: { label: string; children: React.ReactNode }) ); } +function PercentageRow({ label, value, unit = "%" }: { label: string; value?: number; unit?: string }) { + if (value === undefined) return null; + return ( + + {`${(value * 100).toFixed(1)}${unit}`} + + ); +} + +function CapabilityForecast({ decision }: { decision: RoutingDecision }) { + const { + classifier_p_solve: raw, + classifier_calibrated_p_solve: calibrated, + classifier_threshold: threshold, + classifier_capability_boundary: boundary, + classifier_primary_rule: rule, + classifier_calibration_version: version, + } = decision; + if ([raw, calibrated, threshold, boundary, rule].every((value) => value === undefined)) return null; + + return ( +
+
Capability estimates
+
Efficient model solve chance
+ + + + {boundary && {boundary}} + {rule && {rule}} + {version && {version}} +
+ ); +} + +function FuseV2Forecast({ decision }: { decision: RoutingDecision }) { + const { + classifier_efficient_p_solve: rawEfficient, + classifier_capable_p_solve: rawCapable, + classifier_calibrated_efficient_p_solve: calibratedEfficient, + classifier_calibrated_capable_p_solve: calibratedCapable, + classifier_max_quality_gap: allowedGap, + classifier_calibration_version: version, + } = decision; + if ([rawEfficient, rawCapable, calibratedEfficient, calibratedCapable, allowedGap].every((v) => v === undefined)) { + return null; + } + const isCalibrated = [calibratedEfficient, calibratedCapable, version].some((value) => value !== undefined); + const efficient = isCalibrated ? calibratedEfficient : rawEfficient; + const capable = isCalibrated ? calibratedCapable : rawCapable; + const gap = efficient !== undefined && capable !== undefined ? capable - efficient : undefined; + + return ( +
+
FUSE v2 estimates
+ + + + + + + {version && {version}} +
+ ); +} + export function RoutingDecisionCard({ decision, className, @@ -266,6 +346,9 @@ export function RoutingDecisionCard({
)} + + + {signals && signals.length > 0 && ( diff --git a/ui/litellm-dashboard/src/components/view_logs/RequestLogsFilters.test.tsx b/ui/litellm-dashboard/src/components/view_logs/RequestLogsFilters.test.tsx index 5a35c7ae16b..8d1847e0121 100644 --- a/ui/litellm-dashboard/src/components/view_logs/RequestLogsFilters.test.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/RequestLogsFilters.test.tsx @@ -344,4 +344,15 @@ describe("RequestLogsFilters", () => { expect(set).toHaveBeenCalledWith(LOG_FILTER_IDS.CACHE_STATUS, undefined); }); + + it("clears the raw Error Code combobox through the undefined filter contract", async () => { + const user = userEvent.setup(); + const { set } = renderFilters({ [LOG_FILTER_IDS.ERROR_CODE]: "429" }); + const input = await screen.findByPlaceholderText("Select or type an error code"); + + await user.click(input); + await user.click(screen.getByRole("button", { name: "Clear", hidden: true })); + + expect(set).toHaveBeenCalledWith(LOG_FILTER_IDS.ERROR_CODE, undefined); + }); }); diff --git a/ui/litellm-dashboard/src/contexts/AuthContext.tsx b/ui/litellm-dashboard/src/contexts/AuthContext.tsx index 123feb18a6c..c68c8b9d81a 100644 --- a/ui/litellm-dashboard/src/contexts/AuthContext.tsx +++ b/ui/litellm-dashboard/src/contexts/AuthContext.tsx @@ -24,6 +24,7 @@ type AuthContextValue = { premiumUser: boolean; disabledPersonalKeyCreation: boolean; showSSOBanner: boolean; + passwordResetRequired: boolean; setToken: React.Dispatch>; setUserID: React.Dispatch>; @@ -46,6 +47,7 @@ export function AuthProvider({ children }: { children: React.ReactNode }) { const [premiumUser, setPremiumUser] = useState(false); const [disabledPersonalKeyCreation, setDisabledPersonalKeyCreation] = useState(false); const [showSSOBanner, setShowSSOBanner] = useState(true); + const [passwordResetRequired, setPasswordResetRequired] = useState(false); // Load runtime UI config (populates proxyBaseUrl etc.) before clearing // authLoading, so any consumer that builds proxy-rooted URLs from authLoading=false @@ -124,6 +126,7 @@ export function AuthProvider({ children }: { children: React.ReactNode }) { if (decoded.user_id) { setUserID(decoded.user_id); } + setPasswordResetRequired(decoded.password_reset_required === true); }, [token]); const value: AuthContextValue = { @@ -136,6 +139,7 @@ export function AuthProvider({ children }: { children: React.ReactNode }) { premiumUser, disabledPersonalKeyCreation, showSSOBanner, + passwordResetRequired, setToken, setUserID, setUserRole, diff --git a/ui/litellm-dashboard/src/lib/autorouter_presets.test.ts b/ui/litellm-dashboard/src/lib/autorouter_presets.test.ts index e1d9e6ab5ac..87efb2b1d5e 100644 --- a/ui/litellm-dashboard/src/lib/autorouter_presets.test.ts +++ b/ui/litellm-dashboard/src/lib/autorouter_presets.test.ts @@ -746,7 +746,7 @@ describe("autorouter_presets", () => { expect(prefill.escalationKeywords).toEqual([]); }); - it("carries a preset's context-window escalation opt-out and buffer through the prefill", () => { + it.each([undefined, false, true])("preserves a preset's context-window escalation setting: %s", (enabled) => { const prefill = buildPresetPrefill( { tiers: { SIMPLE: ["gpt-5-nano"], MEDIUM: [], COMPLEX: [], REASONING: [] }, @@ -754,12 +754,12 @@ describe("autorouter_presets", () => { classification_mode: "every_request", session_affinity: false, deployment_affinity: true, - enable_context_window_escalation: false, + enable_context_window_escalation: enabled, context_window_escalation_buffer: 0.9, }, groupsOnly(["gpt-5-nano"]), ); - expect(prefill.complexityRouterConfig.enable_context_window_escalation).toBe(false); + expect(prefill.complexityRouterConfig.enable_context_window_escalation).toBe(enabled); expect(prefill.complexityRouterConfig.context_window_escalation_buffer).toBe(0.9); }); diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 46d4e45e85f..37138f2f65b 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -1246,9 +1246,10 @@ export interface paths { * @description Benchmarks for the auto-router dashboard: session shape, savings against the configured * baseline, and prompt-caching behaviour bucketed by what the router did. * - * Reads the LiteLLM_AutoRouterSession rollup, folded once per request at spend-write time, - * so this endpoint never scans LiteLLM_SpendLogs. A session is in the window when it - * overlaps it: its last turn is on or after start_date and its first turn is on or before + * Reads session rollups folded once per request at spend-write time, so this endpoint + * never scans LiteLLM_SpendLogs. A user filter selects only turns attributed to that + * internal user when written; older key-only history remains outside user views. A session + * is in the window when it overlaps it: its last turn is on or after start_date and its first turn is on or before * end_date. Overall hit rate is over telemetry-bearing turns; each bucket's hit rate is * over that bucket's turns. * @@ -4871,6 +4872,27 @@ export interface paths { patch: operations["assemblyai_proxy_route_eu_assemblyai__endpoint__patch"]; trace?: never; }; + "/fal_ai/{endpoint}": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** Fal Ai Proxy Route */ + get: operations["fal_ai_proxy_route_fal_ai__endpoint__get"]; + /** Fal Ai Proxy Route */ + put: operations["fal_ai_proxy_route_fal_ai__endpoint__put"]; + /** Fal Ai Proxy Route */ + post: operations["fal_ai_proxy_route_fal_ai__endpoint__post"]; + /** Fal Ai Proxy Route */ + delete: operations["fal_ai_proxy_route_fal_ai__endpoint__delete"]; + options?: never; + head?: never; + /** Fal Ai Proxy Route */ + patch: operations["fal_ai_proxy_route_fal_ai__endpoint__patch"]; + trace?: never; + }; "/fallback": { parameters: { query?: never; @@ -10687,6 +10709,27 @@ export interface paths { patch: operations["openai_passthrough_route_openai_passthrough__endpoint__patch"]; trace?: never; }; + "/openrouter/{endpoint}": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** Openrouter Proxy Route */ + get: operations["openrouter_proxy_route_openrouter__endpoint__get"]; + /** Openrouter Proxy Route */ + put: operations["openrouter_proxy_route_openrouter__endpoint__put"]; + /** Openrouter Proxy Route */ + post: operations["openrouter_proxy_route_openrouter__endpoint__post"]; + /** Openrouter Proxy Route */ + delete: operations["openrouter_proxy_route_openrouter__endpoint__delete"]; + options?: never; + head?: never; + /** Openrouter Proxy Route */ + patch: operations["openrouter_proxy_route_openrouter__endpoint__patch"]; + trace?: never; + }; "/organization/daily/activity": { parameters: { query?: never; @@ -17392,6 +17435,7 @@ export interface paths { * - prompts: Optional[List[str]] - List of allowed prompts for the user. If specified, the user will only be able to use these specific prompts. * - organizations: List[str] - List of organization id's the user is a member of * - budget_limits: Optional[list] - List of concurrent budget windows for the user. Each window specifies a budget_limit, time_period, and optional budget_duration. Example - [{"budget_limit": 10.0, "time_period": "1d"}, {"budget_limit": 50.0, "time_period": "7d"}]. + * - password: Optional[str] - Not supported; any value is rejected with a 422. Users set their own password through an invitation link (POST /invitation/new). * Returns: * - key: (str) The generated api key for the user * - expires: (datetime) Datetime object for when key expires. @@ -17414,6 +17458,39 @@ export interface paths { patch?: never; trace?: never; }; + "/user/password/change": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + /** + * Change Password + * @description Change the calling user's own password. + * + * Only callable with the dashboard session issued by a username/password + * login; SSO sessions and virtual keys are rejected with 403. Requires the + * current password. The new password must differ from the + * current one and satisfy the configured password policy + * (`general_settings.password_policy_*`: minimum length, character classes, + * and, when enabled, breached-password screening via haveibeenpwned.com). + * A successful change lifts any pending forced password reset + * (`password_reset_required`) on the account. + * + * Parameters: + * - current_password: str - The user's current password. + * - new_password: str - The password to change to. + */ + post: operations["change_password_user_password_change_post"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/user/spend/report": { parameters: { query?: never; @@ -17461,7 +17538,7 @@ export interface paths { * Parameters: * - user_id: Optional[str] - Specify a user id. If not set, a unique id will be generated. * - user_email: Optional[str] - Specify a user email. - * - password: Optional[str] - Specify a user password. + * - password: Optional[str] - Set the user's password (admin only). Must satisfy the configured password policy. The user is required to change it at their next login. Users change their own password with POST /user/password/change. * - user_alias: Optional[str] - A descriptive name for you to know who this user id refers to. * - teams: Optional[list] - specify a list of team id's a user belongs to. * - send_invite_email: Optional[bool] - Specify if an invite email should be sent. @@ -23647,6 +23724,8 @@ export interface components { }; /** AgentConfig */ AgentConfig: { + /** Access Group Ids */ + access_group_ids?: string[] | null; agent_card_params: components["schemas"]["AgentCard"]; /** Agent Name */ agent_name: string; @@ -23794,6 +23873,8 @@ export interface components { }; /** AgentResponse */ AgentResponse: { + /** Access Group Ids */ + access_group_ids?: string[] | null; /** Agent Card Params */ agent_card_params: { [key: string]: unknown; @@ -25349,6 +25430,8 @@ export interface components { object_permission?: components["schemas"]["LiteLLM_ObjectPermissionBase"] | null; /** Organizations */ organizations?: string[] | null; + /** Password */ + password?: string | null; /** * Permissions * @default {} @@ -25848,7 +25931,7 @@ export interface components { * CallTypes * @enum {string} */ - CallTypes: "embedding" | "aembedding" | "completion" | "acompletion" | "atext_completion" | "text_completion" | "image_generation" | "aimage_generation" | "image_edit" | "aimage_edit" | "moderation" | "amoderation" | "atranscription" | "transcription" | "aspeech" | "speech" | "rerank" | "arerank" | "search" | "asearch" | "_arealtime" | "_aresponses_websocket" | "create_batch" | "acreate_batch" | "aretrieve_batch" | "retrieve_batch" | "acancel_batch" | "cancel_batch" | "pass_through_endpoint" | "anthropic_messages" | "aanthropic_messages" | "get_assistants" | "aget_assistants" | "create_assistants" | "acreate_assistants" | "delete_assistant" | "adelete_assistant" | "acreate_thread" | "create_thread" | "aget_thread" | "get_thread" | "a_add_message" | "add_message" | "aget_messages" | "get_messages" | "arun_thread" | "run_thread" | "arun_thread_stream" | "run_thread_stream" | "afile_retrieve" | "file_retrieve" | "afile_delete" | "file_delete" | "afile_list" | "file_list" | "acreate_file" | "create_file" | "afile_content" | "file_content" | "create_fine_tuning_job" | "acreate_fine_tuning_job" | "create_video" | "acreate_video" | "avideo_retrieve" | "video_retrieve" | "avideo_content" | "video_content" | "video_remix" | "avideo_remix" | "video_list" | "avideo_list" | "video_retrieve_job" | "avideo_retrieve_job" | "video_delete" | "avideo_delete" | "video_create_character" | "avideo_create_character" | "video_get_character" | "avideo_get_character" | "video_edit" | "avideo_edit" | "video_extension" | "avideo_extension" | "vector_store_file_create" | "avector_store_file_create" | "vector_store_file_list" | "avector_store_file_list" | "vector_store_file_retrieve" | "avector_store_file_retrieve" | "vector_store_file_content" | "avector_store_file_content" | "vector_store_file_update" | "avector_store_file_update" | "vector_store_file_delete" | "avector_store_file_delete" | "vector_store_create" | "avector_store_create" | "vector_store_search" | "avector_store_search" | "ingest" | "aingest" | "query" | "aquery" | "create_interaction" | "acreate_interaction" | "create_container" | "acreate_container" | "list_containers" | "alist_containers" | "retrieve_container" | "aretrieve_container" | "delete_container" | "adelete_container" | "list_container_files" | "alist_container_files" | "upload_container_file" | "aupload_container_file" | "create_sandbox" | "acreate_sandbox" | "delete_sandbox" | "adelete_sandbox" | "run_code" | "arun_code" | "code_interpreter_tool" | "acode_interpreter_tool" | "acancel_fine_tuning_job" | "cancel_fine_tuning_job" | "alist_fine_tuning_jobs" | "list_fine_tuning_jobs" | "aretrieve_fine_tuning_job" | "retrieve_fine_tuning_job" | "responses" | "aresponses" | "alist_input_items" | "llm_passthrough_route" | "allm_passthrough_route" | "generate_content" | "agenerate_content" | "generate_content_stream" | "agenerate_content_stream" | "ocr" | "aocr" | "call_mcp_tool" | "list_mcp_tools" | "asend_message" | "send_message" | "acreate_skill"; + CallTypes: "embedding" | "aembedding" | "completion" | "acompletion" | "atext_completion" | "text_completion" | "image_generation" | "aimage_generation" | "image_edit" | "aimage_edit" | "moderation" | "amoderation" | "atranscription" | "transcription" | "aspeech" | "speech" | "rerank" | "arerank" | "search" | "asearch" | "_arealtime" | "_aresponses_websocket" | "create_batch" | "acreate_batch" | "aretrieve_batch" | "retrieve_batch" | "acancel_batch" | "cancel_batch" | "pass_through_endpoint" | "anthropic_messages" | "aanthropic_messages" | "get_assistants" | "aget_assistants" | "create_assistants" | "acreate_assistants" | "delete_assistant" | "adelete_assistant" | "acreate_thread" | "create_thread" | "aget_thread" | "get_thread" | "a_add_message" | "add_message" | "aget_messages" | "get_messages" | "arun_thread" | "run_thread" | "arun_thread_stream" | "run_thread_stream" | "afile_retrieve" | "file_retrieve" | "afile_delete" | "file_delete" | "afile_list" | "file_list" | "acreate_file" | "create_file" | "afile_content" | "file_content" | "create_fine_tuning_job" | "acreate_fine_tuning_job" | "create_video" | "acreate_video" | "video_generation" | "avideo_generation" | "avideo_retrieve" | "video_retrieve" | "avideo_content" | "video_content" | "video_remix" | "avideo_remix" | "video_list" | "avideo_list" | "video_retrieve_job" | "avideo_retrieve_job" | "video_delete" | "avideo_delete" | "video_create_character" | "avideo_create_character" | "video_get_character" | "avideo_get_character" | "video_edit" | "avideo_edit" | "video_extension" | "avideo_extension" | "vector_store_file_create" | "avector_store_file_create" | "vector_store_file_list" | "avector_store_file_list" | "vector_store_file_retrieve" | "avector_store_file_retrieve" | "vector_store_file_content" | "avector_store_file_content" | "vector_store_file_update" | "avector_store_file_update" | "vector_store_file_delete" | "avector_store_file_delete" | "vector_store_create" | "avector_store_create" | "vector_store_search" | "avector_store_search" | "ingest" | "aingest" | "query" | "aquery" | "create_interaction" | "acreate_interaction" | "create_container" | "acreate_container" | "list_containers" | "alist_containers" | "retrieve_container" | "aretrieve_container" | "delete_container" | "adelete_container" | "list_container_files" | "alist_container_files" | "upload_container_file" | "aupload_container_file" | "create_sandbox" | "acreate_sandbox" | "delete_sandbox" | "adelete_sandbox" | "run_code" | "arun_code" | "code_interpreter_tool" | "acode_interpreter_tool" | "acancel_fine_tuning_job" | "cancel_fine_tuning_job" | "alist_fine_tuning_jobs" | "list_fine_tuning_jobs" | "aretrieve_fine_tuning_job" | "retrieve_fine_tuning_job" | "responses" | "aresponses" | "alist_input_items" | "llm_passthrough_route" | "allm_passthrough_route" | "generate_content" | "agenerate_content" | "generate_content_stream" | "agenerate_content_stream" | "ocr" | "aocr" | "call_mcp_tool" | "list_mcp_tools" | "asend_message" | "send_message" | "acreate_skill"; /** CallbackDelete */ CallbackDelete: { /** Callback Name */ @@ -25998,6 +26081,20 @@ export interface components { */ threshold_step: number; }; + /** ChangePasswordRequest */ + ChangePasswordRequest: { + /** Current Password */ + current_password: string; + /** New Password */ + new_password: string; + }; + /** ChangePasswordResponse */ + ChangePasswordResponse: { + /** Message */ + message: string; + /** User Id */ + user_id: string; + }; /** ChatCompletionAnnotation */ ChatCompletionAnnotation: { /** @@ -27118,6 +27215,11 @@ export interface components { * @default false */ health_check_skip_disabled_background_models: boolean; + /** + * Include Call Id In Error Body + * @description opt-in to copy the x-litellm-call-id response header's value into JSON error bodies, as error.litellm_call_id on the OpenAI-shaped and /v1/messages routes and as a top-level litellm_call_id on pass-through routes, so an error a client prints names the request to look up. Off by default + */ + include_call_id_in_error_body?: boolean | null; /** * Infer Model From Keys * @description for `/models` endpoint, infers available model based on environment keys (e.g. OPENAI_API_KEY) @@ -27982,7 +28084,9 @@ export interface components { /** Jwt Issuer */ jwt_issuer?: string | null; /** Key */ - key: string; + key?: string | null; + /** Token */ + token?: string | null; }; /** CreateSearchToolRequest */ CreateSearchToolRequest: { @@ -31255,6 +31359,12 @@ export interface components { output_cost_per_character_above_128k_tokens?: number | null; /** Output Cost Per Image */ output_cost_per_image?: number | null; + /** Output Cost Per Image 1024 */ + output_cost_per_image_1024?: number | null; + /** Output Cost Per Image 1536 */ + output_cost_per_image_1536?: number | null; + /** Output Cost Per Image 512 */ + output_cost_per_image_512?: number | null; /** Output Cost Per Image Token */ output_cost_per_image_token?: number | null; /** Output Cost Per Pixel */ @@ -31750,6 +31860,8 @@ export interface components { budget_reset_at?: string | null; /** Created At */ created_at?: string | null; + /** Last Breach Check At */ + last_breach_check_at?: string | null; /** Max Budget */ max_budget?: number | null; /** Max Parallel Requests */ @@ -31784,6 +31896,8 @@ export interface components { organization_id?: string | null; /** Organization Memberships */ organization_memberships?: components["schemas"]["LiteLLM_OrganizationMembershipTable"][] | null; + /** Password Reset Required */ + password_reset_required?: boolean | null; /** * Policies * @default [] @@ -31843,6 +31957,8 @@ export interface components { * @default 0 */ key_count: number; + /** Last Breach Check At */ + last_breach_check_at?: string | null; /** Max Budget */ max_budget?: number | null; /** Max Parallel Requests */ @@ -31877,6 +31993,8 @@ export interface components { organization_id?: string | null; /** Organization Memberships */ organization_memberships?: components["schemas"]["LiteLLM_OrganizationMembershipTable"][] | null; + /** Password Reset Required */ + password_reset_required?: boolean | null; /** * Policies * @default [] @@ -34494,6 +34612,8 @@ export interface components { object_permission?: components["schemas"]["LiteLLM_ObjectPermissionBase"] | null; /** Organizations */ organizations?: string[] | null; + /** Password */ + password?: string | null; /** * Permissions * @default {} @@ -35023,6 +35143,8 @@ export interface components { }; /** PatchAgentRequest */ PatchAgentRequest: { + /** Access Group Ids */ + access_group_ids?: string[] | null; agent_card_params?: components["schemas"]["AgentCard"]; /** Agent Name */ agent_name?: string; @@ -36939,8 +37061,8 @@ export interface components { embedding_model?: string | null; /** * Enable Context Window Escalation - * @description Escalate a request off a tier whose models provably cannot hold its prompt, before dispatch. The classifier scores complexity and never prompt size, so a long agentic session whose newest ask is trivial lands on a small-window tier and the provider rejects it with a context-window 400 that nothing retries. When every model of the decided tier has a declared window smaller than the estimated prompt, the request moves to the lowest configured tier with a model whose declared window fits; when only some of the tier's models fit, the pick is restricted to those and the tier keeps the request. Models with no resolvable window are never escalated away from and never escalated onto. Set false to dispatch on complexity alone, as before. - * @default true + * @description Escalate a request off a tier whose models provably cannot hold its prompt, before dispatch. The classifier scores complexity and never prompt size, so a long agentic session whose newest ask is trivial lands on a small-window tier and the provider rejects it with a context-window 400 that nothing retries. When every model of the decided tier has a declared window smaller than the estimated prompt, the request moves to the lowest configured tier with a model whose declared window fits; when only some of the tier's models fit, the pick is restricted to those and the tier keeps the request. Models with no resolvable window are never escalated away from and never escalated onto. Disabled by default: omit or set false to dispatch on complexity alone; set true to enable context-window escalation. + * @default false */ enable_context_window_escalation: boolean; /** @@ -37347,6 +37469,13 @@ export interface components { routing_strategy_descriptions: { [key: string]: string; }; + /** + * Source + * @description Source of each current router setting + */ + source: { + [key: string]: "config" | "db" | "env" | "default" | "unset"; + }; }; /** * RoutingGroup @@ -39810,6 +39939,10 @@ export interface components { field_schema: { [key: string]: unknown; }; + /** Source */ + source: { + [key: string]: "config" | "db" | "env" | "default" | "unset"; + }; /** Values */ values: { [key: string]: unknown; @@ -39941,6 +40074,8 @@ export interface components { jwt_issuer?: string | null; /** Key */ key?: string | null; + /** Token */ + token?: string | null; }; /** UpdateKeyRequest */ UpdateKeyRequest: { @@ -42142,6 +42277,12 @@ export interface components { output_cost_per_character_above_128k_tokens?: number | null; /** Output Cost Per Image */ output_cost_per_image?: number | null; + /** Output Cost Per Image 1024 */ + output_cost_per_image_1024?: number | null; + /** Output Cost Per Image 1536 */ + output_cost_per_image_1536?: number | null; + /** Output Cost Per Image 512 */ + output_cost_per_image_512?: number | null; /** Output Cost Per Image Token */ output_cost_per_image_token?: number | null; /** Output Cost Per Pixel */ @@ -43984,6 +44125,8 @@ export interface operations { end_date?: string | null; /** @description Filter to one virtual key token hash */ api_key?: string | null; + /** @description Filter to one canonical internal user recorded on each turn */ + user_id?: string | null; }; header?: never; path?: never; @@ -49430,6 +49573,161 @@ export interface operations { }; }; }; + fal_ai_proxy_route_fal_ai__endpoint__get: { + parameters: { + query?: never; + header?: never; + path: { + endpoint: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": unknown; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; + fal_ai_proxy_route_fal_ai__endpoint__put: { + parameters: { + query?: never; + header?: never; + path: { + endpoint: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": unknown; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; + fal_ai_proxy_route_fal_ai__endpoint__post: { + parameters: { + query?: never; + header?: never; + path: { + endpoint: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": unknown; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; + fal_ai_proxy_route_fal_ai__endpoint__delete: { + parameters: { + query?: never; + header?: never; + path: { + endpoint: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": unknown; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; + fal_ai_proxy_route_fal_ai__endpoint__patch: { + parameters: { + query?: never; + header?: never; + path: { + endpoint: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": unknown; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; create_fallback_fallback_post: { parameters: { query?: never; @@ -56483,6 +56781,161 @@ export interface operations { }; }; }; + openrouter_proxy_route_openrouter__endpoint__get: { + parameters: { + query?: never; + header?: never; + path: { + endpoint: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": unknown; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; + openrouter_proxy_route_openrouter__endpoint__put: { + parameters: { + query?: never; + header?: never; + path: { + endpoint: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": unknown; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; + openrouter_proxy_route_openrouter__endpoint__post: { + parameters: { + query?: never; + header?: never; + path: { + endpoint: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": unknown; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; + openrouter_proxy_route_openrouter__endpoint__delete: { + parameters: { + query?: never; + header?: never; + path: { + endpoint: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": unknown; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; + openrouter_proxy_route_openrouter__endpoint__patch: { + parameters: { + query?: never; + header?: never; + path: { + endpoint: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": unknown; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; get_organization_daily_activity_organization_daily_activity_get: { parameters: { query?: { @@ -63982,6 +64435,39 @@ export interface operations { }; }; }; + change_password_user_password_change_post: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["ChangePasswordRequest"]; + }; + }; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ChangePasswordResponse"]; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; get_user_spend_report_user_spend_report_get: { parameters: { query?: { diff --git a/uv.lock b/uv.lock index 543581cbc23..18d57aa991b 100644 --- a/uv.lock +++ b/uv.lock @@ -4516,11 +4516,13 @@ dependencies = [ { name = "boto3" }, { name = "click" }, { name = "fastuuid" }, + { name = "filelock" }, { name = "httpx", extra = ["http2"] }, { name = "importlib-metadata" }, { name = "jinja2" }, { name = "jsonschema" }, { name = "openai" }, + { name = "packaging" }, { name = "pydantic" }, { name = "pydantic-settings" }, { name = "python-dotenv" }, @@ -4769,6 +4771,7 @@ requires-dist = [ { name = "fastapi", marker = "extra == 'proxy'", specifier = ">=0.136.3,<1.0" }, { name = "fastapi-sso", marker = "extra == 'proxy'", specifier = ">=0.19.0,<1.0" }, { name = "fastuuid", specifier = ">=0.14.0,<1.0" }, + { name = "filelock", specifier = ">=3.16.1,<4.0" }, { name = "google-cloud-aiplatform", marker = "extra == 'google'", specifier = ">=1.133.0,<2.0" }, { name = "google-cloud-aiplatform", marker = "extra == 'proxy-runtime'", specifier = ">=1.133.0,<2.0" }, { name = "google-cloud-iam", marker = "extra == 'extra-proxy'", specifier = ">=2.19.1,<3.0" }, @@ -4802,6 +4805,7 @@ requires-dist = [ { name = "numpydoc", marker = "extra == 'utils'", specifier = ">=1.8.0,<2.0" }, { name = "nvidia-riva-client", marker = "extra == 'stt-nvidia-riva'", specifier = ">=2.15.0" }, { name = "openai", specifier = ">=2.20.0,<3.0.0" }, + { name = "packaging", specifier = ">=24.0" }, { name = "opentelemetry-api", marker = "extra == 'proxy-runtime'", specifier = "==1.28.0" }, { name = "opentelemetry-exporter-otlp", marker = "extra == 'proxy-runtime'", specifier = "==1.28.0" }, { name = "opentelemetry-instrumentation-fastapi", marker = "extra == 'proxy-runtime'", specifier = "==0.49b0" },