diff --git a/.gitignore b/.gitignore index 3329f39ca10..9b552a8c269 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ .python-version .venv +tests/e2e/.fixtures/ .venv-typecheck .venv_policy_test .env diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20260817000000_shadow_eval_multi_key/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260817000000_shadow_eval_multi_key/migration.sql new file mode 100644 index 00000000000..18ef5c40662 --- /dev/null +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260817000000_shadow_eval_multi_key/migration.sql @@ -0,0 +1,7 @@ +ALTER TABLE "LiteLLM_ShadowEvalJob" ADD COLUMN IF NOT EXISTS "group_id" TEXT; + +UPDATE "LiteLLM_ShadowEvalJob" SET "group_id" = "id" WHERE "group_id" IS NULL; + +ALTER TABLE "LiteLLM_ShadowEvalJob" ALTER COLUMN "group_id" SET NOT NULL; + +CREATE INDEX IF NOT EXISTS "LiteLLM_ShadowEvalJob_group_id_idx" ON "LiteLLM_ShadowEvalJob"("group_id"); diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20260818224500_add_shadow_eval_stopped_by/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260818224500_add_shadow_eval_stopped_by/migration.sql new file mode 100644 index 00000000000..9efa3fdd052 --- /dev/null +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260818224500_add_shadow_eval_stopped_by/migration.sql @@ -0,0 +1,5 @@ +-- AlterTable +ALTER TABLE "LiteLLM_ShadowEvalJob" ADD COLUMN "stopped_by" TEXT; + +UPDATE "LiteLLM_ShadowEvalJob" SET stopped_by = 'unknown' +WHERE stopped_at IS NOT NULL AND ends_at > (NOW() AT TIME ZONE 'utc'); diff --git a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma index 52fb447157b..f79e2bb0c18 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma +++ b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma @@ -1467,28 +1467,38 @@ model LiteLLM_AutoRouterSession { @@index([last_turn_at], map: "idx_autorouter_session_last_turn") } -// Shadow eval: evaluation of an auto-router against a key's live traffic, in either -// direction. forward duplicates the requests the key did not route through the router -// through it, answering whether the key should adopt it; reverse duplicates the requests -// the router did serve against a fixed baseline model, answering whether a key already on -// it still benefits. Either way a sampled slice runs in a detached task and an LLM judge -// compares real vs shadow responses blind. The job row is immutable config plus -// stopped_at; every count, status, and spend figure is derived from the append-only -// attempt rows, so nothing can disagree across pods or stop races. +// Shadow eval: evaluation of an auto-router against one or more keys' live traffic, in +// either direction. forward duplicates the requests the keys did not route through the +// router through it, answering whether they should adopt it; reverse duplicates the +// requests the router did serve against a fixed baseline model, answering whether a key +// already on it still benefits. Either way a sampled slice runs in a detached task and an +// LLM judge compares real vs shadow responses blind. Each row is ONE key's leg of a job: +// immutable config plus that key's own turn budget and stop state, so one key exhausting +// its budget never ends a sibling's sampling. A job is the set of legs sharing group_id +// (the id the API reports), written together by one atomic create_many with identical +// config; single-key jobs predating group_id were backfilled group_id = id. "One active +// job per (key, direction)" is a partial unique index on (api_key_id, direction) WHERE +// stopped_at IS NULL, expressed only in the migration because schema.prisma cannot state +// partial indexes; it is what makes a concurrent start on another pod race-safe rather +// than read-then-create. Every count, status, and spend figure is derived from the +// append-only attempt rows, so nothing can disagree across pods or stop races. model LiteLLM_ShadowEvalJob { id String @id @default(cuid()) - api_key_id String // hashed virtual key whose traffic is shadowed + group_id String // legs of one job share this; the API's job id + api_key_id String // hashed virtual key whose traffic this leg shadows router_name String // the auto-router under evaluation, in either direction direction String @default("forward") // forward | reverse baseline_model String? // reverse only: the fixed model the router is judged against judge_model String shadow_percentage Float - max_turns Int // sample budget: judge at most this many turns + max_turns Int // this key's sample budget: judge at most this many turns created_at DateTime @default(now()) created_by String? ends_at DateTime stopped_at DateTime? + stopped_by String? // operator who stopped it early; null when it ended on its own + @@index([group_id]) @@index([api_key_id]) @@index([created_at]) } diff --git a/litellm/litellm_core_utils/streaming_handler.py b/litellm/litellm_core_utils/streaming_handler.py index 43bcf892865..485091bccd0 100644 --- a/litellm/litellm_core_utils/streaming_handler.py +++ b/litellm/litellm_core_utils/streaming_handler.py @@ -1830,6 +1830,20 @@ class CustomStreamWrapper: return self.chunks.append(model_response.model_copy(update={"choices": []})) + @staticmethod + def _resolve_provider_reported_cost(usage_cost: object) -> float | None: + """ + Providers report usage.cost either as a number or, for Perplexity, as a + breakdown object whose total lives under ``total_cost``. + """ + if isinstance(usage_cost, bool): + return None + if isinstance(usage_cost, (int, float)): + return float(usage_cost) + if isinstance(usage_cost, dict): + return CustomStreamWrapper._resolve_provider_reported_cost(usage_cost.get("total_cost")) + return None + @staticmethod def _propagate_usage_cost_to_hidden_params( response: "ModelResponse", @@ -1840,10 +1854,11 @@ class CustomStreamWrapper: calculator uses it instead of a token-based estimate. """ _usage: Final[Usage | None] = getattr(response, "usage", None) - if _usage is not None and hasattr(_usage, "cost") and _usage.cost is not None: + _cost: Final = CustomStreamWrapper._resolve_provider_reported_cost(getattr(_usage, "cost", None)) + if _cost is not None: if "additional_headers" not in response._hidden_params: response._hidden_params["additional_headers"] = {} - response._hidden_params["additional_headers"]["llm_provider-x-litellm-response-cost"] = float(_usage.cost) + response._hidden_params["additional_headers"]["llm_provider-x-litellm-response-cost"] = _cost def __next__(self) -> "ModelResponseStream": cache_hit = False diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 8c07ca35443..1ffd0a00a22 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -54,6 +54,7 @@ "output_cost_per_image": 0.04 }, "1024-x-1024/dall-e-2": { + "deprecation_date": "2026-05-12", "input_cost_per_pixel": 1.9e-08, "litellm_provider": "openai", "mode": "image_generation", @@ -67,6 +68,7 @@ "output_cost_per_image": 0.08 }, "256-x-256/dall-e-2": { + "deprecation_date": "2026-05-12", "input_cost_per_pixel": 2.4414e-07, "litellm_provider": "openai", "mode": "image_generation", @@ -80,6 +82,7 @@ "output_cost_per_image": 0.018 }, "512-x-512/dall-e-2": { + "deprecation_date": "2026-05-12", "input_cost_per_pixel": 6.86e-08, "litellm_provider": "openai", "mode": "image_generation", @@ -2887,6 +2890,7 @@ "supports_function_calling": true }, "azure_ai/claude-haiku-4-5": { + "deprecation_date": "2026-10-19", "cache_creation_input_token_cost": 1.25e-06, "cache_creation_input_token_cost_above_1hr": 2e-06, "cache_read_input_token_cost": 1e-07, @@ -2908,6 +2912,7 @@ "supports_vision": true }, "azure_ai/claude-opus-4-5": { + "deprecation_date": "2026-10-19", "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, @@ -2930,6 +2935,7 @@ "supports_output_config": true }, "azure_ai/claude-opus-4-6": { + "deprecation_date": "2027-02-02", "supports_adaptive_thinking": true, "input_cost_per_token": 5e-06, "output_cost_per_token": 2.5e-05, @@ -2959,6 +2965,7 @@ "supports_max_reasoning_effort": true }, "azure_ai/claude-opus-4-7": { + "deprecation_date": "2027-04-06", "supports_adaptive_thinking": true, "input_cost_per_token": 5e-06, "output_cost_per_token": 2.5e-05, @@ -3083,6 +3090,7 @@ "supports_max_reasoning_effort": true }, "azure_ai/claude-opus-4-1": { + "deprecation_date": "2026-08-05", "cache_creation_input_token_cost": 1.875e-05, "cache_creation_input_token_cost_above_1hr": 3e-05, "cache_read_input_token_cost": 1.5e-06, @@ -3104,6 +3112,7 @@ "supports_vision": true }, "azure_ai/claude-sonnet-4-5": { + "deprecation_date": "2026-10-19", "cache_creation_input_token_cost": 3.75e-06, "cache_creation_input_token_cost_above_1hr": 6e-06, "cache_read_input_token_cost": 3e-07, @@ -3156,6 +3165,7 @@ "supports_max_reasoning_effort": true }, "azure_ai/claude-sonnet-4-6": { + "deprecation_date": "2027-02-10", "supports_adaptive_thinking": true, "cache_creation_input_token_cost": 3.75e-06, "cache_creation_input_token_cost_above_1hr": 6e-06, @@ -3226,6 +3236,7 @@ "supports_tool_choice": true }, "azure_ai/gpt-5.5": { + "deprecation_date": "2027-10-26", "cache_read_input_token_cost": 5e-07, "cache_read_input_token_cost_above_272k_tokens": 1e-06, "cache_read_input_token_cost_priority": 1e-06, @@ -3318,6 +3329,7 @@ "supports_minimal_reasoning_effort": false }, "azure_ai/gpt-5.4": { + "deprecation_date": "2027-09-02", "cache_read_input_token_cost": 2.5e-07, "cache_read_input_token_cost_above_272k_tokens": 5e-07, "cache_read_input_token_cost_priority": 5e-07, @@ -3364,6 +3376,7 @@ "supports_minimal_reasoning_effort": true }, "azure_ai/gpt-5.4-2026-03-05": { + "deprecation_date": "2027-09-02", "cache_read_input_token_cost": 2.5e-07, "cache_read_input_token_cost_above_272k_tokens": 5e-07, "cache_read_input_token_cost_priority": 5e-07, @@ -3410,6 +3423,7 @@ "supports_minimal_reasoning_effort": true }, "azure_ai/gpt-5.4-pro": { + "deprecation_date": "2027-09-07", "cache_read_input_token_cost": 3e-06, "cache_read_input_token_cost_above_272k_tokens": 6e-06, "cache_read_input_token_cost_priority": 6e-06, @@ -3455,6 +3469,7 @@ "supports_minimal_reasoning_effort": true }, "azure_ai/gpt-5.4-pro-2026-03-05": { + "deprecation_date": "2027-09-07", "cache_read_input_token_cost": 3e-06, "cache_read_input_token_cost_above_272k_tokens": 6e-06, "cache_read_input_token_cost_priority": 6e-06, @@ -3500,6 +3515,7 @@ "supports_minimal_reasoning_effort": true }, "azure_ai/gpt-5.4-mini": { + "deprecation_date": "2027-09-21", "cache_read_input_token_cost": 7.5e-08, "cache_read_input_token_cost_priority": 1.5e-07, "input_cost_per_token": 7.5e-07, @@ -3540,6 +3556,7 @@ "supports_minimal_reasoning_effort": false }, "azure_ai/gpt-5.4-mini-2026-03-17": { + "deprecation_date": "2027-09-21", "cache_read_input_token_cost": 7.5e-08, "cache_read_input_token_cost_priority": 1.5e-07, "input_cost_per_token": 7.5e-07, @@ -3580,6 +3597,7 @@ "supports_minimal_reasoning_effort": false }, "azure_ai/gpt-5.4-nano": { + "deprecation_date": "2027-09-21", "cache_read_input_token_cost": 2e-08, "cache_read_input_token_cost_priority": 4e-08, "input_cost_per_token": 2e-07, @@ -3620,6 +3638,7 @@ "supports_minimal_reasoning_effort": false }, "azure_ai/gpt-5.4-nano-2026-03-17": { + "deprecation_date": "2027-09-21", "cache_read_input_token_cost": 2e-08, "cache_read_input_token_cost_priority": 4e-08, "input_cost_per_token": 2e-07, @@ -3849,6 +3868,7 @@ "supports_vision": true }, "azure/eu/gpt-5.1": { + "deprecation_date": "2027-05-15", "cache_read_input_token_cost": 1.4e-07, "input_cost_per_token": 1.38e-06, "litellm_provider": "azure", @@ -3918,6 +3938,7 @@ "supports_none_reasoning_effort": true }, "azure/eu/gpt-5.1-codex": { + "deprecation_date": "2027-05-15", "cache_read_input_token_cost": 1.4e-07, "input_cost_per_token": 1.38e-06, "litellm_provider": "azure", @@ -3948,6 +3969,7 @@ "supports_vision": true }, "azure/eu/gpt-5.1-codex-mini": { + "deprecation_date": "2027-05-15", "cache_read_input_token_cost": 2.8e-08, "input_cost_per_token": 2.75e-07, "litellm_provider": "azure", @@ -4107,6 +4129,7 @@ "supports_vision": true }, "azure/global-standard/gpt-4o-mini": { + "deprecation_date": "2027-04-14", "input_cost_per_token": 1.5e-07, "litellm_provider": "azure", "max_input_tokens": 128000, @@ -4155,6 +4178,7 @@ "supports_vision": true }, "azure/global/gpt-5.1": { + "deprecation_date": "2027-05-15", "cache_read_input_token_cost": 1.25e-07, "input_cost_per_token": 1.25e-06, "litellm_provider": "azure", @@ -4224,6 +4248,7 @@ "supports_none_reasoning_effort": true }, "azure/global/gpt-5.1-codex": { + "deprecation_date": "2027-05-15", "cache_read_input_token_cost": 1.25e-07, "input_cost_per_token": 1.25e-06, "litellm_provider": "azure", @@ -4254,6 +4279,7 @@ "supports_vision": true }, "azure/global/gpt-5.1-codex-mini": { + "deprecation_date": "2027-05-15", "cache_read_input_token_cost": 2.5e-08, "input_cost_per_token": 2.5e-07, "litellm_provider": "azure", @@ -4492,6 +4518,7 @@ "supports_vision": true }, "azure/gpt-4.1": { + "deprecation_date": "2027-04-14", "cache_read_input_token_cost": 5e-07, "input_cost_per_token": 2e-06, "input_cost_per_token_batches": 1e-06, @@ -4559,6 +4586,7 @@ "supports_web_search": false }, "azure/gpt-4.1-mini": { + "deprecation_date": "2027-04-14", "cache_read_input_token_cost": 1e-07, "input_cost_per_token": 4e-07, "input_cost_per_token_batches": 2e-07, @@ -4626,6 +4654,7 @@ "supports_web_search": false }, "azure/gpt-4.1-nano": { + "deprecation_date": "2026-10-14", "cache_read_input_token_cost": 2.5e-08, "input_cost_per_token": 1e-07, "input_cost_per_token_batches": 5e-08, @@ -4902,6 +4931,7 @@ "supports_vision": false }, "azure/gpt-4o-mini": { + "deprecation_date": "2027-04-14", "cache_read_input_token_cost": 7.5e-08, "input_cost_per_token": 1.65e-07, "litellm_provider": "azure", @@ -5344,6 +5374,7 @@ "supports_vision": true }, "azure/gpt-5": { + "deprecation_date": "2027-02-09", "cache_read_input_token_cost": 1.25e-07, "input_cost_per_token": 1.25e-06, "litellm_provider": "azure", @@ -5507,6 +5538,7 @@ "supports_vision": true }, "azure/gpt-5-mini": { + "deprecation_date": "2027-02-09", "cache_read_input_token_cost": 2.5e-08, "input_cost_per_token": 2.5e-07, "litellm_provider": "azure", @@ -5572,6 +5604,7 @@ "supports_vision": true }, "azure/gpt-5-nano": { + "deprecation_date": "2027-02-09", "cache_read_input_token_cost": 5e-09, "input_cost_per_token": 5e-08, "litellm_provider": "azure", @@ -5667,6 +5700,7 @@ "supports_vision": true }, "azure/gpt-5.1": { + "deprecation_date": "2027-05-15", "cache_read_input_token_cost": 1.25e-07, "input_cost_per_token": 1.25e-06, "litellm_provider": "azure", @@ -5736,6 +5770,7 @@ "supports_none_reasoning_effort": true }, "azure/gpt-5.1-codex": { + "deprecation_date": "2027-05-15", "cache_read_input_token_cost": 1.25e-07, "input_cost_per_token": 1.25e-06, "litellm_provider": "azure", @@ -5797,6 +5832,7 @@ "supports_vision": true }, "azure/gpt-5.1-codex-mini": { + "deprecation_date": "2027-05-15", "cache_read_input_token_cost": 2.5e-08, "input_cost_per_token": 2.5e-07, "litellm_provider": "azure", @@ -5827,6 +5863,7 @@ "supports_vision": true }, "azure/gpt-5.2": { + "deprecation_date": "2027-06-08", "cache_read_input_token_cost": 1.75e-07, "input_cost_per_token": 1.75e-06, "litellm_provider": "azure", @@ -6136,6 +6173,7 @@ "supports_web_search": true }, "azure/gpt-5.4": { + "deprecation_date": "2027-09-02", "cache_read_input_token_cost": 2.5e-07, "cache_read_input_token_cost_above_272k_tokens": 5e-07, "cache_read_input_token_cost_priority": 5e-07, @@ -6180,6 +6218,7 @@ "supports_minimal_reasoning_effort": true }, "azure/us/gpt-5.4": { + "deprecation_date": "2027-09-02", "cache_read_input_token_cost": 2.8e-07, "cache_read_input_token_cost_priority": 5.5e-07, "input_cost_per_token": 2.75e-06, @@ -6218,6 +6257,7 @@ "supports_minimal_reasoning_effort": true }, "azure/eu/gpt-5.4": { + "deprecation_date": "2027-09-02", "cache_read_input_token_cost": 2.8e-07, "cache_read_input_token_cost_priority": 5.5e-07, "input_cost_per_token": 2.75e-06, @@ -6379,6 +6419,7 @@ "supports_minimal_reasoning_effort": true }, "azure/gpt-5.4-pro": { + "deprecation_date": "2027-09-07", "cache_read_input_token_cost": 3e-06, "cache_read_input_token_cost_above_272k_tokens": 6e-06, "input_cost_per_token": 3e-05, @@ -7045,6 +7086,7 @@ "supports_minimal_reasoning_effort": false }, "azure/gpt-5.5": { + "deprecation_date": "2027-10-26", "cache_read_input_token_cost": 5e-07, "cache_read_input_token_cost_above_272k_tokens": 1e-06, "cache_read_input_token_cost_priority": 1e-06, @@ -7095,6 +7137,7 @@ "supports_minimal_reasoning_effort": false }, "azure/us/gpt-5.5": { + "deprecation_date": "2027-10-26", "cache_read_input_token_cost": 5.5e-07, "cache_read_input_token_cost_above_272k_tokens": 1.1e-06, "cache_read_input_token_cost_priority": 1.38e-06, @@ -7142,6 +7185,7 @@ "supports_minimal_reasoning_effort": false }, "azure/eu/gpt-5.5": { + "deprecation_date": "2027-10-26", "cache_read_input_token_cost": 5.5e-07, "cache_read_input_token_cost_above_272k_tokens": 1.1e-06, "cache_read_input_token_cost_priority": 1.38e-06, @@ -7408,6 +7452,7 @@ "supports_web_search": true }, "azure/gpt-5.4-mini": { + "deprecation_date": "2027-09-21", "cache_read_input_token_cost": 7.5e-08, "input_cost_per_token": 7.5e-07, "litellm_provider": "azure", @@ -7489,6 +7534,7 @@ "supports_xhigh_reasoning_effort": true }, "azure/gpt-5.4-nano": { + "deprecation_date": "2027-09-21", "cache_read_input_token_cost": 2e-08, "input_cost_per_token": 2e-07, "litellm_provider": "azure", @@ -7601,6 +7647,7 @@ "output_cost_per_token": 0.0 }, "azure/high/1024-x-1024/gpt-image-1": { + "deprecation_date": "2026-10-23", "input_cost_per_pixel": 1.59263611e-07, "litellm_provider": "azure", "mode": "image_generation", @@ -7610,6 +7657,7 @@ ] }, "azure/high/1024-x-1536/gpt-image-1": { + "deprecation_date": "2026-10-23", "input_cost_per_pixel": 1.58945719e-07, "litellm_provider": "azure", "mode": "image_generation", @@ -7619,6 +7667,7 @@ ] }, "azure/high/1536-x-1024/gpt-image-1": { + "deprecation_date": "2026-10-23", "input_cost_per_pixel": 1.58945719e-07, "litellm_provider": "azure", "mode": "image_generation", @@ -7628,6 +7677,7 @@ ] }, "azure/low/1024-x-1024/gpt-image-1": { + "deprecation_date": "2026-10-23", "input_cost_per_pixel": 1.0490417e-08, "litellm_provider": "azure", "mode": "image_generation", @@ -7637,6 +7687,7 @@ ] }, "azure/low/1024-x-1536/gpt-image-1": { + "deprecation_date": "2026-10-23", "input_cost_per_pixel": 1.0172526e-08, "litellm_provider": "azure", "mode": "image_generation", @@ -7646,6 +7697,7 @@ ] }, "azure/low/1536-x-1024/gpt-image-1": { + "deprecation_date": "2026-10-23", "input_cost_per_pixel": 1.0172526e-08, "litellm_provider": "azure", "mode": "image_generation", @@ -7655,6 +7707,7 @@ ] }, "azure/medium/1024-x-1024/gpt-image-1": { + "deprecation_date": "2026-10-23", "input_cost_per_pixel": 4.0054321e-08, "litellm_provider": "azure", "mode": "image_generation", @@ -7664,6 +7717,7 @@ ] }, "azure/medium/1024-x-1536/gpt-image-1": { + "deprecation_date": "2026-10-23", "input_cost_per_pixel": 4.0054321e-08, "litellm_provider": "azure", "mode": "image_generation", @@ -7673,6 +7727,7 @@ ] }, "azure/medium/1536-x-1024/gpt-image-1": { + "deprecation_date": "2026-10-23", "input_cost_per_pixel": 4.0054321e-08, "litellm_provider": "azure", "mode": "image_generation", @@ -7695,6 +7750,7 @@ ] }, "azure/gpt-image-1.5": { + "deprecation_date": "2027-06-16", "cache_read_input_token_cost": 1.25e-06, "input_cost_per_token": 5e-06, "input_cost_per_image_token": 8e-06, @@ -7720,6 +7776,7 @@ ] }, "azure/gpt-image-2": { + "deprecation_date": "2027-10-21", "cache_read_input_token_cost": 1.25e-06, "input_cost_per_token": 5e-06, "input_cost_per_image_token": 8e-06, @@ -7751,6 +7808,7 @@ "supports_pdf_input": true }, "azure/low/1024-x-1024/gpt-image-1-mini": { + "deprecation_date": "2027-04-07", "input_cost_per_pixel": 2.0751953125e-09, "litellm_provider": "azure", "mode": "image_generation", @@ -7760,6 +7818,7 @@ ] }, "azure/low/1024-x-1536/gpt-image-1-mini": { + "deprecation_date": "2027-04-07", "input_cost_per_pixel": 2.0751953125e-09, "litellm_provider": "azure", "mode": "image_generation", @@ -7769,6 +7828,7 @@ ] }, "azure/low/1536-x-1024/gpt-image-1-mini": { + "deprecation_date": "2027-04-07", "input_cost_per_pixel": 2.0345052083e-09, "litellm_provider": "azure", "mode": "image_generation", @@ -7778,6 +7838,7 @@ ] }, "azure/medium/1024-x-1024/gpt-image-1-mini": { + "deprecation_date": "2027-04-07", "input_cost_per_pixel": 8.056640625e-09, "litellm_provider": "azure", "mode": "image_generation", @@ -7787,6 +7848,7 @@ ] }, "azure/medium/1024-x-1536/gpt-image-1-mini": { + "deprecation_date": "2027-04-07", "input_cost_per_pixel": 8.056640625e-09, "litellm_provider": "azure", "mode": "image_generation", @@ -7796,6 +7858,7 @@ ] }, "azure/medium/1536-x-1024/gpt-image-1-mini": { + "deprecation_date": "2027-04-07", "input_cost_per_pixel": 7.9752604167e-09, "litellm_provider": "azure", "mode": "image_generation", @@ -7805,6 +7868,7 @@ ] }, "azure/high/1024-x-1024/gpt-image-1-mini": { + "deprecation_date": "2027-04-07", "input_cost_per_pixel": 3.173828125e-08, "litellm_provider": "azure", "mode": "image_generation", @@ -7814,6 +7878,7 @@ ] }, "azure/high/1024-x-1536/gpt-image-1-mini": { + "deprecation_date": "2027-04-07", "input_cost_per_pixel": 3.173828125e-08, "litellm_provider": "azure", "mode": "image_generation", @@ -7823,6 +7888,7 @@ ] }, "azure/high/1536-x-1024/gpt-image-1-mini": { + "deprecation_date": "2027-04-07", "input_cost_per_pixel": 3.1575520833e-08, "litellm_provider": "azure", "mode": "image_generation", @@ -7850,6 +7916,7 @@ "supports_function_calling": true }, "azure/o1": { + "deprecation_date": "2026-10-21", "cache_read_input_token_cost": 7.5e-06, "input_cost_per_token": 1.5e-05, "litellm_provider": "azure", @@ -7944,6 +8011,7 @@ "supports_vision": false }, "azure/o3": { + "deprecation_date": "2026-10-21", "cache_read_input_token_cost": 5e-07, "input_cost_per_token": 2e-06, "litellm_provider": "azure", @@ -8041,6 +8109,7 @@ "supports_web_search": true }, "azure/o3-mini": { + "deprecation_date": "2026-10-01", "cache_read_input_token_cost": 5.5e-07, "input_cost_per_token": 1.1e-06, "litellm_provider": "azure", @@ -8071,6 +8140,7 @@ "supports_vision": false }, "azure/o3-pro": { + "deprecation_date": "2026-12-17", "input_cost_per_token": 2e-05, "input_cost_per_token_batches": 1e-05, "litellm_provider": "azure", @@ -8132,6 +8202,7 @@ "supports_vision": true }, "azure/o4-mini": { + "deprecation_date": "2026-10-16", "cache_read_input_token_cost": 2.75e-07, "input_cost_per_token": 1.1e-06, "litellm_provider": "azure", @@ -8580,6 +8651,7 @@ "supports_vision": true }, "azure/us/gpt-5.1": { + "deprecation_date": "2027-05-15", "cache_read_input_token_cost": 1.4e-07, "input_cost_per_token": 1.38e-06, "litellm_provider": "azure", @@ -8649,6 +8721,7 @@ "supports_none_reasoning_effort": true }, "azure/us/gpt-5.1-codex": { + "deprecation_date": "2027-05-15", "cache_read_input_token_cost": 1.4e-07, "input_cost_per_token": 1.38e-06, "litellm_provider": "azure", @@ -8679,6 +8752,7 @@ "supports_vision": true }, "azure/us/gpt-5.1-codex-mini": { + "deprecation_date": "2027-05-15", "cache_read_input_token_cost": 2.8e-08, "input_cost_per_token": 2.75e-07, "litellm_provider": "azure", @@ -8876,6 +8950,7 @@ ] }, "azure_ai/FW-DeepSeek-V3.2": { + "deprecation_date": "2027-07-01", "cache_read_input_token_cost": 3.1e-07, "input_cost_per_token": 6.2e-07, "litellm_provider": "azure_ai", @@ -8906,6 +8981,7 @@ "supports_tool_choice": true }, "azure_ai/FW-GLM-5": { + "deprecation_date": "2027-07-01", "cache_read_input_token_cost": 2.2e-07, "input_cost_per_token": 1.1e-06, "litellm_provider": "azure_ai", @@ -8921,6 +8997,7 @@ "supports_tool_choice": true }, "azure_ai/FW-GLM-5.1": { + "deprecation_date": "2027-07-01", "cache_read_input_token_cost": 2.86e-07, "input_cost_per_token": 1.54e-06, "litellm_provider": "azure_ai", @@ -8987,6 +9064,7 @@ "supports_tool_choice": true }, "azure_ai/FW-Kimi-K2.5": { + "deprecation_date": "2027-07-01", "cache_read_input_token_cost": 1.1e-07, "input_cost_per_token": 6.6e-07, "litellm_provider": "azure_ai", @@ -9079,6 +9157,7 @@ "supports_vision": true }, "azure_ai/FW-MiniMax-M2.5": { + "deprecation_date": "2027-07-01", "cache_read_input_token_cost": 3.3e-08, "input_cost_per_token": 3.3e-07, "litellm_provider": "azure_ai", @@ -9164,6 +9243,7 @@ ] }, "azure_ai/MAI-Image-2e": { + "deprecation_date": "2026-08-15", "input_cost_per_token": 5e-06, "litellm_provider": "azure_ai", "mode": "image_generation", @@ -9175,6 +9255,7 @@ ] }, "azure_ai/Llama-3.2-11B-Vision-Instruct": { + "deprecation_date": "2026-06-13", "input_cost_per_token": 3.7e-07, "litellm_provider": "azure_ai", "max_input_tokens": 128000, @@ -9188,6 +9269,7 @@ "supports_vision": true }, "azure_ai/Llama-3.2-90B-Vision-Instruct": { + "deprecation_date": "2026-06-13", "input_cost_per_token": 2.04e-06, "litellm_provider": "azure_ai", "max_input_tokens": 128000, @@ -9249,6 +9331,7 @@ "supports_tool_choice": true }, "azure_ai/Meta-Llama-3.1-405B-Instruct": { + "deprecation_date": "2026-06-13", "input_cost_per_token": 5.33e-06, "litellm_provider": "azure_ai", "max_input_tokens": 128000, @@ -9271,6 +9354,7 @@ "supports_tool_choice": true }, "azure_ai/Meta-Llama-3.1-8B-Instruct": { + "deprecation_date": "2026-06-13", "input_cost_per_token": 3e-07, "litellm_provider": "azure_ai", "max_input_tokens": 128000, @@ -9452,6 +9536,7 @@ "supports_reasoning": true }, "azure_ai/mistral-document-ai-2505": { + "deprecation_date": "2026-07-20", "litellm_provider": "azure_ai", "ocr_cost_per_page": 0.003, "mode": "ocr", @@ -9529,6 +9614,7 @@ "output_cost_per_token": 0.0 }, "azure_ai/cohere-rerank-v3.5": { + "deprecation_date": "2026-05-14", "input_cost_per_query": 0.002, "input_cost_per_token": 0.0, "litellm_provider": "azure_ai", @@ -9591,6 +9677,7 @@ "supports_tool_choice": true }, "azure_ai/deepseek-r1": { + "deprecation_date": "2026-08-13", "input_cost_per_token": 1.35e-06, "litellm_provider": "azure_ai", "max_input_tokens": 128000, @@ -9614,6 +9701,7 @@ "supports_tool_choice": true }, "azure_ai/deepseek-v3-0324": { + "deprecation_date": "2026-07-13", "input_cost_per_token": 1.14e-06, "litellm_provider": "azure_ai", "max_input_tokens": 128000, @@ -9626,6 +9714,7 @@ "supports_tool_choice": true }, "azure_ai/deepseek-v3.1": { + "deprecation_date": "2026-07-13", "input_cost_per_token": 1.23e-06, "litellm_provider": "azure_ai", "max_input_tokens": 131072, @@ -9639,6 +9728,7 @@ "supports_tool_choice": true }, "azure_ai/deepseek-v4-pro": { + "deprecation_date": "2028-02-20", "input_cost_per_token": 1.74e-06, "litellm_provider": "azure_ai", "max_input_tokens": 1000000, @@ -9652,6 +9742,7 @@ "supports_tool_choice": true }, "azure_ai/deepseek-v4-flash": { + "deprecation_date": "2028-02-20", "input_cost_per_token": 1.9e-07, "litellm_provider": "azure_ai", "max_input_tokens": 1000000, @@ -9683,6 +9774,7 @@ "supports_embedding_image_input": true }, "azure_ai/global/grok-3": { + "deprecation_date": "2026-05-01", "input_cost_per_token": 3e-06, "litellm_provider": "azure_ai", "max_input_tokens": 131072, @@ -9697,6 +9789,7 @@ "supports_web_search": true }, "azure_ai/global/grok-3-mini": { + "deprecation_date": "2026-05-01", "input_cost_per_token": 2.5e-07, "litellm_provider": "azure_ai", "max_input_tokens": 131072, @@ -9712,6 +9805,7 @@ "supports_web_search": true }, "azure_ai/grok-3": { + "deprecation_date": "2026-05-01", "input_cost_per_token": 3e-06, "litellm_provider": "azure_ai", "max_input_tokens": 131072, @@ -9726,6 +9820,7 @@ "supports_web_search": true }, "azure_ai/grok-3-mini": { + "deprecation_date": "2026-05-01", "input_cost_per_token": 2.5e-07, "litellm_provider": "azure_ai", "max_input_tokens": 131072, @@ -9773,6 +9868,7 @@ "supports_web_search": true }, "azure_ai/grok-4-fast-non-reasoning": { + "deprecation_date": "2026-05-01", "input_cost_per_token": 2e-07, "output_cost_per_token": 5e-07, "litellm_provider": "azure_ai", @@ -9786,6 +9882,7 @@ "supports_web_search": true }, "azure_ai/grok-4-fast-reasoning": { + "deprecation_date": "2026-05-01", "input_cost_per_token": 2e-07, "output_cost_per_token": 5e-07, "litellm_provider": "azure_ai", @@ -9863,6 +9960,7 @@ "supports_tool_choice": true }, "azure_ai/kimi-k2.5": { + "deprecation_date": "2027-01-26", "input_cost_per_token": 6e-07, "litellm_provider": "azure_ai", "max_input_tokens": 262144, @@ -9877,6 +9975,7 @@ "supports_vision": true }, "azure_ai/kimi-k2.6": { + "deprecation_date": "2027-04-16", "input_cost_per_token": 9.5e-07, "litellm_provider": "azure_ai", "max_input_tokens": 262144, @@ -10004,6 +10103,7 @@ "supports_vision": true }, "babbage-002": { + "deprecation_date": "2026-09-28", "input_cost_per_token": 4e-07, "litellm_provider": "text-completion-openai", "max_input_tokens": 16384, @@ -12014,6 +12114,7 @@ ] }, "claude-haiku-4-5-20251001": { + "deprecation_date": "2026-10-15", "cache_creation_input_token_cost": 1.25e-06, "cache_creation_input_token_cost_above_1hr": 2e-06, "cache_read_input_token_cost": 1e-07, @@ -12037,6 +12138,7 @@ "prompt_cache_min_tokens": 4096 }, "claude-haiku-4-5": { + "deprecation_date": "2026-10-15", "cache_creation_input_token_cost": 1.25e-06, "cache_creation_input_token_cost_above_1hr": 2e-06, "cache_read_input_token_cost": 1e-07, @@ -12185,6 +12287,7 @@ "prompt_cache_min_tokens": 1024 }, "claude-sonnet-4-5": { + "deprecation_date": "2026-09-29", "cache_creation_input_token_cost": 3.75e-06, "cache_creation_input_token_cost_above_1hr": 6e-06, "cache_creation_input_token_cost_above_1hr_above_200k_tokens": 1.2e-05, @@ -12218,6 +12321,7 @@ "prompt_cache_min_tokens": 1024 }, "claude-sonnet-4-5-20250929": { + "deprecation_date": "2026-09-29", "cache_creation_input_token_cost": 3.75e-06, "cache_creation_input_token_cost_above_1hr": 6e-06, "cache_creation_input_token_cost_above_1hr_above_200k_tokens": 1.2e-05, @@ -12252,6 +12356,7 @@ "prompt_cache_min_tokens": 1024 }, "claude-sonnet-5": { + "deprecation_date": "2027-06-30", "cache_creation_input_token_cost": 2.5e-06, "cache_creation_input_token_cost_above_1hr": 4e-06, "cache_read_input_token_cost": 2e-07, @@ -12288,6 +12393,7 @@ "prompt_cache_min_tokens": 1024 }, "claude-sonnet-4-6": { + "deprecation_date": "2027-02-17", "cache_creation_input_token_cost": 3.75e-06, "cache_creation_input_token_cost_above_1hr": 6e-06, "cache_read_input_token_cost": 3e-07, @@ -12434,6 +12540,7 @@ "prompt_cache_min_tokens": 1024 }, "claude-opus-4-5-20251101": { + "deprecation_date": "2026-11-24", "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, @@ -12463,6 +12570,7 @@ "prompt_cache_min_tokens": 4096 }, "claude-opus-4-5": { + "deprecation_date": "2026-11-24", "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, @@ -12492,6 +12600,7 @@ "prompt_cache_min_tokens": 4096 }, "claude-opus-4-6": { + "deprecation_date": "2027-02-05", "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, @@ -12528,6 +12637,7 @@ "prompt_cache_min_tokens": 4096 }, "claude-opus-4-6-20260205": { + "deprecation_date": "2027-02-05", "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, @@ -12564,6 +12674,7 @@ "prompt_cache_min_tokens": 4096 }, "claude-opus-4-7": { + "deprecation_date": "2027-04-16", "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, @@ -12602,6 +12713,7 @@ "prompt_cache_min_tokens": 2048 }, "claude-opus-4-7-20260416": { + "deprecation_date": "2027-04-16", "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, @@ -12640,6 +12752,7 @@ "prompt_cache_min_tokens": 2048 }, "claude-fable-5": { + "deprecation_date": "2027-06-09", "cache_creation_input_token_cost": 1.25e-05, "cache_creation_input_token_cost_above_1hr": 2e-05, "cache_read_input_token_cost": 1e-06, @@ -12675,6 +12788,7 @@ "prompt_cache_min_tokens": 512 }, "claude-opus-5": { + "deprecation_date": "2027-07-24", "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, @@ -12713,6 +12827,7 @@ "prompt_cache_min_tokens": 512 }, "claude-opus-4-8": { + "deprecation_date": "2027-05-28", "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, @@ -15042,6 +15157,7 @@ "mode": "search" }, "davinci-002": { + "deprecation_date": "2026-09-28", "input_cost_per_token": 2e-06, "litellm_provider": "text-completion-openai", "max_input_tokens": 16384, @@ -18594,6 +18710,7 @@ } }, "gemini-2.5-flash": { + "deprecation_date": "2026-10-20", "cache_read_input_token_cost": 3e-08, "input_cost_per_audio_token": 1e-06, "input_cost_per_token": 3e-07, @@ -18639,6 +18756,7 @@ "supports_image_size": false }, "gemini-2.5-flash-image": { + "deprecation_date": "2026-10-02", "cache_read_input_token_cost": 3e-08, "input_cost_per_audio_token": 1e-06, "input_cost_per_token": 3e-07, @@ -18683,6 +18801,7 @@ "supports_image_size": false }, "gemini-3-pro-image": { + "deprecation_date": "2027-05-28", "input_cost_per_image": 0.0011, "input_cost_per_token": 2e-06, "input_cost_per_token_batches": 1e-06, @@ -18763,6 +18882,7 @@ "web_search_billing_unit": "per_query" }, "gemini-3.1-flash-image": { + "deprecation_date": "2027-05-28", "input_cost_per_image": 0.00056, "input_cost_per_token": 5e-07, "litellm_provider": "vertex_ai-language-models", @@ -18887,6 +19007,7 @@ "web_search_billing_unit": "per_query" }, "gemini-3.1-flash-lite": { + "deprecation_date": "2027-05-07", "cache_read_input_token_cost": 2.5e-08, "cache_read_input_token_cost_flex": 1.25e-08, "cache_read_input_token_cost_priority": 4.5e-08, @@ -18943,6 +19064,7 @@ "web_search_billing_unit": "per_query" }, "gemini-3.5-flash-lite": { + "deprecation_date": "2027-07-21", "cache_read_input_token_cost": 3e-08, "cache_read_input_token_cost_flex": 2e-08, "cache_read_input_token_cost_priority": 5e-08, @@ -19032,6 +19154,7 @@ "supports_web_search": true }, "gemini-2.5-flash-lite": { + "deprecation_date": "2026-10-20", "cache_read_input_token_cost": 1e-08, "input_cost_per_audio_token": 3e-07, "input_cost_per_token": 1e-07, @@ -19303,6 +19426,7 @@ "supports_image_size": false }, "gemini-2.5-pro": { + "deprecation_date": "2026-10-20", "cache_read_input_token_cost": 1.25e-07, "cache_read_input_token_cost_above_200k_tokens": 2.5e-07, "cache_creation_input_token_cost_above_200k_tokens": 2.5e-07, @@ -19617,6 +19741,7 @@ }, "vertex_ai/gemini-3.5-flash": { "prompt_cache_min_tokens": 4096, + "deprecation_date": "2027-05-19", "cache_read_input_token_cost": 1.5e-07, "input_cost_per_token": 1.5e-06, "input_cost_per_audio_token": 1e-06, @@ -20057,6 +20182,7 @@ "web_search_billing_unit": "per_query" }, "gemini/gemini-robotics-er-1.6-preview": { + "deprecation_date": "2026-08-31", "input_cost_per_audio_token": 2e-06, "input_cost_per_token": 1e-06, "litellm_provider": "gemini", @@ -20127,6 +20253,7 @@ "supports_vision": true }, "gemini-embedding-001": { + "deprecation_date": "2028-05-20", "input_cost_per_token": 1.5e-07, "litellm_provider": "vertex_ai-embedding-models", "max_input_tokens": 2048, @@ -21746,6 +21873,7 @@ }, "gemini-3.5-flash": { "prompt_cache_min_tokens": 4096, + "deprecation_date": "2027-05-19", "cache_read_input_token_cost": 1.5e-07, "input_cost_per_audio_token": 1e-06, "input_cost_per_token": 1.5e-06, @@ -23260,6 +23388,7 @@ "supports_tool_choice": true }, "gpt-3.5-turbo-instruct": { + "deprecation_date": "2026-09-28", "input_cost_per_token": 1.5e-06, "litellm_provider": "text-completion-openai", "max_input_tokens": 8192, @@ -24391,6 +24520,7 @@ "supports_pdf_input": true }, "low/1024-x-1024/gpt-image-1.5": { + "deprecation_date": "2026-12-01", "input_cost_per_image": 0.009, "litellm_provider": "openai", "mode": "image_generation", @@ -24402,6 +24532,7 @@ "supports_pdf_input": true }, "low/1024-x-1536/gpt-image-1.5": { + "deprecation_date": "2026-12-01", "input_cost_per_image": 0.013, "litellm_provider": "openai", "mode": "image_generation", @@ -24413,6 +24544,7 @@ "supports_pdf_input": true }, "low/1536-x-1024/gpt-image-1.5": { + "deprecation_date": "2026-12-01", "input_cost_per_image": 0.013, "litellm_provider": "openai", "mode": "image_generation", @@ -24424,6 +24556,7 @@ "supports_pdf_input": true }, "medium/1024-x-1024/gpt-image-1.5": { + "deprecation_date": "2026-12-01", "input_cost_per_image": 0.034, "litellm_provider": "openai", "mode": "image_generation", @@ -24435,6 +24568,7 @@ "supports_pdf_input": true }, "medium/1024-x-1536/gpt-image-1.5": { + "deprecation_date": "2026-12-01", "input_cost_per_image": 0.05, "litellm_provider": "openai", "mode": "image_generation", @@ -24446,6 +24580,7 @@ "supports_pdf_input": true }, "medium/1536-x-1024/gpt-image-1.5": { + "deprecation_date": "2026-12-01", "input_cost_per_image": 0.05, "litellm_provider": "openai", "mode": "image_generation", @@ -24457,6 +24592,7 @@ "supports_pdf_input": true }, "high/1024-x-1024/gpt-image-1.5": { + "deprecation_date": "2026-12-01", "input_cost_per_image": 0.133, "litellm_provider": "openai", "mode": "image_generation", @@ -24468,6 +24604,7 @@ "supports_pdf_input": true }, "high/1024-x-1536/gpt-image-1.5": { + "deprecation_date": "2026-12-01", "input_cost_per_image": 0.2, "litellm_provider": "openai", "mode": "image_generation", @@ -24479,6 +24616,7 @@ "supports_pdf_input": true }, "high/1536-x-1024/gpt-image-1.5": { + "deprecation_date": "2026-12-01", "input_cost_per_image": 0.2, "litellm_provider": "openai", "mode": "image_generation", @@ -24490,6 +24628,7 @@ "supports_pdf_input": true }, "standard/1024-x-1024/gpt-image-1.5": { + "deprecation_date": "2026-12-01", "input_cost_per_image": 0.009, "litellm_provider": "openai", "mode": "image_generation", @@ -24501,6 +24640,7 @@ "supports_pdf_input": true }, "standard/1024-x-1536/gpt-image-1.5": { + "deprecation_date": "2026-12-01", "input_cost_per_image": 0.013, "litellm_provider": "openai", "mode": "image_generation", @@ -24512,6 +24652,7 @@ "supports_pdf_input": true }, "standard/1536-x-1024/gpt-image-1.5": { + "deprecation_date": "2026-12-01", "input_cost_per_image": 0.013, "litellm_provider": "openai", "mode": "image_generation", @@ -24523,6 +24664,7 @@ "supports_pdf_input": true }, "1024-x-1024/gpt-image-1.5": { + "deprecation_date": "2026-12-01", "input_cost_per_image": 0.009, "litellm_provider": "openai", "mode": "image_generation", @@ -24534,6 +24676,7 @@ "supports_pdf_input": true }, "1024-x-1536/gpt-image-1.5": { + "deprecation_date": "2026-12-01", "input_cost_per_image": 0.013, "litellm_provider": "openai", "mode": "image_generation", @@ -24545,6 +24688,7 @@ "supports_pdf_input": true }, "1536-x-1024/gpt-image-1.5": { + "deprecation_date": "2026-12-01", "input_cost_per_image": 0.013, "litellm_provider": "openai", "mode": "image_generation", @@ -27458,18 +27602,21 @@ "output_cost_per_second": 0.0 }, "hd/1024-x-1024/dall-e-3": { + "deprecation_date": "2026-05-12", "input_cost_per_pixel": 7.629e-08, "litellm_provider": "openai", "mode": "image_generation", "output_cost_per_pixel": 0.0 }, "hd/1024-x-1792/dall-e-3": { + "deprecation_date": "2026-05-12", "input_cost_per_pixel": 6.539e-08, "litellm_provider": "openai", "mode": "image_generation", "output_cost_per_pixel": 0.0 }, "hd/1792-x-1024/dall-e-3": { + "deprecation_date": "2026-05-12", "input_cost_per_pixel": 6.539e-08, "litellm_provider": "openai", "mode": "image_generation", @@ -27516,6 +27663,7 @@ "max_output_tokens": 8192 }, "high/1024-x-1024/gpt-image-1": { + "deprecation_date": "2026-10-23", "input_cost_per_image": 0.167, "input_cost_per_pixel": 1.59263611e-07, "litellm_provider": "openai", @@ -27526,6 +27674,7 @@ ] }, "high/1024-x-1536/gpt-image-1": { + "deprecation_date": "2026-10-23", "input_cost_per_image": 0.25, "input_cost_per_pixel": 1.58945719e-07, "litellm_provider": "openai", @@ -27536,6 +27685,7 @@ ] }, "high/1536-x-1024/gpt-image-1": { + "deprecation_date": "2026-10-23", "input_cost_per_image": 0.25, "input_cost_per_pixel": 1.58945719e-07, "litellm_provider": "openai", @@ -28323,6 +28473,7 @@ "supports_tool_choice": true }, "low/1024-x-1024/gpt-image-1": { + "deprecation_date": "2026-10-23", "input_cost_per_image": 0.011, "input_cost_per_pixel": 1.0490417e-08, "litellm_provider": "openai", @@ -28333,6 +28484,7 @@ ] }, "low/1024-x-1536/gpt-image-1": { + "deprecation_date": "2026-10-23", "input_cost_per_image": 0.016, "input_cost_per_pixel": 1.0172526e-08, "litellm_provider": "openai", @@ -28343,6 +28495,7 @@ ] }, "low/1536-x-1024/gpt-image-1": { + "deprecation_date": "2026-10-23", "input_cost_per_image": 0.016, "input_cost_per_pixel": 1.0172526e-08, "litellm_provider": "openai", @@ -28367,6 +28520,7 @@ "output_cost_per_image": 0.072 }, "medium/1024-x-1024/gpt-image-1": { + "deprecation_date": "2026-10-23", "input_cost_per_image": 0.042, "input_cost_per_pixel": 4.0054321e-08, "litellm_provider": "openai", @@ -28377,6 +28531,7 @@ ] }, "medium/1024-x-1536/gpt-image-1": { + "deprecation_date": "2026-10-23", "input_cost_per_image": 0.063, "input_cost_per_pixel": 4.0054321e-08, "litellm_provider": "openai", @@ -28387,6 +28542,7 @@ ] }, "medium/1536-x-1024/gpt-image-1": { + "deprecation_date": "2026-10-23", "input_cost_per_image": 0.063, "input_cost_per_pixel": 4.0054321e-08, "litellm_provider": "openai", @@ -28397,6 +28553,7 @@ ] }, "low/1024-x-1024/gpt-image-1-mini": { + "deprecation_date": "2026-12-01", "input_cost_per_image": 0.005, "litellm_provider": "openai", "mode": "image_generation", @@ -28405,6 +28562,7 @@ ] }, "low/1024-x-1536/gpt-image-1-mini": { + "deprecation_date": "2026-12-01", "input_cost_per_image": 0.006, "litellm_provider": "openai", "mode": "image_generation", @@ -28413,6 +28571,7 @@ ] }, "low/1536-x-1024/gpt-image-1-mini": { + "deprecation_date": "2026-12-01", "input_cost_per_image": 0.006, "litellm_provider": "openai", "mode": "image_generation", @@ -28421,6 +28580,7 @@ ] }, "medium/1024-x-1024/gpt-image-1-mini": { + "deprecation_date": "2026-12-01", "input_cost_per_image": 0.011, "litellm_provider": "openai", "mode": "image_generation", @@ -28429,6 +28589,7 @@ ] }, "medium/1024-x-1536/gpt-image-1-mini": { + "deprecation_date": "2026-12-01", "input_cost_per_image": 0.015, "litellm_provider": "openai", "mode": "image_generation", @@ -28437,6 +28598,7 @@ ] }, "medium/1536-x-1024/gpt-image-1-mini": { + "deprecation_date": "2026-12-01", "input_cost_per_image": 0.015, "litellm_provider": "openai", "mode": "image_generation", @@ -30330,6 +30492,7 @@ ] }, "multimodalembedding@001": { + "deprecation_date": "2027-04-01", "input_cost_per_character": 2e-07, "input_cost_per_image": 0.0001, "input_cost_per_token": 8e-07, @@ -36028,18 +36191,21 @@ "output_cost_per_image": 0.14 }, "standard/1024-x-1024/dall-e-3": { + "deprecation_date": "2026-05-12", "input_cost_per_pixel": 3.81469e-08, "litellm_provider": "openai", "mode": "image_generation", "output_cost_per_pixel": 0.0 }, "standard/1024-x-1792/dall-e-3": { + "deprecation_date": "2026-05-12", "input_cost_per_pixel": 4.359e-08, "litellm_provider": "openai", "mode": "image_generation", "output_cost_per_pixel": 0.0 }, "standard/1792-x-1024/dall-e-3": { + "deprecation_date": "2026-05-12", "input_cost_per_pixel": 4.359e-08, "litellm_provider": "openai", "mode": "image_generation", @@ -36103,6 +36269,7 @@ "source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models" }, "text-embedding-005": { + "deprecation_date": "2027-04-01", "input_cost_per_character": 2.5e-08, "input_cost_per_token": 1e-07, "litellm_provider": "vertex_ai-embedding-models", @@ -36176,6 +36343,7 @@ "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing" }, "text-moderation-007": { + "deprecation_date": "2025-10-27", "input_cost_per_token": 0.0, "litellm_provider": "openai", "max_input_tokens": 32768, @@ -36185,6 +36353,7 @@ "output_cost_per_token": 0.0 }, "text-moderation-latest": { + "deprecation_date": "2025-10-27", "input_cost_per_token": 0.0, "litellm_provider": "openai", "max_input_tokens": 32768, @@ -36194,6 +36363,7 @@ "output_cost_per_token": 0.0 }, "text-moderation-stable": { + "deprecation_date": "2025-10-27", "input_cost_per_token": 0.0, "litellm_provider": "openai", "max_input_tokens": 32768, @@ -36203,6 +36373,7 @@ "output_cost_per_token": 0.0 }, "text-multilingual-embedding-002": { + "deprecation_date": "2027-04-01", "input_cost_per_character": 2.5e-08, "input_cost_per_token": 1e-07, "litellm_provider": "vertex_ai-embedding-models", @@ -38690,6 +38861,7 @@ "supports_tool_choice": true }, "vertex_ai/claude-haiku-4-5": { + "deprecation_date": "2026-10-15", "cache_creation_input_token_cost": 1.25e-06, "cache_creation_input_token_cost_above_1hr": 2e-06, "cache_read_input_token_cost": 1e-07, @@ -38713,6 +38885,7 @@ "prompt_cache_min_tokens": 4096 }, "vertex_ai/claude-haiku-4-5@20251001": { + "deprecation_date": "2026-10-15", "cache_creation_input_token_cost": 1.25e-06, "cache_creation_input_token_cost_above_1hr": 2e-06, "cache_read_input_token_cost": 1e-07, @@ -38865,6 +39038,7 @@ "supports_vision": true }, "vertex_ai/claude-opus-4": { + "deprecation_date": "2026-05-14", "cache_creation_input_token_cost": 1.875e-05, "cache_creation_input_token_cost_above_1hr": 3e-05, "cache_read_input_token_cost": 1.5e-06, @@ -38892,6 +39066,7 @@ "prompt_cache_min_tokens": 1024 }, "vertex_ai/claude-opus-4-1": { + "deprecation_date": "2026-08-05", "cache_creation_input_token_cost": 1.875e-05, "cache_creation_input_token_cost_above_1hr": 3e-05, "cache_read_input_token_cost": 1.5e-06, @@ -38910,6 +39085,7 @@ "supports_vision": true }, "vertex_ai/claude-opus-4-1@20250805": { + "deprecation_date": "2026-08-05", "cache_creation_input_token_cost": 1.875e-05, "cache_creation_input_token_cost_above_1hr": 3e-05, "cache_read_input_token_cost": 1.5e-06, @@ -38928,6 +39104,7 @@ "supports_vision": true }, "vertex_ai/claude-opus-4-5": { + "deprecation_date": "2026-11-24", "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, @@ -38956,6 +39133,7 @@ "prompt_cache_min_tokens": 4096 }, "vertex_ai/claude-opus-4-5@20251101": { + "deprecation_date": "2026-11-24", "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, @@ -38985,6 +39163,7 @@ "prompt_cache_min_tokens": 4096 }, "vertex_ai/claude-opus-4-6": { + "deprecation_date": "2027-02-05", "supports_adaptive_thinking": true, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, @@ -39015,6 +39194,7 @@ "prompt_cache_min_tokens": 4096 }, "vertex_ai/claude-opus-4-6@default": { + "deprecation_date": "2027-02-05", "supports_adaptive_thinking": true, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, @@ -39045,6 +39225,7 @@ "prompt_cache_min_tokens": 4096 }, "vertex_ai/claude-opus-4-7": { + "deprecation_date": "2027-04-16", "supports_adaptive_thinking": true, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, @@ -39076,6 +39257,7 @@ "prompt_cache_min_tokens": 2048 }, "vertex_ai/claude-opus-4-7@default": { + "deprecation_date": "2027-04-16", "supports_adaptive_thinking": true, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, @@ -39107,6 +39289,7 @@ "prompt_cache_min_tokens": 2048 }, "vertex_ai/claude-fable-5": { + "deprecation_date": "2027-06-08", "supports_mid_conversation_system": true, "cache_creation_input_token_cost": 1.25e-05, "cache_creation_input_token_cost_above_1hr": 2e-05, @@ -39138,6 +39321,7 @@ "supports_max_reasoning_effort": true }, "vertex_ai/claude-fable-5@default": { + "deprecation_date": "2027-06-08", "supports_mid_conversation_system": true, "cache_creation_input_token_cost": 1.25e-05, "cache_creation_input_token_cost_above_1hr": 2e-05, @@ -39169,6 +39353,7 @@ "supports_max_reasoning_effort": true }, "vertex_ai/claude-opus-5": { + "deprecation_date": "2027-01-24", "supports_mid_conversation_system": true, "supports_adaptive_thinking": true, "cache_creation_input_token_cost": 6.25e-06, @@ -39201,6 +39386,7 @@ "prompt_cache_min_tokens": 512 }, "vertex_ai/claude-opus-5@default": { + "deprecation_date": "2027-01-24", "supports_mid_conversation_system": true, "supports_adaptive_thinking": true, "cache_creation_input_token_cost": 6.25e-06, @@ -39233,6 +39419,7 @@ "prompt_cache_min_tokens": 512 }, "vertex_ai/claude-opus-4-8": { + "deprecation_date": "2027-05-28", "supports_mid_conversation_system": true, "supports_adaptive_thinking": true, "cache_creation_input_token_cost": 6.25e-06, @@ -39265,6 +39452,7 @@ "prompt_cache_min_tokens": 1024 }, "vertex_ai/claude-opus-4-8@default": { + "deprecation_date": "2027-05-28", "supports_mid_conversation_system": true, "supports_adaptive_thinking": true, "cache_creation_input_token_cost": 6.25e-06, @@ -39297,6 +39485,7 @@ "prompt_cache_min_tokens": 1024 }, "vertex_ai/claude-sonnet-4-5": { + "deprecation_date": "2026-09-29", "cache_creation_input_token_cost": 3.75e-06, "cache_creation_input_token_cost_above_1hr": 6e-06, "cache_read_input_token_cost": 3e-07, @@ -39325,6 +39514,7 @@ "prompt_cache_min_tokens": 1024 }, "vertex_ai/claude-sonnet-5": { + "deprecation_date": "2026-12-24", "supports_mid_conversation_system": true, "cache_creation_input_token_cost": 2.5e-06, "cache_creation_input_token_cost_above_1hr": 4e-06, @@ -39387,6 +39577,7 @@ "prompt_cache_min_tokens": 1024 }, "vertex_ai/claude-sonnet-4-5@20250929": { + "deprecation_date": "2026-09-29", "cache_creation_input_token_cost": 3.75e-06, "cache_creation_input_token_cost_above_1hr": 6e-06, "cache_read_input_token_cost": 3e-07, @@ -39416,6 +39607,7 @@ "prompt_cache_min_tokens": 1024 }, "vertex_ai/claude-opus-4@20250514": { + "deprecation_date": "2026-05-14", "cache_creation_input_token_cost": 1.875e-05, "cache_creation_input_token_cost_above_1hr": 3e-05, "cache_read_input_token_cost": 1.5e-06, @@ -39443,6 +39635,7 @@ "prompt_cache_min_tokens": 1024 }, "vertex_ai/claude-sonnet-4": { + "deprecation_date": "2026-05-14", "cache_creation_input_token_cost": 3.75e-06, "cache_creation_input_token_cost_above_1hr": 6e-06, "cache_read_input_token_cost": 3e-07, @@ -39474,6 +39667,7 @@ "prompt_cache_min_tokens": 1024 }, "vertex_ai/claude-sonnet-4@20250514": { + "deprecation_date": "2026-05-14", "cache_creation_input_token_cost": 3.75e-06, "cache_creation_input_token_cost_above_1hr": 6e-06, "cache_read_input_token_cost": 3e-07, @@ -39638,6 +39832,7 @@ "supports_tool_choice": true }, "vertex_ai/gemini-2.5-flash-image": { + "deprecation_date": "2026-10-02", "cache_read_input_token_cost": 3e-08, "input_cost_per_audio_token": 1e-06, "input_cost_per_token": 3e-07, @@ -39683,6 +39878,7 @@ "supports_image_size": false }, "vertex_ai/gemini-3-pro-image": { + "deprecation_date": "2027-05-28", "input_cost_per_image": 0.0011, "input_cost_per_token": 2e-06, "input_cost_per_token_batches": 1e-06, @@ -39715,6 +39911,7 @@ "source": "https://docs.cloud.google.com/vertex-ai/generative-ai/docs/models/gemini/3-pro-image" }, "vertex_ai/gemini-3.1-flash-image": { + "deprecation_date": "2027-05-28", "input_cost_per_image": 0.00056, "input_cost_per_token": 5e-07, "litellm_provider": "vertex_ai-language-models", @@ -39791,6 +39988,7 @@ "web_search_billing_unit": "per_query" }, "vertex_ai/gemini-3.1-flash-lite": { + "deprecation_date": "2027-05-07", "cache_read_input_token_cost": 2.5e-08, "cache_read_input_token_cost_flex": 1.25e-08, "cache_read_input_token_cost_priority": 4.5e-08, @@ -39847,6 +40045,7 @@ "web_search_billing_unit": "per_query" }, "vertex_ai/gemini-3.5-flash-lite": { + "deprecation_date": "2027-07-21", "cache_read_input_token_cost": 3e-08, "cache_read_input_token_cost_flex": 2e-08, "cache_read_input_token_cost_priority": 5e-08, @@ -40564,6 +40763,7 @@ "supports_tool_choice": true }, "vertex_ai/veo-2.0-generate-001": { + "deprecation_date": "2026-06-30", "litellm_provider": "vertex_ai-video-models", "max_input_tokens": 1024, "max_tokens": 1024, @@ -40578,6 +40778,7 @@ ] }, "vertex_ai/veo-3.0-fast-generate-001": { + "deprecation_date": "2026-06-30", "litellm_provider": "vertex_ai-video-models", "max_input_tokens": 1024, "max_tokens": 1024, @@ -40592,6 +40793,7 @@ ] }, "vertex_ai/veo-3.0-generate-001": { + "deprecation_date": "2026-06-30", "litellm_provider": "vertex_ai-video-models", "max_input_tokens": 1024, "max_tokens": 1024, @@ -40634,6 +40836,7 @@ ] }, "vertex_ai/veo-3.1-generate-001": { + "deprecation_date": "2026-11-17", "litellm_provider": "vertex_ai-video-models", "max_input_tokens": 1024, "max_tokens": 1024, @@ -40648,6 +40851,7 @@ ] }, "vertex_ai/veo-3.1-fast-generate-001": { + "deprecation_date": "2026-11-17", "litellm_provider": "vertex_ai-video-models", "max_input_tokens": 1024, "max_tokens": 1024, @@ -47029,6 +47233,7 @@ } }, "vertex_ai/claude-sonnet-5@default": { + "deprecation_date": "2026-12-24", "supports_mid_conversation_system": true, "cache_creation_input_token_cost": 2.5e-06, "cache_creation_input_token_cost_above_1hr": 4e-06, diff --git a/litellm/proxy/_experimental/mcp_server/exceptions.py b/litellm/proxy/_experimental/mcp_server/exceptions.py index 8c704c0fe93..a1b3b167a4a 100644 --- a/litellm/proxy/_experimental/mcp_server/exceptions.py +++ b/litellm/proxy/_experimental/mcp_server/exceptions.py @@ -75,6 +75,24 @@ class MCPUpstreamAuthError(Exception): ) +class MCPOpenApiUpstreamError(Exception): + """An OpenAPI-backed MCP tool's upstream answered with a non-2xx that is not a 401. + + Carries the status only. The upstream's response body is deliberately dropped rather than served + as tool content: it crosses a trust boundary and may hold prose, urls, or an error document that + reads as data, which is how these failures came to be reported as successful tool output. This + matches ``outcome_wire_value``'s contract for listing faults, category and status and nothing + else. A 401 is raised as ``MCPUpstreamAuthError`` instead, so the caller learns to + re-authenticate; every other status stays here, mirroring the regular MCP path where a 403 + deliberately does not produce a challenge. + """ + + def __init__(self, status_code: int, server_name: str) -> None: + self.status_code = status_code + self.server_name = server_name + super().__init__(f"upstream returned HTTP {status_code}") + + class MCPToolResultError(Exception): """An MCP tool call completed with ``isError=True`` in its result. diff --git a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py index 65855df89f6..26a6f8d1251 100644 --- a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py +++ b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py @@ -2330,7 +2330,15 @@ class MCPServerManager: input_schema = build_input_schema(resolved_operation) # Create tool function with headers using imported function - tool_func = create_tool_function(path, method, resolved_operation, base_url, headers=headers) + tool_func = create_tool_function( + path, + method, + resolved_operation, + base_url, + headers=headers, + server_label=server.name or server.server_name or server.alias or server.server_id, + relays_upstream_auth=server.is_client_forwarded_token, + ) tool_func.__name__ = prefixed_tool_name tool_func.__doc__ = description @@ -4979,6 +4987,12 @@ class MCPServerManager: return result + except MCPUpstreamAuthError: + # The caller must re-authenticate upstream, so this keeps its type all the way to the + # renderers: the streamable path turns it into an isError result naming the status, and + # the REST path relays a real 401 with the upstream's WWW-Authenticate. Flattening it + # into the generic message below would lose both. + raise except Exception as e: error_msg = f"Error calling OpenAPI tool {tool_name}: {e}" verbose_logger.error(error_msg) diff --git a/litellm/proxy/_experimental/mcp_server/openapi_to_mcp_generator.py b/litellm/proxy/_experimental/mcp_server/openapi_to_mcp_generator.py index eb78aaeca0b..083a98cdd36 100644 --- a/litellm/proxy/_experimental/mcp_server/openapi_to_mcp_generator.py +++ b/litellm/proxy/_experimental/mcp_server/openapi_to_mcp_generator.py @@ -15,6 +15,12 @@ from urllib.parse import quote import httpx from typing_extensions import ReadOnly, Required +from litellm.llms.custom_httpx.http_handler import MaskedHTTPStatusError +from litellm.proxy._experimental.mcp_server.exceptions import ( + MCPOpenApiUpstreamError, + MCPUpstreamAuthError, +) + # Tool names emitted from OpenAPI specs must work across all major LLM providers. # OpenAI/Anthropic/Bedrock all enforce a character class roughly equivalent to # ^[a-zA-Z0-9_-]+$ on tool names. Many specs (notably GitHub's REST API) use @@ -392,12 +398,40 @@ def _merge_openapi_tool_request_headers( return effective_headers +def _raise_for_upstream_failure( + response: httpx.Response, + upstream: str, + relays_upstream_auth: bool, +) -> None: + """Turn a non-2xx upstream response into the right typed failure, or return for a 2xx. + + Both call sites feed this: ``get`` hands back the response for a 4xx, while post/put/patch/delete + raise ``MaskedHTTPStatusError`` from inside the HTTP handler, so without one classifier the + non-GET tools would keep serving an error body as tool output. + + Only the client-forwarded modes carry the caller's own upstream token, so only they can act on a + 401 by re-authenticating; ``_call_regular_mcp_tool`` gates its re-auth signal the same way. Every + other status carries the code alone, never the upstream's body, which crosses a trust boundary. + """ + if response.status_code < 400: + return + if response.status_code == 401 and relays_upstream_auth: + raise MCPUpstreamAuthError( + status_code=response.status_code, + www_authenticate=response.headers.get("www-authenticate"), + server_name=upstream, + ) + raise MCPOpenApiUpstreamError(response.status_code, upstream) + + def create_tool_function( path: str, method: str, operation: _OpenAPIOperation, base_url: str, headers: dict[str, str] | None = None, + server_label: str | None = None, + relays_upstream_auth: bool = False, ): """Create a tool function for an OpenAPI operation. @@ -477,20 +511,26 @@ def create_tool_function( json_body = {"data": body_value} client: Final = get_async_httpx_client(llm_provider=httpxSpecialProvider.MCP) + upstream: Final = server_label or f"{original_method.upper()} {path}" - if original_method == "get": - response = await client.get(url, params=params, headers=effective_headers) - elif original_method == "post": - response = await client.post(url, params=params, json=json_body, headers=effective_headers) - elif original_method == "put": - response = await client.put(url, params=params, json=json_body, headers=effective_headers) - elif original_method == "delete": - response = await client.delete(url, params=params, headers=effective_headers) - elif original_method == "patch": - response = await client.patch(url, params=params, json=json_body, headers=effective_headers) - else: - return f"Unsupported HTTP method: {original_method}" + try: + if original_method == "get": + response = await client.get(url, params=params, headers=effective_headers) + elif original_method == "post": + response = await client.post(url, params=params, json=json_body, headers=effective_headers) + elif original_method == "put": + response = await client.put(url, params=params, json=json_body, headers=effective_headers) + elif original_method == "delete": + response = await client.delete(url, params=params, headers=effective_headers) + elif original_method == "patch": + response = await client.patch(url, params=params, json=json_body, headers=effective_headers) + else: + return f"Unsupported HTTP method: {original_method}" + except MaskedHTTPStatusError as e: + _raise_for_upstream_failure(e.response, upstream, relays_upstream_auth) + raise + _raise_for_upstream_failure(response, upstream, relays_upstream_auth) return response.text return tool_function diff --git a/litellm/proxy/_experimental/mcp_server/server.py b/litellm/proxy/_experimental/mcp_server/server.py index 8d69d84e492..0dc85c0318c 100644 --- a/litellm/proxy/_experimental/mcp_server/server.py +++ b/litellm/proxy/_experimental/mcp_server/server.py @@ -407,8 +407,6 @@ if MCP_AVAILABLE: StreamableHTTPSessionManager = None from mcp.types import ( CallToolResult, - EmbeddedResource, - ImageContent, ListToolsResult, Prompt, TextContent, @@ -2861,12 +2859,11 @@ if MCP_AVAILABLE: _extra_token: Final = _request_extra_headers.set(forwarded_headers) _resolved_token: Final = _request_resolved_auth_headers.set(resolved_auth_headers) try: - local_content = await _handle_local_mcp_tool(name, arguments) + response = await _handle_local_mcp_tool(name, arguments) finally: _request_auth_header.reset(_auth_token) _request_extra_headers.reset(_extra_token) _request_resolved_auth_headers.reset(_resolved_token) - response = CallToolResult(content=local_content, isError=False) # Try managed MCP server tool (the name is bare; the prefix boundary was # already resolved above against this server's registered prefixes) @@ -2940,8 +2937,7 @@ if MCP_AVAILABLE: if "arguments" in hook_result: arguments = hook_result["arguments"] # pyright: ignore[reportAny] # hook returns untyped args - local_content = await _handle_local_mcp_tool(original_tool_name, arguments) - response = CallToolResult(content=local_content, isError=False) + response = await _handle_local_mcp_tool(original_tool_name, arguments) return await _run_post_mcp_call_guardrails( result=response, @@ -3319,11 +3315,18 @@ if MCP_AVAILABLE: verbose_logger.debug("CALL TOOL RESULT: %s", call_tool_result) return call_tool_result - async def _handle_local_mcp_tool( - name: str, arguments: dict[str, object] - ) -> list[TextContent | ImageContent | EmbeddedResource]: - """ - Handle tool execution for local registry tools + async def _handle_local_mcp_tool(name: str, arguments: dict[str, object]) -> CallToolResult: + """Execute a local-registry tool and report whether it succeeded. + + Returns the result rather than bare content because the verdict is part of it: the content + alone cannot say whether the handler failed, so callers used to stamp isError=False on every + outcome and an upstream rejection was served as tool output. + + A failure is reported as ``isError=True`` here rather than raised, because the REST surface + turns an unrecognized exception into a 500 and an upstream 403 or 429 is not a gateway crash. + ``MCPUpstreamAuthError`` is the exception: it propagates so the caller is told to + re-authenticate, which both renderers already know how to say. + Note: Local tools don't use prefixes, so we use the original name """ import inspect @@ -3333,15 +3336,16 @@ if MCP_AVAILABLE: raise HTTPException(status_code=404, detail=f"Tool '{name}' not found") try: - # Check if handler is async or sync if inspect.iscoroutinefunction(tool.handler): result = await tool.handler(**arguments) else: result = tool.handler(**arguments) - return [TextContent(text=str(result), type="text")] + except MCPUpstreamAuthError: + raise except Exception as e: verbose_logger.exception("Error executing local tool %s: %s", name, e) - return [TextContent(text=f"Error: {e}", type="text")] + return CallToolResult(content=[TextContent(text=f"Error: {e}", type="text")], isError=True) + return CallToolResult(content=[TextContent(text=str(result), type="text")], isError=False) def _get_mcp_servers_in_path(path: str) -> list[str] | None: """ diff --git a/litellm/proxy/management_endpoints/auto_router_endpoints.py b/litellm/proxy/management_endpoints/auto_router_endpoints.py index d8b7414f32c..0112ad1f6ed 100644 --- a/litellm/proxy/management_endpoints/auto_router_endpoints.py +++ b/litellm/proxy/management_endpoints/auto_router_endpoints.py @@ -6,10 +6,13 @@ POST /auto_router/test_routing - Route one prompt through an unsaved complexity- from collections.abc import Mapping, Sequence from datetime import datetime, timedelta, timezone +from itertools import groupby +from operator import attrgetter from types import MappingProxyType from typing import TYPE_CHECKING, Annotated, Final, Protocol +from uuid import uuid4 -from pydantic import BaseModel, TypeAdapter +from pydantic import BaseModel, ConfigDict, TypeAdapter, field_validator from litellm._logging import verbose_proxy_logger from litellm.exceptions import BudgetExceededError @@ -41,6 +44,8 @@ from litellm.types.management_endpoints.auto_router_endpoints import ( AutoRouterRoutingTestRequest, AutoRouterRoutingTestResponse, RequestComplexityRouterConfig, + ShadowEvalDirection, + ShadowEvalJobKeyResponse, ShadowEvalJobResponse, ShadowEvalResult, ShadowEvalSlice, @@ -89,17 +94,9 @@ class _ShadowEvalJobRow(Protocol): class _ShadowEvalJobTable(Protocol): - async def find_unique(self, *, where: Mapping[str, object]) -> _ShadowEvalJobRow | None: ... + async def find_many(self, *, where: Mapping[str, object]) -> Sequence[_ShadowEvalJobRow]: ... - async def find_first(self, *, where: Mapping[str, object]) -> _ShadowEvalJobRow | None: ... - - async def find_many( - self, *, where: Mapping[str, object], order: Mapping[str, str], take: int - ) -> Sequence[_ShadowEvalJobRow]: ... - - async def create(self, data: Mapping[str, object]) -> _ShadowEvalJobRow: ... - - async def update(self, *, where: Mapping[str, object], data: Mapping[str, object]) -> _ShadowEvalJobRow | None: ... + async def create_many(self, data: Sequence[Mapping[str, object]]) -> int: ... class _ShadowEvalAttemptRow(Protocol): @@ -606,18 +603,19 @@ _ATTEMPT_AGG_SELECT: Final = """ COUNT(*) FILTER (WHERE outcome = 'tie')::int AS ties, AVG(confidence)::float AS avg_confidence FROM "LiteLLM_ShadowEvalAttempt" -WHERE job_id = $1 AND outcome != 'error' +WHERE job_id = ANY($1::text[]) AND outcome != 'error' GROUP BY 1 """ _ATTEMPT_AGG_BY_TIER_SQL: Final = "SELECT COALESCE(tier, 'UNCLASSIFIED') AS grp," + _ATTEMPT_AGG_SELECT _ATTEMPT_AGG_BY_MODEL_SQL: Final = "SELECT COALESCE(real_model, 'unknown') AS grp," + _ATTEMPT_AGG_SELECT +_ATTEMPT_AGG_BY_LEG_SQL: Final = "SELECT job_id AS grp," + _ATTEMPT_AGG_SELECT _SWEEP_FINISHED_JOBS_SQL: Final = """ -UPDATE "LiteLLM_ShadowEvalJob" j SET stopped_at = NOW() -WHERE j.api_key_id = $1 AND j.stopped_at IS NULL +UPDATE "LiteLLM_ShadowEvalJob" j SET stopped_at = (NOW() AT TIME ZONE 'utc') +WHERE j.api_key_id = ANY($1::text[]) AND j.stopped_at IS NULL AND ( - j.ends_at <= NOW() + j.ends_at <= (NOW() AT TIME ZONE 'utc') OR (SELECT COUNT(*) FROM "LiteLLM_ShadowEvalAttempt" a WHERE a.job_id = j.id) >= j.max_turns ) """ @@ -628,7 +626,52 @@ SELECT COUNT(*) FILTER (WHERE outcome = 'error')::int AS error_count, COALESCE(SUM(judge_cost), 0)::float AS judge_spend FROM "LiteLLM_ShadowEvalAttempt" -WHERE job_id = $1 +WHERE job_id = ANY($1::text[]) +""" + +_ATTEMPT_COUNTS_SQL: Final = """ +SELECT a.job_id, COUNT(*)::int AS attempt_count +FROM "LiteLLM_ShadowEvalAttempt" a +JOIN "LiteLLM_ShadowEvalJob" j ON j.id = a.job_id +WHERE a.job_id = ANY($1::text[]) AND (j.stopped_at IS NULL OR a.created_at <= j.stopped_at) +GROUP BY a.job_id +""" + +_STOP_JOB_SQL: Final = """ +UPDATE "LiteLLM_ShadowEvalJob" +SET stopped_by = $2, stopped_at = COALESCE(stopped_at, $3::timestamp) +WHERE group_id = $1 AND stopped_by IS NULL + AND ends_at > (NOW() AT TIME ZONE 'utc') + AND EXISTS ( + SELECT 1 FROM "LiteLLM_ShadowEvalJob" k + WHERE k.group_id = $1 AND k.stopped_at IS NULL + AND (SELECT COUNT(*) FROM "LiteLLM_ShadowEvalAttempt" a WHERE a.job_id = k.id) < k.max_turns + ) +""" + + +class _AttemptCountRow(BaseModel): + job_id: str + attempt_count: int + + +_ATTEMPT_COUNT_ROWS: Final = TypeAdapter(list[_AttemptCountRow]) + + +_LIST_LEGS_SQL: Final = """ +SELECT * FROM "LiteLLM_ShadowEvalJob" +WHERE group_id IN ( + SELECT group_id FROM "LiteLLM_ShadowEvalJob" + GROUP BY group_id ORDER BY MAX(created_at) DESC LIMIT $1::int +) +""" + +_LIST_LEGS_BY_KEY_SQL: Final = """ +SELECT * FROM "LiteLLM_ShadowEvalJob" +WHERE group_id IN ( + SELECT group_id FROM "LiteLLM_ShadowEvalJob" WHERE api_key_id = $2 + GROUP BY group_id ORDER BY MAX(created_at) DESC LIMIT $1::int +) """ @@ -659,18 +702,98 @@ def _slices(rows: Sequence[_AttemptAggRow]) -> tuple[ShadowEvalSlice, ...]: ) +class _LegRow(BaseModel): + """One LiteLLM_ShadowEvalJob row, validated off the untyped prisma record. A row is + one key's leg of a job; the legs of a job share group_id and identical config, written + together by one create_many. The API's job id is the group id, so leg ids never leave + the server (attempts reference them internally).""" + + model_config = ConfigDict(from_attributes=True) + + id: str + group_id: str + api_key_id: str + router_name: str + direction: ShadowEvalDirection + baseline_model: str | None = None + judge_model: str + shadow_percentage: float + max_turns: int + created_at: datetime + ends_at: datetime + stopped_at: datetime | None = None + stopped_by: str | None = None + + @field_validator("created_at", "ends_at", "stopped_at") + @classmethod + def _as_aware_utc(cls, value: datetime | None) -> datetime | None: + """The columns store naive UTC wall time (prisma's convention); prisma reads hand + back aware datetimes while raw SQL reads hand back naive ones, so this boundary + makes every read aware UTC before anything compares or serializes them.""" + if value is None or value.tzinfo is not None: + return value + return value.replace(tzinfo=timezone.utc) + + +_LEG_ROWS: Final = TypeAdapter(list[_LegRow]) + + +async def _leg_attempt_counts(prisma_client: "PrismaClient", legs: Sequence[_LegRow]) -> Mapping[str, int]: + """Each leg's attempt count by leg id, judged and errored alike, in one grouped read. + It is the same count the sampler budgets against max_turns, so the derived status + flips to completed exactly when sampling actually ends. A stamped leg's count freezes + at its stopped_at: in-flight attempts that land after the stamp are excluded, so they + can never reclassify a leg that was stopped under budget as budget-spent.""" + if not legs: + return MappingProxyType({}) + rows: Final = _ATTEMPT_COUNT_ROWS.validate_python( + await _query_raw(prisma_client, _ATTEMPT_COUNTS_SQL, [leg.id for leg in legs]) # mutable-ok: query param + or () + ) + return MappingProxyType({row.job_id: row.attempt_count for row in rows}) + + +def _group_response(group_id: str, legs: Sequence[_LegRow], attempt_counts: Mapping[str, int]) -> ShadowEvalJobResponse: + """The one constructor of a job response: the caller names the group and passes that + group's legs. Config is read off the first leg because every leg carries the same copy, + written by one create_many. No caller may serialize a raw row (that would leak a leg id + as the job id).""" + first: Final = legs[0] + return ShadowEvalJobResponse( + job_id=group_id, + keys=tuple( + ShadowEvalJobKeyResponse( + api_key_id=leg.api_key_id, + max_turns=leg.max_turns, + stopped_at=leg.stopped_at, + attempt_count=attempt_counts.get(leg.id, 0), + ) + for leg in sorted(legs, key=lambda leg: leg.api_key_id) + ), + router_name=first.router_name, + direction=first.direction, + baseline_model=first.baseline_model, + judge_model=first.judge_model, + shadow_percentage=first.shadow_percentage, + created_at=first.created_at, + ends_at=first.ends_at, + stopped_by=next((leg.stopped_by for leg in legs if leg.stopped_by is not None), None), + ) + + _NO_KEY_LABELS: Final[tuple[str | None, str | None]] = (None, None) async def _with_key_labels( prisma_client: "PrismaClient", responses: Sequence[ShadowEvalJobResponse] ) -> tuple[ShadowEvalJobResponse, ...]: - """Resolve each job's key hash to the key's alias and masked name in one batched read, + """Resolve every scoped key's hash to its alias and masked name in one batched read, so the UI can say whose traffic a job shadows. Deleted keys resolve to None.""" if not responses: return () + tokens: Final = sorted(frozenset(key.api_key_id for response in responses for key in response.keys)) key_rows: Final = await _verification_tokens(prisma_client).find_many( - where={"token": {"in": sorted({response.api_key_id for response in responses})}} # mutable-ok: Prisma filter + where={"token": {"in": tokens}} # mutable-ok: Prisma filter ) labels: Final[Mapping[str, tuple[str | None, str | None]]] = { row.token: (row.key_alias, row.key_name) for row in key_rows or () @@ -678,32 +801,50 @@ async def _with_key_labels( return tuple( response.model_copy( update={ # mutable-ok: pydantic update payload - "key_alias": labels.get(response.api_key_id, _NO_KEY_LABELS)[0], - "key_name": labels.get(response.api_key_id, _NO_KEY_LABELS)[1], + "keys": tuple( + key.model_copy( + update={ # mutable-ok: pydantic update payload + "key_alias": labels.get(key.api_key_id, _NO_KEY_LABELS)[0], + "key_name": labels.get(key.api_key_id, _NO_KEY_LABELS)[1], + } + ) + for key in response.keys + ) } ) for response in responses ) -async def _shadow_eval_results(prisma_client: "PrismaClient", job_id: str) -> ShadowEvalResult | None: - """Both stratifications of one job's verdicts. Tier answers "where does the router do - well"; the model stratification groups by whichever model served the real arm, so it - answers "which of the models this key uses today would the router beat" forward, and - "for the turns the router sent to X, did X beat the baseline" in reverse. Reads are - bounded by the job's own attempts (<= max_turns) via the job_id index.""" +async def _shadow_eval_results(prisma_client: "PrismaClient", legs: Sequence[_LegRow]) -> ShadowEvalResult | None: + """All three stratifications of one job's verdicts. Tier answers "where does the router + do well"; the model stratification groups by whichever model served the real arm, so it + answers "which of the models these keys use today would the router beat" forward, and + "for the turns the router sent to X, did X beat the baseline" in reverse; key answers + "which key's traffic does the router suit". Reads are bounded by the job's own attempts + (<= the sum of its keys' max_turns) via the job_id index.""" + leg_ids: Final = [leg.id for leg in legs] # mutable-ok: query param by_tier: Final = _ATTEMPT_AGG_ROWS.validate_python( - await _query_raw(prisma_client, _ATTEMPT_AGG_BY_TIER_SQL, job_id) or () + await _query_raw(prisma_client, _ATTEMPT_AGG_BY_TIER_SQL, leg_ids) or () ) if not by_tier: return None by_model: Final = _ATTEMPT_AGG_ROWS.validate_python( - await _query_raw(prisma_client, _ATTEMPT_AGG_BY_MODEL_SQL, job_id) or () + await _query_raw(prisma_client, _ATTEMPT_AGG_BY_MODEL_SQL, leg_ids) or () + ) + key_by_leg: Final = MappingProxyType({leg.id: leg.api_key_id for leg in legs}) + by_leg: Final = _ATTEMPT_AGG_ROWS.validate_python( + await _query_raw(prisma_client, _ATTEMPT_AGG_BY_LEG_SQL, leg_ids) or () + ) + by_key: Final = tuple( + row.model_copy(update={"grp": key_by_leg[row.grp]}) # mutable-ok: pydantic update payload + for row in by_leg ) total_turns: Final = sum(r.turn_count for r in by_tier) return ShadowEvalResult( by_tier=_slices(by_tier), by_current_model=_slices(by_model), + by_key=_slices(by_key), overall_shadow_win_rate_pct=_pct_of(sum(r.shadow_wins for r in by_tier), total_turns), overall_tie_rate_pct=_pct_of(sum(r.ties for r in by_tier), total_turns), ) @@ -721,20 +862,21 @@ async def start_shadow_eval( user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], ) -> ShadowEvalJobResponse: """ - Start a shadow eval: duplicate a sampled slice of a key's live traffic against a second - arm, judge the two responses blind, and stratify win rates by tier and by the model that - served the real arm. + Start a shadow eval: duplicate a sampled slice of one or more keys' live traffic against + a second arm, judge the two responses blind, and stratify win rates by tier, by the model + that served the real arm, and by key. - A forward job answers whether the key should adopt router_name: it samples the requests + A forward job answers whether the keys should adopt router_name: it samples the requests the router did not serve and duplicates them through it. A reverse job answers whether a key already on the router still gains from it: it samples the requests the router did serve and duplicates them against baseline_model. A key can hold one active job per direction, so both questions can run at once. - Shadow responses are never served to users. The job samples until it has judged - max_turns turns, reaches the end of its window, or is stopped; sampling changes - propagate to pods within about 10 seconds. Shadow and judge calls bill to the - shadowed key but are excluded from request counts and auto-router adoption metrics. + Shadow responses are never served to users. Each key samples until it has judged + max_turns turns of its own traffic, the job's window ends, or the job is stopped, so one + key running out of budget does not end sampling for the others; sampling changes + propagate to pods within about 10 seconds. Shadow and judge calls bill to the shadowed + key but are excluded from request counts and auto-router adoption metrics. """ from litellm.proxy.proxy_server import llm_router, prisma_client @@ -746,48 +888,58 @@ async def start_shadow_eval( _validate_plain_model(llm_router, data.judge_model, "judge_model") if data.baseline_model is not None: _validate_plain_model(llm_router, data.baseline_model, "baseline_model") - key_row: Final = await _verification_tokens(prisma_client).find_unique( - where={"token": data.api_key_id} # mutable-ok: Prisma filter + token_rows: Final = await _verification_tokens(prisma_client).find_many( + where={"token": {"in": list(data.api_key_ids)}} # mutable-ok: Prisma filter ) - if key_row is None: + unknown: Final = tuple(sorted(frozenset(data.api_key_ids) - frozenset(row.token for row in token_rows or ()))) + if unknown: raise HTTPException( status_code=400, detail=( - f"api_key_id '{data.api_key_id}' is not a key on this proxy; pass the key's token hash, " + f"api_key_ids not on this proxy: {', '.join(unknown)}; pass each key's token hash, " "the value the key list and key info endpoints report" ), ) - # A job that expired or exhausted its turn budget stopped sampling on its own, but - # still holds its slot in the per-key, per-direction partial unique index until - # stamped; free it so a new eval can start. Sweeping both directions is deliberate. - await prisma_client.db.execute_raw(_SWEEP_FINISHED_JOBS_SQL, data.api_key_id) - active: Final = await _shadow_eval_jobs(prisma_client).find_first( + # A job whose window passed or whose turn budget ran out stopped sampling on its own, + # but its legs still hold their slots in the per-key, per-direction partial unique index + # until stamped; free them so a new eval can start. Sweeping both directions is deliberate. + requested: Final = list(data.api_key_ids) # mutable-ok: query param + await prisma_client.db.execute_raw(_SWEEP_FINISHED_JOBS_SQL, requested) + claimed: Final = await _shadow_eval_jobs(prisma_client).find_many( where={ # mutable-ok: Prisma filter - "api_key_id": data.api_key_id, + "api_key_id": {"in": requested}, # mutable-ok: Prisma filter "direction": data.direction, "stopped_at": None, }, ) - if active is not None: + if claimed: raise HTTPException( status_code=409, - detail=f"Key already has an active {data.direction} shadow eval job ({active.id}). Stop it first.", + detail=( + f"Already in an active {data.direction} shadow eval job: " + + ", ".join(sorted(f"{row.api_key_id} (job {row.group_id})" for row in claimed)) + + ". Stop it first." + ), ) now: Final = datetime.now(timezone.utc) + group_id: Final = str(uuid4()) + ends_at: Final = now + timedelta(days=data.duration_days) + shared_config: Final = { # mutable-ok: Prisma payload + "group_id": group_id, + "router_name": data.router_name, + "direction": data.direction, + "baseline_model": data.baseline_model, + "judge_model": data.judge_model, + "shadow_percentage": data.shadow_percentage, + "max_turns": data.max_turns, + "created_by": user_api_key_dict.user_id, + "created_at": now, + "ends_at": ends_at, + } try: - job: Final = await _shadow_eval_jobs(prisma_client).create( - data={ # mutable-ok: Prisma payload - "api_key_id": data.api_key_id, - "router_name": data.router_name, - "direction": data.direction, - "baseline_model": data.baseline_model, - "judge_model": data.judge_model, - "shadow_percentage": data.shadow_percentage, - "max_turns": data.max_turns, - "created_by": user_api_key_dict.user_id, - "ends_at": now + timedelta(days=data.duration_days), - } + await _shadow_eval_jobs(prisma_client).create_many( + data=[{**shared_config, "api_key_id": key} for key in data.api_key_ids] # mutable-ok: Prisma payload ) except Exception as e: if not _is_unique_violation(e): @@ -795,11 +947,28 @@ async def start_shadow_eval( raise HTTPException( status_code=409, detail=( - f"Key already has an active {data.direction} shadow eval job (started concurrently). Stop it first." + f"A requested key was claimed by another {data.direction} shadow eval job concurrently. Stop it first." ), ) from e - return ShadowEvalJobResponse.model_validate(job, from_attributes=True).model_copy( - update={"key_alias": key_row.key_alias, "key_name": key_row.key_name} # mutable-ok: pydantic update payload + labels: Final = MappingProxyType({row.token: row for row in token_rows}) + return ShadowEvalJobResponse( + job_id=group_id, + keys=tuple( + ShadowEvalJobKeyResponse( + api_key_id=api_key_id, + max_turns=data.max_turns, + key_alias=labels[api_key_id].key_alias, + key_name=labels[api_key_id].key_name, + ) + for api_key_id in sorted(data.api_key_ids) + ), + router_name=data.router_name, + direction=data.direction, + baseline_model=data.baseline_model, + judge_model=data.judge_model, + shadow_percentage=data.shadow_percentage, + created_at=now, + ends_at=ends_at, ) @@ -811,23 +980,38 @@ async def start_shadow_eval( ) async def list_shadow_eval_jobs( user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], - api_key_id: Annotated[str | None, Query(description="Filter to jobs shadowing this key")] = None, + api_key_id: Annotated[ + str | None, Query(description="Filter to jobs that shadow this key, alone or alongside others") + ] = None, limit: Annotated[int, Query(ge=1, le=200, description="Newest jobs to return")] = 50, ) -> tuple[ShadowEvalJobResponse, ...]: - """List shadow eval jobs, newest first. Counts and results ride the detail endpoint only.""" + """List shadow eval jobs, newest first, each key with its attempt count so status is + accurate. Judged counts, spend, and results ride the detail endpoint only.""" from litellm.proxy.proxy_server import prisma_client _require_admin_viewer(user_api_key_dict, "view shadow evals") if prisma_client is None: raise HTTPException(status_code=500, detail=CommonProxyErrors.db_not_connected_error.value) - records: Final = await _shadow_eval_jobs(prisma_client).find_many( - where={"api_key_id": api_key_id} if api_key_id else {}, # mutable-ok: Prisma filter - order={"created_at": "desc"}, # mutable-ok: Prisma order - take=limit, + legs: Final = _LEG_ROWS.validate_python( + ( + await _query_raw(prisma_client, _LIST_LEGS_BY_KEY_SQL, limit, api_key_id) + if api_key_id + else await _query_raw(prisma_client, _LIST_LEGS_SQL, limit) + ) + or () ) + by_group: Final[Mapping[str, tuple[_LegRow, ...]]] = MappingProxyType( + { + group_id: tuple(group) + for group_id, group in groupby(sorted(legs, key=attrgetter("group_id")), key=attrgetter("group_id")) + } + ) + newest_first: Final = sorted( + by_group, key=lambda group_id: max(leg.created_at for leg in by_group[group_id]), reverse=True + ) + counts: Final = await _leg_attempt_counts(prisma_client, legs) return await _with_key_labels( - prisma_client, - tuple(ShadowEvalJobResponse.model_validate(record, from_attributes=True) for record in records or ()), + prisma_client, tuple(_group_response(group_id, by_group[group_id], counts) for group_id in newest_first) ) @@ -847,20 +1031,24 @@ async def get_shadow_eval_job( _require_admin_viewer(user_api_key_dict, "view shadow evals") if prisma_client is None: raise HTTPException(status_code=500, detail=CommonProxyErrors.db_not_connected_error.value) - record: Final = await _shadow_eval_jobs(prisma_client).find_unique( - where={"id": job_id} # mutable-ok: Prisma filter + legs: Final = _LEG_ROWS.validate_python( + await _shadow_eval_jobs(prisma_client).find_many( + where={"group_id": job_id} # mutable-ok: Prisma filter + ) + or () ) - if record is None: + if not legs: raise HTTPException(status_code=404, detail=f"No shadow eval job {job_id}") + leg_ids: Final = [leg.id for leg in legs] # mutable-ok: query param totals: Final = _ATTEMPT_TOTALS_ROWS.validate_python( - await _query_raw(prisma_client, _ATTEMPT_TOTALS_SQL, job_id) or () + await _query_raw(prisma_client, _ATTEMPT_TOTALS_SQL, leg_ids) or () ) latest_error: Final = await _shadow_eval_attempts(prisma_client).find_first( - where={"job_id": job_id, "outcome": "error"}, # mutable-ok: Prisma filter + where={"job_id": {"in": leg_ids}, "outcome": "error"}, # mutable-ok: Prisma filter order={"created_at": "desc"}, # mutable-ok: Prisma order ) labeled: Final = await _with_key_labels( - prisma_client, (ShadowEvalJobResponse.model_validate(record, from_attributes=True),) + prisma_client, (_group_response(job_id, legs, await _leg_attempt_counts(prisma_client, legs)),) ) return labeled[0].model_copy( update={ # mutable-ok: pydantic update payload @@ -868,7 +1056,7 @@ async def get_shadow_eval_job( "error_count": totals[0].error_count if totals else 0, "judge_spend": round(totals[0].judge_spend, 6) if totals else 0.0, "last_error": latest_error.error if latest_error else None, - "results": await _shadow_eval_results(prisma_client, job_id), + "results": await _shadow_eval_results(prisma_client, legs), } ) @@ -883,25 +1071,33 @@ async def stop_shadow_eval_job( job_id: str, user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], ) -> ShadowEvalJobResponse: - """Stop an active shadow eval job. Attempts are kept; sampling halts within ~10s.""" + """Stop an active shadow eval job, every key it scopes at once. Attempts are kept; + sampling halts within ~10s. Keys that already stopped on their own budget keep the + stopped_at they earned. The statement is the whole state machine: it claims the job + only while a leg still samples inside the window with no stop recorded, so a racing + operator, a same-instant budget spend, and a repeat stop all read the same 400 with + the status the job actually holds.""" from litellm.proxy.proxy_server import prisma_client _require_admin_writer(user_api_key_dict, "stop a shadow eval") if prisma_client is None: raise HTTPException(status_code=500, detail=CommonProxyErrors.db_not_connected_error.value) - record: Final = await _shadow_eval_jobs(prisma_client).find_unique( - where={"id": job_id} # mutable-ok: Prisma filter + stamp: Final = datetime.now(timezone.utc) + operator: Final = user_api_key_dict.user_id or "operator" + claimed: Final = await prisma_client.db.execute_raw( + _STOP_JOB_SQL, job_id, operator, stamp.replace(tzinfo=None).isoformat() ) - if record is None: + legs: Final = _LEG_ROWS.validate_python( + await _shadow_eval_jobs(prisma_client).find_many( + where={"group_id": job_id} # mutable-ok: Prisma filter + ) + or () + ) + if not legs: raise HTTPException(status_code=404, detail=f"No shadow eval job {job_id}") - current: Final = ShadowEvalJobResponse.model_validate(record, from_attributes=True) - if current.status != "running": + counts: Final = await _leg_attempt_counts(prisma_client, legs) + current: Final = _group_response(job_id, legs, counts) + if claimed == 0: raise HTTPException(status_code=400, detail=f"Job {job_id} is already {current.status}") - updated: Final = await _shadow_eval_jobs(prisma_client).update( - where={"id": job_id}, # mutable-ok: Prisma filter - data={"stopped_at": datetime.now(timezone.utc)}, # mutable-ok: Prisma payload - ) - labeled: Final = await _with_key_labels( - prisma_client, (ShadowEvalJobResponse.model_validate(updated, from_attributes=True),) - ) + labeled: Final = await _with_key_labels(prisma_client, (current,)) return labeled[0] diff --git a/litellm/proxy/management_endpoints/key_management_endpoints.py b/litellm/proxy/management_endpoints/key_management_endpoints.py index 67a836b8c92..ab343cbde2d 100644 --- a/litellm/proxy/management_endpoints/key_management_endpoints.py +++ b/litellm/proxy/management_endpoints/key_management_endpoints.py @@ -1416,6 +1416,11 @@ async def _check_team_key_limits( ) +_INHERITED_MODEL_SENTINELS: Final = frozenset( + {SpecialModelNames.all_team_models.value, SpecialModelNames.all_proxy_models.value} +) + + async def _check_project_key_limits( project_id: str, data: GenerateKeyRequest | UpdateKeyRequest, @@ -1425,7 +1430,8 @@ async def _check_project_key_limits( """ Validate that key's models and budget respect its project's limits. - - Key models must be a subset of project models + - Key models must be a subset of project models, except the all-team-models / all-proxy-models + sentinels, which inherit a parent scope and are narrowed by the project at request time - Key max_budget must be <= project max_budget """ project_obj: Final = await get_project_object( @@ -1443,7 +1449,7 @@ async def _check_project_key_limits( # Validate key models are a subset of project models if data.models and len(project_obj.models) > 0: for m in data.models: - if m not in project_obj.models: + if m not in project_obj.models and m not in _INHERITED_MODEL_SENTINELS: raise HTTPException( status_code=400, detail={ diff --git a/litellm/proxy/schema.prisma b/litellm/proxy/schema.prisma index 52fb447157b..f79e2bb0c18 100644 --- a/litellm/proxy/schema.prisma +++ b/litellm/proxy/schema.prisma @@ -1467,28 +1467,38 @@ model LiteLLM_AutoRouterSession { @@index([last_turn_at], map: "idx_autorouter_session_last_turn") } -// Shadow eval: evaluation of an auto-router against a key's live traffic, in either -// direction. forward duplicates the requests the key did not route through the router -// through it, answering whether the key should adopt it; reverse duplicates the requests -// the router did serve against a fixed baseline model, answering whether a key already on -// it still benefits. Either way a sampled slice runs in a detached task and an LLM judge -// compares real vs shadow responses blind. The job row is immutable config plus -// stopped_at; every count, status, and spend figure is derived from the append-only -// attempt rows, so nothing can disagree across pods or stop races. +// Shadow eval: evaluation of an auto-router against one or more keys' live traffic, in +// either direction. forward duplicates the requests the keys did not route through the +// router through it, answering whether they should adopt it; reverse duplicates the +// requests the router did serve against a fixed baseline model, answering whether a key +// already on it still benefits. Either way a sampled slice runs in a detached task and an +// LLM judge compares real vs shadow responses blind. Each row is ONE key's leg of a job: +// immutable config plus that key's own turn budget and stop state, so one key exhausting +// its budget never ends a sibling's sampling. A job is the set of legs sharing group_id +// (the id the API reports), written together by one atomic create_many with identical +// config; single-key jobs predating group_id were backfilled group_id = id. "One active +// job per (key, direction)" is a partial unique index on (api_key_id, direction) WHERE +// stopped_at IS NULL, expressed only in the migration because schema.prisma cannot state +// partial indexes; it is what makes a concurrent start on another pod race-safe rather +// than read-then-create. Every count, status, and spend figure is derived from the +// append-only attempt rows, so nothing can disagree across pods or stop races. model LiteLLM_ShadowEvalJob { id String @id @default(cuid()) - api_key_id String // hashed virtual key whose traffic is shadowed + group_id String // legs of one job share this; the API's job id + api_key_id String // hashed virtual key whose traffic this leg shadows router_name String // the auto-router under evaluation, in either direction direction String @default("forward") // forward | reverse baseline_model String? // reverse only: the fixed model the router is judged against judge_model String shadow_percentage Float - max_turns Int // sample budget: judge at most this many turns + max_turns Int // this key's sample budget: judge at most this many turns created_at DateTime @default(now()) created_by String? ends_at DateTime stopped_at DateTime? + stopped_by String? // operator who stopped it early; null when it ended on its own + @@index([group_id]) @@index([api_key_id]) @@index([created_at]) } diff --git a/litellm/proxy/spend_tracking/ptu_flat_cost_rollup.py b/litellm/proxy/spend_tracking/ptu_flat_cost_rollup.py index efdbda47fdc..25de9f6d065 100644 --- a/litellm/proxy/spend_tracking/ptu_flat_cost_rollup.py +++ b/litellm/proxy/spend_tracking/ptu_flat_cost_rollup.py @@ -17,6 +17,7 @@ import json from collections.abc import Awaitable, Callable, Mapping from dataclasses import dataclass from datetime import date, datetime, time, timedelta, timezone +from types import MappingProxyType from typing import TYPE_CHECKING, Final from litellm._logging import verbose_proxy_logger @@ -110,17 +111,56 @@ def _public_model_name(row: object, model_info: Mapping[str, object]) -> str: def _decode_model_info(raw: object) -> "Mapping[str, object] | None": - """A deployment's model_info as a dict, decoding a JSON string, else None.""" + """A deployment's model_info as a mapping, decoding a JSON string, else None. + + Valid JSON that is not an object decodes to a list or a scalar, which every caller + would then read fields off, so it is rejected here rather than raised past them. + """ if isinstance(raw, str): try: - return json.loads(raw) + decoded: Final = json.loads(raw) except (TypeError, ValueError): return None - if isinstance(raw, dict): + return decoded if isinstance(decoded, dict) else None + if isinstance(raw, Mapping): return raw return None +@dataclass(frozen=True, slots=True) +class _PTUDeployment: + """A deployment in the shape ``_parse_ptu_model`` reads, whatever declared it. + + A ``LiteLLM_ProxyModelTable`` row already has it. A router entry does not: its id + lives in ``model_info.id`` rather than on the entry itself. + """ + + model_id: str + model_name: str + model_info: Mapping[str, object] + + +def _router_deployment(deployment: Mapping[str, object]) -> _PTUDeployment | None: + """A router ``model_list`` entry in the shape the parser reads, else None. + + An id is required rather than defaulted because it keys the sentinel row: every + deployment without one would collapse onto a single row per team and only the last + would be billed. The mapping is copied because the router rewrites entries in place + while the rollup runs. + """ + model_info: Final = _decode_model_info(deployment.get("model_info")) + if model_info is None: + return None + model_id: Final = model_info.get("id") + if not isinstance(model_id, str) or not model_id: + return None + return _PTUDeployment( + model_id=model_id, + model_name=str(deployment.get("model_name") or ""), + model_info=MappingProxyType(dict(model_info)), + ) + + def _parse_ptu_model(row: object) -> PTUModel | None: """Return a PTUModel when the deployment carries valid manual PTU config, else None. diff --git a/litellm/router_strategy/complexity_router/complexity_router.py b/litellm/router_strategy/complexity_router/complexity_router.py index d16063b9bd4..0573f8acf18 100644 --- a/litellm/router_strategy/complexity_router/complexity_router.py +++ b/litellm/router_strategy/complexity_router/complexity_router.py @@ -1020,13 +1020,14 @@ class ComplexityRouter(CustomLogger): weights: Final = self.config.dimension_weights weighted_score: Final = sum(d.score * weights.get(d.name, 0) for d in dimensions) - # Check for reasoning override (2+ reasoning markers) + boundaries: Final = self._effective_tier_boundaries() + scored_above_simple: Final = weighted_score >= boundaries["simple_medium"] + # Reuse match count from _score_keyword_match to avoid scanning twice - if reasoning_match_count >= 2: + if reasoning_match_count >= 2 and scored_above_simple: return ComplexityTier.REASONING, weighted_score, tuple(signals), "reasoning_override" # Map score to tier - boundaries: Final = self._effective_tier_boundaries() if weighted_score < boundaries["simple_medium"]: tier = ComplexityTier.SIMPLE elif weighted_score < boundaries["medium_complex"]: diff --git a/litellm/types/management_endpoints/auto_router_endpoints.py b/litellm/types/management_endpoints/auto_router_endpoints.py index 9461297feca..63c93e0f268 100644 --- a/litellm/types/management_endpoints/auto_router_endpoints.py +++ b/litellm/types/management_endpoints/auto_router_endpoints.py @@ -6,7 +6,7 @@ from collections.abc import Mapping from datetime import datetime, timezone from typing import Final, Literal, TypeAlias -from pydantic import AliasChoices, BaseModel, ConfigDict, Field, computed_field, field_validator, model_validator +from pydantic import BaseModel, Field, computed_field, field_validator, model_validator from litellm.router_strategy.complexity_router.config import ComplexityRouterConfig from litellm.types.utils import StandardLoggingRoutingDecision @@ -155,13 +155,17 @@ DEFAULT_SHADOW_EVAL_JUDGE_MODEL: Final[str] = "anthropic/claude-sonnet-5" class StartShadowEvalRequest(BaseModel): - """Start duplicating a key's traffic for blind comparison against an auto-router.""" + """Start duplicating one or more keys' traffic for blind comparison against an auto-router.""" - api_key_id: str = Field( + api_key_ids: tuple[str, ...] = Field( + min_length=1, + max_length=100, description=( - "The hashed virtual key whose traffic will be shadowed. Shadow evaluation runs ONLY on this " - "key's traffic; requests made with any other key are not sampled." - ) + "The hashed virtual keys whose traffic will be shadowed. Shadow evaluation runs ONLY on these " + "keys' traffic; requests made with any other key are not sampled. Each key carries its own " + "max_turns budget, so one key exhausting its budget leaves the others sampling. At most 100 " + "keys per job, which also bounds every read the job's endpoints make." + ), ) router_name: str = Field(description="The auto-router under evaluation, in either direction") direction: ShadowEvalDirection = Field( @@ -204,8 +208,9 @@ class StartShadowEvalRequest(BaseModel): ge=1, le=2000, description=( - "Sample budget: the job judges at most this many turns, then completes. This is also the spend " - "bound; expected judge cost is roughly max_turns times one judge call" + "Per-key sample budget: the job judges at most this many turns of EACH scoped key's traffic, " + "so a job over N keys judges at most N times max_turns turns. This is also the spend bound; " + "expected judge cost is roughly that turn ceiling times one judge call" ), ) @@ -214,6 +219,12 @@ class StartShadowEvalRequest(BaseModel): def _round_percentage(cls, value: float) -> float: return round(value, 2) + @field_validator("api_key_ids") + @classmethod + def _dedupe_keys(cls, value: tuple[str, ...]) -> tuple[str, ...]: + """A key named twice would collide with itself on the one-active-per-(key, direction) index.""" + return tuple(dict.fromkeys(value)) + @model_validator(mode="after") def _baseline_model_matches_direction(self) -> "StartShadowEvalRequest": if self.direction == "reverse" and self.baseline_model is None: @@ -251,24 +262,46 @@ class ShadowEvalResult(BaseModel): by_tier: tuple[ShadowEvalSlice, ...] by_current_model: tuple[ShadowEvalSlice, ...] = Field( description=( - "Sliced by the model that served the real arm: the key's incumbent models in forward mode, " + "Sliced by the model that served the real arm: the keys' incumbent models in forward mode, " "and in reverse the models the router itself picked" ) ) + by_key: tuple[ShadowEvalSlice, ...] = Field( + description=( + "One slice per scoped key that has judged verdicts, grouped on the raw key hash. Keys the job " + "scopes but has not judged a turn for yet are absent rather than reported as zero" + ), + ) overall_shadow_win_rate_pct: float overall_tie_rate_pct: float -class ShadowEvalJobResponse(BaseModel): - """A shadow-eval job. Validates directly from the prisma record (job_id reads the - row's id); status is derived from stopped_at and ends_at, never stored, so no writer - anywhere can produce an inconsistent one. Aggregate fields are populated by the - detail endpoint only and stay None on list responses.""" +class ShadowEvalJobKeyResponse(BaseModel): + """One key a job shadows, with its own budget and stop state.""" - model_config = ConfigDict(from_attributes=True, populate_by_name=True) + api_key_id: str = Field(description="The hashed virtual key whose traffic this entry scopes") + max_turns: int = Field(description="This key's own sample budget, independent of its siblings'") + stopped_at: datetime | None = Field( + default=None, + description=( + "When this key's slot was stamped free, whether its own budget ran out, the window closed, " + "or an operator stopped the job; status is derived, so a spent budget reads completed even " + "while this is still unset" + ), + ) + attempt_count: int | None = Field( + default=None, + description=( + "This key's sampled attempts so far, judged and errored alike, the same count the sampler " + "budgets against max_turns; populated on list and detail responses. Frozen at stopped_at " + "once the key is stamped, so in-flight attempts landing after a stop never reclassify it" + ), + ) + + @property + def budget_spent(self) -> bool: + return self.attempt_count is not None and self.attempt_count >= self.max_turns - job_id: str = Field(validation_alias=AliasChoices("id", "job_id")) - api_key_id: str = Field(description="The hashed virtual key whose traffic this job evaluates, and only that key's") key_alias: str | None = Field( default=None, description="Alias of the shadowed key, resolved from the key row at read time; None when unset or deleted", @@ -277,15 +310,34 @@ class ShadowEvalJobResponse(BaseModel): default=None, description="Masked display name (sk-...) of the shadowed key, resolved at read time like key_alias", ) + + +class ShadowEvalJobResponse(BaseModel): + """A shadow-eval job over one or more keys, each with its own budget and stop state; + status is derived from stopped_by, the keys' stop and budget state, and ends_at, + never stored, so no writer anywhere can produce an inconsistent one. Aggregate + fields are populated by the detail endpoint only and stay None on list responses.""" + + job_id: str + keys: tuple[ShadowEvalJobKeyResponse, ...] = Field( + min_length=1, + description="The keys whose traffic this job evaluates, and only those keys', each with its own budget", + ) router_name: str direction: ShadowEvalDirection = "forward" baseline_model: str | None = None judge_model: str shadow_percentage: float - max_turns: int created_at: datetime ends_at: datetime - stopped_at: datetime | None = None + stopped_by: str | None = Field( + default=None, + description=( + "The operator who stopped the job early, recorded by the stop endpoint; 'unknown' backfilled " + "by migration for jobs that displayed stopped when the column arrived; None when the job " + "ended on its own. Its presence is what makes a job read stopped rather than completed" + ), + ) judged_count: int | None = Field(default=None, description="Verdicts recorded; detail endpoint only") error_count: int | None = Field(default=None, description="Sampled attempts that errored; detail endpoint only") @@ -296,12 +348,19 @@ class ShadowEvalJobResponse(BaseModel): @computed_field @property def status(self) -> ShadowEvalStatus: - """A job whose window has passed reads completed even if a later sweep stamped - stopped_at; stopped means sampling ended before the window did.""" + """Three recorded facts, no history-guessing: a stop is stopped_by (the migration + backfills it for every job that displayed stopped when the column arrived, so the + pre-column population is closed), completion is the window passing or every key + spending its budget, and anything else is running. The all-keys-stamped fallback + covers only stops written by pre-column pods during a rolling deploy.""" + if self.stopped_by is not None: + return "stopped" if datetime.now(timezone.utc) >= ( self.ends_at if self.ends_at.tzinfo else self.ends_at.replace(tzinfo=timezone.utc) ): return "completed" - if self.stopped_at is not None: + if all(key.budget_spent for key in self.keys): + return "completed" + if all(key.stopped_at is not None for key in self.keys): return "stopped" return "running" diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 8c07ca35443..1ffd0a00a22 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -54,6 +54,7 @@ "output_cost_per_image": 0.04 }, "1024-x-1024/dall-e-2": { + "deprecation_date": "2026-05-12", "input_cost_per_pixel": 1.9e-08, "litellm_provider": "openai", "mode": "image_generation", @@ -67,6 +68,7 @@ "output_cost_per_image": 0.08 }, "256-x-256/dall-e-2": { + "deprecation_date": "2026-05-12", "input_cost_per_pixel": 2.4414e-07, "litellm_provider": "openai", "mode": "image_generation", @@ -80,6 +82,7 @@ "output_cost_per_image": 0.018 }, "512-x-512/dall-e-2": { + "deprecation_date": "2026-05-12", "input_cost_per_pixel": 6.86e-08, "litellm_provider": "openai", "mode": "image_generation", @@ -2887,6 +2890,7 @@ "supports_function_calling": true }, "azure_ai/claude-haiku-4-5": { + "deprecation_date": "2026-10-19", "cache_creation_input_token_cost": 1.25e-06, "cache_creation_input_token_cost_above_1hr": 2e-06, "cache_read_input_token_cost": 1e-07, @@ -2908,6 +2912,7 @@ "supports_vision": true }, "azure_ai/claude-opus-4-5": { + "deprecation_date": "2026-10-19", "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, @@ -2930,6 +2935,7 @@ "supports_output_config": true }, "azure_ai/claude-opus-4-6": { + "deprecation_date": "2027-02-02", "supports_adaptive_thinking": true, "input_cost_per_token": 5e-06, "output_cost_per_token": 2.5e-05, @@ -2959,6 +2965,7 @@ "supports_max_reasoning_effort": true }, "azure_ai/claude-opus-4-7": { + "deprecation_date": "2027-04-06", "supports_adaptive_thinking": true, "input_cost_per_token": 5e-06, "output_cost_per_token": 2.5e-05, @@ -3083,6 +3090,7 @@ "supports_max_reasoning_effort": true }, "azure_ai/claude-opus-4-1": { + "deprecation_date": "2026-08-05", "cache_creation_input_token_cost": 1.875e-05, "cache_creation_input_token_cost_above_1hr": 3e-05, "cache_read_input_token_cost": 1.5e-06, @@ -3104,6 +3112,7 @@ "supports_vision": true }, "azure_ai/claude-sonnet-4-5": { + "deprecation_date": "2026-10-19", "cache_creation_input_token_cost": 3.75e-06, "cache_creation_input_token_cost_above_1hr": 6e-06, "cache_read_input_token_cost": 3e-07, @@ -3156,6 +3165,7 @@ "supports_max_reasoning_effort": true }, "azure_ai/claude-sonnet-4-6": { + "deprecation_date": "2027-02-10", "supports_adaptive_thinking": true, "cache_creation_input_token_cost": 3.75e-06, "cache_creation_input_token_cost_above_1hr": 6e-06, @@ -3226,6 +3236,7 @@ "supports_tool_choice": true }, "azure_ai/gpt-5.5": { + "deprecation_date": "2027-10-26", "cache_read_input_token_cost": 5e-07, "cache_read_input_token_cost_above_272k_tokens": 1e-06, "cache_read_input_token_cost_priority": 1e-06, @@ -3318,6 +3329,7 @@ "supports_minimal_reasoning_effort": false }, "azure_ai/gpt-5.4": { + "deprecation_date": "2027-09-02", "cache_read_input_token_cost": 2.5e-07, "cache_read_input_token_cost_above_272k_tokens": 5e-07, "cache_read_input_token_cost_priority": 5e-07, @@ -3364,6 +3376,7 @@ "supports_minimal_reasoning_effort": true }, "azure_ai/gpt-5.4-2026-03-05": { + "deprecation_date": "2027-09-02", "cache_read_input_token_cost": 2.5e-07, "cache_read_input_token_cost_above_272k_tokens": 5e-07, "cache_read_input_token_cost_priority": 5e-07, @@ -3410,6 +3423,7 @@ "supports_minimal_reasoning_effort": true }, "azure_ai/gpt-5.4-pro": { + "deprecation_date": "2027-09-07", "cache_read_input_token_cost": 3e-06, "cache_read_input_token_cost_above_272k_tokens": 6e-06, "cache_read_input_token_cost_priority": 6e-06, @@ -3455,6 +3469,7 @@ "supports_minimal_reasoning_effort": true }, "azure_ai/gpt-5.4-pro-2026-03-05": { + "deprecation_date": "2027-09-07", "cache_read_input_token_cost": 3e-06, "cache_read_input_token_cost_above_272k_tokens": 6e-06, "cache_read_input_token_cost_priority": 6e-06, @@ -3500,6 +3515,7 @@ "supports_minimal_reasoning_effort": true }, "azure_ai/gpt-5.4-mini": { + "deprecation_date": "2027-09-21", "cache_read_input_token_cost": 7.5e-08, "cache_read_input_token_cost_priority": 1.5e-07, "input_cost_per_token": 7.5e-07, @@ -3540,6 +3556,7 @@ "supports_minimal_reasoning_effort": false }, "azure_ai/gpt-5.4-mini-2026-03-17": { + "deprecation_date": "2027-09-21", "cache_read_input_token_cost": 7.5e-08, "cache_read_input_token_cost_priority": 1.5e-07, "input_cost_per_token": 7.5e-07, @@ -3580,6 +3597,7 @@ "supports_minimal_reasoning_effort": false }, "azure_ai/gpt-5.4-nano": { + "deprecation_date": "2027-09-21", "cache_read_input_token_cost": 2e-08, "cache_read_input_token_cost_priority": 4e-08, "input_cost_per_token": 2e-07, @@ -3620,6 +3638,7 @@ "supports_minimal_reasoning_effort": false }, "azure_ai/gpt-5.4-nano-2026-03-17": { + "deprecation_date": "2027-09-21", "cache_read_input_token_cost": 2e-08, "cache_read_input_token_cost_priority": 4e-08, "input_cost_per_token": 2e-07, @@ -3849,6 +3868,7 @@ "supports_vision": true }, "azure/eu/gpt-5.1": { + "deprecation_date": "2027-05-15", "cache_read_input_token_cost": 1.4e-07, "input_cost_per_token": 1.38e-06, "litellm_provider": "azure", @@ -3918,6 +3938,7 @@ "supports_none_reasoning_effort": true }, "azure/eu/gpt-5.1-codex": { + "deprecation_date": "2027-05-15", "cache_read_input_token_cost": 1.4e-07, "input_cost_per_token": 1.38e-06, "litellm_provider": "azure", @@ -3948,6 +3969,7 @@ "supports_vision": true }, "azure/eu/gpt-5.1-codex-mini": { + "deprecation_date": "2027-05-15", "cache_read_input_token_cost": 2.8e-08, "input_cost_per_token": 2.75e-07, "litellm_provider": "azure", @@ -4107,6 +4129,7 @@ "supports_vision": true }, "azure/global-standard/gpt-4o-mini": { + "deprecation_date": "2027-04-14", "input_cost_per_token": 1.5e-07, "litellm_provider": "azure", "max_input_tokens": 128000, @@ -4155,6 +4178,7 @@ "supports_vision": true }, "azure/global/gpt-5.1": { + "deprecation_date": "2027-05-15", "cache_read_input_token_cost": 1.25e-07, "input_cost_per_token": 1.25e-06, "litellm_provider": "azure", @@ -4224,6 +4248,7 @@ "supports_none_reasoning_effort": true }, "azure/global/gpt-5.1-codex": { + "deprecation_date": "2027-05-15", "cache_read_input_token_cost": 1.25e-07, "input_cost_per_token": 1.25e-06, "litellm_provider": "azure", @@ -4254,6 +4279,7 @@ "supports_vision": true }, "azure/global/gpt-5.1-codex-mini": { + "deprecation_date": "2027-05-15", "cache_read_input_token_cost": 2.5e-08, "input_cost_per_token": 2.5e-07, "litellm_provider": "azure", @@ -4492,6 +4518,7 @@ "supports_vision": true }, "azure/gpt-4.1": { + "deprecation_date": "2027-04-14", "cache_read_input_token_cost": 5e-07, "input_cost_per_token": 2e-06, "input_cost_per_token_batches": 1e-06, @@ -4559,6 +4586,7 @@ "supports_web_search": false }, "azure/gpt-4.1-mini": { + "deprecation_date": "2027-04-14", "cache_read_input_token_cost": 1e-07, "input_cost_per_token": 4e-07, "input_cost_per_token_batches": 2e-07, @@ -4626,6 +4654,7 @@ "supports_web_search": false }, "azure/gpt-4.1-nano": { + "deprecation_date": "2026-10-14", "cache_read_input_token_cost": 2.5e-08, "input_cost_per_token": 1e-07, "input_cost_per_token_batches": 5e-08, @@ -4902,6 +4931,7 @@ "supports_vision": false }, "azure/gpt-4o-mini": { + "deprecation_date": "2027-04-14", "cache_read_input_token_cost": 7.5e-08, "input_cost_per_token": 1.65e-07, "litellm_provider": "azure", @@ -5344,6 +5374,7 @@ "supports_vision": true }, "azure/gpt-5": { + "deprecation_date": "2027-02-09", "cache_read_input_token_cost": 1.25e-07, "input_cost_per_token": 1.25e-06, "litellm_provider": "azure", @@ -5507,6 +5538,7 @@ "supports_vision": true }, "azure/gpt-5-mini": { + "deprecation_date": "2027-02-09", "cache_read_input_token_cost": 2.5e-08, "input_cost_per_token": 2.5e-07, "litellm_provider": "azure", @@ -5572,6 +5604,7 @@ "supports_vision": true }, "azure/gpt-5-nano": { + "deprecation_date": "2027-02-09", "cache_read_input_token_cost": 5e-09, "input_cost_per_token": 5e-08, "litellm_provider": "azure", @@ -5667,6 +5700,7 @@ "supports_vision": true }, "azure/gpt-5.1": { + "deprecation_date": "2027-05-15", "cache_read_input_token_cost": 1.25e-07, "input_cost_per_token": 1.25e-06, "litellm_provider": "azure", @@ -5736,6 +5770,7 @@ "supports_none_reasoning_effort": true }, "azure/gpt-5.1-codex": { + "deprecation_date": "2027-05-15", "cache_read_input_token_cost": 1.25e-07, "input_cost_per_token": 1.25e-06, "litellm_provider": "azure", @@ -5797,6 +5832,7 @@ "supports_vision": true }, "azure/gpt-5.1-codex-mini": { + "deprecation_date": "2027-05-15", "cache_read_input_token_cost": 2.5e-08, "input_cost_per_token": 2.5e-07, "litellm_provider": "azure", @@ -5827,6 +5863,7 @@ "supports_vision": true }, "azure/gpt-5.2": { + "deprecation_date": "2027-06-08", "cache_read_input_token_cost": 1.75e-07, "input_cost_per_token": 1.75e-06, "litellm_provider": "azure", @@ -6136,6 +6173,7 @@ "supports_web_search": true }, "azure/gpt-5.4": { + "deprecation_date": "2027-09-02", "cache_read_input_token_cost": 2.5e-07, "cache_read_input_token_cost_above_272k_tokens": 5e-07, "cache_read_input_token_cost_priority": 5e-07, @@ -6180,6 +6218,7 @@ "supports_minimal_reasoning_effort": true }, "azure/us/gpt-5.4": { + "deprecation_date": "2027-09-02", "cache_read_input_token_cost": 2.8e-07, "cache_read_input_token_cost_priority": 5.5e-07, "input_cost_per_token": 2.75e-06, @@ -6218,6 +6257,7 @@ "supports_minimal_reasoning_effort": true }, "azure/eu/gpt-5.4": { + "deprecation_date": "2027-09-02", "cache_read_input_token_cost": 2.8e-07, "cache_read_input_token_cost_priority": 5.5e-07, "input_cost_per_token": 2.75e-06, @@ -6379,6 +6419,7 @@ "supports_minimal_reasoning_effort": true }, "azure/gpt-5.4-pro": { + "deprecation_date": "2027-09-07", "cache_read_input_token_cost": 3e-06, "cache_read_input_token_cost_above_272k_tokens": 6e-06, "input_cost_per_token": 3e-05, @@ -7045,6 +7086,7 @@ "supports_minimal_reasoning_effort": false }, "azure/gpt-5.5": { + "deprecation_date": "2027-10-26", "cache_read_input_token_cost": 5e-07, "cache_read_input_token_cost_above_272k_tokens": 1e-06, "cache_read_input_token_cost_priority": 1e-06, @@ -7095,6 +7137,7 @@ "supports_minimal_reasoning_effort": false }, "azure/us/gpt-5.5": { + "deprecation_date": "2027-10-26", "cache_read_input_token_cost": 5.5e-07, "cache_read_input_token_cost_above_272k_tokens": 1.1e-06, "cache_read_input_token_cost_priority": 1.38e-06, @@ -7142,6 +7185,7 @@ "supports_minimal_reasoning_effort": false }, "azure/eu/gpt-5.5": { + "deprecation_date": "2027-10-26", "cache_read_input_token_cost": 5.5e-07, "cache_read_input_token_cost_above_272k_tokens": 1.1e-06, "cache_read_input_token_cost_priority": 1.38e-06, @@ -7408,6 +7452,7 @@ "supports_web_search": true }, "azure/gpt-5.4-mini": { + "deprecation_date": "2027-09-21", "cache_read_input_token_cost": 7.5e-08, "input_cost_per_token": 7.5e-07, "litellm_provider": "azure", @@ -7489,6 +7534,7 @@ "supports_xhigh_reasoning_effort": true }, "azure/gpt-5.4-nano": { + "deprecation_date": "2027-09-21", "cache_read_input_token_cost": 2e-08, "input_cost_per_token": 2e-07, "litellm_provider": "azure", @@ -7601,6 +7647,7 @@ "output_cost_per_token": 0.0 }, "azure/high/1024-x-1024/gpt-image-1": { + "deprecation_date": "2026-10-23", "input_cost_per_pixel": 1.59263611e-07, "litellm_provider": "azure", "mode": "image_generation", @@ -7610,6 +7657,7 @@ ] }, "azure/high/1024-x-1536/gpt-image-1": { + "deprecation_date": "2026-10-23", "input_cost_per_pixel": 1.58945719e-07, "litellm_provider": "azure", "mode": "image_generation", @@ -7619,6 +7667,7 @@ ] }, "azure/high/1536-x-1024/gpt-image-1": { + "deprecation_date": "2026-10-23", "input_cost_per_pixel": 1.58945719e-07, "litellm_provider": "azure", "mode": "image_generation", @@ -7628,6 +7677,7 @@ ] }, "azure/low/1024-x-1024/gpt-image-1": { + "deprecation_date": "2026-10-23", "input_cost_per_pixel": 1.0490417e-08, "litellm_provider": "azure", "mode": "image_generation", @@ -7637,6 +7687,7 @@ ] }, "azure/low/1024-x-1536/gpt-image-1": { + "deprecation_date": "2026-10-23", "input_cost_per_pixel": 1.0172526e-08, "litellm_provider": "azure", "mode": "image_generation", @@ -7646,6 +7697,7 @@ ] }, "azure/low/1536-x-1024/gpt-image-1": { + "deprecation_date": "2026-10-23", "input_cost_per_pixel": 1.0172526e-08, "litellm_provider": "azure", "mode": "image_generation", @@ -7655,6 +7707,7 @@ ] }, "azure/medium/1024-x-1024/gpt-image-1": { + "deprecation_date": "2026-10-23", "input_cost_per_pixel": 4.0054321e-08, "litellm_provider": "azure", "mode": "image_generation", @@ -7664,6 +7717,7 @@ ] }, "azure/medium/1024-x-1536/gpt-image-1": { + "deprecation_date": "2026-10-23", "input_cost_per_pixel": 4.0054321e-08, "litellm_provider": "azure", "mode": "image_generation", @@ -7673,6 +7727,7 @@ ] }, "azure/medium/1536-x-1024/gpt-image-1": { + "deprecation_date": "2026-10-23", "input_cost_per_pixel": 4.0054321e-08, "litellm_provider": "azure", "mode": "image_generation", @@ -7695,6 +7750,7 @@ ] }, "azure/gpt-image-1.5": { + "deprecation_date": "2027-06-16", "cache_read_input_token_cost": 1.25e-06, "input_cost_per_token": 5e-06, "input_cost_per_image_token": 8e-06, @@ -7720,6 +7776,7 @@ ] }, "azure/gpt-image-2": { + "deprecation_date": "2027-10-21", "cache_read_input_token_cost": 1.25e-06, "input_cost_per_token": 5e-06, "input_cost_per_image_token": 8e-06, @@ -7751,6 +7808,7 @@ "supports_pdf_input": true }, "azure/low/1024-x-1024/gpt-image-1-mini": { + "deprecation_date": "2027-04-07", "input_cost_per_pixel": 2.0751953125e-09, "litellm_provider": "azure", "mode": "image_generation", @@ -7760,6 +7818,7 @@ ] }, "azure/low/1024-x-1536/gpt-image-1-mini": { + "deprecation_date": "2027-04-07", "input_cost_per_pixel": 2.0751953125e-09, "litellm_provider": "azure", "mode": "image_generation", @@ -7769,6 +7828,7 @@ ] }, "azure/low/1536-x-1024/gpt-image-1-mini": { + "deprecation_date": "2027-04-07", "input_cost_per_pixel": 2.0345052083e-09, "litellm_provider": "azure", "mode": "image_generation", @@ -7778,6 +7838,7 @@ ] }, "azure/medium/1024-x-1024/gpt-image-1-mini": { + "deprecation_date": "2027-04-07", "input_cost_per_pixel": 8.056640625e-09, "litellm_provider": "azure", "mode": "image_generation", @@ -7787,6 +7848,7 @@ ] }, "azure/medium/1024-x-1536/gpt-image-1-mini": { + "deprecation_date": "2027-04-07", "input_cost_per_pixel": 8.056640625e-09, "litellm_provider": "azure", "mode": "image_generation", @@ -7796,6 +7858,7 @@ ] }, "azure/medium/1536-x-1024/gpt-image-1-mini": { + "deprecation_date": "2027-04-07", "input_cost_per_pixel": 7.9752604167e-09, "litellm_provider": "azure", "mode": "image_generation", @@ -7805,6 +7868,7 @@ ] }, "azure/high/1024-x-1024/gpt-image-1-mini": { + "deprecation_date": "2027-04-07", "input_cost_per_pixel": 3.173828125e-08, "litellm_provider": "azure", "mode": "image_generation", @@ -7814,6 +7878,7 @@ ] }, "azure/high/1024-x-1536/gpt-image-1-mini": { + "deprecation_date": "2027-04-07", "input_cost_per_pixel": 3.173828125e-08, "litellm_provider": "azure", "mode": "image_generation", @@ -7823,6 +7888,7 @@ ] }, "azure/high/1536-x-1024/gpt-image-1-mini": { + "deprecation_date": "2027-04-07", "input_cost_per_pixel": 3.1575520833e-08, "litellm_provider": "azure", "mode": "image_generation", @@ -7850,6 +7916,7 @@ "supports_function_calling": true }, "azure/o1": { + "deprecation_date": "2026-10-21", "cache_read_input_token_cost": 7.5e-06, "input_cost_per_token": 1.5e-05, "litellm_provider": "azure", @@ -7944,6 +8011,7 @@ "supports_vision": false }, "azure/o3": { + "deprecation_date": "2026-10-21", "cache_read_input_token_cost": 5e-07, "input_cost_per_token": 2e-06, "litellm_provider": "azure", @@ -8041,6 +8109,7 @@ "supports_web_search": true }, "azure/o3-mini": { + "deprecation_date": "2026-10-01", "cache_read_input_token_cost": 5.5e-07, "input_cost_per_token": 1.1e-06, "litellm_provider": "azure", @@ -8071,6 +8140,7 @@ "supports_vision": false }, "azure/o3-pro": { + "deprecation_date": "2026-12-17", "input_cost_per_token": 2e-05, "input_cost_per_token_batches": 1e-05, "litellm_provider": "azure", @@ -8132,6 +8202,7 @@ "supports_vision": true }, "azure/o4-mini": { + "deprecation_date": "2026-10-16", "cache_read_input_token_cost": 2.75e-07, "input_cost_per_token": 1.1e-06, "litellm_provider": "azure", @@ -8580,6 +8651,7 @@ "supports_vision": true }, "azure/us/gpt-5.1": { + "deprecation_date": "2027-05-15", "cache_read_input_token_cost": 1.4e-07, "input_cost_per_token": 1.38e-06, "litellm_provider": "azure", @@ -8649,6 +8721,7 @@ "supports_none_reasoning_effort": true }, "azure/us/gpt-5.1-codex": { + "deprecation_date": "2027-05-15", "cache_read_input_token_cost": 1.4e-07, "input_cost_per_token": 1.38e-06, "litellm_provider": "azure", @@ -8679,6 +8752,7 @@ "supports_vision": true }, "azure/us/gpt-5.1-codex-mini": { + "deprecation_date": "2027-05-15", "cache_read_input_token_cost": 2.8e-08, "input_cost_per_token": 2.75e-07, "litellm_provider": "azure", @@ -8876,6 +8950,7 @@ ] }, "azure_ai/FW-DeepSeek-V3.2": { + "deprecation_date": "2027-07-01", "cache_read_input_token_cost": 3.1e-07, "input_cost_per_token": 6.2e-07, "litellm_provider": "azure_ai", @@ -8906,6 +8981,7 @@ "supports_tool_choice": true }, "azure_ai/FW-GLM-5": { + "deprecation_date": "2027-07-01", "cache_read_input_token_cost": 2.2e-07, "input_cost_per_token": 1.1e-06, "litellm_provider": "azure_ai", @@ -8921,6 +8997,7 @@ "supports_tool_choice": true }, "azure_ai/FW-GLM-5.1": { + "deprecation_date": "2027-07-01", "cache_read_input_token_cost": 2.86e-07, "input_cost_per_token": 1.54e-06, "litellm_provider": "azure_ai", @@ -8987,6 +9064,7 @@ "supports_tool_choice": true }, "azure_ai/FW-Kimi-K2.5": { + "deprecation_date": "2027-07-01", "cache_read_input_token_cost": 1.1e-07, "input_cost_per_token": 6.6e-07, "litellm_provider": "azure_ai", @@ -9079,6 +9157,7 @@ "supports_vision": true }, "azure_ai/FW-MiniMax-M2.5": { + "deprecation_date": "2027-07-01", "cache_read_input_token_cost": 3.3e-08, "input_cost_per_token": 3.3e-07, "litellm_provider": "azure_ai", @@ -9164,6 +9243,7 @@ ] }, "azure_ai/MAI-Image-2e": { + "deprecation_date": "2026-08-15", "input_cost_per_token": 5e-06, "litellm_provider": "azure_ai", "mode": "image_generation", @@ -9175,6 +9255,7 @@ ] }, "azure_ai/Llama-3.2-11B-Vision-Instruct": { + "deprecation_date": "2026-06-13", "input_cost_per_token": 3.7e-07, "litellm_provider": "azure_ai", "max_input_tokens": 128000, @@ -9188,6 +9269,7 @@ "supports_vision": true }, "azure_ai/Llama-3.2-90B-Vision-Instruct": { + "deprecation_date": "2026-06-13", "input_cost_per_token": 2.04e-06, "litellm_provider": "azure_ai", "max_input_tokens": 128000, @@ -9249,6 +9331,7 @@ "supports_tool_choice": true }, "azure_ai/Meta-Llama-3.1-405B-Instruct": { + "deprecation_date": "2026-06-13", "input_cost_per_token": 5.33e-06, "litellm_provider": "azure_ai", "max_input_tokens": 128000, @@ -9271,6 +9354,7 @@ "supports_tool_choice": true }, "azure_ai/Meta-Llama-3.1-8B-Instruct": { + "deprecation_date": "2026-06-13", "input_cost_per_token": 3e-07, "litellm_provider": "azure_ai", "max_input_tokens": 128000, @@ -9452,6 +9536,7 @@ "supports_reasoning": true }, "azure_ai/mistral-document-ai-2505": { + "deprecation_date": "2026-07-20", "litellm_provider": "azure_ai", "ocr_cost_per_page": 0.003, "mode": "ocr", @@ -9529,6 +9614,7 @@ "output_cost_per_token": 0.0 }, "azure_ai/cohere-rerank-v3.5": { + "deprecation_date": "2026-05-14", "input_cost_per_query": 0.002, "input_cost_per_token": 0.0, "litellm_provider": "azure_ai", @@ -9591,6 +9677,7 @@ "supports_tool_choice": true }, "azure_ai/deepseek-r1": { + "deprecation_date": "2026-08-13", "input_cost_per_token": 1.35e-06, "litellm_provider": "azure_ai", "max_input_tokens": 128000, @@ -9614,6 +9701,7 @@ "supports_tool_choice": true }, "azure_ai/deepseek-v3-0324": { + "deprecation_date": "2026-07-13", "input_cost_per_token": 1.14e-06, "litellm_provider": "azure_ai", "max_input_tokens": 128000, @@ -9626,6 +9714,7 @@ "supports_tool_choice": true }, "azure_ai/deepseek-v3.1": { + "deprecation_date": "2026-07-13", "input_cost_per_token": 1.23e-06, "litellm_provider": "azure_ai", "max_input_tokens": 131072, @@ -9639,6 +9728,7 @@ "supports_tool_choice": true }, "azure_ai/deepseek-v4-pro": { + "deprecation_date": "2028-02-20", "input_cost_per_token": 1.74e-06, "litellm_provider": "azure_ai", "max_input_tokens": 1000000, @@ -9652,6 +9742,7 @@ "supports_tool_choice": true }, "azure_ai/deepseek-v4-flash": { + "deprecation_date": "2028-02-20", "input_cost_per_token": 1.9e-07, "litellm_provider": "azure_ai", "max_input_tokens": 1000000, @@ -9683,6 +9774,7 @@ "supports_embedding_image_input": true }, "azure_ai/global/grok-3": { + "deprecation_date": "2026-05-01", "input_cost_per_token": 3e-06, "litellm_provider": "azure_ai", "max_input_tokens": 131072, @@ -9697,6 +9789,7 @@ "supports_web_search": true }, "azure_ai/global/grok-3-mini": { + "deprecation_date": "2026-05-01", "input_cost_per_token": 2.5e-07, "litellm_provider": "azure_ai", "max_input_tokens": 131072, @@ -9712,6 +9805,7 @@ "supports_web_search": true }, "azure_ai/grok-3": { + "deprecation_date": "2026-05-01", "input_cost_per_token": 3e-06, "litellm_provider": "azure_ai", "max_input_tokens": 131072, @@ -9726,6 +9820,7 @@ "supports_web_search": true }, "azure_ai/grok-3-mini": { + "deprecation_date": "2026-05-01", "input_cost_per_token": 2.5e-07, "litellm_provider": "azure_ai", "max_input_tokens": 131072, @@ -9773,6 +9868,7 @@ "supports_web_search": true }, "azure_ai/grok-4-fast-non-reasoning": { + "deprecation_date": "2026-05-01", "input_cost_per_token": 2e-07, "output_cost_per_token": 5e-07, "litellm_provider": "azure_ai", @@ -9786,6 +9882,7 @@ "supports_web_search": true }, "azure_ai/grok-4-fast-reasoning": { + "deprecation_date": "2026-05-01", "input_cost_per_token": 2e-07, "output_cost_per_token": 5e-07, "litellm_provider": "azure_ai", @@ -9863,6 +9960,7 @@ "supports_tool_choice": true }, "azure_ai/kimi-k2.5": { + "deprecation_date": "2027-01-26", "input_cost_per_token": 6e-07, "litellm_provider": "azure_ai", "max_input_tokens": 262144, @@ -9877,6 +9975,7 @@ "supports_vision": true }, "azure_ai/kimi-k2.6": { + "deprecation_date": "2027-04-16", "input_cost_per_token": 9.5e-07, "litellm_provider": "azure_ai", "max_input_tokens": 262144, @@ -10004,6 +10103,7 @@ "supports_vision": true }, "babbage-002": { + "deprecation_date": "2026-09-28", "input_cost_per_token": 4e-07, "litellm_provider": "text-completion-openai", "max_input_tokens": 16384, @@ -12014,6 +12114,7 @@ ] }, "claude-haiku-4-5-20251001": { + "deprecation_date": "2026-10-15", "cache_creation_input_token_cost": 1.25e-06, "cache_creation_input_token_cost_above_1hr": 2e-06, "cache_read_input_token_cost": 1e-07, @@ -12037,6 +12138,7 @@ "prompt_cache_min_tokens": 4096 }, "claude-haiku-4-5": { + "deprecation_date": "2026-10-15", "cache_creation_input_token_cost": 1.25e-06, "cache_creation_input_token_cost_above_1hr": 2e-06, "cache_read_input_token_cost": 1e-07, @@ -12185,6 +12287,7 @@ "prompt_cache_min_tokens": 1024 }, "claude-sonnet-4-5": { + "deprecation_date": "2026-09-29", "cache_creation_input_token_cost": 3.75e-06, "cache_creation_input_token_cost_above_1hr": 6e-06, "cache_creation_input_token_cost_above_1hr_above_200k_tokens": 1.2e-05, @@ -12218,6 +12321,7 @@ "prompt_cache_min_tokens": 1024 }, "claude-sonnet-4-5-20250929": { + "deprecation_date": "2026-09-29", "cache_creation_input_token_cost": 3.75e-06, "cache_creation_input_token_cost_above_1hr": 6e-06, "cache_creation_input_token_cost_above_1hr_above_200k_tokens": 1.2e-05, @@ -12252,6 +12356,7 @@ "prompt_cache_min_tokens": 1024 }, "claude-sonnet-5": { + "deprecation_date": "2027-06-30", "cache_creation_input_token_cost": 2.5e-06, "cache_creation_input_token_cost_above_1hr": 4e-06, "cache_read_input_token_cost": 2e-07, @@ -12288,6 +12393,7 @@ "prompt_cache_min_tokens": 1024 }, "claude-sonnet-4-6": { + "deprecation_date": "2027-02-17", "cache_creation_input_token_cost": 3.75e-06, "cache_creation_input_token_cost_above_1hr": 6e-06, "cache_read_input_token_cost": 3e-07, @@ -12434,6 +12540,7 @@ "prompt_cache_min_tokens": 1024 }, "claude-opus-4-5-20251101": { + "deprecation_date": "2026-11-24", "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, @@ -12463,6 +12570,7 @@ "prompt_cache_min_tokens": 4096 }, "claude-opus-4-5": { + "deprecation_date": "2026-11-24", "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, @@ -12492,6 +12600,7 @@ "prompt_cache_min_tokens": 4096 }, "claude-opus-4-6": { + "deprecation_date": "2027-02-05", "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, @@ -12528,6 +12637,7 @@ "prompt_cache_min_tokens": 4096 }, "claude-opus-4-6-20260205": { + "deprecation_date": "2027-02-05", "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, @@ -12564,6 +12674,7 @@ "prompt_cache_min_tokens": 4096 }, "claude-opus-4-7": { + "deprecation_date": "2027-04-16", "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, @@ -12602,6 +12713,7 @@ "prompt_cache_min_tokens": 2048 }, "claude-opus-4-7-20260416": { + "deprecation_date": "2027-04-16", "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, @@ -12640,6 +12752,7 @@ "prompt_cache_min_tokens": 2048 }, "claude-fable-5": { + "deprecation_date": "2027-06-09", "cache_creation_input_token_cost": 1.25e-05, "cache_creation_input_token_cost_above_1hr": 2e-05, "cache_read_input_token_cost": 1e-06, @@ -12675,6 +12788,7 @@ "prompt_cache_min_tokens": 512 }, "claude-opus-5": { + "deprecation_date": "2027-07-24", "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, @@ -12713,6 +12827,7 @@ "prompt_cache_min_tokens": 512 }, "claude-opus-4-8": { + "deprecation_date": "2027-05-28", "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, @@ -15042,6 +15157,7 @@ "mode": "search" }, "davinci-002": { + "deprecation_date": "2026-09-28", "input_cost_per_token": 2e-06, "litellm_provider": "text-completion-openai", "max_input_tokens": 16384, @@ -18594,6 +18710,7 @@ } }, "gemini-2.5-flash": { + "deprecation_date": "2026-10-20", "cache_read_input_token_cost": 3e-08, "input_cost_per_audio_token": 1e-06, "input_cost_per_token": 3e-07, @@ -18639,6 +18756,7 @@ "supports_image_size": false }, "gemini-2.5-flash-image": { + "deprecation_date": "2026-10-02", "cache_read_input_token_cost": 3e-08, "input_cost_per_audio_token": 1e-06, "input_cost_per_token": 3e-07, @@ -18683,6 +18801,7 @@ "supports_image_size": false }, "gemini-3-pro-image": { + "deprecation_date": "2027-05-28", "input_cost_per_image": 0.0011, "input_cost_per_token": 2e-06, "input_cost_per_token_batches": 1e-06, @@ -18763,6 +18882,7 @@ "web_search_billing_unit": "per_query" }, "gemini-3.1-flash-image": { + "deprecation_date": "2027-05-28", "input_cost_per_image": 0.00056, "input_cost_per_token": 5e-07, "litellm_provider": "vertex_ai-language-models", @@ -18887,6 +19007,7 @@ "web_search_billing_unit": "per_query" }, "gemini-3.1-flash-lite": { + "deprecation_date": "2027-05-07", "cache_read_input_token_cost": 2.5e-08, "cache_read_input_token_cost_flex": 1.25e-08, "cache_read_input_token_cost_priority": 4.5e-08, @@ -18943,6 +19064,7 @@ "web_search_billing_unit": "per_query" }, "gemini-3.5-flash-lite": { + "deprecation_date": "2027-07-21", "cache_read_input_token_cost": 3e-08, "cache_read_input_token_cost_flex": 2e-08, "cache_read_input_token_cost_priority": 5e-08, @@ -19032,6 +19154,7 @@ "supports_web_search": true }, "gemini-2.5-flash-lite": { + "deprecation_date": "2026-10-20", "cache_read_input_token_cost": 1e-08, "input_cost_per_audio_token": 3e-07, "input_cost_per_token": 1e-07, @@ -19303,6 +19426,7 @@ "supports_image_size": false }, "gemini-2.5-pro": { + "deprecation_date": "2026-10-20", "cache_read_input_token_cost": 1.25e-07, "cache_read_input_token_cost_above_200k_tokens": 2.5e-07, "cache_creation_input_token_cost_above_200k_tokens": 2.5e-07, @@ -19617,6 +19741,7 @@ }, "vertex_ai/gemini-3.5-flash": { "prompt_cache_min_tokens": 4096, + "deprecation_date": "2027-05-19", "cache_read_input_token_cost": 1.5e-07, "input_cost_per_token": 1.5e-06, "input_cost_per_audio_token": 1e-06, @@ -20057,6 +20182,7 @@ "web_search_billing_unit": "per_query" }, "gemini/gemini-robotics-er-1.6-preview": { + "deprecation_date": "2026-08-31", "input_cost_per_audio_token": 2e-06, "input_cost_per_token": 1e-06, "litellm_provider": "gemini", @@ -20127,6 +20253,7 @@ "supports_vision": true }, "gemini-embedding-001": { + "deprecation_date": "2028-05-20", "input_cost_per_token": 1.5e-07, "litellm_provider": "vertex_ai-embedding-models", "max_input_tokens": 2048, @@ -21746,6 +21873,7 @@ }, "gemini-3.5-flash": { "prompt_cache_min_tokens": 4096, + "deprecation_date": "2027-05-19", "cache_read_input_token_cost": 1.5e-07, "input_cost_per_audio_token": 1e-06, "input_cost_per_token": 1.5e-06, @@ -23260,6 +23388,7 @@ "supports_tool_choice": true }, "gpt-3.5-turbo-instruct": { + "deprecation_date": "2026-09-28", "input_cost_per_token": 1.5e-06, "litellm_provider": "text-completion-openai", "max_input_tokens": 8192, @@ -24391,6 +24520,7 @@ "supports_pdf_input": true }, "low/1024-x-1024/gpt-image-1.5": { + "deprecation_date": "2026-12-01", "input_cost_per_image": 0.009, "litellm_provider": "openai", "mode": "image_generation", @@ -24402,6 +24532,7 @@ "supports_pdf_input": true }, "low/1024-x-1536/gpt-image-1.5": { + "deprecation_date": "2026-12-01", "input_cost_per_image": 0.013, "litellm_provider": "openai", "mode": "image_generation", @@ -24413,6 +24544,7 @@ "supports_pdf_input": true }, "low/1536-x-1024/gpt-image-1.5": { + "deprecation_date": "2026-12-01", "input_cost_per_image": 0.013, "litellm_provider": "openai", "mode": "image_generation", @@ -24424,6 +24556,7 @@ "supports_pdf_input": true }, "medium/1024-x-1024/gpt-image-1.5": { + "deprecation_date": "2026-12-01", "input_cost_per_image": 0.034, "litellm_provider": "openai", "mode": "image_generation", @@ -24435,6 +24568,7 @@ "supports_pdf_input": true }, "medium/1024-x-1536/gpt-image-1.5": { + "deprecation_date": "2026-12-01", "input_cost_per_image": 0.05, "litellm_provider": "openai", "mode": "image_generation", @@ -24446,6 +24580,7 @@ "supports_pdf_input": true }, "medium/1536-x-1024/gpt-image-1.5": { + "deprecation_date": "2026-12-01", "input_cost_per_image": 0.05, "litellm_provider": "openai", "mode": "image_generation", @@ -24457,6 +24592,7 @@ "supports_pdf_input": true }, "high/1024-x-1024/gpt-image-1.5": { + "deprecation_date": "2026-12-01", "input_cost_per_image": 0.133, "litellm_provider": "openai", "mode": "image_generation", @@ -24468,6 +24604,7 @@ "supports_pdf_input": true }, "high/1024-x-1536/gpt-image-1.5": { + "deprecation_date": "2026-12-01", "input_cost_per_image": 0.2, "litellm_provider": "openai", "mode": "image_generation", @@ -24479,6 +24616,7 @@ "supports_pdf_input": true }, "high/1536-x-1024/gpt-image-1.5": { + "deprecation_date": "2026-12-01", "input_cost_per_image": 0.2, "litellm_provider": "openai", "mode": "image_generation", @@ -24490,6 +24628,7 @@ "supports_pdf_input": true }, "standard/1024-x-1024/gpt-image-1.5": { + "deprecation_date": "2026-12-01", "input_cost_per_image": 0.009, "litellm_provider": "openai", "mode": "image_generation", @@ -24501,6 +24640,7 @@ "supports_pdf_input": true }, "standard/1024-x-1536/gpt-image-1.5": { + "deprecation_date": "2026-12-01", "input_cost_per_image": 0.013, "litellm_provider": "openai", "mode": "image_generation", @@ -24512,6 +24652,7 @@ "supports_pdf_input": true }, "standard/1536-x-1024/gpt-image-1.5": { + "deprecation_date": "2026-12-01", "input_cost_per_image": 0.013, "litellm_provider": "openai", "mode": "image_generation", @@ -24523,6 +24664,7 @@ "supports_pdf_input": true }, "1024-x-1024/gpt-image-1.5": { + "deprecation_date": "2026-12-01", "input_cost_per_image": 0.009, "litellm_provider": "openai", "mode": "image_generation", @@ -24534,6 +24676,7 @@ "supports_pdf_input": true }, "1024-x-1536/gpt-image-1.5": { + "deprecation_date": "2026-12-01", "input_cost_per_image": 0.013, "litellm_provider": "openai", "mode": "image_generation", @@ -24545,6 +24688,7 @@ "supports_pdf_input": true }, "1536-x-1024/gpt-image-1.5": { + "deprecation_date": "2026-12-01", "input_cost_per_image": 0.013, "litellm_provider": "openai", "mode": "image_generation", @@ -27458,18 +27602,21 @@ "output_cost_per_second": 0.0 }, "hd/1024-x-1024/dall-e-3": { + "deprecation_date": "2026-05-12", "input_cost_per_pixel": 7.629e-08, "litellm_provider": "openai", "mode": "image_generation", "output_cost_per_pixel": 0.0 }, "hd/1024-x-1792/dall-e-3": { + "deprecation_date": "2026-05-12", "input_cost_per_pixel": 6.539e-08, "litellm_provider": "openai", "mode": "image_generation", "output_cost_per_pixel": 0.0 }, "hd/1792-x-1024/dall-e-3": { + "deprecation_date": "2026-05-12", "input_cost_per_pixel": 6.539e-08, "litellm_provider": "openai", "mode": "image_generation", @@ -27516,6 +27663,7 @@ "max_output_tokens": 8192 }, "high/1024-x-1024/gpt-image-1": { + "deprecation_date": "2026-10-23", "input_cost_per_image": 0.167, "input_cost_per_pixel": 1.59263611e-07, "litellm_provider": "openai", @@ -27526,6 +27674,7 @@ ] }, "high/1024-x-1536/gpt-image-1": { + "deprecation_date": "2026-10-23", "input_cost_per_image": 0.25, "input_cost_per_pixel": 1.58945719e-07, "litellm_provider": "openai", @@ -27536,6 +27685,7 @@ ] }, "high/1536-x-1024/gpt-image-1": { + "deprecation_date": "2026-10-23", "input_cost_per_image": 0.25, "input_cost_per_pixel": 1.58945719e-07, "litellm_provider": "openai", @@ -28323,6 +28473,7 @@ "supports_tool_choice": true }, "low/1024-x-1024/gpt-image-1": { + "deprecation_date": "2026-10-23", "input_cost_per_image": 0.011, "input_cost_per_pixel": 1.0490417e-08, "litellm_provider": "openai", @@ -28333,6 +28484,7 @@ ] }, "low/1024-x-1536/gpt-image-1": { + "deprecation_date": "2026-10-23", "input_cost_per_image": 0.016, "input_cost_per_pixel": 1.0172526e-08, "litellm_provider": "openai", @@ -28343,6 +28495,7 @@ ] }, "low/1536-x-1024/gpt-image-1": { + "deprecation_date": "2026-10-23", "input_cost_per_image": 0.016, "input_cost_per_pixel": 1.0172526e-08, "litellm_provider": "openai", @@ -28367,6 +28520,7 @@ "output_cost_per_image": 0.072 }, "medium/1024-x-1024/gpt-image-1": { + "deprecation_date": "2026-10-23", "input_cost_per_image": 0.042, "input_cost_per_pixel": 4.0054321e-08, "litellm_provider": "openai", @@ -28377,6 +28531,7 @@ ] }, "medium/1024-x-1536/gpt-image-1": { + "deprecation_date": "2026-10-23", "input_cost_per_image": 0.063, "input_cost_per_pixel": 4.0054321e-08, "litellm_provider": "openai", @@ -28387,6 +28542,7 @@ ] }, "medium/1536-x-1024/gpt-image-1": { + "deprecation_date": "2026-10-23", "input_cost_per_image": 0.063, "input_cost_per_pixel": 4.0054321e-08, "litellm_provider": "openai", @@ -28397,6 +28553,7 @@ ] }, "low/1024-x-1024/gpt-image-1-mini": { + "deprecation_date": "2026-12-01", "input_cost_per_image": 0.005, "litellm_provider": "openai", "mode": "image_generation", @@ -28405,6 +28562,7 @@ ] }, "low/1024-x-1536/gpt-image-1-mini": { + "deprecation_date": "2026-12-01", "input_cost_per_image": 0.006, "litellm_provider": "openai", "mode": "image_generation", @@ -28413,6 +28571,7 @@ ] }, "low/1536-x-1024/gpt-image-1-mini": { + "deprecation_date": "2026-12-01", "input_cost_per_image": 0.006, "litellm_provider": "openai", "mode": "image_generation", @@ -28421,6 +28580,7 @@ ] }, "medium/1024-x-1024/gpt-image-1-mini": { + "deprecation_date": "2026-12-01", "input_cost_per_image": 0.011, "litellm_provider": "openai", "mode": "image_generation", @@ -28429,6 +28589,7 @@ ] }, "medium/1024-x-1536/gpt-image-1-mini": { + "deprecation_date": "2026-12-01", "input_cost_per_image": 0.015, "litellm_provider": "openai", "mode": "image_generation", @@ -28437,6 +28598,7 @@ ] }, "medium/1536-x-1024/gpt-image-1-mini": { + "deprecation_date": "2026-12-01", "input_cost_per_image": 0.015, "litellm_provider": "openai", "mode": "image_generation", @@ -30330,6 +30492,7 @@ ] }, "multimodalembedding@001": { + "deprecation_date": "2027-04-01", "input_cost_per_character": 2e-07, "input_cost_per_image": 0.0001, "input_cost_per_token": 8e-07, @@ -36028,18 +36191,21 @@ "output_cost_per_image": 0.14 }, "standard/1024-x-1024/dall-e-3": { + "deprecation_date": "2026-05-12", "input_cost_per_pixel": 3.81469e-08, "litellm_provider": "openai", "mode": "image_generation", "output_cost_per_pixel": 0.0 }, "standard/1024-x-1792/dall-e-3": { + "deprecation_date": "2026-05-12", "input_cost_per_pixel": 4.359e-08, "litellm_provider": "openai", "mode": "image_generation", "output_cost_per_pixel": 0.0 }, "standard/1792-x-1024/dall-e-3": { + "deprecation_date": "2026-05-12", "input_cost_per_pixel": 4.359e-08, "litellm_provider": "openai", "mode": "image_generation", @@ -36103,6 +36269,7 @@ "source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models" }, "text-embedding-005": { + "deprecation_date": "2027-04-01", "input_cost_per_character": 2.5e-08, "input_cost_per_token": 1e-07, "litellm_provider": "vertex_ai-embedding-models", @@ -36176,6 +36343,7 @@ "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing" }, "text-moderation-007": { + "deprecation_date": "2025-10-27", "input_cost_per_token": 0.0, "litellm_provider": "openai", "max_input_tokens": 32768, @@ -36185,6 +36353,7 @@ "output_cost_per_token": 0.0 }, "text-moderation-latest": { + "deprecation_date": "2025-10-27", "input_cost_per_token": 0.0, "litellm_provider": "openai", "max_input_tokens": 32768, @@ -36194,6 +36363,7 @@ "output_cost_per_token": 0.0 }, "text-moderation-stable": { + "deprecation_date": "2025-10-27", "input_cost_per_token": 0.0, "litellm_provider": "openai", "max_input_tokens": 32768, @@ -36203,6 +36373,7 @@ "output_cost_per_token": 0.0 }, "text-multilingual-embedding-002": { + "deprecation_date": "2027-04-01", "input_cost_per_character": 2.5e-08, "input_cost_per_token": 1e-07, "litellm_provider": "vertex_ai-embedding-models", @@ -38690,6 +38861,7 @@ "supports_tool_choice": true }, "vertex_ai/claude-haiku-4-5": { + "deprecation_date": "2026-10-15", "cache_creation_input_token_cost": 1.25e-06, "cache_creation_input_token_cost_above_1hr": 2e-06, "cache_read_input_token_cost": 1e-07, @@ -38713,6 +38885,7 @@ "prompt_cache_min_tokens": 4096 }, "vertex_ai/claude-haiku-4-5@20251001": { + "deprecation_date": "2026-10-15", "cache_creation_input_token_cost": 1.25e-06, "cache_creation_input_token_cost_above_1hr": 2e-06, "cache_read_input_token_cost": 1e-07, @@ -38865,6 +39038,7 @@ "supports_vision": true }, "vertex_ai/claude-opus-4": { + "deprecation_date": "2026-05-14", "cache_creation_input_token_cost": 1.875e-05, "cache_creation_input_token_cost_above_1hr": 3e-05, "cache_read_input_token_cost": 1.5e-06, @@ -38892,6 +39066,7 @@ "prompt_cache_min_tokens": 1024 }, "vertex_ai/claude-opus-4-1": { + "deprecation_date": "2026-08-05", "cache_creation_input_token_cost": 1.875e-05, "cache_creation_input_token_cost_above_1hr": 3e-05, "cache_read_input_token_cost": 1.5e-06, @@ -38910,6 +39085,7 @@ "supports_vision": true }, "vertex_ai/claude-opus-4-1@20250805": { + "deprecation_date": "2026-08-05", "cache_creation_input_token_cost": 1.875e-05, "cache_creation_input_token_cost_above_1hr": 3e-05, "cache_read_input_token_cost": 1.5e-06, @@ -38928,6 +39104,7 @@ "supports_vision": true }, "vertex_ai/claude-opus-4-5": { + "deprecation_date": "2026-11-24", "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, @@ -38956,6 +39133,7 @@ "prompt_cache_min_tokens": 4096 }, "vertex_ai/claude-opus-4-5@20251101": { + "deprecation_date": "2026-11-24", "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, @@ -38985,6 +39163,7 @@ "prompt_cache_min_tokens": 4096 }, "vertex_ai/claude-opus-4-6": { + "deprecation_date": "2027-02-05", "supports_adaptive_thinking": true, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, @@ -39015,6 +39194,7 @@ "prompt_cache_min_tokens": 4096 }, "vertex_ai/claude-opus-4-6@default": { + "deprecation_date": "2027-02-05", "supports_adaptive_thinking": true, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, @@ -39045,6 +39225,7 @@ "prompt_cache_min_tokens": 4096 }, "vertex_ai/claude-opus-4-7": { + "deprecation_date": "2027-04-16", "supports_adaptive_thinking": true, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, @@ -39076,6 +39257,7 @@ "prompt_cache_min_tokens": 2048 }, "vertex_ai/claude-opus-4-7@default": { + "deprecation_date": "2027-04-16", "supports_adaptive_thinking": true, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, @@ -39107,6 +39289,7 @@ "prompt_cache_min_tokens": 2048 }, "vertex_ai/claude-fable-5": { + "deprecation_date": "2027-06-08", "supports_mid_conversation_system": true, "cache_creation_input_token_cost": 1.25e-05, "cache_creation_input_token_cost_above_1hr": 2e-05, @@ -39138,6 +39321,7 @@ "supports_max_reasoning_effort": true }, "vertex_ai/claude-fable-5@default": { + "deprecation_date": "2027-06-08", "supports_mid_conversation_system": true, "cache_creation_input_token_cost": 1.25e-05, "cache_creation_input_token_cost_above_1hr": 2e-05, @@ -39169,6 +39353,7 @@ "supports_max_reasoning_effort": true }, "vertex_ai/claude-opus-5": { + "deprecation_date": "2027-01-24", "supports_mid_conversation_system": true, "supports_adaptive_thinking": true, "cache_creation_input_token_cost": 6.25e-06, @@ -39201,6 +39386,7 @@ "prompt_cache_min_tokens": 512 }, "vertex_ai/claude-opus-5@default": { + "deprecation_date": "2027-01-24", "supports_mid_conversation_system": true, "supports_adaptive_thinking": true, "cache_creation_input_token_cost": 6.25e-06, @@ -39233,6 +39419,7 @@ "prompt_cache_min_tokens": 512 }, "vertex_ai/claude-opus-4-8": { + "deprecation_date": "2027-05-28", "supports_mid_conversation_system": true, "supports_adaptive_thinking": true, "cache_creation_input_token_cost": 6.25e-06, @@ -39265,6 +39452,7 @@ "prompt_cache_min_tokens": 1024 }, "vertex_ai/claude-opus-4-8@default": { + "deprecation_date": "2027-05-28", "supports_mid_conversation_system": true, "supports_adaptive_thinking": true, "cache_creation_input_token_cost": 6.25e-06, @@ -39297,6 +39485,7 @@ "prompt_cache_min_tokens": 1024 }, "vertex_ai/claude-sonnet-4-5": { + "deprecation_date": "2026-09-29", "cache_creation_input_token_cost": 3.75e-06, "cache_creation_input_token_cost_above_1hr": 6e-06, "cache_read_input_token_cost": 3e-07, @@ -39325,6 +39514,7 @@ "prompt_cache_min_tokens": 1024 }, "vertex_ai/claude-sonnet-5": { + "deprecation_date": "2026-12-24", "supports_mid_conversation_system": true, "cache_creation_input_token_cost": 2.5e-06, "cache_creation_input_token_cost_above_1hr": 4e-06, @@ -39387,6 +39577,7 @@ "prompt_cache_min_tokens": 1024 }, "vertex_ai/claude-sonnet-4-5@20250929": { + "deprecation_date": "2026-09-29", "cache_creation_input_token_cost": 3.75e-06, "cache_creation_input_token_cost_above_1hr": 6e-06, "cache_read_input_token_cost": 3e-07, @@ -39416,6 +39607,7 @@ "prompt_cache_min_tokens": 1024 }, "vertex_ai/claude-opus-4@20250514": { + "deprecation_date": "2026-05-14", "cache_creation_input_token_cost": 1.875e-05, "cache_creation_input_token_cost_above_1hr": 3e-05, "cache_read_input_token_cost": 1.5e-06, @@ -39443,6 +39635,7 @@ "prompt_cache_min_tokens": 1024 }, "vertex_ai/claude-sonnet-4": { + "deprecation_date": "2026-05-14", "cache_creation_input_token_cost": 3.75e-06, "cache_creation_input_token_cost_above_1hr": 6e-06, "cache_read_input_token_cost": 3e-07, @@ -39474,6 +39667,7 @@ "prompt_cache_min_tokens": 1024 }, "vertex_ai/claude-sonnet-4@20250514": { + "deprecation_date": "2026-05-14", "cache_creation_input_token_cost": 3.75e-06, "cache_creation_input_token_cost_above_1hr": 6e-06, "cache_read_input_token_cost": 3e-07, @@ -39638,6 +39832,7 @@ "supports_tool_choice": true }, "vertex_ai/gemini-2.5-flash-image": { + "deprecation_date": "2026-10-02", "cache_read_input_token_cost": 3e-08, "input_cost_per_audio_token": 1e-06, "input_cost_per_token": 3e-07, @@ -39683,6 +39878,7 @@ "supports_image_size": false }, "vertex_ai/gemini-3-pro-image": { + "deprecation_date": "2027-05-28", "input_cost_per_image": 0.0011, "input_cost_per_token": 2e-06, "input_cost_per_token_batches": 1e-06, @@ -39715,6 +39911,7 @@ "source": "https://docs.cloud.google.com/vertex-ai/generative-ai/docs/models/gemini/3-pro-image" }, "vertex_ai/gemini-3.1-flash-image": { + "deprecation_date": "2027-05-28", "input_cost_per_image": 0.00056, "input_cost_per_token": 5e-07, "litellm_provider": "vertex_ai-language-models", @@ -39791,6 +39988,7 @@ "web_search_billing_unit": "per_query" }, "vertex_ai/gemini-3.1-flash-lite": { + "deprecation_date": "2027-05-07", "cache_read_input_token_cost": 2.5e-08, "cache_read_input_token_cost_flex": 1.25e-08, "cache_read_input_token_cost_priority": 4.5e-08, @@ -39847,6 +40045,7 @@ "web_search_billing_unit": "per_query" }, "vertex_ai/gemini-3.5-flash-lite": { + "deprecation_date": "2027-07-21", "cache_read_input_token_cost": 3e-08, "cache_read_input_token_cost_flex": 2e-08, "cache_read_input_token_cost_priority": 5e-08, @@ -40564,6 +40763,7 @@ "supports_tool_choice": true }, "vertex_ai/veo-2.0-generate-001": { + "deprecation_date": "2026-06-30", "litellm_provider": "vertex_ai-video-models", "max_input_tokens": 1024, "max_tokens": 1024, @@ -40578,6 +40778,7 @@ ] }, "vertex_ai/veo-3.0-fast-generate-001": { + "deprecation_date": "2026-06-30", "litellm_provider": "vertex_ai-video-models", "max_input_tokens": 1024, "max_tokens": 1024, @@ -40592,6 +40793,7 @@ ] }, "vertex_ai/veo-3.0-generate-001": { + "deprecation_date": "2026-06-30", "litellm_provider": "vertex_ai-video-models", "max_input_tokens": 1024, "max_tokens": 1024, @@ -40634,6 +40836,7 @@ ] }, "vertex_ai/veo-3.1-generate-001": { + "deprecation_date": "2026-11-17", "litellm_provider": "vertex_ai-video-models", "max_input_tokens": 1024, "max_tokens": 1024, @@ -40648,6 +40851,7 @@ ] }, "vertex_ai/veo-3.1-fast-generate-001": { + "deprecation_date": "2026-11-17", "litellm_provider": "vertex_ai-video-models", "max_input_tokens": 1024, "max_tokens": 1024, @@ -47029,6 +47233,7 @@ } }, "vertex_ai/claude-sonnet-5@default": { + "deprecation_date": "2026-12-24", "supports_mid_conversation_system": true, "cache_creation_input_token_cost": 2.5e-06, "cache_creation_input_token_cost_above_1hr": 4e-06, diff --git a/schema.prisma b/schema.prisma index 52fb447157b..f79e2bb0c18 100644 --- a/schema.prisma +++ b/schema.prisma @@ -1467,28 +1467,38 @@ model LiteLLM_AutoRouterSession { @@index([last_turn_at], map: "idx_autorouter_session_last_turn") } -// Shadow eval: evaluation of an auto-router against a key's live traffic, in either -// direction. forward duplicates the requests the key did not route through the router -// through it, answering whether the key should adopt it; reverse duplicates the requests -// the router did serve against a fixed baseline model, answering whether a key already on -// it still benefits. Either way a sampled slice runs in a detached task and an LLM judge -// compares real vs shadow responses blind. The job row is immutable config plus -// stopped_at; every count, status, and spend figure is derived from the append-only -// attempt rows, so nothing can disagree across pods or stop races. +// Shadow eval: evaluation of an auto-router against one or more keys' live traffic, in +// either direction. forward duplicates the requests the keys did not route through the +// router through it, answering whether they should adopt it; reverse duplicates the +// requests the router did serve against a fixed baseline model, answering whether a key +// already on it still benefits. Either way a sampled slice runs in a detached task and an +// LLM judge compares real vs shadow responses blind. Each row is ONE key's leg of a job: +// immutable config plus that key's own turn budget and stop state, so one key exhausting +// its budget never ends a sibling's sampling. A job is the set of legs sharing group_id +// (the id the API reports), written together by one atomic create_many with identical +// config; single-key jobs predating group_id were backfilled group_id = id. "One active +// job per (key, direction)" is a partial unique index on (api_key_id, direction) WHERE +// stopped_at IS NULL, expressed only in the migration because schema.prisma cannot state +// partial indexes; it is what makes a concurrent start on another pod race-safe rather +// than read-then-create. Every count, status, and spend figure is derived from the +// append-only attempt rows, so nothing can disagree across pods or stop races. model LiteLLM_ShadowEvalJob { id String @id @default(cuid()) - api_key_id String // hashed virtual key whose traffic is shadowed + group_id String // legs of one job share this; the API's job id + api_key_id String // hashed virtual key whose traffic this leg shadows router_name String // the auto-router under evaluation, in either direction direction String @default("forward") // forward | reverse baseline_model String? // reverse only: the fixed model the router is judged against judge_model String shadow_percentage Float - max_turns Int // sample budget: judge at most this many turns + max_turns Int // this key's sample budget: judge at most this many turns created_at DateTime @default(now()) created_by String? ends_at DateTime stopped_at DateTime? + stopped_by String? // operator who stopped it early; null when it ended on its own + @@index([group_id]) @@index([api_key_id]) @@index([created_at]) } diff --git a/tests/e2e/CLAUDE.md b/tests/e2e/CLAUDE.md index 680e0dff67b..9969ed10308 100644 --- a/tests/e2e/CLAUDE.md +++ b/tests/e2e/CLAUDE.md @@ -71,6 +71,16 @@ Request and response bodies are typed pydantic models in `models.py`; only the f Mark live tests with `@pytest.mark.e2e` (on the class or the module). Pure coverage of the harness itself carries no marker and runs regardless. Use `scoped_key` for a fresh all-models key that auto-deletes, `resources` when you need to create and tear down more than a key, and `unique_marker()` from `e2e_config` to keep prompts, tags, and customer ids from colliding across concurrent runs and the shared response cache +## Record and replay fixtures + +`E2E_FIXTURE_MODE` selects the transport every client is built on: `live` (the default, and what an unset variable means: nothing changes), `record` (run against the live proxy and write every interaction to a fixture bundle), or `replay` (serve every interaction back from the bundle with no HTTP at all, so a replay run needs no proxy and cannot bill a provider). The seam is `select_transport` in `fixture_transport.py`, applied inside `build_proxy_client`; both transports fulfil the same `Transport` protocol, so no test or client changes shape in any mode + +A bundle (default `tests/e2e/.fixtures`, override with `E2E_FIXTURE_DIR`) is a directory: `manifest.json` carries the record timestamp, harness git version, and format version, and each test gets a subdirectory holding one JSON file per transport call in call order (`0000-post-chat-completions.json`). Auth header values are redacted on write, and file uploads store a sha256 digest instead of the bytes; response bodies are stored verbatim (a /key/generate response keeps the ephemeral virtual key it minted), which is part of why bundles are gitignored. `fixture_bundle.py` owns the format + +Replay matches calls per test by transport verb and path in recorded order and raises `ReplayMiss` on any drift, naming the recorded and the actual call; a passed test must also consume its whole recording, or teardown fails it naming the first leftover interaction. Either way the fix is always to re-record with `E2E_FIXTURE_MODE=record`. Record starts fresh every time: it wipes the previous bundle (refusing to wipe a directory that is not a bundle) and never reads it. A replay bundle whose manifest is older than seven days hard-fails at collection time naming the bundle's age, so replay can never certify against fixtures that have drifted more than a week from the live proxy + +Deliberately not here yet: canonical content-based match keys (LIT-5741), streaming chunk fidelity (LIT-5742), and scoping record/replay to provider-bound traffic (LIT-5745) + ## Typing The harness is fully typed with no error budget: `make lint-e2e-basedpyright` must report zero basedpyright errors, and CI enforces that on any PR touching `tests/e2e/**/*.py`. When a response field is untyped, model it in `models.py` (just the fields you read) and let pydantic validate it, rather than threading a `dict` or `Any` through the test diff --git a/tests/e2e/CONTRIBUTING.md b/tests/e2e/CONTRIBUTING.md index dc69bd42171..67da1be9562 100644 --- a/tests/e2e/CONTRIBUTING.md +++ b/tests/e2e/CONTRIBUTING.md @@ -52,6 +52,17 @@ The suites run against a live proxy, so bring one up first by running the litell Some suites need extra services the bare proxy does not start. The `logging/` OTEL trace-completeness tests read spans back from a jaeger query API at `http://localhost:16686` (override with `E2E_OTEL_QUERY_URL`); run a `jaegertracing/all-in-one` and point `PHOENIX_COLLECTOR_HTTP_ENDPOINT` at its OTLP ingest. The `mcp/` suite needs the deterministic upstream MCP server in `mcp_tests/mcp_e2e_upstream_server.py` reachable by the proxy +### Record and replay + +`E2E_FIXTURE_MODE=record` runs a suite against the live proxy as usual while writing every request/response pair to a fixture bundle (default `tests/e2e/.fixtures`, override with `E2E_FIXTURE_DIR`); `E2E_FIXTURE_MODE=replay` then runs the same suite entirely from that bundle, with no proxy traffic and no provider spend; the proxy liveness gate is skipped, so replay runs with no proxy up at all. Unset (or `live`) behaves exactly as before the knob existed + +```bash +E2E_FIXTURE_MODE=record uv run pytest tests/e2e/llm_translation/ -v +E2E_FIXTURE_MODE=replay uv run pytest tests/e2e/llm_translation/ -v +``` + +Replay fails hard (`ReplayMiss`) when the tests drift from the recording, and a bundle older than seven days fails at collection time naming its age; either way the fix is to re-record. See `CLAUDE.md` in this directory for the bundle format and the transport seam + Tests marked `@pytest.mark.e2e` hard-fail when no proxy answers `/health/liveliness`, so a run that goes red with `No live proxy` at setup means the proxy isn't up; they never skip for a missing proxy, so an absent proxy can't be mistaken for a pass ## What a complete test looks like diff --git a/tests/e2e/conftest.py b/tests/e2e/conftest.py index eff3b4ddf58..da2a7da0bfa 100644 --- a/tests/e2e/conftest.py +++ b/tests/e2e/conftest.py @@ -15,19 +15,27 @@ shared fixtures build on it. import functools import os -from collections.abc import Iterator +from collections.abc import Generator, Iterator +from datetime import datetime, timezone import pytest import requests -from e2e_config import CONTROL_PLANE_BASE_URL, PROXY_BASE_URL +from e2e_config import CONTROL_PLANE_BASE_URL, FIXTURE_DIR, FIXTURE_MODE_RAW, PROXY_BASE_URL from e2e_db import RESET_OPT_IN_ENV, reset_spend_logs, run_spend_log_cleanup +from fixture_transport import ( + fixture_mode_collection_error, + fixture_report_lines, + parse_fixture_mode, + replay_leftover_error, +) from junit_properties import attach_result_properties from lifecycle import ProxyClientProvider, ResourceManager from proxy_client import ProxyClient, build_proxy_client _E2E_TEST_RAN = pytest.StashKey[bool]() +_CALL_PASSED = pytest.StashKey[bool]() def pytest_configure(config: pytest.Config) -> None: @@ -49,6 +57,21 @@ def pytest_configure(config: pytest.Config) -> None: ) +def pytest_sessionstart(session: pytest.Session) -> None: + """Abort before collection when E2E_FIXTURE_MODE can never work: an unknown + mode value, or replay against a missing, unreadable, or stale bundle (the + stale message names the bundle's age). Live and record modes pass through.""" + reason = fixture_mode_collection_error( + FIXTURE_MODE_RAW, FIXTURE_DIR, now=datetime.now(timezone.utc) + ) + if reason is not None: + raise pytest.UsageError(reason) + + +def pytest_report_header(config: pytest.Config) -> list[str]: + return fixture_report_lines(FIXTURE_MODE_RAW, FIXTURE_DIR, now=datetime.now(timezone.utc)) + + def pytest_collection_modifyitems(items: list[pytest.Item]) -> None: """Attach the two custom signals (suite package and covered cell ids) to every test's user_properties so the standard JUnit report (`--junitxml`) records them @@ -91,9 +114,12 @@ def _proxy_fail_reason() -> str | None: def pytest_runtest_setup(item: pytest.Item) -> None: """Hard-fail `e2e`-marked tests unless a proxy answers its liveness probe. Unmarked tests (unit coverage of the harness) don't touch the proxy, so they - run even when none is up. Never skip for a missing proxy.""" + run even when none is up. Never skip for a missing proxy. Replay mode serves + every call from the fixture bundle, so it needs no live proxy either.""" if item.get_closest_marker("e2e") is None: return + if parse_fixture_mode(FIXTURE_MODE_RAW) == "replay": + return reason = _proxy_fail_reason() if reason is not None: pytest.fail(reason) @@ -110,6 +136,36 @@ def pytest_runtest_call(item: pytest.Item) -> None: item.session.stash[_E2E_TEST_RAN] = True +@pytest.hookimpl(wrapper=True) +def pytest_runtest_makereport( + item: pytest.Item, call: pytest.CallInfo[None] +) -> Generator[None, pytest.TestReport, pytest.TestReport]: + """Stash the call-phase outcome so teardown can tell a passed test from a + failed one without re-deriving it.""" + report = yield + if report.when == "call": + item.stash[_CALL_PASSED] = report.passed + return report + + +@pytest.hookimpl(wrapper=True) +def pytest_runtest_teardown(item: pytest.Item) -> Generator[None, None, None]: + """In replay mode a passing test must consume its whole recording: leftover + interactions mean the test now makes fewer calls than it did at record time, + so the replay proved less than the bundle claims. The check runs after the + yield so fixture finalizers replay their recorded calls first. Failed tests + are left alone - their own failure already explains any unconsumed tail.""" + result = yield + if not item.stash.get(_CALL_PASSED, False): + return result + reason = replay_leftover_error( + mode_raw=FIXTURE_MODE_RAW, bundle_dir=FIXTURE_DIR, test_key=item.nodeid + ) + if reason is not None: + pytest.fail(reason) + return result + + def pytest_sessionfinish(session: pytest.Session, exitstatus: int) -> None: """Once the whole e2e session is done (all suites), optionally truncate the spend logs so the DB doesn't accumulate test rows. The truncate is destructive diff --git a/tests/e2e/e2e_config.py b/tests/e2e/e2e_config.py index 277478eebaf..a5c3729f4be 100644 --- a/tests/e2e/e2e_config.py +++ b/tests/e2e/e2e_config.py @@ -13,6 +13,8 @@ from pathlib import Path from dotenv import load_dotenv +from fixture_transport import deterministic_marker, parse_fixture_mode + # Local runs keep provider / DataDog keys in tests/e2e/.env (see CONTRIBUTING.md). # Compose injects them into the proxy container, but pytest on the host does not # inherit that file unless we load it. override=False so a real shell export wins. @@ -90,6 +92,15 @@ PROPAGATION_TIMEOUT = float(os.environ.get("E2E_PROPAGATION_TIMEOUT", "15")) EXPECT_RUST = os.environ.get("E2E_EXPECT_RUST", "").strip().lower() in ("1", "true", "yes") +# Record/replay fixture selection (see fixture_transport.py). The raw mode value +# is parsed and validated there; "live" (the default, also for empty values) +# means the harness behaves exactly as before this knob existed. +FIXTURE_MODE_RAW = os.environ.get("E2E_FIXTURE_MODE", "live") +FIXTURE_DIR = Path( + os.environ.get("E2E_FIXTURE_DIR", "").strip() + or str(Path(__file__).resolve().parent / ".fixtures") +) + # Deliberately modest concurrency. The suite shares its proxy with every other # suite in the run, and 750 users at spawn rate 50 saturated the request path hard # enough to distort latency-sensitive neighbours (and to spend real provider money @@ -148,7 +159,11 @@ def datadog_mcp_url(*, toolsets: str = "core") -> str: def unique_marker() -> str: """A short unique token per call/run, so concurrent runs and the shared - response cache never collide on prompts, tags, or customer ids.""" + response cache never collide on prompts, tags, or customer ids. In record + and replay modes the token is deterministic per test instead, so a replay + run regenerates the exact requests the record run sent.""" + if parse_fixture_mode(FIXTURE_MODE_RAW) in ("record", "replay"): + return deterministic_marker() return uuid.uuid4().hex[:12] diff --git a/tests/e2e/fixture_bundle.py b/tests/e2e/fixture_bundle.py new file mode 100644 index 00000000000..5eff2cf2876 --- /dev/null +++ b/tests/e2e/fixture_bundle.py @@ -0,0 +1,314 @@ +"""On-disk fixture bundle format for record/replay e2e runs (LIT-5729). + +A bundle is a directory: one ``manifest.json`` (record timestamp + harness +version + format version) plus one subdirectory per test, holding one JSON file +per transport interaction in call order. Bundles older than +``MAX_BUNDLE_AGE`` hard-fail replay at collection time (see conftest), so a +green replay run can never certify against fixtures that have drifted more than +a week from the live proxy. + +This module owns the format only. The transports that produce and consume it +live in fixture_transport.py; canonical request matching, streaming chunk +fidelity, and provider-scoping are follow-ups (LIT-5741/5742/5745) and are +deliberately absent here, which is why every interaction file stores the full +redacted request even though replay today matches by call order. +""" + +from __future__ import annotations + +import hashlib +import re +import shutil +import subprocess +from dataclasses import dataclass, field +from datetime import datetime, timedelta, timezone +from pathlib import Path +from typing import Annotated, Final, Literal + +from pydantic import BaseModel, Field, JsonValue, TypeAdapter + +from e2e_http import ( + BinaryStream, + NetworkError, + ProbeResult, + RateLimitedError, + Result, + StreamingResponse, + Success, + UnauthorizedError, + UnknownApiError, + ValidationError, +) + +BUNDLE_FORMAT_VERSION: Final = 1 +MAX_BUNDLE_AGE: Final = timedelta(days=7) +MANIFEST_FILENAME: Final = "manifest.json" + +_JSON: Final[TypeAdapter[JsonValue]] = TypeAdapter(JsonValue) + + +class Manifest(BaseModel): + format_version: int + recorded_at: datetime + harness_version: str + + +class RecordedRequest(BaseModel): + """The request as the transport saw it, auth header values redacted. + + Replay today only matches ``method`` (the transport verb, not the HTTP verb) + and ``path`` in call order; the rest is stored so LIT-5741 can move to + content-based match keys without re-recording. File uploads store a content + digest instead of the bytes.""" + + method: str + path: str + headers: dict[str, str] + params: dict[str, str] = {} + body: JsonValue | None = None + form: dict[str, str] | None = None + file_name: str | None = None + file_sha256: str | None = None + file_bytes: int | None = None + + +class RecordedResult(BaseModel): + """A ``Result[R]`` flattened for disk. ``data`` holds the success payload as + raw JSON; replay re-validates it against the ``response_type`` the caller + passes, exactly like a live response body.""" + + shape: Literal["result"] = "result" + kind: Literal["success", "network", "unauthorized", "rate_limited", "validation", "unknown"] + status_code: int | None = None + data: JsonValue | None = None + message: str | None = None + body: str | None = None + retry_after_seconds: int | None = None + + +class RecordedStreaming(BaseModel): + shape: Literal["streaming"] = "streaming" + payload: StreamingResponse + + +class RecordedBinary(BaseModel): + shape: Literal["binary"] = "binary" + payload: BinaryStream + + +class RecordedProbe(BaseModel): + shape: Literal["probe"] = "probe" + payload: ProbeResult + + +type RecordedResponse = RecordedResult | RecordedStreaming | RecordedBinary | RecordedProbe + + +class Interaction(BaseModel): + request: RecordedRequest + response: Annotated[ + RecordedResult | RecordedStreaming | RecordedBinary | RecordedProbe, + Field(discriminator="shape"), + ] + + +def to_json_value(model: BaseModel) -> JsonValue: + return _JSON.validate_json(model.model_dump_json(by_alias=True)) + + +def from_result[R: BaseModel](result: Result[R]) -> RecordedResult: + match result: + case Success(status_code=status_code, data=data): + return RecordedResult(kind="success", status_code=status_code, data=to_json_value(data)) + case NetworkError(message=message): + return RecordedResult(kind="network", message=message) + case UnauthorizedError(): + return RecordedResult(kind="unauthorized") + case RateLimitedError(retry_after_seconds=retry_after_seconds, body=body): + return RecordedResult(kind="rate_limited", retry_after_seconds=retry_after_seconds, body=body) + case ValidationError(message=message): + return RecordedResult(kind="validation", message=message) + case UnknownApiError(status_code=status_code, body=body): + return RecordedResult(kind="unknown", status_code=status_code, body=body) + + +def to_result[R: BaseModel](recorded: RecordedResult, response_type: type[R]) -> Result[R]: + match recorded.kind: + case "success": + return Success( + status_code=recorded.status_code or 200, + data=response_type.model_validate(recorded.data), + ) + case "network": + return NetworkError(message=recorded.message or "") + case "unauthorized": + return UnauthorizedError() + case "rate_limited": + return RateLimitedError( + retry_after_seconds=recorded.retry_after_seconds, body=recorded.body or "" + ) + case "validation": + return ValidationError(message=recorded.message or "") + case "unknown": + return UnknownApiError(status_code=recorded.status_code or 0, body=recorded.body or "") + + +def slugify(raw: str, *, limit: int = 60) -> str: + clean = re.sub(r"[^A-Za-z0-9_.-]+", "-", raw).strip("-") + return clean[:limit].rstrip("-") + + +def slug_for_test(test_key: str) -> str: + """Directory name for one test's interactions: a readable tail plus a short + digest of the full node id, so same-named methods in different classes or + files never collide.""" + digest = hashlib.sha1(test_key.encode()).hexdigest()[:8] + tail = slugify(test_key.rsplit("::", 1)[-1]) + return f"{tail}-{digest}" if tail else digest + + +def interaction_filename(ordinal: int, request: RecordedRequest) -> str: + path_part = slugify(request.path, limit=40) or "root" + return f"{ordinal:04d}-{request.method}-{path_part}.json" + + +def harness_version() -> str: + try: + proc = subprocess.run( + ("git", "rev-parse", "--short", "HEAD"), + cwd=Path(__file__).resolve().parent, + capture_output=True, + text=True, + timeout=10, + check=False, + ) + except (OSError, subprocess.SubprocessError): + return "unknown" + return proc.stdout.strip() or "unknown" + + +@dataclass(slots=True) +class BundleRecorder: + """Appends interaction files under ``root``, one subdirectory per test, with + a per-test ordinal that fixes replay order. ``prepare_bundle`` is the only + constructor: it guarantees the directory started empty with a fresh + manifest, so record mode never reads (or merges into) an existing bundle.""" + + root: Path + _ordinals: dict[str, int] = field(default_factory=dict) + + def record(self, *, test_key: str, request: RecordedRequest, response: RecordedResponse) -> None: + slug = slug_for_test(test_key) + ordinal = self._ordinals.get(slug, 0) + self._ordinals[slug] = ordinal + 1 + directory = self.root / slug + directory.mkdir(parents=True, exist_ok=True) + interaction = Interaction(request=request, response=response) + target = directory / interaction_filename(ordinal, request) + target.write_text(interaction.model_dump_json(indent=2), encoding="utf-8") + + +@dataclass(frozen=True, slots=True) +class UnsafeBundleDir: + path: Path + reason: str + + +def prepare_bundle(root: Path) -> BundleRecorder | UnsafeBundleDir: + """Start a fresh bundle at ``root`` for record mode: wipe whatever bundle is + there and write a new manifest. Refuses to wipe a directory that is neither + empty nor a bundle (no manifest.json), so a mistyped E2E_FIXTURE_DIR can + never delete unrelated files.""" + if root.exists(): + if not root.is_dir(): + return UnsafeBundleDir(path=root, reason="exists and is not a directory") + entries = tuple(root.iterdir()) + if entries and not (root / MANIFEST_FILENAME).is_file(): + return UnsafeBundleDir( + path=root, + reason=f"is not empty and has no {MANIFEST_FILENAME}; refusing to wipe a non-bundle directory", + ) + shutil.rmtree(root) + root.mkdir(parents=True) + manifest = Manifest( + format_version=BUNDLE_FORMAT_VERSION, + recorded_at=datetime.now(timezone.utc), + harness_version=harness_version(), + ) + (root / MANIFEST_FILENAME).write_text(manifest.model_dump_json(indent=2), encoding="utf-8") + return BundleRecorder(root=root) + + +@dataclass(frozen=True, slots=True) +class FreshBundle: + manifest: Manifest + + +@dataclass(frozen=True, slots=True) +class StaleBundle: + recorded_at: datetime + age: timedelta + limit: timedelta + + +@dataclass(frozen=True, slots=True) +class UnreadableBundle: + reason: str + + +type BundleFreshness = FreshBundle | StaleBundle | UnreadableBundle + + +def _read_manifest(root: Path) -> Manifest | UnreadableBundle: + manifest_path = root / MANIFEST_FILENAME + if not manifest_path.is_file(): + return UnreadableBundle(reason=f"no {MANIFEST_FILENAME} found (record one with E2E_FIXTURE_MODE=record)") + try: + return Manifest.model_validate_json(manifest_path.read_text(encoding="utf-8")) + except ValueError as exc: + return UnreadableBundle(reason=f"{MANIFEST_FILENAME} is invalid: {exc}") + + +def check_freshness(root: Path, *, now: datetime) -> BundleFreshness: + manifest = _read_manifest(root) + if isinstance(manifest, UnreadableBundle): + return manifest + if manifest.format_version != BUNDLE_FORMAT_VERSION: + return UnreadableBundle( + reason=f"format_version {manifest.format_version} != supported {BUNDLE_FORMAT_VERSION}" + ) + recorded_at = ( + manifest.recorded_at + if manifest.recorded_at.tzinfo is not None + else manifest.recorded_at.replace(tzinfo=timezone.utc) + ) + age = now - recorded_at + if age > MAX_BUNDLE_AGE: + return StaleBundle(recorded_at=recorded_at, age=age, limit=MAX_BUNDLE_AGE) + return FreshBundle(manifest=manifest) + + +def format_age(age: timedelta) -> str: + total_hours = int(age.total_seconds()) // 3600 + return f"{total_hours // 24}d{total_hours % 24}h" + + +@dataclass(frozen=True, slots=True) +class LoadedBundle: + manifest: Manifest + interactions: dict[str, tuple[Interaction, ...]] + + +def load_bundle(root: Path) -> LoadedBundle | UnreadableBundle: + manifest = _read_manifest(root) + if isinstance(manifest, UnreadableBundle): + return manifest + interactions = { + directory.name: tuple( + Interaction.model_validate_json(file.read_text(encoding="utf-8")) + for file in sorted(directory.glob("*.json")) + ) + for directory in sorted(root.iterdir()) + if directory.is_dir() + } + return LoadedBundle(manifest=manifest, interactions=interactions) diff --git a/tests/e2e/fixture_transport.py b/tests/e2e/fixture_transport.py new file mode 100644 index 00000000000..b99b0d2d80d --- /dev/null +++ b/tests/e2e/fixture_transport.py @@ -0,0 +1,574 @@ +"""Record/replay transports behind the same ``Transport`` protocol (LIT-5729). + +``RecordingTransport`` decorates the live transport: every call passes through +unchanged and its request/response pair is appended to the fixture bundle. +``ReplayTransport`` implements the protocol from a recorded bundle alone: no +HTTP, no proxy, no provider spend. Because both fulfil ``Transport``, no test +or client changes shape; ``build_proxy_client`` picks the transport from +``E2E_FIXTURE_MODE`` (live | record | replay, default live). + +Replay matches each call by test node id and call order, verifying transport +verb + path and failing hard on any drift (``ReplayMiss``). Canonical +content-based match keys are LIT-5741; streaming chunk fidelity is LIT-5742; +scoping record/replay to provider-bound traffic is LIT-5745. +""" + +from __future__ import annotations + +import functools +import hashlib +import os +from dataclasses import dataclass, field +from datetime import datetime +from pathlib import Path +from typing import Final, Literal, assert_never + +from pydantic import BaseModel + +from e2e_http import AuthHeaders, BinaryStream, ProbeResult, Result, StreamingResponse +from fixture_bundle import ( + BundleRecorder, + FreshBundle, + Interaction, + LoadedBundle, + RecordedBinary, + RecordedProbe, + RecordedRequest, + RecordedResponse, + RecordedResult, + RecordedStreaming, + StaleBundle, + UnreadableBundle, + UnsafeBundleDir, + check_freshness, + format_age, + from_result, + load_bundle, + prepare_bundle, + slug_for_test, + to_json_value, + to_result, +) +from transport import Transport + +type FixtureMode = Literal["live", "record", "replay"] + +FIXTURE_MODES: Final[tuple[FixtureMode, ...]] = ("live", "record", "replay") + +SESSION_TEST_KEY: Final = "session" + +REDACTED_HEADER_NAMES: Final[frozenset[str]] = frozenset({"authorization", "x-litellm-api-key"}) +REDACTED_VALUE: Final = "" + + +@dataclass(frozen=True, slots=True) +class InvalidFixtureMode: + value: str + + +def parse_fixture_mode(raw: str) -> FixtureMode | InvalidFixtureMode: + normalized = raw.strip().lower() or "live" + match normalized: + case "live" | "record" | "replay": + return normalized + case _: + return InvalidFixtureMode(value=raw) + + +def current_test_key() -> str: + """The pytest node id of the running test, from the PYTEST_CURRENT_TEST env + var pytest maintains (`` (setup|call|teardown)``); ``session`` for + calls outside any test (e.g. session-finish cleanup).""" + raw = os.environ.get("PYTEST_CURRENT_TEST", "") + if not raw: + return SESSION_TEST_KEY + return raw.rsplit(" (", 1)[0] + + +class ReplayMiss(AssertionError): + """Replay had no recorded interaction for a call the suite made. The test + drifted from the bundle (or the bundle from the suite): re-record.""" + + +_marker_ordinals: Final[dict[str, int]] = {} + + +def deterministic_marker() -> str: + """Stable stand-in for uuid-based unique markers in record and replay modes: + the Nth marker of a test is a pure function of the test's node id and N, so a + replay run regenerates exactly the model names, prompts, and tags the record + run sent and every recorded poll response still satisfies its predicate.""" + test_key = current_test_key() + ordinal = _marker_ordinals.get(test_key, 0) + _marker_ordinals[test_key] = ordinal + 1 + return hashlib.sha1(f"{test_key}#{ordinal}".encode()).hexdigest()[:12] + + +def _dump_flat(model: BaseModel | None) -> dict[str, str]: + if model is None: + return {} + dumped: dict[str, object] = model.model_dump(by_alias=True, exclude_none=True) + return {key: str(value) for key, value in dumped.items()} + + +def _redact(headers: dict[str, str]) -> dict[str, str]: + return { + name: REDACTED_VALUE if name.lower() in REDACTED_HEADER_NAMES else value + for name, value in headers.items() + } + + +def recorded_request( + method: str, + path: str, + *, + headers: BaseModel, + body: BaseModel | None = None, + params: BaseModel | None = None, + form: BaseModel | None = None, + file_name: str | None = None, + file_content: bytes | None = None, +) -> RecordedRequest: + return RecordedRequest( + method=method, + path=path, + headers=_redact(_dump_flat(headers)), + params=_dump_flat(params), + body=None if body is None else to_json_value(body), + form=None if form is None else _dump_flat(form), + file_name=file_name, + file_sha256=None if file_content is None else hashlib.sha256(file_content).hexdigest(), + file_bytes=None if file_content is None else len(file_content), + ) + + +@dataclass(frozen=True, slots=True) +class RecordingTransport: + """Decorator over the live transport: forwards every call and appends the + interaction to the bundle, so a green live run leaves behind exactly the + traffic replay needs.""" + + inner: Transport + recorder: BundleRecorder + + def _record(self, request: RecordedRequest, response: RecordedResponse) -> None: + self.recorder.record(test_key=current_test_key(), request=request, response=response) + + def bearer(self, key: str) -> AuthHeaders: + return self.inner.bearer(key) + + @property + def master(self) -> AuthHeaders: + return self.inner.master + + def post[R: BaseModel]( + self, path: str, *, headers: BaseModel, json: BaseModel, response_type: type[R] + ) -> Result[R]: + result = self.inner.post(path, headers=headers, json=json, response_type=response_type) + self._record(recorded_request("post", path, headers=headers, body=json), from_result(result)) + return result + + def get[R: BaseModel]( + self, + path: str, + *, + headers: BaseModel, + params: BaseModel, + response_type: type[R], + timeout: float | None = None, + ) -> Result[R]: + result = self.inner.get( + path, headers=headers, params=params, response_type=response_type, timeout=timeout + ) + self._record(recorded_request("get", path, headers=headers, params=params), from_result(result)) + return result + + def delete[R: BaseModel]( + self, + path: str, + *, + headers: BaseModel, + json: BaseModel, + response_type: type[R], + params: BaseModel | None = None, + ) -> Result[R]: + result = self.inner.delete( + path, headers=headers, json=json, response_type=response_type, params=params + ) + self._record( + recorded_request("delete", path, headers=headers, body=json, params=params), + from_result(result), + ) + return result + + def patch[R: BaseModel]( + self, path: str, *, headers: BaseModel, json: BaseModel, response_type: type[R] + ) -> Result[R]: + result = self.inner.patch(path, headers=headers, json=json, response_type=response_type) + self._record(recorded_request("patch", path, headers=headers, body=json), from_result(result)) + return result + + def put[R: BaseModel]( + self, path: str, *, headers: BaseModel, json: BaseModel, response_type: type[R] + ) -> Result[R]: + result = self.inner.put(path, headers=headers, json=json, response_type=response_type) + self._record(recorded_request("put", path, headers=headers, body=json), from_result(result)) + return result + + def stream(self, path: str, *, headers: BaseModel, json: BaseModel) -> StreamingResponse: + response = self.inner.stream(path, headers=headers, json=json) + self._record( + recorded_request("stream", path, headers=headers, body=json), + RecordedStreaming(payload=response), + ) + return response + + def stream_binary( + self, path: str, *, headers: BaseModel, json: BaseModel, chunk_size: int = 8192 + ) -> BinaryStream: + response = self.inner.stream_binary(path, headers=headers, json=json, chunk_size=chunk_size) + self._record( + recorded_request("stream_binary", path, headers=headers, body=json), + RecordedBinary(payload=response), + ) + return response + + def send( + self, + path: str, + *, + headers: BaseModel, + json: BaseModel, + params: BaseModel | None = None, + stream: bool = False, + ) -> StreamingResponse: + response = self.inner.send(path, headers=headers, json=json, params=params, stream=stream) + self._record( + recorded_request("send", path, headers=headers, body=json, params=params), + RecordedStreaming(payload=response), + ) + return response + + def probe(self, path: str, *, params: BaseModel) -> ProbeResult: + response = self.inner.probe(path, params=params) + self._record( + recorded_request("probe", path, headers=self.master, params=params), + RecordedProbe(payload=response), + ) + return response + + def upload[R: BaseModel]( + self, + path: str, + *, + headers: BaseModel, + form: BaseModel, + filename: str, + content: bytes, + file_content_type: str = "application/jsonl", + file_field: str = "file", + params: BaseModel | None = None, + response_type: type[R], + ) -> Result[R]: + result = self.inner.upload( + path, + headers=headers, + form=form, + filename=filename, + content=content, + file_content_type=file_content_type, + file_field=file_field, + params=params, + response_type=response_type, + ) + self._record( + recorded_request( + "upload", + path, + headers=headers, + params=params, + form=form, + file_name=filename, + file_content=content, + ), + from_result(result), + ) + return result + + def download(self, path: str, *, headers: BaseModel) -> StreamingResponse: + response = self.inner.download(path, headers=headers) + self._record( + recorded_request("download", path, headers=headers), + RecordedStreaming(payload=response), + ) + return response + + +@dataclass(slots=True) +class ReplaySource: + """One shared cursor set over a loaded bundle, so every client built in the + session consumes the same recorded sequence per test.""" + + bundle: LoadedBundle + _cursors: dict[str, int] = field(default_factory=dict) + + def next_interaction(self, method: str, path: str) -> Interaction: + test_key = current_test_key() + slug = slug_for_test(test_key) + recorded = self.bundle.interactions.get(slug, ()) + index = self._cursors.get(slug, 0) + if index >= len(recorded): + raise ReplayMiss( + f"replay exhausted for {test_key}: call #{index + 1} ({method} {path}) has no recorded " + f"interaction ({len(recorded)} recorded under {slug}); re-record with E2E_FIXTURE_MODE=record" + ) + interaction = recorded[index] + if interaction.request.method != method or interaction.request.path != path: + raise ReplayMiss( + f"replay mismatch for {test_key} at call #{index + 1}: recorded " + f"{interaction.request.method} {interaction.request.path}, test made {method} {path}; " + "re-record with E2E_FIXTURE_MODE=record" + ) + self._cursors[slug] = index + 1 + return interaction + + def leftover_error(self, test_key: str) -> str | None: + """Non-None when the test consumed fewer interactions than were recorded, + meaning a passing replay proved less than the bundle claims.""" + slug = slug_for_test(test_key) + recorded = self.bundle.interactions.get(slug, ()) + consumed = self._cursors.get(slug, 0) + if consumed >= len(recorded): + return None + pending = recorded[consumed] + return ( + f"replay incomplete for {test_key}: {len(recorded) - consumed} of {len(recorded)} recorded " + f"interactions never consumed, next is {pending.request.method} {pending.request.path}; " + "re-record with E2E_FIXTURE_MODE=record" + ) + + +def _expect_result(interaction: Interaction) -> RecordedResult: + match interaction.response: + case RecordedResult() as recorded: + return recorded + case RecordedStreaming() | RecordedBinary() | RecordedProbe(): + raise ReplayMiss( + f"recorded {interaction.request.method} {interaction.request.path} is not a typed result" + ) + + +def _expect_streaming(interaction: Interaction) -> StreamingResponse: + match interaction.response: + case RecordedStreaming(payload=payload): + return payload + case RecordedResult() | RecordedBinary() | RecordedProbe(): + raise ReplayMiss( + f"recorded {interaction.request.method} {interaction.request.path} is not a streaming response" + ) + + +@dataclass(frozen=True, slots=True) +class ReplayTransport: + """A ``Transport`` served entirely from a recorded bundle: never opens a + connection, so a replay run cannot bill a provider.""" + + source: ReplaySource + master_key: str + + def bearer(self, key: str) -> AuthHeaders: + return AuthHeaders(authorization=f"Bearer {key}") + + @property + def master(self) -> AuthHeaders: + return self.bearer(self.master_key) + + def post[R: BaseModel]( + self, path: str, *, headers: BaseModel, json: BaseModel, response_type: type[R] + ) -> Result[R]: + return to_result(_expect_result(self.source.next_interaction("post", path)), response_type) + + def get[R: BaseModel]( + self, + path: str, + *, + headers: BaseModel, + params: BaseModel, + response_type: type[R], + timeout: float | None = None, + ) -> Result[R]: + return to_result(_expect_result(self.source.next_interaction("get", path)), response_type) + + def delete[R: BaseModel]( + self, + path: str, + *, + headers: BaseModel, + json: BaseModel, + response_type: type[R], + params: BaseModel | None = None, + ) -> Result[R]: + return to_result(_expect_result(self.source.next_interaction("delete", path)), response_type) + + def patch[R: BaseModel]( + self, path: str, *, headers: BaseModel, json: BaseModel, response_type: type[R] + ) -> Result[R]: + return to_result(_expect_result(self.source.next_interaction("patch", path)), response_type) + + def put[R: BaseModel]( + self, path: str, *, headers: BaseModel, json: BaseModel, response_type: type[R] + ) -> Result[R]: + return to_result(_expect_result(self.source.next_interaction("put", path)), response_type) + + def stream(self, path: str, *, headers: BaseModel, json: BaseModel) -> StreamingResponse: + return _expect_streaming(self.source.next_interaction("stream", path)) + + def stream_binary( + self, path: str, *, headers: BaseModel, json: BaseModel, chunk_size: int = 8192 + ) -> BinaryStream: + interaction = self.source.next_interaction("stream_binary", path) + match interaction.response: + case RecordedBinary(payload=payload): + return payload + case RecordedResult() | RecordedStreaming() | RecordedProbe(): + raise ReplayMiss( + f"recorded stream_binary {interaction.request.path} is not a binary stream" + ) + + def send( + self, + path: str, + *, + headers: BaseModel, + json: BaseModel, + params: BaseModel | None = None, + stream: bool = False, + ) -> StreamingResponse: + return _expect_streaming(self.source.next_interaction("send", path)) + + def probe(self, path: str, *, params: BaseModel) -> ProbeResult: + interaction = self.source.next_interaction("probe", path) + match interaction.response: + case RecordedProbe(payload=payload): + return payload + case RecordedResult() | RecordedStreaming() | RecordedBinary(): + raise ReplayMiss(f"recorded probe {interaction.request.path} is not a probe result") + + def upload[R: BaseModel]( + self, + path: str, + *, + headers: BaseModel, + form: BaseModel, + filename: str, + content: bytes, + file_content_type: str = "application/jsonl", + file_field: str = "file", + params: BaseModel | None = None, + response_type: type[R], + ) -> Result[R]: + return to_result(_expect_result(self.source.next_interaction("upload", path)), response_type) + + def download(self, path: str, *, headers: BaseModel) -> StreamingResponse: + return _expect_streaming(self.source.next_interaction("download", path)) + + +@functools.lru_cache(maxsize=8) +def _shared_recorder(root: Path) -> BundleRecorder: + prepared = prepare_bundle(root) + if isinstance(prepared, UnsafeBundleDir): + raise ValueError(f"E2E_FIXTURE_DIR {prepared.path} {prepared.reason}") + return prepared + + +@functools.lru_cache(maxsize=8) +def _shared_replay_source(root: Path) -> ReplaySource: + loaded = load_bundle(root) + if isinstance(loaded, UnreadableBundle): + raise ValueError(f"cannot replay from {root}: {loaded.reason}") + return ReplaySource(bundle=loaded) + + +def replay_leftover_error(*, mode_raw: str, bundle_dir: Path, test_key: str) -> str | None: + """Teardown-time completeness check: in replay mode a passed test with + unconsumed recorded interactions must fail instead of passing against a + recording it no longer matches. Inert in every other mode.""" + if parse_fixture_mode(mode_raw) != "replay": + return None + return _shared_replay_source(bundle_dir).leftover_error(test_key) + + +def select_transport( + live: Transport, *, mode_raw: str, bundle_dir: Path, master_key: str +) -> Transport: + """The one seam every client build goes through: wraps (record), replaces + (replay), or passes through (live) the transport per E2E_FIXTURE_MODE. The + recorder and replay cursors are process-wide singletons per bundle dir, so + every client in a session shares one bundle and one recorded sequence.""" + mode = parse_fixture_mode(mode_raw) + match mode: + case InvalidFixtureMode(value=value): + raise ValueError(f"E2E_FIXTURE_MODE={value!r} is not one of {', '.join(FIXTURE_MODES)}") + case "live": + return live + case "record": + return RecordingTransport(inner=live, recorder=_shared_recorder(bundle_dir)) + case "replay": + return ReplayTransport(source=_shared_replay_source(bundle_dir), master_key=master_key) + case _: + assert_never(mode) + + +def fixture_mode_collection_error(mode_raw: str, bundle_dir: Path, *, now: datetime) -> str | None: + """Session-abort reason for a fixture-mode setup that can never work, or None. + Called at collection time (conftest pytest_sessionstart) so a stale or missing + bundle fails the whole run up front, naming the bundle age, instead of failing + every test individually.""" + mode = parse_fixture_mode(mode_raw) + match mode: + case InvalidFixtureMode(value=value): + return f"E2E_FIXTURE_MODE={value!r} is not one of {', '.join(FIXTURE_MODES)}" + case "live" | "record": + return None + case "replay": + freshness = check_freshness(bundle_dir, now=now) + match freshness: + case FreshBundle(): + return None + case StaleBundle(recorded_at=recorded_at, age=age, limit=limit): + return ( + f"fixture bundle at {bundle_dir} is stale: recorded {recorded_at.isoformat()}, " + f"age {format_age(age)} exceeds the {limit.days}-day limit; " + "re-record with E2E_FIXTURE_MODE=record" + ) + case UnreadableBundle(reason=reason): + return f"E2E_FIXTURE_MODE=replay cannot use bundle at {bundle_dir}: {reason}" + case _: + assert_never(freshness) + case _: + assert_never(mode) + + +def fixture_report_lines(mode_raw: str, bundle_dir: Path, *, now: datetime) -> list[str]: + """pytest report-header lines; empty in live mode so an unset + E2E_FIXTURE_MODE keeps today's output byte-identical.""" + mode = parse_fixture_mode(mode_raw) + match mode: + case InvalidFixtureMode() | "live": + return [] + case "record": + return [f"e2e fixture mode: record -> {bundle_dir}"] + case "replay": + freshness = check_freshness(bundle_dir, now=now) + match freshness: + case FreshBundle(manifest=manifest): + return [ + f"e2e fixture mode: replay <- {bundle_dir} " + f"(recorded {manifest.recorded_at.isoformat()}, harness {manifest.harness_version})" + ] + case StaleBundle() | UnreadableBundle(): + return [f"e2e fixture mode: replay <- {bundle_dir}"] + case _: + assert_never(freshness) + case _: + assert_never(mode) diff --git a/tests/e2e/proxy_client.py b/tests/e2e/proxy_client.py index 5050b6fce68..843799ede6c 100644 --- a/tests/e2e/proxy_client.py +++ b/tests/e2e/proxy_client.py @@ -65,6 +65,8 @@ from models import ( ) from e2e_config import ( CONTROL_PLANE_BASE_URL, + FIXTURE_DIR, + FIXTURE_MODE_RAW, MASTER_KEY, POLL_INTERVAL, POLL_TIMEOUT, @@ -72,6 +74,7 @@ from e2e_config import ( REQUEST_TIMEOUT, settle_propagation, ) +from fixture_transport import select_transport from transport import HttpTransport, SplitTransport, Transport RowsPredicate = Callable[[list[SpendLogRow]], bool] @@ -531,19 +534,29 @@ def build_proxy_client( The endpoints are injectable for callers that resolve the proxy some other way than ``e2e_config``'s env names (see ``claude_code/_env.py``); they must pass all three together, since a caller that overrides only the data plane - would leave management calls pointed at the env default.""" + would leave management calls pointed at the env default. + + E2E_FIXTURE_MODE wraps (record) or replaces (replay) the transport here, so + every client built from this seam records or replays without changing shape; + unset it stays the plain SplitTransport (see fixture_transport.py).""" + split = SplitTransport( + data=HttpTransport( + base_url=base_url, + master_key=master_key, + request_timeout=REQUEST_TIMEOUT, + ), + control=HttpTransport( + base_url=control_plane_base_url, + master_key=master_key, + request_timeout=REQUEST_TIMEOUT, + ), + ) return ProxyClient( - transport=SplitTransport( - data=HttpTransport( - base_url=base_url, - master_key=master_key, - request_timeout=REQUEST_TIMEOUT, - ), - control=HttpTransport( - base_url=control_plane_base_url, - master_key=master_key, - request_timeout=REQUEST_TIMEOUT, - ), + transport=select_transport( + split, + mode_raw=FIXTURE_MODE_RAW, + bundle_dir=FIXTURE_DIR, + master_key=master_key, ), poll_timeout=POLL_TIMEOUT, poll_interval=POLL_INTERVAL, diff --git a/tests/e2e/test_fixture_bundle.py b/tests/e2e/test_fixture_bundle.py new file mode 100644 index 00000000000..fd4cca6451f --- /dev/null +++ b/tests/e2e/test_fixture_bundle.py @@ -0,0 +1,218 @@ +"""Harness coverage for the on-disk fixture bundle format (LIT-5729). + +No proxy and no ``e2e`` marker: these pin the bundle CONTRACT - the seven-day +freshness gate that names the bundle's age, record mode's wipe safety (never +delete a directory that is not a bundle), collision-free per-test slugs, and +lossless Result round-trips - so replay can never silently drift from what +record wrote. +""" + +from __future__ import annotations + +from datetime import datetime, timedelta, timezone +from pathlib import Path + +import pytest +from pydantic import BaseModel + +from e2e_http import ( + NetworkError, + RateLimitedError, + Result, + Success, + UnauthorizedError, + UnknownApiError, + ValidationError, +) +from fixture_bundle import ( + BUNDLE_FORMAT_VERSION, + MANIFEST_FILENAME, + MAX_BUNDLE_AGE, + BundleRecorder, + FreshBundle, + LoadedBundle, + Manifest, + RecordedRequest, + RecordedResult, + StaleBundle, + UnreadableBundle, + UnsafeBundleDir, + check_freshness, + format_age, + from_result, + interaction_filename, + load_bundle, + prepare_bundle, + slug_for_test, + to_result, +) + +NOW = datetime(2026, 8, 18, 12, 0, 0, tzinfo=timezone.utc) + + +class Payload(BaseModel): + value: str + + +def write_manifest( + root: Path, recorded_at: datetime, *, format_version: int = BUNDLE_FORMAT_VERSION +) -> None: + root.mkdir(parents=True, exist_ok=True) + manifest = Manifest( + format_version=format_version, recorded_at=recorded_at, harness_version="abc1234" + ) + (root / MANIFEST_FILENAME).write_text(manifest.model_dump_json(), encoding="utf-8") + + +def prepared(root: Path) -> BundleRecorder: + recorder = prepare_bundle(root) + assert isinstance(recorder, BundleRecorder) + return recorder + + +def plain_request(path: str) -> RecordedRequest: + return RecordedRequest(method="post", path=path, headers={}) + + +class TestResultRoundTrip: + @pytest.mark.parametrize( + "result", + [ + Success(status_code=201, data=Payload(value="ok")), + NetworkError(message="connection refused"), + UnauthorizedError(), + RateLimitedError(retry_after_seconds=7, body="slow down"), + ValidationError(message="bad shape"), + UnknownApiError(status_code=502, body="upstream exploded"), + ], + ) + def test_every_result_kind_survives_disk_and_back(self, result: Result[Payload]) -> None: + assert to_result(from_result(result), Payload) == result + + +class TestFreshness: + def test_bundle_at_the_limit_is_still_fresh(self, tmp_path: Path) -> None: + root = tmp_path / "bundle" + write_manifest(root, NOW - MAX_BUNDLE_AGE) + assert isinstance(check_freshness(root, now=NOW), FreshBundle) + + def test_stale_bundle_reports_age_and_limit(self, tmp_path: Path) -> None: + root = tmp_path / "bundle" + write_manifest(root, NOW - timedelta(days=8, hours=3)) + freshness = check_freshness(root, now=NOW) + assert isinstance(freshness, StaleBundle) + assert freshness.age == timedelta(days=8, hours=3) + assert format_age(freshness.age) == "8d3h" + assert freshness.limit == MAX_BUNDLE_AGE + + def test_naive_recorded_at_is_read_as_utc(self, tmp_path: Path) -> None: + root = tmp_path / "bundle" + write_manifest(root, (NOW - timedelta(days=1)).replace(tzinfo=None)) + assert isinstance(check_freshness(root, now=NOW), FreshBundle) + + def test_missing_manifest_is_unreadable_with_recording_hint(self, tmp_path: Path) -> None: + freshness = check_freshness(tmp_path / "absent", now=NOW) + assert isinstance(freshness, UnreadableBundle) + assert MANIFEST_FILENAME in freshness.reason + assert "E2E_FIXTURE_MODE=record" in freshness.reason + + def test_corrupt_manifest_is_unreadable(self, tmp_path: Path) -> None: + root = tmp_path / "bundle" + root.mkdir() + (root / MANIFEST_FILENAME).write_text("{not json", encoding="utf-8") + assert isinstance(check_freshness(root, now=NOW), UnreadableBundle) + + def test_unknown_format_version_is_unreadable(self, tmp_path: Path) -> None: + root = tmp_path / "bundle" + write_manifest(root, NOW, format_version=BUNDLE_FORMAT_VERSION + 1) + freshness = check_freshness(root, now=NOW) + assert isinstance(freshness, UnreadableBundle) + assert f"format_version {BUNDLE_FORMAT_VERSION + 1}" in freshness.reason + + +class TestPrepareBundle: + def test_fresh_directory_gets_a_fresh_manifest(self, tmp_path: Path) -> None: + root = tmp_path / "bundle" + prepared(root) + freshness = check_freshness(root, now=datetime.now(timezone.utc)) + assert isinstance(freshness, FreshBundle) + assert freshness.manifest.format_version == BUNDLE_FORMAT_VERSION + assert freshness.manifest.harness_version + + def test_record_wipes_the_previous_bundle_instead_of_reading_it(self, tmp_path: Path) -> None: + root = tmp_path / "bundle" + prepared(root).record( + test_key="old.py::test_old", + request=plain_request("/stale"), + response=RecordedResult(kind="unauthorized"), + ) + assert any(entry.is_dir() for entry in root.iterdir()) + prepared(root) + assert {entry.name for entry in root.iterdir()} == {MANIFEST_FILENAME} + + def test_refuses_to_wipe_a_directory_that_is_not_a_bundle(self, tmp_path: Path) -> None: + root = tmp_path / "precious" + root.mkdir() + (root / "notes.txt").write_text("keep me", encoding="utf-8") + outcome = prepare_bundle(root) + assert isinstance(outcome, UnsafeBundleDir) + assert MANIFEST_FILENAME in outcome.reason + assert (root / "notes.txt").read_text(encoding="utf-8") == "keep me" + + def test_refuses_a_path_that_is_a_file(self, tmp_path: Path) -> None: + target = tmp_path / "not-a-dir" + target.write_text("x", encoding="utf-8") + outcome = prepare_bundle(target) + assert isinstance(outcome, UnsafeBundleDir) + assert "not a directory" in outcome.reason + + +class TestSlugs: + def test_slug_for_test_is_deterministic(self) -> None: + key = "tests/e2e/suite/test_mod.py::TestX::test_case" + assert slug_for_test(key) == slug_for_test(key) + + def test_same_tail_in_different_files_never_collides(self) -> None: + first = slug_for_test("tests/e2e/a/test_a.py::test_case") + second = slug_for_test("tests/e2e/b/test_b.py::test_case") + assert first != second + assert first.startswith("test_case-") + assert second.startswith("test_case-") + + def test_interaction_filename_orders_and_slugs(self) -> None: + request = RecordedRequest(method="post", path="/chat/completions", headers={}) + assert interaction_filename(3, request) == "0003-post-chat-completions.json" + + +class TestRecordAndLoad: + def test_load_returns_interactions_in_recorded_order(self, tmp_path: Path) -> None: + root = tmp_path / "bundle" + recorder = prepared(root) + key = "suite/test_mod.py::test_ordered" + for path in ("/first", "/second", "/third"): + recorder.record( + test_key=key, + request=plain_request(path), + response=RecordedResult(kind="unauthorized"), + ) + loaded = load_bundle(root) + assert isinstance(loaded, LoadedBundle) + assert [ + interaction.request.path for interaction in loaded.interactions[slug_for_test(key)] + ] == ["/first", "/second", "/third"] + + def test_interactions_group_per_test(self, tmp_path: Path) -> None: + root = tmp_path / "bundle" + recorder = prepared(root) + for key in ("suite/test_a.py::test_one", "suite/test_b.py::test_two"): + recorder.record( + test_key=key, + request=plain_request(f"/{key[-3:]}"), + response=RecordedResult(kind="unauthorized"), + ) + loaded = load_bundle(root) + assert isinstance(loaded, LoadedBundle) + assert set(loaded.interactions) == { + slug_for_test("suite/test_a.py::test_one"), + slug_for_test("suite/test_b.py::test_two"), + } diff --git a/tests/e2e/test_fixture_transport.py b/tests/e2e/test_fixture_transport.py new file mode 100644 index 00000000000..5c7201cca37 --- /dev/null +++ b/tests/e2e/test_fixture_transport.py @@ -0,0 +1,485 @@ +"""Harness coverage for the record/replay transports (LIT-5729). + +No proxy and no ``e2e`` marker. A fake in-memory ``Transport`` stands in for +the live one (dependency injection, no monkeypatching): recording must pass +every value through unchanged while writing one redacted interaction file per +call, and replay must serve identical values from the bundle alone - the +fake's call log proves nothing reaches the inner transport - failing hard +(``ReplayMiss``) on any drift in order, verb, or path. The collection-time +gate and report header are pinned here too, including the stale message that +names the bundle's age. +""" + +from __future__ import annotations + +import hashlib +from dataclasses import dataclass, field +from datetime import datetime, timedelta, timezone +from pathlib import Path + +import pytest +from pydantic import BaseModel + +from e2e_http import ( + AuthHeaders, + BinaryStream, + ProbeResult, + Result, + StreamingResponse, + Success, +) +from fixture_bundle import ( + BUNDLE_FORMAT_VERSION, + MANIFEST_FILENAME, + BundleRecorder, + Interaction, + LoadedBundle, + Manifest, + load_bundle, + prepare_bundle, + slug_for_test, +) +from fixture_transport import ( + InvalidFixtureMode, + RecordingTransport, + ReplayMiss, + ReplaySource, + ReplayTransport, + current_test_key, + deterministic_marker, + fixture_mode_collection_error, + fixture_report_lines, + parse_fixture_mode, + replay_leftover_error, + select_transport, +) +from transport import Transport + +NOW = datetime(2026, 8, 18, 12, 0, 0, tzinfo=timezone.utc) + + +class Payload(BaseModel): + value: str + + +class Body(BaseModel): + prompt: str + + +class Query(BaseModel): + q: str + + +STREAMING = StreamingResponse( + status_code=200, + body="", + content_type="text/event-stream", + chunks=2, + stream_events=["one", "two"], + stream_done=True, +) +BINARY = BinaryStream(status_code=200, content_type="audio/mpeg", chunk_count=3, total_bytes=42) +PROBE = ProbeResult(status_code=200, body="alive") + + +@dataclass +class FakeTransport: + calls: list[str] = field(default_factory=list) + + def bearer(self, key: str) -> AuthHeaders: + return AuthHeaders(authorization=f"Bearer {key}") + + @property + def master(self) -> AuthHeaders: + return self.bearer("sk-fake-master") + + def _success[R: BaseModel](self, response_type: type[R]) -> Result[R]: + return Success(status_code=200, data=response_type.model_validate({"value": "live"})) + + def post[R: BaseModel]( + self, path: str, *, headers: BaseModel, json: BaseModel, response_type: type[R] + ) -> Result[R]: + self.calls.append(f"post {path}") + return self._success(response_type) + + def get[R: BaseModel]( + self, + path: str, + *, + headers: BaseModel, + params: BaseModel, + response_type: type[R], + timeout: float | None = None, + ) -> Result[R]: + self.calls.append(f"get {path}") + return self._success(response_type) + + def delete[R: BaseModel]( + self, + path: str, + *, + headers: BaseModel, + json: BaseModel, + response_type: type[R], + params: BaseModel | None = None, + ) -> Result[R]: + self.calls.append(f"delete {path}") + return self._success(response_type) + + def patch[R: BaseModel]( + self, path: str, *, headers: BaseModel, json: BaseModel, response_type: type[R] + ) -> Result[R]: + self.calls.append(f"patch {path}") + return self._success(response_type) + + def put[R: BaseModel]( + self, path: str, *, headers: BaseModel, json: BaseModel, response_type: type[R] + ) -> Result[R]: + self.calls.append(f"put {path}") + return self._success(response_type) + + def stream(self, path: str, *, headers: BaseModel, json: BaseModel) -> StreamingResponse: + self.calls.append(f"stream {path}") + return STREAMING + + def stream_binary( + self, path: str, *, headers: BaseModel, json: BaseModel, chunk_size: int = 8192 + ) -> BinaryStream: + self.calls.append(f"stream_binary {path}") + return BINARY + + def send( + self, + path: str, + *, + headers: BaseModel, + json: BaseModel, + params: BaseModel | None = None, + stream: bool = False, + ) -> StreamingResponse: + self.calls.append(f"send {path}") + return STREAMING + + def probe(self, path: str, *, params: BaseModel) -> ProbeResult: + self.calls.append(f"probe {path}") + return PROBE + + def upload[R: BaseModel]( + self, + path: str, + *, + headers: BaseModel, + form: BaseModel, + filename: str, + content: bytes, + file_content_type: str = "application/jsonl", + file_field: str = "file", + params: BaseModel | None = None, + response_type: type[R], + ) -> Result[R]: + self.calls.append(f"upload {path}") + return self._success(response_type) + + def download(self, path: str, *, headers: BaseModel) -> StreamingResponse: + self.calls.append(f"download {path}") + return STREAMING + + +def make_recorder(root: Path) -> BundleRecorder: + recorder = prepare_bundle(root) + assert isinstance(recorder, BundleRecorder) + return recorder + + +def replay_source(root: Path) -> ReplaySource: + loaded = load_bundle(root) + assert isinstance(loaded, LoadedBundle) + return ReplaySource(bundle=loaded) + + +def this_tests_files(root: Path) -> list[Path]: + slug_dir = root / slug_for_test(current_test_key()) + return sorted(slug_dir.glob("*.json")) if slug_dir.is_dir() else [] + + +def write_manifest(root: Path, recorded_at: datetime) -> None: + root.mkdir(parents=True, exist_ok=True) + manifest = Manifest( + format_version=BUNDLE_FORMAT_VERSION, recorded_at=recorded_at, harness_version="abc1234" + ) + (root / MANIFEST_FILENAME).write_text(manifest.model_dump_json(), encoding="utf-8") + + +class TestParseFixtureMode: + @pytest.mark.parametrize( + ("raw", "expected"), + [("live", "live"), ("record", "record"), ("replay", "replay"), ("", "live"), (" REPLAY ", "replay")], + ) + def test_known_values_normalize(self, raw: str, expected: str) -> None: + assert parse_fixture_mode(raw) == expected + + def test_unknown_value_is_invalid_with_the_original_spelling(self) -> None: + assert parse_fixture_mode("cached") == InvalidFixtureMode(value="cached") + + +class TestDeterministicMarker: + def test_sequence_is_a_pure_function_of_test_and_ordinal(self) -> None: + """A replay process must regenerate exactly the markers the record + process generated, so the Nth marker of a test is pinned to a pure + function of the node id and N.""" + key = current_test_key() + assert deterministic_marker() == hashlib.sha1(f"{key}#0".encode()).hexdigest()[:12] + assert deterministic_marker() == hashlib.sha1(f"{key}#1".encode()).hexdigest()[:12] + + +class TestCurrentTestKey: + def test_names_this_test_and_strips_the_phase(self) -> None: + key = current_test_key() + assert key.endswith("TestCurrentTestKey::test_names_this_test_and_strips_the_phase") + assert "(call)" not in key + + +class TestRecordingTransport: + def test_passes_the_result_through_and_writes_one_file_per_call(self, tmp_path: Path) -> None: + fake = FakeTransport() + root = tmp_path / "bundle" + recording: Transport = RecordingTransport(inner=fake, recorder=make_recorder(root)) + result = recording.post( + "/model/new", headers=fake.master, json=Body(prompt="x"), response_type=Payload + ) + assert result == Success(status_code=200, data=Payload(value="live")) + assert fake.calls == ["post /model/new"] + files = this_tests_files(root) + assert [file.name for file in files] == ["0000-post-model-new.json"] + interaction = Interaction.model_validate_json(files[0].read_text(encoding="utf-8")) + assert interaction.request.method == "post" + assert interaction.request.path == "/model/new" + + def test_redacts_auth_header_values_in_the_recorded_request(self, tmp_path: Path) -> None: + fake = FakeTransport() + root = tmp_path / "bundle" + recording: Transport = RecordingTransport(inner=fake, recorder=make_recorder(root)) + headers = AuthHeaders.model_validate( + {"authorization": "Bearer sk-secret", "x-litellm-api-key": "sk-other"} + ) + recording.post("/key/generate", headers=headers, json=Body(prompt="x"), response_type=Payload) + interaction = Interaction.model_validate_json( + this_tests_files(root)[0].read_text(encoding="utf-8") + ) + assert interaction.request.headers == { + "authorization": "", + "x-litellm-api-key": "", + } + assert "sk-secret" not in this_tests_files(root)[0].read_text(encoding="utf-8") + + def test_upload_records_a_content_digest_not_the_bytes(self, tmp_path: Path) -> None: + fake = FakeTransport() + root = tmp_path / "bundle" + recording: Transport = RecordingTransport(inner=fake, recorder=make_recorder(root)) + recording.upload( + "/v1/files", + headers=fake.master, + form=Query(q="batch"), + filename="batch.jsonl", + content=b'{"custom_id": "1"}', + response_type=Payload, + ) + interaction = Interaction.model_validate_json( + this_tests_files(root)[0].read_text(encoding="utf-8") + ) + assert interaction.request.file_name == "batch.jsonl" + assert interaction.request.file_bytes == len(b'{"custom_id": "1"}') + assert interaction.request.file_sha256 is not None + assert "custom_id" not in interaction.request.model_dump_json() + + +class TestReplayTransport: + def test_serves_recorded_values_without_touching_the_inner_transport( + self, tmp_path: Path + ) -> None: + fake = FakeTransport() + root = tmp_path / "bundle" + recording: Transport = RecordingTransport(inner=fake, recorder=make_recorder(root)) + recorded_post = recording.post( + "/model/new", headers=fake.master, json=Body(prompt="x"), response_type=Payload + ) + recorded_get = recording.get( + "/v1/models", headers=fake.master, params=Query(q="all"), response_type=Payload + ) + recorded_stream = recording.stream( + "/chat/completions", headers=fake.master, json=Body(prompt="hi") + ) + recorded_probe = recording.probe("/health/liveliness", params=Query(q="1")) + recorded_binary = recording.stream_binary( + "/v1/audio/speech", headers=fake.master, json=Body(prompt="say") + ) + calls_after_record = list(fake.calls) + + replay: Transport = ReplayTransport(source=replay_source(root), master_key="sk-1234") + assert ( + replay.post("/model/new", headers=replay.master, json=Body(prompt="x"), response_type=Payload) + == recorded_post + ) + assert ( + replay.get("/v1/models", headers=replay.master, params=Query(q="all"), response_type=Payload) + == recorded_get + ) + assert ( + replay.stream("/chat/completions", headers=replay.master, json=Body(prompt="hi")) + == recorded_stream + ) + assert replay.probe("/health/liveliness", params=Query(q="1")) == recorded_probe + assert ( + replay.stream_binary("/v1/audio/speech", headers=replay.master, json=Body(prompt="say")) + == recorded_binary + ) + assert fake.calls == calls_after_record + + def test_mismatched_call_names_recorded_and_actual(self, tmp_path: Path) -> None: + fake = FakeTransport() + root = tmp_path / "bundle" + recording: Transport = RecordingTransport(inner=fake, recorder=make_recorder(root)) + recording.post("/model/new", headers=fake.master, json=Body(prompt="x"), response_type=Payload) + replay: Transport = ReplayTransport(source=replay_source(root), master_key="sk-1234") + with pytest.raises(ReplayMiss, match=r"recorded post /model/new, test made get /v1/models"): + replay.get("/v1/models", headers=replay.master, params=Query(q="all"), response_type=Payload) + + def test_exhausted_recording_names_the_call_count(self, tmp_path: Path) -> None: + fake = FakeTransport() + root = tmp_path / "bundle" + recording: Transport = RecordingTransport(inner=fake, recorder=make_recorder(root)) + recording.post("/model/new", headers=fake.master, json=Body(prompt="x"), response_type=Payload) + replay: Transport = ReplayTransport(source=replay_source(root), master_key="sk-1234") + replay.post("/model/new", headers=replay.master, json=Body(prompt="x"), response_type=Payload) + with pytest.raises(ReplayMiss, match=r"call #2 \(post /model/new\) has no recorded interaction \(1 recorded"): + replay.post("/model/new", headers=replay.master, json=Body(prompt="x"), response_type=Payload) + + +class TestReplayLeftover: + def test_fully_consumed_recording_leaves_nothing(self, tmp_path: Path) -> None: + fake = FakeTransport() + root = tmp_path / "bundle" + recording: Transport = RecordingTransport(inner=fake, recorder=make_recorder(root)) + recording.post("/model/new", headers=fake.master, json=Body(prompt="x"), response_type=Payload) + source = replay_source(root) + replay: Transport = ReplayTransport(source=source, master_key="sk-1234") + replay.post("/model/new", headers=replay.master, json=Body(prompt="x"), response_type=Payload) + assert source.leftover_error(current_test_key()) is None + + def test_unconsumed_trailing_interactions_name_the_next_call(self, tmp_path: Path) -> None: + fake = FakeTransport() + root = tmp_path / "bundle" + recording: Transport = RecordingTransport(inner=fake, recorder=make_recorder(root)) + recording.post("/model/new", headers=fake.master, json=Body(prompt="x"), response_type=Payload) + recording.probe("/health/liveliness", params=Query(q="1")) + source = replay_source(root) + replay: Transport = ReplayTransport(source=source, master_key="sk-1234") + replay.post("/model/new", headers=replay.master, json=Body(prompt="x"), response_type=Payload) + error = source.leftover_error(current_test_key()) + assert error is not None + assert "1 of 2 recorded interactions never consumed" in error + assert "next is probe /health/liveliness" in error + assert "re-record with E2E_FIXTURE_MODE=record" in error + + def test_test_without_recordings_has_no_leftover(self, tmp_path: Path) -> None: + root = tmp_path / "bundle" + make_recorder(root) + assert replay_source(root).leftover_error("suite.py::test_never_recorded") is None + + def test_inert_outside_replay_mode(self, tmp_path: Path) -> None: + missing = tmp_path / "missing" + assert replay_leftover_error(mode_raw="", bundle_dir=missing, test_key="k") is None + assert replay_leftover_error(mode_raw="record", bundle_dir=missing, test_key="k") is None + + def test_replay_mode_reads_the_shared_bundle(self, tmp_path: Path) -> None: + fake = FakeTransport() + root = tmp_path / "bundle" + recording: Transport = RecordingTransport(inner=fake, recorder=make_recorder(root)) + recording.post("/model/new", headers=fake.master, json=Body(prompt="x"), response_type=Payload) + error = replay_leftover_error(mode_raw="replay", bundle_dir=root, test_key=current_test_key()) + assert error is not None + assert "1 of 1 recorded interactions never consumed" in error + + +class TestSelectTransport: + def test_live_returns_the_live_transport_untouched(self, tmp_path: Path) -> None: + fake = FakeTransport() + for mode_raw in ("live", ""): + assert ( + select_transport(fake, mode_raw=mode_raw, bundle_dir=tmp_path / "b", master_key="sk") + is fake + ) + + def test_record_wraps_live_and_starts_a_fresh_bundle(self, tmp_path: Path) -> None: + fake = FakeTransport() + root = tmp_path / "bundle" + write_manifest(root, NOW - timedelta(days=30)) + (root / "old-test-slug").mkdir() + (root / "old-test-slug" / "0000-post-old.json").write_text("{}", encoding="utf-8") + selected = select_transport(fake, mode_raw="record", bundle_dir=root, master_key="sk") + assert isinstance(selected, RecordingTransport) + assert selected.inner is fake + assert {entry.name for entry in root.iterdir()} == {MANIFEST_FILENAME} + + def test_replay_builds_a_transport_from_the_bundle_alone(self, tmp_path: Path) -> None: + fake = FakeTransport() + root = tmp_path / "bundle" + make_recorder(root) + selected = select_transport(fake, mode_raw="replay", bundle_dir=root, master_key="sk-master") + assert isinstance(selected, ReplayTransport) + assert selected.master == AuthHeaders(authorization="Bearer sk-master") + + def test_invalid_mode_raises_naming_the_value(self, tmp_path: Path) -> None: + with pytest.raises(ValueError, match="cached"): + select_transport( + FakeTransport(), mode_raw="cached", bundle_dir=tmp_path / "b", master_key="sk" + ) + + +class TestCollectionGate: + def test_invalid_mode_names_the_value_and_the_choices(self, tmp_path: Path) -> None: + assert ( + fixture_mode_collection_error("cached", tmp_path, now=NOW) + == "E2E_FIXTURE_MODE='cached' is not one of live, record, replay" + ) + + @pytest.mark.parametrize("mode_raw", ["live", "", "record"]) + def test_live_and_record_never_block_collection(self, mode_raw: str, tmp_path: Path) -> None: + assert fixture_mode_collection_error(mode_raw, tmp_path / "missing", now=NOW) is None + + def test_replay_with_no_bundle_says_how_to_record_one(self, tmp_path: Path) -> None: + reason = fixture_mode_collection_error("replay", tmp_path / "missing", now=NOW) + assert reason is not None + assert f"no {MANIFEST_FILENAME}" in reason + assert "E2E_FIXTURE_MODE=record" in reason + + def test_stale_replay_bundle_fails_naming_its_age(self, tmp_path: Path) -> None: + root = tmp_path / "bundle" + write_manifest(root, NOW - timedelta(days=9, hours=5)) + reason = fixture_mode_collection_error("replay", root, now=NOW) + assert reason is not None + assert "age 9d5h exceeds the 7-day limit" in reason + assert "re-record with E2E_FIXTURE_MODE=record" in reason + + def test_fresh_replay_bundle_collects(self, tmp_path: Path) -> None: + root = tmp_path / "bundle" + write_manifest(root, NOW - timedelta(days=2)) + assert fixture_mode_collection_error("replay", root, now=NOW) is None + + +class TestReportHeader: + def test_live_mode_prints_nothing(self, tmp_path: Path) -> None: + assert fixture_report_lines("live", tmp_path, now=NOW) == [] + assert fixture_report_lines("", tmp_path, now=NOW) == [] + + def test_record_and_replay_name_the_bundle(self, tmp_path: Path) -> None: + root = tmp_path / "bundle" + recorded_at = NOW - timedelta(days=1) + write_manifest(root, recorded_at) + assert fixture_report_lines("record", root, now=NOW) == [ + f"e2e fixture mode: record -> {root}" + ] + replay_lines = fixture_report_lines("replay", root, now=NOW) + assert len(replay_lines) == 1 + assert "replay" in replay_lines[0] + assert recorded_at.isoformat() in replay_lines[0] diff --git a/tests/test_litellm/litellm_core_utils/test_streaming_handler.py b/tests/test_litellm/litellm_core_utils/test_streaming_handler.py index c1612aff3d3..05b44fffbc5 100644 --- a/tests/test_litellm/litellm_core_utils/test_streaming_handler.py +++ b/tests/test_litellm/litellm_core_utils/test_streaming_handler.py @@ -1774,6 +1774,80 @@ def test_openrouter_streaming_cost_propagates_to_hidden_params(): assert provider_cost == 0.00025 +def test_perplexity_streaming_dict_cost_propagates_to_hidden_params(): + """ + Regression: Perplexity reports usage.cost as a breakdown object, which used to + blow up the end of the stream with + `float() argument must be a string or a real number, not 'dict'`. + """ + import litellm + from litellm.cost_calculator import get_response_cost_from_hidden_params + + chunks = [ + ModelResponseStream( + id="chatcmpl-pplx", + created=1742056047, + model="perplexity/sonar", + choices=[ + StreamingChoices( + finish_reason=None, + index=0, + delta=Delta(content="Hi", role="assistant"), + ) + ], + usage=None, + ), + ModelResponseStream( + id="chatcmpl-pplx", + created=1742056048, + model="perplexity/sonar", + choices=[ + StreamingChoices(finish_reason="stop", index=0, delta=Delta(content="")) + ], + usage=None, + ), + ModelResponseStream( + id="chatcmpl-pplx", + created=1742056049, + model="perplexity/sonar", + choices=[ + StreamingChoices(finish_reason=None, index=0, delta=Delta(content="")) + ], + usage=Usage( + completion_tokens=18, + prompt_tokens=12, + total_tokens=30, + cost={ + "input_tokens_cost": 0.000012, + "output_tokens_cost": 0.000018, + "request_cost": 0.005, + "total_cost": 0.00503, + }, + ), + ), + ] + + complete_response = litellm.stream_chunk_builder( + chunks=chunks, messages=[{"role": "user", "content": "test"}] + ) + + assert complete_response is not None + + CustomStreamWrapper._propagate_usage_cost_to_hidden_params(complete_response) + + assert ( + get_response_cost_from_hidden_params(complete_response._hidden_params) + == 0.00503 + ) + + +def test_provider_reported_cost_ignores_unusable_shapes(): + assert CustomStreamWrapper._resolve_provider_reported_cost(None) is None + assert CustomStreamWrapper._resolve_provider_reported_cost({}) is None + assert CustomStreamWrapper._resolve_provider_reported_cost({"total_cost": None}) is None + assert CustomStreamWrapper._resolve_provider_reported_cost(0.5) == 0.5 + + def test_handle_special_delta_attributes( initialized_custom_stream_wrapper: CustomStreamWrapper, ): diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py index 9064312fd6d..5ee8143fb8e 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py @@ -4395,8 +4395,12 @@ class TestMCPServerManager: captured: dict = {} - def fake_create_tool_function(path, method, operation, base_url, headers=None): + def fake_create_tool_function( + path, method, operation, base_url, headers=None, server_label=None, relays_upstream_auth=False + ): captured["headers"] = headers + captured["server_label"] = server_label + captured["relays_upstream_auth"] = relays_upstream_auth async def tool_func(**kwargs): return "ok" @@ -4425,6 +4429,11 @@ class TestMCPServerManager: assert captured["headers"] is not None assert captured["headers"]["Authorization"] == "STATIC token" + # The label names the server in an upstream-failure error, so registration must thread it; + # without this the fake would simply tolerate the argument and prove nothing about it. + assert captured["server_label"] == "openapi-server" + # auth_type is none here, so a 401 from this upstream must not be dressed up as a re-auth signal + assert captured["relays_upstream_auth"] is False @pytest.mark.asyncio async def test_pre_call_tool_check_allowed_tools_list_allows_tool(self): @@ -10765,3 +10774,61 @@ class TestResolveOpenapiToolAuth: ) assert "Authorization" not in (forwarded or {}) + + +class TestOpenApiHandlerRelaysUpstreamAuth: + """`_call_openapi_tool_handler` must not flatten a re-auth signal into a generic message. + + Its catch-all turned every exception into "Error calling OpenAPI tool ...", which is an isError + result but loses the status, so the REST surface could no longer relay a 401 with the upstream's + WWW-Authenticate and the streamable surface could not name the status the caller must act on. + """ + + @staticmethod + def _server() -> MCPServer: + return MCPServer( + server_id="srv-openapi", + name="report_api", + server_name="report_api", + url="https://api.example.com", + transport=MCPTransport.http, + auth_type=MCPAuth.oauth_delegate, + spec_path="https://api.example.com/openapi.json", + ) + + @pytest.mark.asyncio + async def test_upstream_auth_error_keeps_its_type(self): + from litellm.proxy._experimental.mcp_server.exceptions import MCPUpstreamAuthError + from litellm.proxy._experimental.mcp_server.tool_registry import global_mcp_tool_registry + + manager = MCPServerManager() + server = self._server() + + async def raising_handler(**_kwargs): + raise MCPUpstreamAuthError(status_code=401, www_authenticate="Bearer realm=x", server_name="report_api") + + tool = MagicMock() + tool.handler = raising_handler + with patch.object(global_mcp_tool_registry, "get_tool", return_value=tool): + with pytest.raises(MCPUpstreamAuthError) as exc: + await manager._call_openapi_tool_handler(server, "list_reports", {}) + + assert exc.value.status_code == 401 + assert exc.value.www_authenticate == "Bearer realm=x" + + @pytest.mark.asyncio + async def test_other_failures_still_become_an_error_result(self): + from litellm.proxy._experimental.mcp_server.tool_registry import global_mcp_tool_registry + + manager = MCPServerManager() + + async def raising_handler(**_kwargs): + raise RuntimeError("upstream returned HTTP 503") + + tool = MagicMock() + tool.handler = raising_handler + with patch.object(global_mcp_tool_registry, "get_tool", return_value=tool): + result = await manager._call_openapi_tool_handler(self._server(), "list_reports", {}) + + assert result.isError is True + assert "upstream returned HTTP 503" in result.content[0].text diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_openapi_to_mcp_generator.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_openapi_to_mcp_generator.py index 1f9316ee9c8..e59616e53c1 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_openapi_to_mcp_generator.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_openapi_to_mcp_generator.py @@ -27,12 +27,21 @@ from litellm.proxy._experimental.mcp_server.openapi_to_mcp_generator import ( resolve_operation_params, ) +from litellm.proxy._experimental.mcp_server.exceptions import ( + MCPOpenApiUpstreamError, + MCPUpstreamAuthError, +) + GET_ASYNC_CLIENT_TARGET = "litellm.proxy._experimental.mcp_server.openapi_to_mcp_generator.get_async_httpx_client" -def _create_mock_client(method: str, response_text: str) -> AsyncMock: - """Utility to create a mocked async httpx client for the given method.""" - response = SimpleNamespace(text=response_text) +def _create_mock_client(method: str, response_text: str, status_code: int = 200) -> AsyncMock: + """Utility to create a mocked async httpx client for the given method. + + ``status_code`` and ``headers`` are part of the real response the tool function reads, so the + fake carries them too; a fake that omits them cannot observe whether the status is checked. + """ + response = SimpleNamespace(text=response_text, status_code=status_code, headers={}) client = AsyncMock() setattr(client, method, AsyncMock(return_value=response)) return client @@ -1259,3 +1268,113 @@ class TestRequestExtraHeaders: headers_sent = async_client.get.call_args[1]["headers"] assert "Authorization" not in headers_sent + + +class TestUpstreamStatusIsClassified: + """A non-2xx upstream must never be returned as tool output. + + The body used to be returned verbatim whatever the status, so an upstream rejection arrived as a + successful tool result and the request logged as a success. 401 is singled out because it is the + only status the caller can act on by re-authenticating, matching `_call_regular_mcp_tool` where a + 403 deliberately does not produce a challenge. + """ + + @staticmethod + def _tool(status_code: int, text: str = "body", headers: dict | None = None, relays_upstream_auth: bool = True): + response = SimpleNamespace(text=text, status_code=status_code, headers=headers or {}) + client = AsyncMock() + client.get = AsyncMock(return_value=response) + return create_tool_function( + "/reports", + "get", + {"operationId": "list_reports"}, + "https://api.example.com", + server_label="report_api", + relays_upstream_auth=relays_upstream_auth, + ), client + + @pytest.mark.asyncio + async def test_success_still_returns_the_body(self): + tool, client = self._tool(200, text='{"reports": []}') + with patch(GET_ASYNC_CLIENT_TARGET, return_value=client): + assert await tool() == '{"reports": []}' + + @pytest.mark.asyncio + async def test_401_raises_the_reauth_signal_carrying_the_challenge(self): + tool, client = self._tool(401, text='{"error":"invalid_token"}', headers={"www-authenticate": 'Bearer realm="x"'}) + with patch(GET_ASYNC_CLIENT_TARGET, return_value=client): + with pytest.raises(MCPUpstreamAuthError) as exc: + await tool() + + assert exc.value.status_code == 401 + assert exc.value.www_authenticate == 'Bearer realm="x"' + assert exc.value.server_name == "report_api" + + @pytest.mark.asyncio + async def test_401_on_a_non_forwarding_server_is_not_a_reauth_signal(self): + """Only the client-forwarded modes carry the caller's own upstream token, so only they can act + on a 401. `_call_regular_mcp_tool` gates its signal the same way, and without the gate an + api_key server rejecting a token would push clients into an OAuth flow that does not apply.""" + tool, client = self._tool(401, text='{"error":"bad key"}', relays_upstream_auth=False) + with patch(GET_ASYNC_CLIENT_TARGET, return_value=client): + with pytest.raises(MCPOpenApiUpstreamError) as exc: + await tool() + + assert exc.value.status_code == 401 + + @pytest.mark.parametrize("method", ["post", "put", "patch", "delete"]) + @pytest.mark.parametrize("status_code, expected", [(401, "auth"), (500, "other")]) + @pytest.mark.asyncio + async def test_non_get_methods_are_classified_too(self, method: str, status_code: int, expected: str): + """post/put/patch/delete call raise_for_status inside the HTTP handler. + + Only `get` hands a 4xx back to the caller; the others raise `MaskedHTTPStatusError` before any + status check the tool function could do, so classifying the returned response alone would + leave every non-GET tool still serving an upstream error body as successful tool output. A + fake client that simply returns a response cannot observe this, which is why this test builds + the error the real handler raises. + """ + import httpx + + from litellm.llms.custom_httpx.http_handler import MaskedHTTPStatusError + + request = httpx.Request(method.upper(), "https://api.example.com/reports") + raw = httpx.Response( + status_code, + headers={"www-authenticate": 'Bearer realm="x"'}, + text="internal hostname db-prod-7.corp.example.com", + request=request, + ) + masked = MaskedHTTPStatusError(httpx.HTTPStatusError("boom", request=request, response=raw)) + + client = AsyncMock() + setattr(client, method, AsyncMock(side_effect=masked)) + tool = create_tool_function( + "/reports", + method, + {"operationId": "list_reports"}, + "https://api.example.com", + server_label="report_api", + relays_upstream_auth=True, + ) + + expected_type = MCPUpstreamAuthError if expected == "auth" else MCPOpenApiUpstreamError + with patch(GET_ASYNC_CLIENT_TARGET, return_value=client): + with pytest.raises(expected_type) as exc: + await tool() + + assert exc.value.status_code == status_code + assert "db-prod-7" not in str(exc.value) + + @pytest.mark.parametrize("status_code", [403, 404, 429, 500, 503]) + @pytest.mark.asyncio + async def test_other_failures_raise_without_leaking_the_upstream_body(self, status_code: int): + secret_body = "internal hostname db-prod-7.corp.example.com and a stack trace" + tool, client = self._tool(status_code, text=secret_body) + with patch(GET_ASYNC_CLIENT_TARGET, return_value=client): + with pytest.raises(MCPOpenApiUpstreamError) as exc: + await tool() + + assert exc.value.status_code == status_code + assert secret_body not in str(exc.value) + assert str(exc.value) == f"upstream returned HTTP {status_code}" diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_openapi_tool_auth.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_openapi_tool_auth.py index 7bd846aeda4..bd953dc55f3 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_openapi_tool_auth.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_openapi_tool_auth.py @@ -652,3 +652,80 @@ async def test_per_server_auth_header_reaches_both_openapi_dispatch_arms(dispatc assert captured["resolver_credential"] == {"Authorization": OPENAPI_PER_SERVER_TOKEN} assert captured["injected"] == OPENAPI_PER_SERVER_TOKEN assert _request_auth_header.get() is None + + +@pytest.mark.parametrize("failure", ["auth", "other"]) +@pytest.mark.asyncio +async def test_local_dispatch_reports_the_outcome_instead_of_success(failure: str): + """A failing local handler must never be reported as a successful tool result, and only an auth + failure may propagate. + + `_handle_local_mcp_tool` used to catch every exception and return it as TextContent, and both of + its callers then stamped `isError=False`, so an upstream rejection was served as tool output and + `extract_mcp_tool_result_error_message` logged the request as a success. + + The two kinds are split by consequence. `MCPUpstreamAuthError` propagates because both renderers + know it: the streamable path names the status and the REST path relays a real 401 with the + upstream's WWW-Authenticate. Anything else is reported as `isError=True` right here, because + `call_tool_rest_api` turns an unrecognized exception into HTTP 500 and an upstream 403 or 429 is + not a gateway crash. + """ + from litellm.proxy._experimental.mcp_server import server as mcp_module + from litellm.proxy._experimental.mcp_server.exceptions import ( + MCPOpenApiUpstreamError, + MCPUpstreamAuthError, + ) + + error = ( + MCPUpstreamAuthError(status_code=401, www_authenticate=None, server_name="report_api") + if failure == "auth" + else MCPOpenApiUpstreamError(429, "report_api") + ) + + async def raising_handler(**_kwargs): + raise error + + fake_tool = MagicMock() + fake_tool.name = "list_reports" + fake_tool.handler = raising_handler + server = MCPServer( + server_id="srv-openapi", + name="report_api", + server_name="report_api", + url="https://api.example.com", + transport=MCPTransport.http, + auth_type=MCPAuth.oauth_delegate, + spec_path="https://api.example.com/openapi.json", + ) + user = UserAPIKeyAuth(api_key="sk-user", user_id="alice", user_role=LitellmUserRoles.INTERNAL_USER.value) + + with ( + patch.object(mcp_module.global_mcp_server_manager, "_get_mcp_server_from_tool_name", return_value=server), + patch.object(mcp_module.global_mcp_server_manager, "pre_call_tool_check", new=AsyncMock(return_value={})), + patch.object(mcp_module.global_mcp_tool_registry, "get_tool", return_value=fake_tool), + patch.object( + mcp_module.global_mcp_server_manager, + "resolve_openapi_upstream_auth", + new=AsyncMock(return_value=(None, None)), + ), + patch( + "litellm.proxy._experimental.mcp_server.server.MCPRequestHandler.is_tool_allowed", + return_value=True, + ), + ): + call = mcp_module.execute_mcp_tool( + name="list_reports", + arguments={}, + allowed_mcp_servers=[server], + start_time=datetime.now(timezone.utc), + user_api_key_auth=user, + ) + if failure == "auth": + with pytest.raises(MCPUpstreamAuthError): + await call + return + result = await call + + # A non-auth upstream failure stays a 200 with isError, so REST does not report it as a gateway 500 + assert result.isError is True + assert "upstream returned HTTP 429" in result.content[0].text diff --git a/tests/test_litellm/proxy/management_endpoints/test_auto_router_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_auto_router_endpoints.py index 77149457e82..3fd023552f5 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_auto_router_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_auto_router_endpoints.py @@ -4,6 +4,7 @@ Unit tests for auto router management endpoints import os import sys +from pathlib import Path import pytest from fastapi import HTTPException @@ -325,9 +326,7 @@ class TestAutoRouterBenchmarks: from litellm.proxy.management_endpoints.auto_router_endpoints import _benchmark_totals totals = _benchmark_totals(self.ROW) - bucket_hits = ( - totals.cache.same_model.hits + totals.cache.first_visit.hits + totals.cache.return_to_tier.hits - ) + bucket_hits = totals.cache.same_model.hits + totals.cache.first_visit.hits + totals.cache.return_to_tier.hits assert bucket_hits == 27 assert totals.cache.hit_rate_pct == pytest.approx(100.0 * 28 / 38, abs=0.1) @@ -490,7 +489,7 @@ from litellm.proxy.management_endpoints.auto_router_endpoints import ( start_shadow_eval, stop_shadow_eval_job, ) -from litellm.types.management_endpoints.auto_router_endpoints import ShadowEvalJobResponse, StartShadowEvalRequest +from litellm.types.management_endpoints.auto_router_endpoints import StartShadowEvalRequest VIEWER = UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY, api_key="sk-view", user_id="viewer") NON_ADMIN = UserAPIKeyAuth(user_role=LitellmUserRoles.INTERNAL_USER, api_key="sk-user", user_id="user") @@ -507,19 +506,23 @@ def _shadow_router() -> MagicMock: return router -def _job_record(**overrides: object) -> MagicMock: - """Spec'd like a real prisma row: only the table's columns exist as attributes, so - from_attributes validation falls back to model defaults for everything else.""" +def _leg_record(**overrides: object) -> MagicMock: + """Spec'd like a real prisma row: only the table's columns exist as attributes. One + row is one key's leg of a job; legs sharing group_id are one job.""" defaults = { - "id": "job-1", + "id": "leg-1", + "group_id": "job-1", "api_key_id": "key-hash", "router_name": "my-router", + "direction": "forward", + "baseline_model": None, "judge_model": "anthropic/claude-sonnet-5", "shadow_percentage": 10.0, "max_turns": 200, "created_at": datetime(2026, 8, 11, tzinfo=timezone.utc), "ends_at": datetime.now(timezone.utc) + timedelta(days=7), "stopped_at": None, + "stopped_by": None, } fields = {**defaults, **overrides} record = MagicMock(spec=list(fields)) @@ -538,23 +541,90 @@ def _key_record( return record -def _shadow_prisma(active_job=None, agg_rows=None) -> MagicMock: +def _shadow_prisma(legs=(), agg_rows=None, by_leg_rows=None, known_keys=("key-hash", "key-hash-2")) -> MagicMock: + """The job-table fake honours the filters it is handed, so a read that forgets + stopped_at sees rows the partial index would have released, one that forgets + direction sees the opposite-direction legs a key may hold at the same time, and a + group read that matched on a leg id would come back empty.""" prisma = MagicMock() - prisma.db.litellm_verificationtoken.find_unique = AsyncMock(return_value=_key_record()) - prisma.db.litellm_verificationtoken.find_many = AsyncMock(return_value=[_key_record()]) - prisma.db.execute_raw = AsyncMock(return_value=0) - prisma.db.litellm_shadowevaljob.find_first = AsyncMock(return_value=active_job) - prisma.db.litellm_shadowevaljob.find_unique = AsyncMock(return_value=None) - prisma.db.litellm_shadowevaljob.find_many = AsyncMock(return_value=[]) - prisma.db.litellm_shadowevaljob.create = AsyncMock(return_value=_job_record()) - prisma.db.litellm_shadowevaljob.update = AsyncMock( - return_value=_job_record(stopped_at=datetime.now(timezone.utc)) - ) + prisma.db.litellm_verificationtoken.find_many = AsyncMock(return_value=[_key_record(token) for token in known_keys]) + async def execute_raw(sql: str, *params: object): + if "SET stopped_by" in sql: + group = [row for row in stored if row.group_id == params[0]] + counts = {row["job_id"]: row["attempt_count"] for row in prisma.attempt_rows} + sampling = any(row.stopped_at is None and counts.get(row.id, 0) < row.max_turns for row in group) + window_open = bool(group) and group[0].ends_at > datetime.now(timezone.utc) + claimable = [row for row in group if row.stopped_by is None] + if not (claimable and sampling and window_open): + return 0 + for row in claimable: + row.stopped_by = params[1] + if row.stopped_at is None: + row.stopped_at = datetime.fromisoformat(str(params[2])).replace(tzinfo=timezone.utc) + return len(claimable) + return 0 + + prisma.db.execute_raw = AsyncMock(side_effect=execute_raw) + stored = legs if isinstance(legs, list) else list(legs) + + async def find_many_legs(where=None, **_: object): + current = list(stored) + w = dict(where or {}) + if "api_key_id" in w: + wanted = w["api_key_id"]["in"] if isinstance(w["api_key_id"], dict) else [w["api_key_id"]] + current = [row for row in current if row.api_key_id in wanted] + if "direction" in w: + current = [row for row in current if row.direction == w["direction"]] + if "stopped_at" in w: + current = [row for row in current if row.stopped_at is w["stopped_at"]] + if "group_id" in w: + wanted = w["group_id"]["in"] if isinstance(w["group_id"], dict) else [w["group_id"]] + current = [row for row in current if row.group_id in wanted] + return current + + def newest_groups(rows, limit): + latest: dict = {} + for row in rows: + if row.group_id not in latest or row.created_at > latest[row.group_id]: + latest[row.group_id] = row.created_at + ordered = sorted(latest, key=lambda group_id: latest[group_id], reverse=True) + return ordered[: int(limit)] + + def leg_dict(row): + fields = ( + "id", + "group_id", + "api_key_id", + "router_name", + "direction", + "baseline_model", + "judge_model", + "shadow_percentage", + "max_turns", + "created_at", + "ends_at", + "stopped_at", + "stopped_by", + ) + return {field: getattr(row, field) for field in fields} + + prisma.db.litellm_shadowevaljob.find_many = AsyncMock(side_effect=find_many_legs) + prisma.db.litellm_shadowevaljob.create_many = AsyncMock(return_value=1) + prisma.db.litellm_shadowevaljob.update_many = AsyncMock(return_value=1) prisma.db.litellm_shadowevalattempt.find_first = AsyncMock(return_value=None) + prisma.attempt_rows = [] async def query_raw(sql: str, *params: object): + if "AS attempt_count" in sql: + return prisma.attempt_rows + if "GROUP BY group_id" in sql: + scoped = [row for row in stored if "api_key_id = $2" not in sql or row.api_key_id == params[1]] + keep = set(newest_groups(scoped, params[0])) + return [leg_dict(row) for row in stored if row.group_id in keep] if "FILTER (WHERE outcome != 'error')::int AS judged_count" in sql: return [{"judged_count": 10, "error_count": 2, "judge_spend": 0.031}] + if "SELECT job_id AS grp" in sql: + return by_leg_rows if by_leg_rows is not None else [] return agg_rows if agg_rows is not None else [] prisma.db.query_raw = AsyncMock(side_effect=query_raw) @@ -563,7 +633,7 @@ def _shadow_prisma(active_job=None, agg_rows=None) -> MagicMock: def _start_request(**overrides: object) -> StartShadowEvalRequest: payload = { - "api_key_id": "key-hash", + "api_key_ids": ("key-hash",), "router_name": "my-router", "shadow_percentage": 10.0, "judge_model": "anthropic/claude-sonnet-5", @@ -575,44 +645,55 @@ def _start_request(**overrides: object) -> StartShadowEvalRequest: @pytest.mark.asyncio -async def test_start_shadow_eval_creates_job_and_frees_expired_or_exhausted_ones(monkeypatch: pytest.MonkeyPatch): - """Expiry and turn-budget exhaustion both end sampling on their own; either must - release the key's slot in the active-job index so a new eval can start.""" +async def test_start_shadow_eval_writes_one_leg_per_key_in_one_statement(monkeypatch: pytest.MonkeyPatch): + """N keys become N sibling rows sharing group_id and identical config, written by a + single create_many so a unique-index loser rolls back the whole claim, and expiry or + budget exhaustion frees every requested key's slot first.""" import litellm.proxy.proxy_server as proxy_server prisma = _shadow_prisma() monkeypatch.setattr(proxy_server, "prisma_client", prisma) monkeypatch.setattr(proxy_server, "llm_router", _shadow_router()) - response = await start_shadow_eval(_start_request(), ADMIN) + response = await start_shadow_eval(_start_request(api_key_ids=("key-hash", "key-hash-2")), ADMIN) - assert response.status == "running" - assert response.max_turns == 200 - assert response.judged_count is None - sweep_sql, sweep_key = prisma.db.execute_raw.call_args.args + sweep_sql, sweep_keys = prisma.db.execute_raw.call_args.args assert "stopped_at IS NULL" in sweep_sql - assert "ends_at <= NOW()" in sweep_sql + assert "j.ends_at <= (NOW() AT TIME ZONE 'utc')" in sweep_sql + assert "SET stopped_at = (NOW() AT TIME ZONE 'utc')" in sweep_sql assert ">= j.max_turns" in sweep_sql - assert sweep_key == "key-hash" - create_data = prisma.db.litellm_shadowevaljob.create.call_args.kwargs["data"] - assert create_data["api_key_id"] == "key-hash" - assert create_data["created_by"] == "admin" - assert "status" not in create_data + assert "j.api_key_id = ANY($1::text[])" in sweep_sql + assert sweep_keys == ["key-hash", "key-hash-2"] + prisma.db.litellm_shadowevaljob.create_many.assert_awaited_once() + rows = prisma.db.litellm_shadowevaljob.create_many.call_args.kwargs["data"] + assert [row["api_key_id"] for row in rows] == ["key-hash", "key-hash-2"] + assert len({frozenset((k, v) for k, v in row.items() if k != "api_key_id") for row in rows}) == 1 + assert len({row["group_id"] for row in rows}) == 1 + assert all(row["max_turns"] == 200 and row["created_by"] == "admin" for row in rows) + assert all("status" not in row and "id" not in row for row in rows) + assert response.job_id == rows[0]["group_id"] + assert response.status == "running" + assert response.judged_count is None + assert [(key.api_key_id, key.max_turns, key.key_alias) for key in response.keys] == [ + ("key-hash", 200, "prod-alpha"), + ("key-hash-2", 200, "prod-alpha"), + ] @pytest.mark.asyncio @pytest.mark.parametrize( - "caller,request_overrides,active,expected_status", + "caller,request_overrides,claimed,expected_status", [ - (NON_ADMIN, {}, None, 403), - (VIEWER, {}, None, 403), - (ADMIN, {"router_name": "not-a-router"}, None, 400), - (ADMIN, {"judge_model": "not/a real model!"}, None, 400), - (ADMIN, {"judge_model": "my-router"}, None, 400), - (ADMIN, {}, "active", 409), - (ADMIN, {"direction": "reverse", "baseline_model": "my-router"}, None, 400), - (ADMIN, {"direction": "reverse", "baseline_model": "not/a real model!"}, None, 400), - (ADMIN, {"direction": "reverse", "baseline_model": "openai/gpt-4o", "router_name": "not-a-router"}, None, 400), + (NON_ADMIN, {}, (), 403), + (VIEWER, {}, (), 403), + (ADMIN, {"router_name": "not-a-router"}, (), 400), + (ADMIN, {"judge_model": "not/a real model!"}, (), 400), + (ADMIN, {"judge_model": "my-router"}, (), 400), + (ADMIN, {}, ("key-hash",), 409), + (ADMIN, {"api_key_ids": ("key-hash", "key-hash-2")}, ("key-hash-2",), 409), + (ADMIN, {"direction": "reverse", "baseline_model": "my-router"}, (), 400), + (ADMIN, {"direction": "reverse", "baseline_model": "not/a real model!"}, (), 400), + (ADMIN, {"direction": "reverse", "baseline_model": "openai/gpt-4o", "router_name": "not-a-router"}, (), 400), ], ids=[ "non-admin", @@ -621,23 +702,143 @@ async def test_start_shadow_eval_creates_job_and_frees_expired_or_exhausted_ones "unresolvable-judge", "router-as-judge", "already-active", + "one-of-several-keys-already-active", "router-as-baseline", "unresolvable-baseline", "reverse-still-needs-an-auto-router", ], ) async def test_start_shadow_eval_rejections( - monkeypatch: pytest.MonkeyPatch, caller, request_overrides, active, expected_status + monkeypatch: pytest.MonkeyPatch, caller, request_overrides, claimed, expected_status ): import litellm.proxy.proxy_server as proxy_server - prisma = _shadow_prisma(active_job=_job_record() if active else None) + prisma = _shadow_prisma(legs=[_leg_record(id=f"leg-{key}", group_id="job-7", api_key_id=key) for key in claimed]) monkeypatch.setattr(proxy_server, "prisma_client", prisma) monkeypatch.setattr(proxy_server, "llm_router", _shadow_router()) with pytest.raises(HTTPException) as exc: await start_shadow_eval(_start_request(**request_overrides), caller) assert exc.value.status_code == expected_status + prisma.db.litellm_shadowevaljob.create_many.assert_not_called() + + +@pytest.mark.asyncio +async def test_start_shadow_eval_names_the_busy_key_and_its_job(monkeypatch: pytest.MonkeyPatch): + """A key busy elsewhere blocks the whole start rather than being silently dropped from + it, and the 409 names which key and which job so the caller can stop or drop it.""" + import litellm.proxy.proxy_server as proxy_server + + prisma = _shadow_prisma(legs=[_leg_record(id="leg-b", group_id="job-7", api_key_id="key-hash-2")]) + monkeypatch.setattr(proxy_server, "prisma_client", prisma) + monkeypatch.setattr(proxy_server, "llm_router", _shadow_router()) + + with pytest.raises(HTTPException) as exc: + await start_shadow_eval(_start_request(api_key_ids=("key-hash", "key-hash-2")), ADMIN) + assert exc.value.status_code == 409 + assert "key-hash-2 (job job-7)" in exc.value.detail + + +@pytest.mark.asyncio +async def test_start_shadow_eval_reuses_a_key_whose_previous_job_already_stopped(monkeypatch: pytest.MonkeyPatch): + """The claim is held by unstopped legs only, matching the partial unique index. A read + that forgets that would strand every key that has ever finished a job.""" + import litellm.proxy.proxy_server as proxy_server + + prisma = _shadow_prisma(legs=[_leg_record(group_id="job-7", stopped_at=datetime.now(timezone.utc))]) + monkeypatch.setattr(proxy_server, "prisma_client", prisma) + monkeypatch.setattr(proxy_server, "llm_router", _shadow_router()) + + job = await start_shadow_eval(_start_request(), ADMIN) + + assert job.status == "running" + prisma.db.litellm_shadowevaljob.create_many.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_start_shadow_eval_reverse_records_its_arms_and_holds_its_own_slot(monkeypatch: pytest.MonkeyPatch): + """The two directions ask opposite questions of the same key, so a forward job holding + the slot must not block a reverse one. The second reverse start still 409s.""" + import litellm.proxy.proxy_server as proxy_server + + legs = [_leg_record(group_id="job-fwd")] + prisma = _shadow_prisma(legs=legs) + monkeypatch.setattr(proxy_server, "prisma_client", prisma) + monkeypatch.setattr(proxy_server, "llm_router", _shadow_router()) + + reverse = _start_request(direction="reverse", baseline_model="openai/gpt-4o") + response = await start_shadow_eval(reverse, ADMIN) + + assert (response.direction, response.baseline_model) == ("reverse", "openai/gpt-4o") + rows = prisma.db.litellm_shadowevaljob.create_many.call_args.kwargs["data"] + assert rows[0]["direction"] == "reverse" + assert rows[0]["baseline_model"] == "openai/gpt-4o" + + legs.append(_leg_record(id="leg-2", group_id="job-rev", direction="reverse")) + with pytest.raises(HTTPException) as exc: + await start_shadow_eval(reverse, ADMIN) + assert exc.value.status_code == 409 + + +@pytest.mark.asyncio +async def test_start_shadow_eval_forward_leaves_the_baseline_column_empty(monkeypatch: pytest.MonkeyPatch): + import litellm.proxy.proxy_server as proxy_server + + prisma = _shadow_prisma() + monkeypatch.setattr(proxy_server, "prisma_client", prisma) + monkeypatch.setattr(proxy_server, "llm_router", _shadow_router()) + + await start_shadow_eval(_start_request(), ADMIN) + + rows = prisma.db.litellm_shadowevaljob.create_many.call_args.kwargs["data"] + assert rows[0]["direction"] == "forward" + assert rows[0]["baseline_model"] is None + + +@pytest.mark.asyncio +async def test_start_shadow_eval_rejects_keys_this_proxy_does_not_know(monkeypatch: pytest.MonkeyPatch): + """A typo'd api_key_id would otherwise create a leg no traffic can ever match. Every + unknown key is named at once, so a caller passing several fixes them in one round.""" + import litellm.proxy.proxy_server as proxy_server + + prisma = _shadow_prisma(known_keys=("key-hash",)) + monkeypatch.setattr(proxy_server, "prisma_client", prisma) + monkeypatch.setattr(proxy_server, "llm_router", _shadow_router()) + + with pytest.raises(HTTPException) as exc: + await start_shadow_eval(_start_request(api_key_ids=("key-hash", "typo-a", "typo-b")), ADMIN) + assert exc.value.status_code == 400 + assert "typo-a, typo-b" in exc.value.detail + assert "key-hash," not in exc.value.detail + prisma.db.litellm_shadowevaljob.create_many.assert_not_called() + + +def test_start_shadow_eval_request_dedupes_and_bounds_the_key_set(): + """A key named twice would collide with itself on the one-active-per-key index, a job + scoping no key samples nothing, and the key-count cap bounds every downstream read.""" + assert _start_request(api_key_ids=("a", "b", "a")).api_key_ids == ("a", "b") + assert len(_start_request(api_key_ids=tuple(f"k{i}" for i in range(100))).api_key_ids) == 100 + with pytest.raises(ValidationError): + _start_request(api_key_ids=()) + with pytest.raises(ValidationError): + _start_request(api_key_ids=tuple(f"k{i}" for i in range(101))) + + +@pytest.mark.asyncio +async def test_start_shadow_eval_concurrent_unique_violation_is_a_409(monkeypatch: pytest.MonkeyPatch): + import litellm.proxy.proxy_server as proxy_server + from prisma.errors import UniqueViolationError + + prisma = _shadow_prisma() + prisma.db.litellm_shadowevaljob.create_many = AsyncMock( + side_effect=UniqueViolationError(MagicMock(message="unique constraint")) + ) + monkeypatch.setattr(proxy_server, "prisma_client", prisma) + monkeypatch.setattr(proxy_server, "llm_router", _shadow_router()) + + with pytest.raises(HTTPException) as exc: + await start_shadow_eval(_start_request(), ADMIN) + assert exc.value.status_code == 409 @pytest.mark.parametrize( @@ -657,97 +858,25 @@ def test_start_request_pins_baseline_model_to_reverse(overrides): @pytest.mark.asyncio -async def test_start_shadow_eval_reverse_records_its_arms_and_holds_its_own_slot(monkeypatch: pytest.MonkeyPatch): - """The two directions ask opposite questions of the same key, so a forward job holding - the slot must not block a reverse one. The second reverse start still 409s.""" - import litellm.proxy.proxy_server as proxy_server - - prisma = _shadow_prisma() - active = {"forward": _job_record()} - prisma.db.litellm_shadowevaljob.find_first = AsyncMock( - side_effect=lambda where, **_: active.get(str(where.get("direction"))) - ) - prisma.db.litellm_shadowevaljob.create = AsyncMock( - return_value=_job_record(direction="reverse", baseline_model="openai/gpt-4o") - ) - monkeypatch.setattr(proxy_server, "prisma_client", prisma) - monkeypatch.setattr(proxy_server, "llm_router", _shadow_router()) - - reverse = _start_request(direction="reverse", baseline_model="openai/gpt-4o") - response = await start_shadow_eval(reverse, ADMIN) - - assert (response.direction, response.baseline_model) == ("reverse", "openai/gpt-4o") - create_data = prisma.db.litellm_shadowevaljob.create.call_args.kwargs["data"] - assert create_data["direction"] == "reverse" - assert create_data["baseline_model"] == "openai/gpt-4o" - - active["reverse"] = _job_record(id="job-2", direction="reverse") - with pytest.raises(HTTPException) as exc: - await start_shadow_eval(reverse, ADMIN) - assert exc.value.status_code == 409 - - -@pytest.mark.asyncio -async def test_start_shadow_eval_forward_leaves_the_baseline_column_empty(monkeypatch: pytest.MonkeyPatch): - import litellm.proxy.proxy_server as proxy_server - - prisma = _shadow_prisma() - monkeypatch.setattr(proxy_server, "prisma_client", prisma) - monkeypatch.setattr(proxy_server, "llm_router", _shadow_router()) - - await start_shadow_eval(_start_request(), ADMIN) - - create_data = prisma.db.litellm_shadowevaljob.create.call_args.kwargs["data"] - assert create_data["direction"] == "forward" - assert create_data["baseline_model"] is None - - -@pytest.mark.asyncio -async def test_start_shadow_eval_rejects_a_key_this_proxy_does_not_know(monkeypatch: pytest.MonkeyPatch): - """A typo'd api_key_id would otherwise create a job no traffic can ever match.""" - import litellm.proxy.proxy_server as proxy_server - - prisma = _shadow_prisma() - prisma.db.litellm_verificationtoken.find_unique = AsyncMock(return_value=None) - monkeypatch.setattr(proxy_server, "prisma_client", prisma) - monkeypatch.setattr(proxy_server, "llm_router", _shadow_router()) - - with pytest.raises(HTTPException) as exc: - await start_shadow_eval(_start_request(), ADMIN) - assert exc.value.status_code == 400 - assert "not a key on this proxy" in exc.value.detail - - -@pytest.mark.asyncio -async def test_start_shadow_eval_concurrent_unique_violation_is_a_409(monkeypatch: pytest.MonkeyPatch): - import litellm.proxy.proxy_server as proxy_server - from prisma.errors import UniqueViolationError - - prisma = _shadow_prisma() - prisma.db.litellm_shadowevaljob.create = AsyncMock( - side_effect=UniqueViolationError(MagicMock(message="unique constraint")) - ) - monkeypatch.setattr(proxy_server, "prisma_client", prisma) - monkeypatch.setattr(proxy_server, "llm_router", _shadow_router()) - - with pytest.raises(HTTPException) as exc: - await start_shadow_eval(_start_request(), ADMIN) - assert exc.value.status_code == 409 - - -@pytest.mark.asyncio -async def test_get_shadow_eval_job_derives_counts_spend_and_stratified_results(monkeypatch: pytest.MonkeyPatch): +async def test_get_shadow_eval_job_pools_counts_and_slices_results_per_key(monkeypatch: pytest.MonkeyPatch): + """One read answers for every leg: totals and stratifications aggregate over the + group's leg ids, and the by-key slice maps each leg id back to its key hash.""" import litellm.proxy.proxy_server as proxy_server tier_rows = [ {"grp": "SIMPLE", "turn_count": 8, "real_wins": 2, "shadow_wins": 4, "ties": 2, "avg_confidence": 0.8}, {"grp": "REASONING", "turn_count": 2, "real_wins": 2, "shadow_wins": 0, "ties": 0, "avg_confidence": 0.9}, ] - prisma = _shadow_prisma(agg_rows=tier_rows) - prisma.db.litellm_shadowevaljob.find_unique = AsyncMock(return_value=_job_record()) - prisma.db.litellm_shadowevalattempt.find_first = AsyncMock( - return_value=MagicMock(error="judge call failed: boom") + leg_rows = [ + {"grp": "leg-1", "turn_count": 6, "real_wins": 1, "shadow_wins": 4, "ties": 1, "avg_confidence": 0.7}, + {"grp": "leg-2", "turn_count": 4, "real_wins": 3, "shadow_wins": 0, "ties": 1, "avg_confidence": 0.6}, + ] + prisma = _shadow_prisma( + legs=[_leg_record(), _leg_record(id="leg-2", api_key_id="key-hash-2", max_turns=50)], + agg_rows=tier_rows, + by_leg_rows=leg_rows, ) + prisma.db.litellm_shadowevalattempt.find_first = AsyncMock(return_value=MagicMock(error="judge call failed: boom")) monkeypatch.setattr(proxy_server, "prisma_client", prisma) response = await get_shadow_eval_job("job-1", VIEWER) @@ -762,6 +891,13 @@ async def test_get_shadow_eval_job_derives_counts_spend_and_stratified_results(m assert response.results.by_tier[0].shadow_win_rate_pct == 50.0 assert response.results.overall_shadow_win_rate_pct == 40.0 assert response.results.overall_tie_rate_pct == 20.0 + assert [(s.group, s.turn_count) for s in response.results.by_key] == [("key-hash", 6), ("key-hash-2", 4)] + assert response.results.by_key[0].shadow_win_rate_pct == 66.7 + assert [(key.api_key_id, key.max_turns) for key in response.keys] == [("key-hash", 200), ("key-hash-2", 50)] + totals_args = [call.args for call in prisma.db.query_raw.await_args_list if "judged_count" in call.args[0]] + assert totals_args == [(totals_args[0][0], ["leg-1", "leg-2"])] + error_where = prisma.db.litellm_shadowevalattempt.find_first.call_args.kwargs["where"] + assert error_where == {"job_id": {"in": ["leg-1", "leg-2"]}, "outcome": "error"} @pytest.mark.asyncio @@ -780,79 +916,326 @@ async def test_get_shadow_eval_job_404s_and_gates_on_role(monkeypatch: pytest.Mo @pytest.mark.asyncio -async def test_list_shadow_eval_jobs_returns_derived_status_without_aggregates(monkeypatch: pytest.MonkeyPatch): +async def test_list_shadow_eval_jobs_collapses_legs_into_jobs_newest_first(monkeypatch: pytest.MonkeyPatch): + """A job over two keys is one list entry with both keys, not two entries, and a job + whose keys all stopped reads stopped while a half-stopped one still runs.""" import litellm.proxy.proxy_server as proxy_server - prisma = _shadow_prisma() - prisma.db.litellm_shadowevaljob.find_many = AsyncMock( - return_value=[ - _job_record(), - _job_record(id="job-2", ends_at=datetime.now(timezone.utc) - timedelta(days=1)), - _job_record(id="job-3", stopped_at=datetime.now(timezone.utc)), + stamp = datetime.now(timezone.utc) + prisma = _shadow_prisma( + legs=[ + _leg_record(created_at=datetime(2026, 8, 13, tzinfo=timezone.utc)), + _leg_record( + id="leg-2", + api_key_id="key-hash-2", + stopped_at=stamp, + created_at=datetime(2026, 8, 13, tzinfo=timezone.utc), + ), + _leg_record( + id="leg-3", + group_id="job-2", + stopped_at=stamp, + created_at=datetime(2026, 8, 12, tzinfo=timezone.utc), + ), + _leg_record( + id="leg-4", + group_id="job-3", + ends_at=datetime.now(timezone.utc) - timedelta(days=1), + created_at=datetime(2026, 8, 11, tzinfo=timezone.utc), + ), ] ) monkeypatch.setattr(proxy_server, "prisma_client", prisma) jobs = await list_shadow_eval_jobs(VIEWER, api_key_id=None, limit=50) - assert [job.status for job in jobs] == ["running", "completed", "stopped"] - swept = ShadowEvalJobResponse.model_validate( - _job_record( - id="job-4", - ends_at=datetime.now(timezone.utc) - timedelta(days=1), - stopped_at=datetime.now(timezone.utc), - ), - from_attributes=True, - ) - assert swept.status == "completed" + assert [(job.job_id, job.status) for job in jobs] == [ + ("job-1", "running"), + ("job-2", "stopped"), + ("job-3", "completed"), + ] + assert [key.api_key_id for key in jobs[0].keys] == ["key-hash", "key-hash-2"] assert all(job.judged_count is None and job.results is None for job in jobs) - assert prisma.db.query_raw.await_count == 0 + legs_sql, legs_limit = prisma.db.query_raw.await_args_list[0].args + assert "GROUP BY group_id ORDER BY MAX(created_at) DESC LIMIT $1::int" in legs_sql + assert legs_limit == 50 + counts_sql, _ = prisma.db.query_raw.await_args_list[1].args + assert "AS attempt_count" in counts_sql + assert "j.stopped_at IS NULL OR a.created_at <= j.stopped_at" in counts_sql + assert prisma.db.query_raw.await_count == 2 + prisma.db.litellm_shadowevaljob.find_many.assert_not_called() @pytest.mark.asyncio -async def test_shadow_eval_responses_name_the_shadowed_key(monkeypatch: pytest.MonkeyPatch): +async def test_list_shadow_eval_jobs_filters_to_jobs_containing_the_key(monkeypatch: pytest.MonkeyPatch): + """The filter matches a key anywhere in a job's key set and still returns the whole + job, sibling keys included.""" import litellm.proxy.proxy_server as proxy_server - prisma = _shadow_prisma() - prisma.db.litellm_shadowevaljob.find_many = AsyncMock( - return_value=[_job_record(), _job_record(id="job-2", api_key_id="deleted-key-hash")] + prisma = _shadow_prisma( + legs=[ + _leg_record(), + _leg_record(id="leg-2", api_key_id="key-hash-2"), + _leg_record(id="leg-3", group_id="job-2", api_key_id="key-hash-2"), + _leg_record(id="leg-4", group_id="job-3"), + ] + ) + monkeypatch.setattr(proxy_server, "prisma_client", prisma) + + jobs = await list_shadow_eval_jobs(VIEWER, api_key_id="key-hash-2", limit=50) + + assert [job.job_id for job in jobs] == ["job-1", "job-2"] + assert [key.api_key_id for key in jobs[0].keys] == ["key-hash", "key-hash-2"] + + +@pytest.mark.parametrize( + ("stopped_flags", "days_left", "expected"), + [ + ((False, False), 7, "running"), + ((True, False), 7, "running"), + ((True, True), 7, "stopped"), + ((True, True), -1, "completed"), + ((False, False), -1, "completed"), + ], +) +@pytest.mark.asyncio +async def test_job_status_runs_until_every_key_stops_and_completed_outranks_stopped( + monkeypatch: pytest.MonkeyPatch, stopped_flags: tuple[bool, ...], days_left: int, expected: str +): + import litellm.proxy.proxy_server as proxy_server + + stamp = datetime.now(timezone.utc) + prisma = _shadow_prisma( + legs=[ + _leg_record( + id=f"leg-{index}", + api_key_id=f"key-{index}", + stopped_at=stamp if stopped else None, + ends_at=datetime.now(timezone.utc) + timedelta(days=days_left), + ) + for index, stopped in enumerate(stopped_flags) + ] + ) + monkeypatch.setattr(proxy_server, "prisma_client", prisma) + + jobs = await list_shadow_eval_jobs(VIEWER, api_key_id=None, limit=50) + + assert [job.status for job in jobs] == [expected] + + +@pytest.mark.asyncio +async def test_list_reads_completed_once_every_key_spends_its_budget(monkeypatch: pytest.MonkeyPatch): + """A job whose keys all exhausted their turn budgets stopped sampling on its own, so + it must read completed on the very next list, before any sweep stamps its legs; one + key under budget keeps the whole job running. An operator starting an unrelated eval + must never look like it terminated a finished one.""" + import litellm.proxy.proxy_server as proxy_server + + prisma = _shadow_prisma( + legs=[ + _leg_record(max_turns=5), + _leg_record(id="leg-2", api_key_id="key-hash-2", max_turns=5), + _leg_record(id="leg-3", group_id="job-2", api_key_id="key-hash", max_turns=5), + _leg_record(id="leg-4", group_id="job-2", api_key_id="key-hash-2", max_turns=5), + ] + ) + prisma.attempt_rows = [ + {"job_id": "leg-1", "attempt_count": 5}, + {"job_id": "leg-2", "attempt_count": 6}, + {"job_id": "leg-3", "attempt_count": 5}, + {"job_id": "leg-4", "attempt_count": 3}, + ] + monkeypatch.setattr(proxy_server, "prisma_client", prisma) + + jobs = await list_shadow_eval_jobs(VIEWER, api_key_id=None, limit=50) + + by_id = {job.job_id: job for job in jobs} + assert by_id["job-1"].status == "completed" + assert all(key.stopped_at is None for key in by_id["job-1"].keys) + assert by_id["job-2"].status == "running" + assert {key.api_key_id: key.attempt_count for key in by_id["job-2"].keys} == {"key-hash": 5, "key-hash-2": 3} + + +@pytest.mark.asyncio +async def test_recorded_operator_stop_outranks_budget_arithmetic(monkeypatch: pytest.MonkeyPatch): + """A detached attempt can land around the stop and push the raw count past the + budget; the recorded stopped_by must keep the job reading stopped regardless.""" + import litellm.proxy.proxy_server as proxy_server + + stamp = datetime.now(timezone.utc) + prisma = _shadow_prisma(legs=[_leg_record(max_turns=5, stopped_at=stamp, stopped_by="admin")]) + prisma.attempt_rows = [{"job_id": "leg-1", "attempt_count": 6}] + monkeypatch.setattr(proxy_server, "prisma_client", prisma) + + jobs = await list_shadow_eval_jobs(VIEWER, api_key_id=None, limit=50) + assert jobs[0].status == "stopped" + assert jobs[0].stopped_by == "admin" + + detail = await get_shadow_eval_job("job-1", VIEWER) + assert detail.status == "stopped" + + +@pytest.mark.asyncio +async def test_backfilled_legacy_stop_never_reads_as_completion(monkeypatch: pytest.MonkeyPatch): + """Jobs stopped before stopped_by existed are backfilled with 'unknown' by the + migration, so even one whose stray attempts crossed the budget stays stopped.""" + import litellm.proxy.proxy_server as proxy_server + + prisma = _shadow_prisma( + legs=[_leg_record(max_turns=5, stopped_at=datetime.now(timezone.utc), stopped_by="unknown")] + ) + prisma.attempt_rows = [{"job_id": "leg-1", "attempt_count": 6}] + monkeypatch.setattr(proxy_server, "prisma_client", prisma) + + jobs = await list_shadow_eval_jobs(VIEWER, api_key_id=None, limit=50) + assert jobs[0].status == "stopped" + + +def test_stopped_by_migration_backfills_every_job_that_displayed_stopped(): + """The migration must close the pre-column population: without the backfill, a + legacy stop whose stray attempts crossed the budget would read completed.""" + import litellm_proxy_extras + + sql = ( + Path(litellm_proxy_extras.__file__).parent + / "migrations" + / "20260818224500_add_shadow_eval_stopped_by" + / "migration.sql" + ).read_text() + assert 'ADD COLUMN "stopped_by" TEXT' in sql + assert "SET stopped_by = 'unknown'" in sql + assert "WHERE stopped_at IS NOT NULL AND ends_at > (NOW() AT TIME ZONE 'utc')" in sql + + +@pytest.mark.asyncio +async def test_stop_rejects_a_job_that_already_spent_its_budget(monkeypatch: pytest.MonkeyPatch): + import litellm.proxy.proxy_server as proxy_server + + prisma = _shadow_prisma(legs=[_leg_record(max_turns=3)]) + prisma.attempt_rows = [{"job_id": "leg-1", "attempt_count": 3}] + monkeypatch.setattr(proxy_server, "prisma_client", prisma) + + with pytest.raises(HTTPException) as exhausted: + await stop_shadow_eval_job("job-1", ADMIN) + assert exhausted.value.status_code == 400 + assert "completed" in exhausted.value.detail + prisma.db.litellm_shadowevaljob.update_many.assert_not_called() + + +@pytest.mark.asyncio +async def test_shadow_eval_responses_name_every_shadowed_key(monkeypatch: pytest.MonkeyPatch): + import litellm.proxy.proxy_server as proxy_server + + prisma = _shadow_prisma( + legs=[_leg_record(), _leg_record(id="leg-2", api_key_id="deleted-key-hash")], + known_keys=("key-hash", "key-hash-2"), ) monkeypatch.setattr(proxy_server, "llm_router", _shadow_router()) monkeypatch.setattr(proxy_server, "prisma_client", prisma) - started = await start_shadow_eval(_start_request(), ADMIN) - assert (started.key_alias, started.key_name) == ("prod-alpha", "sk-...lpha") - jobs = await list_shadow_eval_jobs(VIEWER, api_key_id=None, limit=50) - assert [(job.key_alias, job.key_name) for job in jobs] == [("prod-alpha", "sk-...lpha"), (None, None)] + assert [(key.key_alias, key.key_name) for key in jobs[0].keys] == [ + (None, None), + ("prod-alpha", "sk-...lpha"), + ] batched_where = prisma.db.litellm_verificationtoken.find_many.call_args.kwargs["where"] assert batched_where == {"token": {"in": ["deleted-key-hash", "key-hash"]}} - prisma.db.litellm_shadowevaljob.find_unique = AsyncMock(return_value=_job_record()) detail = await get_shadow_eval_job("job-1", VIEWER) - assert detail.key_alias == "prod-alpha" + assert [key.key_alias for key in detail.keys] == [None, "prod-alpha"] @pytest.mark.asyncio -async def test_stop_shadow_eval_sets_stopped_at_and_rejects_non_running(monkeypatch: pytest.MonkeyPatch): +async def test_stop_shadow_eval_stops_every_unstopped_leg_and_rejects_non_running( + monkeypatch: pytest.MonkeyPatch, +): + """One stop ends sampling for the whole job, while a leg that already stopped on its + own budget keeps the stopped_at it earned.""" import litellm.proxy.proxy_server as proxy_server - prisma = _shadow_prisma() - prisma.db.litellm_shadowevaljob.find_unique = AsyncMock(return_value=_job_record()) + earned = datetime.now(timezone.utc) - timedelta(hours=1) + prisma = _shadow_prisma(legs=[_leg_record(), _leg_record(id="leg-2", api_key_id="key-hash-2", stopped_at=earned)]) monkeypatch.setattr(proxy_server, "prisma_client", prisma) stopped = await stop_shadow_eval_job("job-1", ADMIN) - assert stopped.status == "stopped" - update = prisma.db.litellm_shadowevaljob.update.call_args.kwargs - assert set(update["data"]) == {"stopped_at"} - prisma.db.litellm_shadowevaljob.find_unique = AsyncMock( - return_value=_job_record(ends_at=datetime.now(timezone.utc) - timedelta(days=1)) - ) + assert stopped.status == "stopped" + assert stopped.stopped_by == "admin" + stop_sql, stop_group, stop_operator, stop_stamp = prisma.db.execute_raw.call_args.args + assert "SET stopped_by = $2, stopped_at = COALESCE(stopped_at, $3::timestamp)" in stop_sql + assert "WHERE group_id = $1 AND stopped_by IS NULL" in stop_sql + assert "ends_at > (NOW() AT TIME ZONE 'utc')" in stop_sql + assert ") < k.max_turns" in stop_sql + assert (stop_group, stop_operator) == ("job-1", "admin") + assert datetime.fromisoformat(stop_stamp).tzinfo is None + assert prisma.db.execute_raw.await_count == 1 + prisma.db.litellm_shadowevaljob.update_many.assert_not_called() + by_key = {key.api_key_id: key.stopped_at for key in stopped.keys} + assert by_key["key-hash-2"] == earned + assert by_key["key-hash"] is not None and by_key["key-hash"] != earned + + done_leg = _leg_record(ends_at=datetime.now(timezone.utc) - timedelta(days=1)) + prisma_done = _shadow_prisma(legs=[done_leg]) + monkeypatch.setattr(proxy_server, "prisma_client", prisma_done) with pytest.raises(HTTPException) as exc: await stop_shadow_eval_job("job-1", ADMIN) assert exc.value.status_code == 400 + assert "already completed" in exc.value.detail + assert done_leg.stopped_by is None with pytest.raises(HTTPException) as forbidden: await stop_shadow_eval_job("job-1", VIEWER) assert forbidden.value.status_code == 403 + + +def test_every_shadow_eval_sql_constant_speaks_naive_utc(): + """The tables store naive UTC wall time (prisma's convention), so SQL-side time must be + NOW() AT TIME ZONE 'utc' and python-side params must cast ::timestamp; a bare NOW() or a + timestamptz cast writes session-local wall time into the naive column and skews every + comparison against prisma-written stamps.""" + import litellm.proxy.management_endpoints.auto_router_endpoints as module + + sql_constants = {name: value for name, value in vars(module).items() if name.endswith("_SQL")} + assert sql_constants + for name, sql in sql_constants.items(): + assert "::timestamptz" not in sql, name + for occurrence in sql.split("NOW()")[1:]: + assert occurrence.startswith(" AT TIME ZONE 'utc'"), name + + +@pytest.mark.asyncio +async def test_a_stop_racing_the_last_budgeted_attempt_reports_completed_not_stopped( + monkeypatch: pytest.MonkeyPatch, +): + """The statement claims the job only while a leg still samples, so a stop landing in + the same instant the budget spends records nothing and the job keeps reading + completed; stamping it would misreport a self-ended job as operator-stopped forever.""" + import litellm.proxy.proxy_server as proxy_server + + prisma = _shadow_prisma(legs=[_leg_record(max_turns=2)]) + prisma.attempt_rows = [{"job_id": "leg-1", "attempt_count": 2}] + monkeypatch.setattr(proxy_server, "prisma_client", prisma) + + with pytest.raises(HTTPException) as exc: + await stop_shadow_eval_job("job-1", ADMIN) + assert exc.value.status_code == 400 + assert "already completed" in exc.value.detail + assert prisma.db.litellm_shadowevaljob.find_many.await_args.kwargs["where"] == {"group_id": "job-1"} + + +@pytest.mark.asyncio +async def test_two_racing_stops_produce_exactly_one_winner(monkeypatch: pytest.MonkeyPatch): + """The statement's stopped_by IS NULL predicate lets only one racer claim rows; the + loser reads the stamped state and gets the same answer a late caller gets.""" + import litellm.proxy.proxy_server as proxy_server + + prisma = _shadow_prisma(legs=[_leg_record()]) + monkeypatch.setattr(proxy_server, "prisma_client", prisma) + + first = await stop_shadow_eval_job("job-1", ADMIN) + assert first.status == "stopped" + + with pytest.raises(HTTPException) as exc: + await stop_shadow_eval_job("job-1", ADMIN) + assert exc.value.status_code == 400 + assert "already stopped" in exc.value.detail diff --git a/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py index bdf09a95e4b..0936a2d1439 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py @@ -22,6 +22,7 @@ from litellm.proxy._types import ( NewUserRequest, LiteLLM_BudgetTable, LiteLLM_OrganizationTable, + LiteLLM_ProjectTableCachedObj, LiteLLM_TeamTableCachedObj, LiteLLM_UserTable, LiteLLM_VerificationToken, @@ -31,9 +32,12 @@ from litellm.proxy._types import ( ResetSpendRequest, UpdateKeyRequest, ) +from litellm.proxy.auth.auth_checks import _project_cache_key from litellm.proxy.auth.user_api_key_auth import UserAPIKeyAuth +from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache from litellm.proxy.management_endpoints.key_management_endpoints import ( _check_org_key_limits, + _check_project_key_limits, _check_team_key_limits, _common_key_generation_helper, _enforce_upperbound_key_params, @@ -16444,3 +16448,49 @@ async def test_regenerate_key_repoints_live_membership_not_the_key_row_it_read( assert await _authorized_models_for_key( access_groups, new_token_hash, ["ag-revoked-since", "ag-attached-since"] ) == ["attached-model"] + + +async def _cache_with_project(project_id: str, project_models: list[str]) -> UserApiKeyCache: + user_api_key_cache = UserApiKeyCache() + await user_api_key_cache.async_set_cache( + key=_project_cache_key(project_id), + value=LiteLLM_ProjectTableCachedObj(project_id=project_id, team_id="team-lit-5823", models=project_models), + model_type=LiteLLM_ProjectTableCachedObj, + ) + return user_api_key_cache + + +@pytest.mark.parametrize("request_cls", [GenerateKeyRequest, UpdateKeyRequest]) +@pytest.mark.parametrize("sentinel", ["all-team-models", "all-proxy-models"]) +@pytest.mark.asyncio +async def test_check_project_key_limits_accepts_inherited_model_sentinels(request_cls, sentinel): + """LIT-5823: the sentinels inherit a parent scope, so a project allowlist must not treat them as model names.""" + user_api_key_cache = await _cache_with_project("proj-lit-5823", ["gpt-5.4-nano"]) + + await _check_project_key_limits( + project_id="proj-lit-5823", + data=request_cls(key="sk-lit-5823", models=[sentinel]), + prisma_client=MagicMock(), + user_api_key_cache=user_api_key_cache, + ) + + +@pytest.mark.parametrize("request_cls", [GenerateKeyRequest, UpdateKeyRequest]) +@pytest.mark.parametrize( + "key_models", + [["gpt-5.4-mini"], ["all-team-models", "gpt-5.4-mini"], ["gpt-5.4-nano", "all-proxy-models", "gpt-5.4-mini"]], +) +@pytest.mark.asyncio +async def test_check_project_key_limits_still_rejects_real_model_outside_project(request_cls, key_models): + user_api_key_cache = await _cache_with_project("proj-lit-5823", ["gpt-5.4-nano"]) + + with pytest.raises(HTTPException) as exc_info: + await _check_project_key_limits( + project_id="proj-lit-5823", + data=request_cls(key="sk-lit-5823", models=key_models), + prisma_client=MagicMock(), + user_api_key_cache=user_api_key_cache, + ) + + assert exc_info.value.status_code == 400 + assert "Model 'gpt-5.4-mini' not in project's allowed models" in exc_info.value.detail["error"] diff --git a/tests/test_litellm/proxy/spend_tracking/test_ptu_flat_cost_rollup.py b/tests/test_litellm/proxy/spend_tracking/test_ptu_flat_cost_rollup.py index 736fc13d137..f41756d2b87 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_ptu_flat_cost_rollup.py +++ b/tests/test_litellm/proxy/spend_tracking/test_ptu_flat_cost_rollup.py @@ -1,5 +1,6 @@ """Tests for the per-model PTU flat-cost daily rollup.""" +import json import types from datetime import date, datetime, timedelta, timezone from unittest.mock import AsyncMock, MagicMock @@ -373,6 +374,182 @@ def test_parse_ptu_model_rejects_an_inverted_window(): assert parsed is None +def _router_entry(model_id="dep-1", model_name="gpt-4o-ptu", model_info=None, with_start=True): + """A deployment as the router stores one: a plain dict whose id lives in model_info.""" + info = dict(model_info or {}) + if ( + with_start + and info.get("ptu_count") is not None + and info.get("cost_per_ptu_per_hour") is not None + and "ptu_effective_from" not in info + ): + info["ptu_effective_from"] = _DEFAULT_PTU_START + if model_id is not None: + info["id"] = model_id + return { + "model_name": model_name, + "litellm_params": {"model": "azure/gpt-4o"}, + "model_info": info, + } + + +# A team-scoped deployment is stored under a synthetic routing key with the operator's name +# in team_public_model_name, while the same deployment in config.yaml carries the operator's +# name directly. Both must resolve to the same PTUModel or the two sources bill differently. +_PARITY_CASES = ( + ( + "an iso string start against the datetime pydantic coerces it to", + {"ptu_effective_from": "2026-07-30T23:00:00Z"}, + {"ptu_effective_from": datetime(2026, 7, 30, 23, 0)}, + datetime(2026, 7, 30, 23, 0, tzinfo=timezone.utc), + None, + ), + ( + "an open window, stored as null and dropped by exclude_none", + {"ptu_effective_from": "2026-07-01T00:00:00Z", "ptu_effective_to": None}, + {"ptu_effective_from": datetime(2026, 7, 1, 0, 0)}, + datetime(2026, 7, 1, tzinfo=timezone.utc), + None, + ), + ( + "a closed window", + {"ptu_effective_from": "2026-07-01T00:00:00Z", "ptu_effective_to": "2026-07-31T00:00:00Z"}, + { + "ptu_effective_from": datetime(2026, 7, 1, 0, 0), + "ptu_effective_to": datetime(2026, 7, 31, 0, 0), + }, + datetime(2026, 7, 1, tzinfo=timezone.utc), + datetime(2026, 7, 31, tzinfo=timezone.utc), + ), +) + + +@pytest.mark.parametrize( + "db_window, router_window, expected_from, expected_to", + [case[1:] for case in _PARITY_CASES], + ids=[case[0] for case in _PARITY_CASES], +) +def test_a_deployment_parses_identically_from_the_db_and_from_config_yaml( + db_window, router_window, expected_from, expected_to +): + # The contract the router union is built on. If these ever diverge, a PTU deployment + # declared in config.yaml is billed differently from the identical one in the database. + from_db = _parse_ptu_model( + _model_row( + model_id="dep-1", + model_name="model_name_t_9f3c2b", + model_info={**_VALID_PTU, **db_window, "team_public_model_name": "gpt-4o-ptu"}, + with_start=False, + ) + ) + from_router = _parse_ptu_model( + ptu_rollup._router_deployment( + _router_entry(model_id="dep-1", model_info={**_VALID_PTU, **router_window}, with_start=False) + ) + ) + expected = PTUModel( + model_id="dep-1", + model_name="gpt-4o-ptu", + team_id="t", + ptu_count=5, + cost_per_ptu_per_hour=2.0, + effective_from=expected_from, + effective_to=expected_to, + ) + assert from_db == from_router == expected + + +@pytest.mark.parametrize("model_id", [None, "", 12345], ids=["absent", "blank", "not a string"]) +def test_a_router_deployment_without_a_usable_id_is_dropped(model_id): + # model_id keys the sentinel row, so an unusable one would file every such deployment + # in a team onto one row and bill for a single reservation. + entry = _router_entry(model_id=None, model_info=dict(_VALID_PTU)) + if model_id is not None: + entry["model_info"]["id"] = model_id + assert ptu_rollup._router_deployment(entry) is None + + +def test_a_router_deployment_keeps_the_name_the_operator_wrote(): + priced = _parse_ptu_model( + ptu_rollup._router_deployment(_router_entry(model_name="gpt-4o-ptu", model_info=dict(_VALID_PTU))) + ) + assert priced is not None + assert priced.model_name == "gpt-4o-ptu" + + +def test_a_team_alias_on_a_router_deployment_still_wins_over_the_routing_name(): + # config.yaml can carry team_public_model_name to expose a team-facing alias, and the + # charge has to file under the name that team calls rather than the routing one. + priced = _parse_ptu_model( + ptu_rollup._router_deployment( + _router_entry( + model_name="routing-name", + model_info={**_VALID_PTU, "team_public_model_name": "public-alias"}, + ) + ) + ) + assert priced is not None + assert priced.model_name == "public-alias" + + +def test_a_router_deployment_with_a_stringified_model_info_still_decodes(): + entry = _router_entry(model_info=dict(_VALID_PTU)) + entry["model_info"] = json.dumps(entry["model_info"]) + parsed = _parse_ptu_model(ptu_rollup._router_deployment(entry)) + assert parsed is not None + assert parsed.model_id == "dep-1" + + +@pytest.mark.parametrize( + "model_info", + [None, "not json", 42, "[1, 2, 3]", '"a string"', "42", "true", "null"], + ids=[ + "absent", + "unparseable", + "not a string or dict", + "json array", + "json string", + "json number", + "json bool", + "json null", + ], +) +def test_a_router_deployment_without_usable_model_info_is_dropped(model_info): + # Valid JSON that is not an object decodes to a list or a scalar, and reading fields + # off one raises rather than dropping the single bad deployment the rollup expects + assert ptu_rollup._router_deployment({"model_name": "x", "model_info": model_info}) is None + + +@pytest.mark.parametrize( + "model_info", + ["[1, 2, 3]", '"a string"', "42", "true", "null"], + ids=["json array", "json string", "json number", "json bool", "json null"], +) +def test_a_db_row_holding_non_object_json_is_dropped(model_info): + assert _parse_ptu_model(_model_row(model_info=model_info, with_start=False)) is None + + +def test_a_router_deployment_does_not_alias_the_routers_own_model_info(): + # The rollup runs on a cron while requests are in flight, and the router rewrites + # model_info in place, so a held record must neither observe nor cause those writes. + live = {"id": "dep-1", **_VALID_PTU} + record = ptu_rollup._router_deployment({"model_name": "x", "model_info": live}) + assert record is not None + + live["ptu_count"] = 999 + assert record.model_info["ptu_count"] == _VALID_PTU["ptu_count"] + + with pytest.raises(TypeError): + record.model_info["ptu_count"] = 1 + + +def test_a_raw_router_dict_is_not_a_deployment_record(): + # The parser reads attributes, so a router dict passed straight to it returns None + # instead of raising. Skipping the factory would silently drop every config.yaml + # deployment while leaving the suite green. + assert _parse_ptu_model(_router_entry(model_info=dict(_VALID_PTU))) is None + + @pytest.mark.asyncio async def test_rollup_returns_empty_when_prisma_client_is_none(): result = await run_ptu_flat_cost_rollup(None, target_date=DAY) diff --git a/tests/test_litellm/router_strategy/test_complexity_router.py b/tests/test_litellm/router_strategy/test_complexity_router.py index e1e8d9553b3..a148da8b675 100644 --- a/tests/test_litellm/router_strategy/test_complexity_router.py +++ b/tests/test_litellm/router_strategy/test_complexity_router.py @@ -267,6 +267,24 @@ class TestReasoningMarkerScoring: # 2+ reasoning markers should force REASONING tier assert tier == ComplexityTier.REASONING + def test_reasoning_override_does_not_rescue_a_simple_score(self, complexity_router): + """Reasoning markers on an otherwise trivial prompt must not reach REASONING.""" + prompt = "hi, step by step, pros and cons" + tier, score, signals = complexity_router.classify(prompt) + assert score < complexity_router.config.tier_boundaries["simple_medium"] + assert any("step by step" in s and "pros and cons" in s for s in signals) + assert tier == ComplexityTier.SIMPLE + + def test_reasoning_override_applies_at_the_simple_medium_boundary(self, complexity_router): + """A score sitting exactly on simple_medium is not SIMPLE, so the override still promotes it.""" + prompt = ( + "Give me the pros and cons, step by step, of moving our checkout service " + "to an event-driven architecture." + ) + tier, score, signals = complexity_router.classify(prompt) + assert score == complexity_router.config.tier_boundaries["simple_medium"] + assert tier == ComplexityTier.REASONING + def test_system_prompt_reasoning_not_counted(self, complexity_router): """Reasoning markers in system prompt should not count for override.""" user_prompt = "What is 2+2?" diff --git a/ui/litellm-dashboard/eslint-suppressions.json b/ui/litellm-dashboard/eslint-suppressions.json index 98225de5d8b..09e602e269b 100644 --- a/ui/litellm-dashboard/eslint-suppressions.json +++ b/ui/litellm-dashboard/eslint-suppressions.json @@ -1404,11 +1404,6 @@ "count": 2 } }, - "src/app/(dashboard)/vector-stores/_components/S3VectorsConfig.tsx": { - "no-restricted-imports": { - "count": 1 - } - }, "src/app/(dashboard)/vector-stores/_components/VectorStoreForm.tsx": { "no-nested-ternary": { "count": 2 @@ -1453,9 +1448,6 @@ "local/no-complex-jsx-arrow": { "count": 1 }, - "no-restricted-imports": { - "count": 1 - }, "react-hooks/set-state-in-effect": { "count": 2 } @@ -1470,11 +1462,6 @@ "count": 1 } }, - "src/app/onboarding/OnboardingFormBody.tsx": { - "no-restricted-imports": { - "count": 1 - } - }, "src/components/AIHub/ModelHubTable.test.tsx": { "max-params": { "count": 1 @@ -1998,11 +1985,6 @@ "count": 1 } }, - "src/components/common_components/PassThroughGuardrailsSection.tsx": { - "no-restricted-imports": { - "count": 1 - } - }, "src/components/common_components/RateLimitTypeFormItem.test.tsx": { "no-restricted-imports": { "count": 1 diff --git a/ui/litellm-dashboard/package-lock.json b/ui/litellm-dashboard/package-lock.json index 186d38234d5..7069450a75c 100644 --- a/ui/litellm-dashboard/package-lock.json +++ b/ui/litellm-dashboard/package-lock.json @@ -32,9 +32,9 @@ "openapi-fetch": "^0.17.0", "openapi-react-query": "^0.5.4", "papaparse": "5.5.3", - "react": "18.3.1", + "react": "19.2.8", "react-copy-to-clipboard": "5.1.1", - "react-dom": "18.3.1", + "react-dom": "19.2.8", "react-hook-form": "7.82.0", "react-json-view-lite": "2.5.0", "react-markdown": "9.1.0", @@ -55,9 +55,9 @@ "@testing-library/react": "16.3.2", "@testing-library/user-event": "14.6.1", "@types/node": "20.19.37", - "@types/react": "18.2.48", + "@types/react": "19.2.18", "@types/react-copy-to-clipboard": "5.0.7", - "@types/react-dom": "18.3.7", + "@types/react-dom": "19.2.4", "@types/react-syntax-highlighter": "15.5.13", "@vitest/coverage-v8": "3.2.6", "@vitest/ui": "3.2.6", @@ -200,9 +200,9 @@ } }, "node_modules/@ant-design/icons-svg": { - "version": "4.4.2", - "resolved": "https://registry.npmjs.org/@ant-design/icons-svg/-/icons-svg-4.4.2.tgz", - "integrity": "sha512-vHbT+zJEVzllwP+CM+ul7reTEfBR0vgxFe7+lREAsAA7YGsYpboiq2sQNeQeRvh09GfQgs/GyFEvZpJ9cLXpXA==", + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/@ant-design/icons-svg/-/icons-svg-4.5.0.tgz", + "integrity": "sha512-1BTUFyKPTBZ53MuTP8s0k5SFEXL7o3VHEOwLgzaoWKwnBeqIcqUtVshc4SKzhI6uACfqhJqBwBUE9FsWR3uULA==", "license": "MIT" }, "node_modules/@ant-design/react-slick": { @@ -606,19 +606,6 @@ } } }, - "node_modules/@base-ui/react/node_modules/@floating-ui/react-dom": { - "version": "2.1.8", - "resolved": "https://registry.npmjs.org/@floating-ui/react-dom/-/react-dom-2.1.8.tgz", - "integrity": "sha512-cC52bHwM/n/CxS87FH0yWdngEZrjdtLW/qVruo68qg+prK7ZQ4YGdut2GyDVpoGeAYe/h899rVeOVm6Oi40k2A==", - "license": "MIT", - "dependencies": { - "@floating-ui/dom": "^1.7.6" - }, - "peerDependencies": { - "react": ">=16.8.0", - "react-dom": ">=16.8.0" - } - }, "node_modules/@base-ui/utils": { "version": "0.3.1", "resolved": "https://registry.npmjs.org/@base-ui/utils/-/utils-0.3.1.tgz", @@ -1441,28 +1428,41 @@ } }, "node_modules/@floating-ui/core": { - "version": "1.7.5", - "resolved": "https://registry.npmjs.org/@floating-ui/core/-/core-1.7.5.tgz", - "integrity": "sha512-1Ih4WTWyw0+lKyFMcBHGbb5U5FtuHJuujoyyr5zTaWS5EYMeT6Jb2AuDeftsCsEuchO+mM2ij5+q9crhydzLhQ==", + "version": "1.8.0", + "resolved": "https://registry.npmjs.org/@floating-ui/core/-/core-1.8.0.tgz", + "integrity": "sha512-0CIZ5itps/8x7BG8dEIhs53BvCUH2PCoogtakwRTut+Arm58sJooJ0AuZhLw2HJYIR5cMLNPBSS728sPho2khQ==", "license": "MIT", "dependencies": { - "@floating-ui/utils": "^0.2.11" + "@floating-ui/utils": "^0.2.12" } }, "node_modules/@floating-ui/dom": { - "version": "1.7.6", - "resolved": "https://registry.npmjs.org/@floating-ui/dom/-/dom-1.7.6.tgz", - "integrity": "sha512-9gZSAI5XM36880PPMm//9dfiEngYoC6Am2izES1FF406YFsjvyBMmeJ2g4SAju3xWwtuynNRFL2s9hgxpLI5SQ==", + "version": "1.8.0", + "resolved": "https://registry.npmjs.org/@floating-ui/dom/-/dom-1.8.0.tgz", + "integrity": "sha512-yXSrzeHZBTZadLOlfyhCkJHNeLJnHRnRInwdZ40L7ZiaAtrBwoYlsDrX3v5zB1Utk7CLfzcOVnVVWoXEky7Ceg==", "license": "MIT", "dependencies": { - "@floating-ui/core": "^1.7.5", - "@floating-ui/utils": "^0.2.11" + "@floating-ui/core": "^1.8.0", + "@floating-ui/utils": "^0.2.12" + } + }, + "node_modules/@floating-ui/react-dom": { + "version": "2.1.9", + "resolved": "https://registry.npmjs.org/@floating-ui/react-dom/-/react-dom-2.1.9.tgz", + "integrity": "sha512-JDjEFGCpImxDCA7JJKviA0M9+RtmJdj0m/NVU5IMgBK+AmZouAQQ7/+2GLH0GXXY0YMw9oXPB8hKdbPYg5QLYg==", + "license": "MIT", + "dependencies": { + "@floating-ui/dom": "^1.8.0" + }, + "peerDependencies": { + "react": ">=16.8.0", + "react-dom": ">=16.8.0" } }, "node_modules/@floating-ui/utils": { - "version": "0.2.11", - "resolved": "https://registry.npmjs.org/@floating-ui/utils/-/utils-0.2.11.tgz", - "integrity": "sha512-RiB/yIh78pcIxl6lLMG0CgBXAZ2Y0eVHqMPYugu+9U0AeT6YBeiJpf7lbdJNIugFP5SIjwNRgo4DhR1Qxi26Gg==", + "version": "0.2.12", + "resolved": "https://registry.npmjs.org/@floating-ui/utils/-/utils-0.2.12.tgz", + "integrity": "sha512-HpCo8tmWzLVad5s2d19EhAz5zqrrQ6s69qd6moPMQvkOuSwDT1YgRfWSVuc4ennqrgv3OHppiOGMQ7oC13yIww==", "license": "MIT" }, "node_modules/@headlessui/tailwindcss": { @@ -2627,9 +2627,9 @@ "license": "MIT" }, "node_modules/@rc-component/async-validator": { - "version": "5.1.0", - "resolved": "https://registry.npmjs.org/@rc-component/async-validator/-/async-validator-5.1.0.tgz", - "integrity": "sha512-n4HcR5siNUXRX23nDizbZBQPO0ZM/5oTtmKZ6/eqL0L2bo747cklFdZGRN2f+c9qWGICwDzrhW0H7tE9PptdcA==", + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/@rc-component/async-validator/-/async-validator-5.1.2.tgz", + "integrity": "sha512-WYbrZSjzznU1ekD0qFq2qRxt309VoS61MTG5npnFQlKYcoy9IzU8T+ZCIhq5bGAXRbXysABFWTspicMfmWFwow==", "license": "MIT", "dependencies": { "@babel/runtime": "^7.24.4" @@ -2669,9 +2669,9 @@ } }, "node_modules/@rc-component/mini-decimal": { - "version": "1.1.3", - "resolved": "https://registry.npmjs.org/@rc-component/mini-decimal/-/mini-decimal-1.1.3.tgz", - "integrity": "sha512-bk/FJ09fLf+NLODMAFll6CfYrHPBioTedhW6lxDBuuWucJEqFUd4l/D/5JgIi3dina6sYahB8iuPAZTNz2pMxw==", + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/@rc-component/mini-decimal/-/mini-decimal-1.1.4.tgz", + "integrity": "sha512-xiuXcaCwyOWpD8a8scdExFl+bntNphAW8XeenL1ig2en0AAZY0Pcp4pC0dI22qJ+NvxKn9RoNIoRdqYU3BLH4w==", "license": "MIT", "dependencies": { "@babel/runtime": "^7.18.0" @@ -2717,9 +2717,9 @@ } }, "node_modules/@rc-component/qrcode": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/@rc-component/qrcode/-/qrcode-1.1.1.tgz", - "integrity": "sha512-LfLGNymzKdUPjXUbRP+xOhIWY4jQ+YMj5MmWAcgcAq1Ij8XP7tRmAXqyuv96XvLUBE/5cA8hLFl9eO1JQMujrA==", + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/@rc-component/qrcode/-/qrcode-1.1.3.tgz", + "integrity": "sha512-aGv6alnn4HbDEsURzKP+jv13rbi1VxmAYfBNZr5GKF1iohMNWy5tAVoJ1E3cOvzMB1kbUPvCXchM6zSFlRGPhA==", "license": "MIT", "dependencies": { "@babel/runtime": "^7.24.7" @@ -3661,12 +3661,12 @@ } }, "node_modules/@tanstack/react-store": { - "version": "0.11.0", - "resolved": "https://registry.npmjs.org/@tanstack/react-store/-/react-store-0.11.0.tgz", - "integrity": "sha512-tX4YXh3PDkmpvGQWkWqKpzs/MSqbtuwY9dWdWhtV9Q50PmO+jOkUKIWIX4G85dwt7lxdHLXsiaEKPdKmC8F41w==", + "version": "0.11.1", + "resolved": "https://registry.npmjs.org/@tanstack/react-store/-/react-store-0.11.1.tgz", + "integrity": "sha512-HaIGKI3YLmjBYIvy5DFDY23oNaYZIsTZfngey07Uh5iLVJgM3bIGCnZeOFOqzjFld9JHWcaHJnasD/bKoGKwJQ==", "license": "MIT", "dependencies": { - "@tanstack/store": "0.11.0", + "@tanstack/store": "0.11.1", "use-sync-external-store": "^1.6.0" }, "funding": { @@ -3699,9 +3699,9 @@ } }, "node_modules/@tanstack/store": { - "version": "0.11.0", - "resolved": "https://registry.npmjs.org/@tanstack/store/-/store-0.11.0.tgz", - "integrity": "sha512-WlzzCt3xi0G6pCAJu1U+2jiECwabETDpQDi3hfkFZvJii9AuZqEKbOiVarX1/bWhTNjU486yQtJCCasi/0q+Cw==", + "version": "0.11.1", + "resolved": "https://registry.npmjs.org/@tanstack/store/-/store-0.11.1.tgz", + "integrity": "sha512-mzTOBhypOuDJAy/D8n2MfUZ1HFkXnmSETviRyhqEC8LUE7/IZQExOTxMANj3KjTofYTkFNpBY67qaVrT41YccA==", "license": "MIT", "funding": { "type": "github", @@ -3999,21 +3999,13 @@ "@types/node": "*" } }, - "node_modules/@types/prop-types": { - "version": "15.7.15", - "resolved": "https://registry.npmjs.org/@types/prop-types/-/prop-types-15.7.15.tgz", - "integrity": "sha512-F6bEyamV9jKGAFBEmlQnesRPGOQqS2+Uwi0Em15xenOxHaf2hv6L8YCVn3rPdPJOiJfPiCnLIRyvwVaqMY3MIw==", - "license": "MIT" - }, "node_modules/@types/react": { - "version": "18.2.48", - "resolved": "https://registry.npmjs.org/@types/react/-/react-18.2.48.tgz", - "integrity": "sha512-qboRCl6Ie70DQQG9hhNREz81jqC1cs9EVNcjQ1AU+jH6NFfSAhVVbrrY/+nSF+Bsk4AOwm9Qa61InvMCyV+H3w==", + "version": "19.2.18", + "resolved": "https://registry.npmjs.org/@types/react/-/react-19.2.18.tgz", + "integrity": "sha512-AnzbBERsrLKtk2XSfTbYRLjQPdy116Sty4q+T+Bp3IC4l6jNBvreVPAHmpq9qhXQM7CXZPjLVmGMw9sy+hxQ3w==", "license": "MIT", "dependencies": { - "@types/prop-types": "*", - "@types/scheduler": "*", - "csstype": "^3.0.2" + "csstype": "^3.2.2" } }, "node_modules/@types/react-copy-to-clipboard": { @@ -4027,13 +4019,13 @@ } }, "node_modules/@types/react-dom": { - "version": "18.3.7", - "resolved": "https://registry.npmjs.org/@types/react-dom/-/react-dom-18.3.7.tgz", - "integrity": "sha512-MEe3UeoENYVFXzoXEWsvcpg6ZvlrFNlOQ7EOsvhI3CfAXwzPfO8Qwuxd40nepsYKqyyVQnTdEfv68q91yLcKrQ==", + "version": "19.2.4", + "resolved": "https://registry.npmjs.org/@types/react-dom/-/react-dom-19.2.4.tgz", + "integrity": "sha512-Bsc+QHgp+P/F02XDzNCY9jnZNCUuLki36KT7VKrTXXLdHf+vHMNZnW1rVu5DNW/rCK+fya3DATySbLM4yhtKUw==", "dev": true, "license": "MIT", "peerDependencies": { - "@types/react": "^18.0.0" + "@types/react": "^19.2.0" } }, "node_modules/@types/react-syntax-highlighter": { @@ -4046,12 +4038,6 @@ "@types/react": "*" } }, - "node_modules/@types/scheduler": { - "version": "0.26.0", - "resolved": "https://registry.npmjs.org/@types/scheduler/-/scheduler-0.26.0.tgz", - "integrity": "sha512-WFHp9YUJQ6CKshqoC37iOlHnQSmxNc795UhB26CyBBttrN9svdIrUjl/NjnNmfcwtncN0h/0PPAFWv9ovP8mLA==", - "license": "MIT" - }, "node_modules/@types/unist": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/@types/unist/-/unist-3.0.3.tgz", @@ -11565,13 +11551,10 @@ } }, "node_modules/react": { - "version": "18.3.1", - "resolved": "https://registry.npmjs.org/react/-/react-18.3.1.tgz", - "integrity": "sha512-wS+hAgJShR0KhEvPJArfuPVN1+Hz1t0Y6n5jLrGQbkb4urgPE/0Rve+1kMB1v/oWgHgm4WIcV+i7F2pTVj+2iQ==", + "version": "19.2.8", + "resolved": "https://registry.npmjs.org/react/-/react-19.2.8.tgz", + "integrity": "sha512-PWaYA1L/q9u2u7xYQi+Y3L3Yfnie7XyLeaJICV1MGD6LprsBxcAqGjYyr0eY3p+QdsA+x/Irkt4Qif8D63+Sbw==", "license": "MIT", - "dependencies": { - "loose-envify": "^1.1.0" - }, "engines": { "node": ">=0.10.0" } @@ -11590,16 +11573,15 @@ } }, "node_modules/react-dom": { - "version": "18.3.1", - "resolved": "https://registry.npmjs.org/react-dom/-/react-dom-18.3.1.tgz", - "integrity": "sha512-5m4nQKp+rZRb09LNH59GM4BxTh9251/ylbKIbpe7TpGxfJ+9kv6BLkLBXIjjspbgbnIBNqlI23tRnTWT0snUIw==", + "version": "19.2.8", + "resolved": "https://registry.npmjs.org/react-dom/-/react-dom-19.2.8.tgz", + "integrity": "sha512-rVprimfGBG3DR+Tq0IQG2DT5PxKth1WIGDmj5yPmlzr4YBe7uyE+Du4oVqTDXZSHGGGXRtTJEGSSePyQCMBglQ==", "license": "MIT", "dependencies": { - "loose-envify": "^1.1.0", - "scheduler": "^0.23.2" + "scheduler": "^0.27.0" }, "peerDependencies": { - "react": "^18.3.1" + "react": "^19.2.8" } }, "node_modules/react-hook-form": { @@ -12204,13 +12186,10 @@ } }, "node_modules/scheduler": { - "version": "0.23.2", - "resolved": "https://registry.npmjs.org/scheduler/-/scheduler-0.23.2.tgz", - "integrity": "sha512-UOShsPwz7NrMUqhR6t0hWjFduvOzbtv7toDH1/hIrfRNIDBnnBWd0CwJTGvTpngVlmwGCdP9/Zl/tVrDqcuYzQ==", - "license": "MIT", - "dependencies": { - "loose-envify": "^1.1.0" - } + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/scheduler/-/scheduler-0.27.0.tgz", + "integrity": "sha512-eNv+WrVbKu1f3vbYJT/xtiF5syA5HPIMtf9IgY/nKg0sWqzAUEvqY/xm7OcZc/qafLx/iO9FgOmeSAp4v5ti/Q==", + "license": "MIT" }, "node_modules/scroll-into-view-if-needed": { "version": "3.1.0", diff --git a/ui/litellm-dashboard/package.json b/ui/litellm-dashboard/package.json index 9407ff25a1f..962cabcba7b 100644 --- a/ui/litellm-dashboard/package.json +++ b/ui/litellm-dashboard/package.json @@ -9,8 +9,11 @@ "start": "next start", "lint": "eslint .", "test": "vitest", + "test:unit": "vitest run --project unit", + "test:component": "vitest run --project component", + "test:integration": "vitest run --project integration", "test:dot": "vitest --reporter=dot", - "test:types": "vitest --run --typecheck.only", + "test:types": "vitest run --project types", "test:watch": "vitest -w", "test:coverage": "vitest run --coverage", "format": "prettier --write .", @@ -45,9 +48,9 @@ "openapi-fetch": "^0.17.0", "openapi-react-query": "^0.5.4", "papaparse": "5.5.3", - "react": "18.3.1", + "react": "19.2.8", "react-copy-to-clipboard": "5.1.1", - "react-dom": "18.3.1", + "react-dom": "19.2.8", "react-hook-form": "7.82.0", "react-json-view-lite": "2.5.0", "react-markdown": "9.1.0", @@ -68,9 +71,9 @@ "@testing-library/react": "16.3.2", "@testing-library/user-event": "14.6.1", "@types/node": "20.19.37", - "@types/react": "18.2.48", + "@types/react": "19.2.18", "@types/react-copy-to-clipboard": "5.0.7", - "@types/react-dom": "18.3.7", + "@types/react-dom": "19.2.4", "@types/react-syntax-highlighter": "15.5.13", "@vitest/coverage-v8": "3.2.6", "@vitest/ui": "3.2.6", diff --git a/ui/litellm-dashboard/src/app/(dashboard)/admin-panel/_components/AdminPanel.tsx b/ui/litellm-dashboard/src/app/(dashboard)/admin-panel/_components/AdminPanel.tsx index 48eaa62e37b..e9d7190abf5 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/admin-panel/_components/AdminPanel.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/admin-panel/_components/AdminPanel.tsx @@ -7,8 +7,8 @@ import { Alert, AlertDescription, AlertTitle } from "@/components/shared/Alert"; import { Button } from "@/components/ui/button"; import { Card } from "@/components/ui/card"; import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from "@/components/ui/table"; -import { Alert as AntdAlert, Modal, Space, Tabs, Typography } from "antd"; -import { Info } from "lucide-react"; +import { Modal, Space, Tabs, Typography } from "antd"; +import { Info, TriangleAlert } from "lucide-react"; import React, { useEffect, useState } from "react"; import NewBadge from "@/components/common_components/NewBadge"; import { useBaseUrl } from "@/components/constants"; @@ -223,12 +223,14 @@ const AdminPanel: React.FC = ({ proxySettings }) => { <> ✨ Security Settings - + + + SSO Configuration Deprecated + + Editing SSO Settings on this page is deprecated and will be removed in a future version. Please use the + SSO Settings tab for SSO configuration. + +
= {}): ShadowEvalJob => ({ baseline_model: null, judge_model: "anthropic/claude-sonnet-5", shadow_percentage: 10, - max_turns: 200, + keys: [ + { + api_key_id: "hashed-key-abc", + max_turns: 200, + stopped_at: null, + key_alias: "prod-alpha", + key_name: "sk-...alpha", + }, + ], judged_count: 42, error_count: 1, judge_spend: 3.21, @@ -110,19 +118,28 @@ const job = (overrides: Partial = {}): ShadowEvalJob => ({ avg_judge_confidence: 0.8, }, ], + by_key: [], overall_shadow_win_rate_pct: 48.0, overall_tie_rate_pct: 22.0, }, created_at: "2026-08-07T00:00:00Z", ends_at: "2026-09-07T00:00:00Z", - stopped_at: null, - api_key_id: "hashed-key-abc", - key_alias: "prod-alpha", - key_name: "sk-...alpha", last_error: null, ...overrides, }); +const keyEntry = ( + api_key_id: string, + overrides: Partial = {}, +): ShadowEvalJob["keys"][number] => ({ + api_key_id, + max_turns: 200, + stopped_at: null, + key_alias: null, + key_name: null, + ...overrides, +}); + const mockHooks = ({ jobs = [], detailsById = {}, @@ -199,8 +216,8 @@ describe("ShadowEvalSection", () => { it("gives every active job its own card with a stop button, with the form still offered", () => { mockHooks({ jobs: [ - job({ job_id: "job-a", status: "running", api_key_id: "key-a" }), - job({ job_id: "job-b", status: "running", api_key_id: "key-b" }), + job({ job_id: "job-a", status: "running", keys: [keyEntry("key-a")] }), + job({ job_id: "job-b", status: "running", keys: [keyEntry("key-b")] }), ], }); render(); @@ -342,7 +359,7 @@ describe("ShadowEvalSection", () => { expect(container).toBeEmptyDOMElement(); }); - it("keeps the start button disabled until key, router, and judge model are picked, then submits them", async () => { + it("keeps the start button disabled until key, router, and judge model are picked, then submits the key as a list", async () => { const user = userEvent.setup(); const { start } = mockHooks({}); render(); @@ -361,7 +378,7 @@ describe("ShadowEvalSection", () => { await user.click(screen.getByText("Start shadow eval")); const expectedBody = { - api_key_id: "hash-alpha", + api_key_ids: ["hash-alpha"], router_name: "gpt-auto", direction: "forward", shadow_percentage: 10, @@ -396,7 +413,7 @@ describe("ShadowEvalSection", () => { await user.click(screen.getByText("Start shadow eval")); const expectedBody = { - api_key_id: "hash-alpha", + api_key_ids: ["hash-alpha"], router_name: "gpt-auto", direction: "reverse", baseline_model: "prod-claude", @@ -429,9 +446,9 @@ describe("ShadowEvalSection", () => { }); it("labels the shadowed key by alias, then masked name, then truncated hash", () => { - expect(shadowedKeyLabel(job())).toBe("prod-alpha"); - expect(shadowedKeyLabel(job({ key_alias: null }))).toBe("sk-...alpha"); - expect(shadowedKeyLabel(job({ key_alias: null, key_name: null }))).toBe("hashed-key…"); + expect(shadowedKeyLabel(job().keys[0])).toBe("prod-alpha"); + expect(shadowedKeyLabel(keyEntry("hashed-key-abc", { key_name: "sk-...alpha" }))).toBe("sk-...alpha"); + expect(shadowedKeyLabel(keyEntry("hashed-key-abc"))).toBe("hashed-key…"); }); it("keeps an older job's verdicts reachable through the previous evaluations list", async () => { diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/ShadowEvalSection.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/ShadowEvalSection.tsx index 005636615b1..6d240a84c98 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/ShadowEvalSection.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/ShadowEvalSection.tsx @@ -24,6 +24,7 @@ import { useStartShadowEval, useStopShadowEval, type ShadowEvalJob, + type ShadowEvalJobKey, type ShadowEvalSlice, } from "./useShadowEval"; @@ -50,19 +51,24 @@ const routerMatchedOrBeatPct = ( ? 100 - results.overall_shadow_win_rate_pct : results.overall_shadow_win_rate_pct + results.overall_tie_rate_pct; -export const shadowedKeyLabel = (job: ShadowEvalJob): string => - job.key_alias || job.key_name || `${job.api_key_id.slice(0, 10)}…`; +export const shadowedKeyLabel = (key: ShadowEvalJobKey): string => + key.key_alias || key.key_name || `${key.api_key_id.slice(0, 10)}…`; + +const shadowedKeysLabel = (job: ShadowEvalJob): string => + job.keys.length === 1 ? shadowedKeyLabel(job.keys[0]) : `${job.keys.length} keys`; + +const totalBudget = (job: ShadowEvalJob): number => job.keys.reduce((sum, key) => sum + key.max_turns, 0); const jobHeadline = (job: ShadowEvalJob): React.ReactNode => job.direction === "reverse" ? ( <> Comparing {job.router_name} to{" "} {job.baseline_model} on {job.shadow_percentage}% of{" "} - {shadowedKeyLabel(job)} traffic + {shadowedKeysLabel(job)} traffic ) : ( <> - Shadowing {job.shadow_percentage}% of {shadowedKeyLabel(job)} traffic + Shadowing {job.shadow_percentage}% of {shadowedKeysLabel(job)} traffic via {job.router_name} ); @@ -178,7 +184,8 @@ const emptyResultsText = (job: ShadowEvalJob, resultsError: boolean): string => const ResultsBody: React.FC<{ job: ShadowEvalJob; resultsError?: boolean }> = ({ job, resultsError = false }) => { const results = job.results; - if (!results || (results.by_tier.length === 0 && results.by_current_model.length === 0)) { + const stratifications = results ? [results.by_tier, results.by_current_model, results.by_key] : []; + if (!results || stratifications.every((slices) => slices.length === 0)) { return

{emptyResultsText(job, resultsError)}

; } return ( @@ -224,7 +231,7 @@ const JobResults: React.FC<{

{jobHeadline(job)}

- {(job.judged_count ?? 0).toLocaleString()} of {job.max_turns.toLocaleString()} turns judged ·{" "} + {(job.judged_count ?? 0).toLocaleString()} of {totalBudget(job).toLocaleString()} turns judged ·{" "} {(job.error_count ?? 0).toLocaleString()} errored · {usd(job.judge_spend ?? 0)} judge spend {active && remaining ? ` · ${remaining}` : ""}

@@ -386,14 +393,13 @@ const StartForm: React.FC = () => { const percentageValid = parsedPct >= 0.1 && parsedPct <= 100; const parsedMaxTurns = Number.parseInt(maxTurns, 10); const maxTurnsValid = parsedMaxTurns >= 1 && parsedMaxTurns <= 2000; - const filled = - [apiKeyId, routerName, judgeModel].every((field) => field !== "") && - (direction === "forward" || baselineModel !== ""); + const baselinePicked = direction === "forward" || baselineModel !== ""; + const filled = [apiKeyId, routerName, judgeModel].every((field) => field !== "") && baselinePicked; const boundsValid = percentageValid && maxTurnsValid; const valid = Boolean(accessToken) && filled && boundsValid; const handleStart = () => { const startBody = { - api_key_id: apiKeyId, + api_key_ids: [apiKeyId], router_name: routerName, direction, ...(direction === "reverse" ? { baseline_model: baselineModel } : {}), diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/useShadowEval.ts b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/useShadowEval.ts index 7645fcc3346..eef98320e67 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/useShadowEval.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/useShadowEval.ts @@ -7,6 +7,7 @@ import { $api, fetchClient } from "@/lib/http/api"; import type { components } from "@/lib/http/schema"; export type ShadowEvalJob = components["schemas"]["ShadowEvalJobResponse"]; +export type ShadowEvalJobKey = components["schemas"]["ShadowEvalJobKeyResponse"]; export type ShadowEvalSlice = components["schemas"]["ShadowEvalSlice"]; export type StartShadowEvalRequest = components["schemas"]["StartShadowEvalRequest"]; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-tracking/_components/pricing_calculator/multi_export_utils.test.ts b/ui/litellm-dashboard/src/app/(dashboard)/cost-tracking/_components/pricing_calculator/multi_export_utils.test.ts index 3c319e5ba18..25b4da869bb 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-tracking/_components/pricing_calculator/multi_export_utils.test.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-tracking/_components/pricing_calculator/multi_export_utils.test.ts @@ -1,3 +1,5 @@ +// @vitest-environment jsdom + import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; import { exportMultiToPDF, exportMultiToCSV } from "./multi_export_utils"; import type { MultiModelResult } from "./types"; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/MCPPermissionManagement.tsx b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/MCPPermissionManagement.tsx index b170157049c..80a82dc22b8 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/MCPPermissionManagement.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/MCPPermissionManagement.tsx @@ -1,5 +1,7 @@ import React, { useEffect } from "react"; -import { Alert, Select, Tooltip, Collapse, Input, Space, Switch } from "antd"; +import { Select, Tooltip, Collapse, Input, Space, Switch } from "antd"; +import { TriangleAlert } from "lucide-react"; +import { Alert, AlertDescription, AlertTitle } from "@/components/shared/Alert"; import { Button } from "@/components/ui/button"; import { InfoCircleOutlined, MinusCircleOutlined, PlusOutlined } from "@ant-design/icons"; import { useFieldArray, useFormContext, useWatch } from "react-hook-form"; @@ -271,13 +273,15 @@ const MCPPermissionManagement: React.FC = ({ )} {showInternalDelegatePkceWarning && ( - + + + Internal server with upstream OAuth delegation + + This MCP server is configured as internal-only but delegates auth to upstream. Anonymous users will be + able to reach the upstream OAuth2 /authorize flow without a LiteLLM session. Ensure your upstream + provider and network enforce access controls. + + )} = ({ server, open, acces
) : isError ? ( - + + + Failed to load env vars + ) : required.length === 0 ? ( - + + + No per-user fields configured for this server. + ) : ( <> diff --git a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/createOAuthUiState.test.ts b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/createOAuthUiState.test.ts index e2e7814f0d7..94544ec6e2d 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/createOAuthUiState.test.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/createOAuthUiState.test.ts @@ -1,3 +1,5 @@ +// @vitest-environment jsdom + import { beforeEach, describe, expect, it, vi } from "vitest"; import { setSecureItem } from "@/utils/secureStorage"; import { CreateUiSnapshot, readCreateUiSnapshot, writeCreateUiSnapshot } from "./createOAuthUiState"; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/mcp_connect.tsx b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/mcp_connect.tsx index bb739901faa..8aeb94404f2 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/mcp_connect.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/mcp_connect.tsx @@ -1,10 +1,22 @@ /* eslint-disable react/no-unescaped-entities */ import React, { useState } from "react"; -import { Card, Typography, Space, Alert, Switch } from "antd"; +import { Card, Typography, Space, Switch } from "antd"; +import { Alert, AlertDescription, AlertTitle } from "@/components/shared/Alert"; import { Button } from "@/components/ui/button"; import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs"; -import { CopyIcon, Code, Terminal, Globe, CheckIcon, ExternalLinkIcon, KeyIcon, ServerIcon, Zap } from "lucide-react"; +import { + CopyIcon, + Code, + Terminal, + Globe, + CheckIcon, + ExternalLinkIcon, + Info, + KeyIcon, + ServerIcon, + Zap, +} from "lucide-react"; import { getProxyBaseUrl } from "@/components/networking"; import { copyToClipboard as utilCopyToClipboard } from "@/utils/dataUtils"; @@ -69,25 +81,21 @@ const FeatureCard: React.FC = ({
{useServerHeader && ( - -

- Option 1: Get a specific server: "{serverName.replace(/\s+/g, "_")}" -

-

- Option 2: Get a group of MCPs: "dev-group" -

-

- You can also mix both: "Server1,dev-group" -

- - } - /> + + + Two Options + +

+ Option 1: Get a specific server: "{serverName.replace(/\s+/g, "_")}" +

+

+ Option 2: Get a group of MCPs: "dev-group" +

+

+ You can also mix both: "Server1,dev-group" +

+ + )} )} diff --git a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/mcp_server_edit.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/mcp_server_edit.test.tsx index 14e26fa5393..96685bb8359 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/mcp_server_edit.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/mcp_server_edit.test.tsx @@ -1939,6 +1939,24 @@ describe("MCPServerEdit OAuth flow prefill display", () => { expect(screen.queryByText("This server has no OAuth flow set")).not.toBeInTheDocument(); }); + it("never flashes the warning while mounting a server that already has a flow", async () => { + const flashes: Node[] = []; + const observer = new MutationObserver((records) => { + for (const record of records) { + for (const node of record.addedNodes) { + if (node.textContent?.includes("This server has no OAuth flow set")) flashes.push(node); + } + } + }); + observer.observe(document.body, { childList: true, subtree: true }); + + renderEdit({ oauth2_flow: "client_credentials" }); + await screen.findByText("Machine-to-Machine (M2M)"); + observer.disconnect(); + + expect(flashes).toHaveLength(0); + }); + it("does not warn for a delegate (PKCE passthrough) server even with no flow set", () => { renderEdit({ oauth2_flow: null, delegate_auth_to_upstream: true }); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/mcp_server_edit.tsx b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/mcp_server_edit.tsx index eb462f788a5..4b092aa3f80 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/mcp_server_edit.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/mcp_server_edit.tsx @@ -1,5 +1,7 @@ import React, { useState, useEffect } from "react"; -import { Select, Tooltip, Input, InputNumber, Alert } from "antd"; +import { Select, Tooltip, Input, InputNumber } from "antd"; +import { TriangleAlert } from "lucide-react"; +import { Alert, AlertDescription, AlertTitle } from "@/components/shared/Alert"; import { FormProvider, useForm } from "react-hook-form"; import { InfoCircleOutlined } from "@ant-design/icons"; import { Button } from "@/components/ui/button"; @@ -171,7 +173,10 @@ const MCPServerEdit: React.FC = ({ const isTokenExchangeAuthType = authType === AUTH_TYPE.OAUTH2_TOKEN_EXCHANGE; const isIdJagAuthType = authType === AUTH_TYPE.OAUTH2_ID_JAG; const isAwsSigV4AuthType = authType === AUTH_TYPE.AWS_SIGV4; - const oauthFlowTypeValue = mountedValues.oauth_flow_type as string | undefined; + // Same fallback as the delegate switch below: the value is undefined until the field mounts, so + // reading it alone flashes the "no OAuth flow set" warning at a server that already has one. + const oauthFlowTypeValue = + (mountedValues.oauth_flow_type as string | undefined) ?? oauth2FlowToFormValue(mcpServer.oauth2_flow); const isM2MFlow = isOAuthAuthType && oauthFlowTypeValue === OAUTH_FLOW.M2M; // Watch reflects a live toggle when the delegate switch is mounted; fall back to // the stored value otherwise (useWatch returns undefined for an unmounted field, @@ -1041,13 +1046,15 @@ const MCPServerEdit: React.FC = ({ {!isStdioTransport && isOAuthAuthType && ( <> {!oauthFlowTypeValue && !isDelegateAuth && ( - + + + This server has no OAuth flow set + + Choose Machine-to-Machine (M2M) or Interactive (PKCE) so LiteLLM authenticates it the way you + intend, then save. Until it is set, LiteLLM falls back to interactive per-user auth and treats + a machine-to-machine credential shape conservatively. + + )} = ({ {selectedMode === "flow_builder" && ( + > + + You'll be redirected to the full-screen Flow Builder to design your policy logic visually. + + )}
@@ -457,35 +461,33 @@ const AddPolicyForm: React.FC = ({ {resolvedGuardrails.length > 0 && ( - - - These are the final guardrails that will be applied (including inheritance): - -
- {resolvedGuardrails.map((g) => ( - - {g} - - ))} -
+ + + Resolved Guardrails + + + These are the final guardrails that will be applied (including inheritance): + +
+ {resolvedGuardrails.map((g) => ( + + {g} + + ))}
- } - type="info" - showIcon - /> + + )} - + + + Model Scope + + By default, this policy will run on all models. You can optionally restrict it to specific models below. + +
Model Condition Type diff --git a/ui/litellm-dashboard/src/app/(dashboard)/policies/_components/policy_test_panel.tsx b/ui/litellm-dashboard/src/app/(dashboard)/policies/_components/policy_test_panel.tsx index 90642d23a46..9c234a7a70c 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/policies/_components/policy_test_panel.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/policies/_components/policy_test_panel.tsx @@ -1,6 +1,8 @@ import React, { useState, useEffect } from "react"; import { useForm } from "react-hook-form"; -import { Alert, Empty } from "antd"; +import { Empty } from "antd"; +import { CircleAlert } from "lucide-react"; +import { Alert, AlertDescription, AlertTitle } from "@/components/shared/Alert"; import { resolvePoliciesCall, teamListCall, keyListCall, modelAvailableCall } from "@/components/networking"; import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; import { FieldGroup } from "@/components/shared/form/field"; @@ -325,7 +327,11 @@ const PolicyTestPanel: React.FC = ({ accessToken }) => { )} {hasSearched && !result && !isLoading && ( - + + + Error + Failed to resolve policies. Check the proxy logs. + )}
); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/prompts/_components/prompt_editor_view/conversation_panel/MessageList.tsx b/ui/litellm-dashboard/src/app/(dashboard)/prompts/_components/prompt_editor_view/conversation_panel/MessageList.tsx index 347553630ae..48d6697a052 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/prompts/_components/prompt_editor_view/conversation_panel/MessageList.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/prompts/_components/prompt_editor_view/conversation_panel/MessageList.tsx @@ -8,7 +8,7 @@ interface MessageListProps { messages: Message[]; isLoading: boolean; hasVariables: boolean; - messagesEndRef: React.RefObject; + messagesEndRef: React.RefObject; } const MessageList: React.FC = ({ messages, isLoading, hasVariables, messagesEndRef }) => { diff --git a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/UsageViewSelect/UsageViewSelect.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/UsageViewSelect/UsageViewSelect.test.tsx index 3d835c98f52..8848ef9f49e 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/UsageViewSelect/UsageViewSelect.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/UsageViewSelect/UsageViewSelect.test.tsx @@ -1,6 +1,7 @@ import { render, screen, within } from "@testing-library/react"; import userEvent from "@testing-library/user-event"; import { beforeEach, describe, expect, it, vi } from "vitest"; +import { chooseSelectOption } from "@/../tests/test-utils"; import { UsageViewSelect } from "./UsageViewSelect"; const openMenu = async (user: ReturnType) => { @@ -35,9 +36,7 @@ describe("UsageViewSelect", () => { const user = userEvent.setup(); render(); - await openMenu(user); - const matches = screen.getAllByText("Team Usage"); - await user.click(matches[matches.length - 1]); + await chooseSelectOption(user, screen.getByRole("combobox"), /^Team Usage/); expect(mockOnChange).toHaveBeenCalled(); expect(mockOnChange.mock.calls[0][0]).toBe("team"); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/vector-stores/_components/CreateVectorStore.tsx b/ui/litellm-dashboard/src/app/(dashboard)/vector-stores/_components/CreateVectorStore.tsx index 93aaa90ef7c..271ef31a68e 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/vector-stores/_components/CreateVectorStore.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/vector-stores/_components/CreateVectorStore.tsx @@ -1,9 +1,10 @@ import React, { useState } from "react"; -import { Upload, Alert } from "antd"; +import { Upload } from "antd"; import { toast } from "@/lib/toast"; import { InboxOutlined } from "@ant-design/icons"; import type { UploadProps } from "antd"; -import { CircleHelp } from "lucide-react"; +import { CircleCheck, CircleHelp, X } from "lucide-react"; +import { Alert, AlertAction, AlertDescription, AlertTitle } from "@/components/shared/Alert"; import { ragIngestCall } from "@/components/networking"; import { DocumentUpload, RAGIngestResponse } from "@/components/vector_store_management/types"; import DocumentsTable from "./DocumentsTable"; @@ -359,22 +360,23 @@ const CreateVectorStore: React.FC = ({ accessToken, onSu {/* Success Message */} {ingestResults.length > 0 && ( - -

- Vector Store ID: {ingestResults[0]?.vector_store_id} -

-

- Documents Ingested: {ingestResults.length} -

-
- } - type="success" - showIcon - closable - /> + + + Vector Store Created Successfully + +

+ Vector Store ID: {ingestResults[0]?.vector_store_id} +

+

+ Documents Ingested: {ingestResults.length} +

+ + + + + )} diff --git a/ui/litellm-dashboard/src/app/(dashboard)/vector-stores/_components/S3VectorsConfig.tsx b/ui/litellm-dashboard/src/app/(dashboard)/vector-stores/_components/S3VectorsConfig.tsx index 3e286ee9b92..cdc63e8b7ec 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/vector-stores/_components/S3VectorsConfig.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/vector-stores/_components/S3VectorsConfig.tsx @@ -1,6 +1,6 @@ import React, { useState, useEffect } from "react"; -import { Alert } from "antd"; -import { CircleHelp } from "lucide-react"; +import { CircleHelp, Info } from "lucide-react"; +import { Alert, AlertDescription, AlertTitle } from "@/components/shared/Alert"; import { fetchAvailableModels, ModelGroup } from "@/components/llm_calls/fetch_models"; import { Field, FieldError, FieldLabel } from "@/components/shared/form/field"; import { @@ -72,32 +72,28 @@ const S3VectorsConfig: React.FC = ({ accessToken, provider return ( - -

AWS S3 Vectors allows you to store and query vector embeddings directly in S3:

-
    -
  • Vector buckets and indexes will be automatically created if they don't exist
    • -
  • Vector dimensions are auto-detected from your selected embedding model
    • -
  • Ensure your AWS credentials have permissions for S3 Vectors operations
    • -
  • - Learn more:{" "} - - AWS S3 Vectors Documentation - -
    • -
- - } - type="info" - showIcon - style={{ marginBottom: "16px" }} - /> + + + AWS S3 Vectors Setup + +

AWS S3 Vectors allows you to store and query vector embeddings directly in S3:

+
    +
  • Vector buckets and indexes will be automatically created if they don't exist
    • +
  • Vector dimensions are auto-detected from your selected embedding model
    • +
  • Ensure your AWS credentials have permissions for S3 Vectors operations
    • +
  • + Learn more:{" "} + + AWS S3 Vectors Documentation + +
    • +
+ + diff --git a/ui/litellm-dashboard/src/app/(dashboard)/vector-stores/_components/VectorStoreForm.tsx b/ui/litellm-dashboard/src/app/(dashboard)/vector-stores/_components/VectorStoreForm.tsx index 05c829d79df..ff6c10bf428 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/vector-stores/_components/VectorStoreForm.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/vector-stores/_components/VectorStoreForm.tsx @@ -1,6 +1,7 @@ import React, { useState, useEffect } from "react"; -import { Modal, Alert } from "antd"; -import { CircleHelp, Eye, EyeOff } from "lucide-react"; +import { Modal } from "antd"; +import { CircleHelp, Eye, EyeOff, Info } from "lucide-react"; +import { Alert, AlertDescription, AlertTitle } from "@/components/shared/Alert"; import { useWatch } from "react-hook-form"; import { z } from "zod/v4"; import { CredentialItem, vectorStoreCreateCall } from "@/components/networking"; @@ -310,142 +311,129 @@ const VectorStoreForm: React.FC = ({ {selectedProvider === "pg_vector" && ( - -

LiteLLM provides a server to connect to PG Vector. To use this provider:

-
    -
  1. - Deploy the litellm-pgvector server from:{" "} - - https://github.com/BerriAI/litellm-pgvector - -
    2. -
  2. Configure your PostgreSQL database with pgvector extension
    3. -
  3. Start the server and note the API base URL and API key
    4. -
  4. Enter those details in the fields below
    5. -
- - } - type="info" - showIcon - /> + + + PG Vector Setup Required + +

LiteLLM provides a server to connect to PG Vector. To use this provider:

+
    +
  1. + Deploy the litellm-pgvector server from:{" "} + + https://github.com/BerriAI/litellm-pgvector + +
    2. +
  2. Configure your PostgreSQL database with pgvector extension
    3. +
  3. Start the server and note the API base URL and API key
    4. +
  4. Enter those details in the fields below
    5. +
+ + )} {selectedProvider === "valkey" && ( - -

- LiteLLM searches documents you have already stored in Valkey. It does not create the index or - upload documents for you. Before creating this vector store, make sure: -

-
    -
  1. - Your Valkey server has vector search enabled (the valkey-search module, included in the - valkey-bundle image and in AWS ElastiCache / MemoryDB for Valkey) -
    2. -
  2. - You have already created a search index and loaded your documents and their embeddings into it. - Enter that index name as the Vector Store ID -
    3. -
  3. - You know which embedding model created those stored embeddings. That model must be added to this - proxy under Models so you can pick it below. Using a different model returns wrong results -
    4. -
  4. - You know the field names your documents use for their text and their embedding. If they are not - "text" and "embedding", set them below -
    5. -
-

- When a query comes in, LiteLLM converts it to an embedding with the model below and returns the - closest matching documents from your index. -

- - } - type="info" - showIcon - /> + + + Valkey Setup Required + +

+ LiteLLM searches documents you have already stored in Valkey. It does not create the index or upload + documents for you. Before creating this vector store, make sure: +

+
    +
  1. + Your Valkey server has vector search enabled (the valkey-search module, included in the + valkey-bundle image and in AWS ElastiCache / MemoryDB for Valkey) +
    2. +
  2. + You have already created a search index and loaded your documents and their embeddings into it. + Enter that index name as the Vector Store ID +
    3. +
  3. + You know which embedding model created those stored embeddings. That model must be added to this + proxy under Models so you can pick it below. Using a different model returns wrong results +
    4. +
  4. + You know the field names your documents use for their text and their embedding. If they are not + "text" and "embedding", set them below +
    5. +
+

+ When a query comes in, LiteLLM converts it to an embedding with the model below and returns the + closest matching documents from your index. +

+ + )} {selectedProvider === "vertex_rag_engine" && ( - -

To use Vertex AI RAG Engine:

-

- Note: Google Cloud has renamed this to "RAG Engine" in its console — the steps below - still apply. -

-
    -
  1. - Set up your Vertex AI RAG Engine corpus following the guide:{" "} - - Vertex AI RAG Engine Overview - -
    2. -
  2. Create a corpus in your Google Cloud project
    3. -
  3. - Note the corpus ID from the Vertex AI console (now labeled "RAG Engine" in Google - Cloud) -
    4. -
  4. Enter the corpus ID in the Vector Store ID field below
    5. -
- - } - type="info" - showIcon - /> + + + Vertex AI RAG Engine Setup + +

To use Vertex AI RAG Engine:

+

+ Note: Google Cloud has renamed this to "RAG Engine" in its console — the steps below still + apply. +

+
    +
  1. + Set up your Vertex AI RAG Engine corpus following the guide:{" "} + + Vertex AI RAG Engine Overview + +
    2. +
  2. Create a corpus in your Google Cloud project
    3. +
  3. + Note the corpus ID from the Vertex AI console (now labeled "RAG Engine" in Google Cloud) +
    4. +
  4. Enter the corpus ID in the Vector Store ID field below
    5. +
+ + )} {selectedProvider === "vertex_ai/search_api" && ( - -

To use Vertex AI Search (Discovery Engine):

-

- Note: Google Cloud has renamed this to "Agent Search" in its console — the steps below - still apply. -

-
    -
  1. - Enable the Discovery Engine API on your Google Cloud project and create a data store following - the guide:{" "} - - Create a Vertex AI Search data store - -
    2. -
  2. Pick a supported location: global, us, or eu
    3. -
  3. - For most data store types (Cloud Storage, BigQuery, Media): copy the data store ID and enter it - in the Vector Store ID field below. -
    4. -
  4. - For website, healthcare, and connector-based sources (Drive, Gmail, Slack, Jira, etc.): create a - search app on top of the data store, then copy the Engine ID and enter it in - the Engine ID field. The Vector Store ID is still required as the LiteLLM-side name for this - record, but it isn't used in the GCP URL when Engine ID is set. -
    5. -
- - } - type="info" - showIcon - /> + + + Vertex AI Search Setup + +

To use Vertex AI Search (Discovery Engine):

+

+ Note: Google Cloud has renamed this to "Agent Search" in its console — the steps below + still apply. +

+
    +
  1. + Enable the Discovery Engine API on your Google Cloud project and create a data store following the + guide:{" "} + + Create a Vertex AI Search data store + +
    2. +
  2. Pick a supported location: global, us, or eu
    3. +
  3. + For most data store types (Cloud Storage, BigQuery, Media): copy the data store ID and enter it in + the Vector Store ID field below. +
    4. +
  4. + For website, healthcare, and connector-based sources (Drive, Gmail, Slack, Jira, etc.): create a + search app on top of the data store, then copy the Engine ID and enter it in the + Engine ID field. The Vector Store ID is still required as the LiteLLM-side name for this record, + but it isn't used in the GCP URL when Engine ID is set. +
    5. +
+ + )} ; +function SsoEnabledNotice() { + const [isDismissed, setIsDismissed] = useState(false); + if (isDismissed) return null; + + return ( + + + + Single Sign-On (SSO) is enabled. LiteLLM no longer automatically redirects to the SSO login flow upon loading + this page. To re-enable auto-redirect-to-SSO, set{" "} + AUTO_REDIRECT_UI_LOGIN_TO_SSO=true in your + environment configuration. + + + + + + ); +} + function LoginPageContent() { const [isLoading, setIsLoading] = useState(true); const { data: uiConfig, isLoading: isConfigLoading } = useUIConfig(); @@ -170,22 +192,19 @@ function LoginPageContent() {

🚅 LiteLLM

- -

- The Admin UI has been disabled by the administrator. To re-enable it, please update the following - environment variable: -

-

- DISABLE_ADMIN_UI=False -

- - } - type="warning" - showIcon - /> + + + Admin UI Disabled + +

+ The Admin UI has been disabled by the administrator. To re-enable it, please update the following + environment variable: +

+

+ DISABLE_ADMIN_UI=False +

+ + @@ -209,31 +228,32 @@ function LoginPageContent() { {!uiConfig?.hide_default_credentials_hint && ( - -

- By default, Username is admin{" "} - and Password is your set LiteLLM Proxy - MASTER_KEY. -

-

- Need to set UI credentials or SSO?{" "} - - Check the documentation - - . -

- - } - type="info" - icon={} - showIcon - /> + + + Default Credentials + +

+ By default, Username is admin and + Password is your set LiteLLM Proxy + MASTER_KEY. +

+

+ Need to set UI credentials or SSO?{" "} + + Check the documentation + + . +

+ + )} - {error && } + {error && ( + + + {error} + + )}
@@ -326,22 +346,7 @@ function LoginPageContent() {
- {uiConfig?.sso_configured && ( - - Single Sign-On (SSO) is enabled. LiteLLM no longer automatically redirects to the SSO login flow - upon loading this page. To re-enable auto-redirect-to-SSO, set{" "} - AUTO_REDIRECT_UI_LOGIN_TO_SSO=true{" "} - in your environment configuration. - - } - /> - )} + {uiConfig?.sso_configured && } diff --git a/ui/litellm-dashboard/src/app/onboarding/OnboardingFormBody.tsx b/ui/litellm-dashboard/src/app/onboarding/OnboardingFormBody.tsx index 1f2441c4870..0a03cc6a591 100644 --- a/ui/litellm-dashboard/src/app/onboarding/OnboardingFormBody.tsx +++ b/ui/litellm-dashboard/src/app/onboarding/OnboardingFormBody.tsx @@ -1,6 +1,7 @@ -import { Alert } from "antd"; +import { CircleAlert, Info } from "lucide-react"; import React from "react"; import { z } from "zod/v4"; +import { Alert, AlertDescription, AlertTitle } from "@/components/shared/Alert"; import { PasswordInput } from "@/components/shared/PasswordInput"; import { Field, FieldLabel, FieldGroup } from "@/components/shared/form/field"; import { FormField } from "@/components/shared/form/FormField"; @@ -46,11 +47,10 @@ export function OnboardingFormBody({ variant, userEmail, isPending, claimError,

{variant === "signup" && ( - + + SSO +
SSO is under the Enterprise Tier.
- } - showIcon - /> + + )}
@@ -84,7 +83,12 @@ export function OnboardingFormBody({ variant, userEmail, isPending, claimError, - {claimError && } + {claimError && ( + + + {claimError} + + )}
+ + )} {updateError && ( - + + Could not update settings + {updateError instanceof Error && {updateError.message}} + )} diff --git a/ui/litellm-dashboard/src/components/ToolPolicies/ToolPoliciesPanel.test.tsx b/ui/litellm-dashboard/src/components/ToolPolicies/ToolPoliciesPanel.test.tsx index aaa70a5e357..a2d3539a58b 100644 --- a/ui/litellm-dashboard/src/components/ToolPolicies/ToolPoliciesPanel.test.tsx +++ b/ui/litellm-dashboard/src/components/ToolPolicies/ToolPoliciesPanel.test.tsx @@ -4,7 +4,7 @@ import { act, render, screen, waitFor, within } from "@testing-library/react"; import userEvent from "@testing-library/user-event"; import { focusManager, QueryClient, QueryClientProvider } from "@tanstack/react-query"; -import { renderWithProviders, testQueryClient } from "../../../tests/test-utils"; +import { chooseSelectOption, renderWithProviders, testQueryClient } from "../../../tests/test-utils"; import type { ToolRow } from "@/components/networking"; import { ToolPoliciesPanel } from "./ToolPoliciesPanel"; import { toast } from "@/lib/toast"; @@ -84,17 +84,6 @@ const policyValue = (toolId: string, kind: "input" | "output"): string => { const isSaving = (toolId: string, kind: "input" | "output"): boolean => policySelect(toolId, kind).hasAttribute("disabled"); -const chooseOption = async (user: ReturnType, trigger: HTMLElement, label: string) => { - await user.click(trigger); - // The label also renders in the trigger once selected, so take the last match: - // the popup is portalled after the table in document order. - const option = await waitFor(() => { - const matches = screen.getAllByText(label); - return matches[matches.length - 1]; - }); - await user.click(option); -}; - const renderPanel = (onSelectTool = vi.fn()) => renderWithProviders(); @@ -202,7 +191,7 @@ describe("ToolPoliciesPanel inline policy editing", () => { renderPanel(); await waitForRows(); - await chooseOption(user, policySelect("tool-1", "input"), "trusted"); + await chooseSelectOption(user, policySelect("tool-1", "input"), "trusted"); expect(updateToolPolicy).toHaveBeenCalledWith("sk-token", "get_weather", { input_policy: "trusted" }); await waitFor(() => expect(policyValue("tool-1", "input")).toBe("trusted")); @@ -214,7 +203,7 @@ describe("ToolPoliciesPanel inline policy editing", () => { renderPanel(); await waitForRows(); - await chooseOption(user, policySelect("tool-1", "output"), "trusted"); + await chooseSelectOption(user, policySelect("tool-1", "output"), "trusted"); expect(updateToolPolicy).toHaveBeenCalledWith("sk-token", "get_weather", { output_policy: "trusted" }); }); @@ -225,8 +214,8 @@ describe("ToolPoliciesPanel inline policy editing", () => { renderPanel(); await waitForRows(); - await chooseOption(user, policySelect("tool-1", "input"), "trusted"); - await chooseOption(user, policySelect("tool-2", "input"), "blocked"); + await chooseSelectOption(user, policySelect("tool-1", "input"), "trusted"); + await chooseSelectOption(user, policySelect("tool-2", "input"), "blocked"); expect(isSaving("tool-2", "input")).toBe(true); expect(isSaving("tool-1", "input")).toBe(true); @@ -241,8 +230,8 @@ describe("ToolPoliciesPanel inline policy editing", () => { renderPanel(); await waitForRows(); - await chooseOption(user, policySelect("tool-1", "input"), "trusted"); - await chooseOption(user, policySelect("tool-2", "input"), "blocked"); + await chooseSelectOption(user, policySelect("tool-1", "input"), "trusted"); + await chooseSelectOption(user, policySelect("tool-2", "input"), "blocked"); await act(async () => { finishFirst(); }); @@ -262,7 +251,7 @@ describe("ToolPoliciesPanel inline policy editing", () => { () => new Promise((resolve) => (landStaleRefresh = () => resolve(TOOLS))), ); await user.click(screen.getByTestId("datatable-refresh")); - await chooseOption(user, policySelect("tool-1", "input"), "trusted"); + await chooseSelectOption(user, policySelect("tool-1", "input"), "trusted"); await waitFor(() => expect(policyValue("tool-1", "input")).toBe("trusted")); await act(async () => { @@ -281,7 +270,7 @@ describe("ToolPoliciesPanel inline policy editing", () => { renderPanel(); await waitForRows(); - await chooseOption(user, policySelect("tool-1", "input"), "trusted"); + await chooseSelectOption(user, policySelect("tool-1", "input"), "trusted"); await waitFor(() => expect(fromBackend).toHaveBeenCalledWith("Failed to update input policy: nope")); expect(policyValue("tool-1", "input")).toBe("untrusted"); @@ -293,7 +282,7 @@ describe("ToolPoliciesPanel inline policy editing", () => { renderPanel(); await waitForRows(); - await chooseOption(user, policySelect("tool-1", "input"), "trusted"); + await chooseSelectOption(user, policySelect("tool-1", "input"), "trusted"); await waitFor(() => expect(isSaving("tool-1", "input")).toBe(true)); expect(isSaving("tool-1", "output")).toBe(false); diff --git a/ui/litellm-dashboard/src/components/add_model/AddModelForm.tsx b/ui/litellm-dashboard/src/components/add_model/AddModelForm.tsx index 4cb60dc116a..c0c73fbb762 100644 --- a/ui/litellm-dashboard/src/components/add_model/AddModelForm.tsx +++ b/ui/litellm-dashboard/src/components/add_model/AddModelForm.tsx @@ -5,7 +5,9 @@ import { all_admin_roles, isUserTeamAdminForAnyTeam } from "@/utils/roles"; import { modelCreationScope } from "@/utils/modelPermissions"; import { Switch } from "@/components/ui/switch"; import { Field, FieldLabel } from "@/components/shared/form/field"; -import { Select as AntdSelect, Card, Col, Modal, Row, Tooltip, Typography, Alert } from "antd"; +import { Select as AntdSelect, Card, Col, Modal, Row, Tooltip, Typography } from "antd"; +import { Info } from "lucide-react"; +import { Alert, AlertDescription, AlertTitle } from "@/components/shared/Alert"; import { Button } from "@/components/ui/button"; import type { UploadProps } from "antd/es/upload"; import React, { useEffect, useMemo, useState } from "react"; @@ -168,13 +170,13 @@ const AddModelForm: React.FC = ({ )} {!teamAdminSelectedTeam && ( - + + + Team Selection Required + + As a team admin, you need to select your team first before adding models. + + )} )} diff --git a/ui/litellm-dashboard/src/components/add_model/ClassificationMethodConfig.tsx b/ui/litellm-dashboard/src/components/add_model/ClassificationMethodConfig.tsx index 8d42b34a14b..da0c50c7bd0 100644 --- a/ui/litellm-dashboard/src/components/add_model/ClassificationMethodConfig.tsx +++ b/ui/litellm-dashboard/src/components/add_model/ClassificationMethodConfig.tsx @@ -93,7 +93,7 @@ const HowClassificationWorks: React.FC<{ value: ComplexityRouterConfigValue }> =
  • {effectiveTierLabel("REASONING", value.tier_labels)}: Score > {ranges.complexReasoning}{" "} - (or 2+ reasoning markers) + (or 2+ reasoning markers with a score of at least {ranges.simpleMedium})
    • )} diff --git a/ui/litellm-dashboard/src/components/add_pass_through.tsx b/ui/litellm-dashboard/src/components/add_pass_through.tsx index 9f95e7f27ac..09ddac2cf42 100644 --- a/ui/litellm-dashboard/src/components/add_pass_through.tsx +++ b/ui/litellm-dashboard/src/components/add_pass_through.tsx @@ -1,8 +1,9 @@ "use client"; import React, { useState } from "react"; -import { Modal, Alert } from "antd"; -import { CircleHelp, Plug } from "lucide-react"; +import { Modal } from "antd"; +import { CircleHelp, Info, Plug } from "lucide-react"; +import { Alert, AlertDescription, AlertTitle } from "@/components/shared/Alert"; import { useWatch } from "react-hook-form"; import { z } from "zod/v4"; @@ -156,13 +157,14 @@ const AddPassThroughEndpoint: React.FC = ({ }} >
    - + + + What is a Pass-Through Endpoint? + + Route requests from your LiteLLM proxy to any external API. Perfect for custom models, image generation + APIs, or any service you want to proxy through LiteLLM. + + diff --git a/ui/litellm-dashboard/src/components/common_components/PassThroughGuardrailsSection.tsx b/ui/litellm-dashboard/src/components/common_components/PassThroughGuardrailsSection.tsx index f51629b7ee1..a544a8be32b 100644 --- a/ui/litellm-dashboard/src/components/common_components/PassThroughGuardrailsSection.tsx +++ b/ui/litellm-dashboard/src/components/common_components/PassThroughGuardrailsSection.tsx @@ -1,6 +1,6 @@ import React from "react"; -import { Alert } from "antd"; -import { CircleHelp } from "lucide-react"; +import { CircleHelp, Info } from "lucide-react"; +import { Alert, AlertDescription, AlertTitle } from "@/components/shared/Alert"; import GuardrailSelector from "../guardrails/GuardrailSelector"; import { TagsInput } from "@/app/(dashboard)/guardrails/_components/content_filter/TagsInput"; @@ -67,21 +67,20 @@ const PassThroughGuardrailsSection: React.FC endpoints.

    - - Field-Level Targeting{" "} -     - (Learn More) - - - } - description={ + + + + Field-Level Targeting{" "} + + (Learn More) + + +
    Optionally specify which fields to check. If left empty, the entire request/response is sent to the @@ -100,11 +99,8 @@ const PassThroughGuardrailsSection: React.FC
    - } - type="info" - showIcon - className="mb-4" - /> + + diff --git a/ui/litellm-dashboard/src/components/common_components/user_search_modal.test.tsx b/ui/litellm-dashboard/src/components/common_components/user_search_modal.test.tsx index cbaf00b7131..946bbdd649b 100644 --- a/ui/litellm-dashboard/src/components/common_components/user_search_modal.test.tsx +++ b/ui/litellm-dashboard/src/components/common_components/user_search_modal.test.tsx @@ -74,7 +74,7 @@ describe("UserSearchModal", () => { expect(notice).toHaveTextContent(/users that already exist/i); expect(notice).toHaveTextContent(/ask a proxy admin to create their account first/i); // info, not warning: a warning here would read as an error state on an empty form - expect(notice.className).toMatch(/ant-alert-info/); + expect(notice.className).toMatch(/text-info/); }); }); diff --git a/ui/litellm-dashboard/src/components/common_components/user_search_modal.tsx b/ui/litellm-dashboard/src/components/common_components/user_search_modal.tsx index aeead9d82e6..e089b15addc 100644 --- a/ui/litellm-dashboard/src/components/common_components/user_search_modal.tsx +++ b/ui/litellm-dashboard/src/components/common_components/user_search_modal.tsx @@ -1,5 +1,7 @@ import { useState } from "react"; -import { Modal, Alert } from "antd"; +import { Modal } from "antd"; +import { Info } from "lucide-react"; +import { Alert, AlertTitle } from "@/components/shared/Alert"; import { UserAddOutlined } from "@ant-design/icons"; import { useDebouncedCallback } from "@tanstack/react-pacer/debouncer"; import { useForm } from "react-hook-form"; @@ -202,13 +204,13 @@ const UserSearchModal: React.FC = ({ - + + + + Search selects from users that already exist. To add someone new, ask a proxy admin to create their + account first. + + diff --git a/ui/litellm-dashboard/src/components/organisms/RegenerateKeyModal.tsx b/ui/litellm-dashboard/src/components/organisms/RegenerateKeyModal.tsx index d717c11de86..88554ddba89 100644 --- a/ui/litellm-dashboard/src/components/organisms/RegenerateKeyModal.tsx +++ b/ui/litellm-dashboard/src/components/organisms/RegenerateKeyModal.tsx @@ -1,8 +1,9 @@ import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; import { CheckOutlined, CopyOutlined, SyncOutlined } from "@ant-design/icons"; -import { Alert, Modal, Space } from "antd"; +import { Alert, AlertTitle } from "@/components/shared/Alert"; +import { Modal, Space } from "antd"; import { Button } from "@/components/ui/button"; -import { CircleHelp } from "lucide-react"; +import { CircleHelp, TriangleAlert } from "lucide-react"; import React, { useEffect, useMemo, useState } from "react"; import { useWatch } from "react-hook-form"; import { CopyToClipboard } from "react-copy-to-clipboard"; @@ -184,7 +185,10 @@ export function RegenerateKeyModal({ selectedToken, visible, onClose, onKeyUpdat > {regeneratedKey ? (
    - + + + Save it now, you will not see it again +
    Key Alias diff --git a/ui/litellm-dashboard/src/components/team/TeamInfo.test.tsx b/ui/litellm-dashboard/src/components/team/TeamInfo.test.tsx index 77f35762528..a7e8e6788dd 100644 --- a/ui/litellm-dashboard/src/components/team/TeamInfo.test.tsx +++ b/ui/litellm-dashboard/src/components/team/TeamInfo.test.tsx @@ -3,7 +3,7 @@ import * as networking from "@/components/networking"; import { fireEvent, screen, waitFor, within } from "@testing-library/react"; import userEvent from "@testing-library/user-event"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -import { renderWithProviders, testQueryClient } from "../../../tests/test-utils"; +import { chooseSelectOption, renderWithProviders, testQueryClient } from "../../../tests/test-utils"; import TeamInfoView from "./TeamInfo"; const authState = vi.hoisted(() => ({ userRole: "Admin" })); @@ -993,8 +993,7 @@ describe("TeamInfoView", () => { const user = userEvent.setup({ delay: null }); const resetBudgetSelect = await openSettingsEditorForTeam(user, { budget_duration: "30d" }); - await user.click(resetBudgetSelect); - await user.click(await screen.findByText("Never resets")); + await chooseSelectOption(user, resetBudgetSelect, "Never resets"); await waitFor(() => { expect(resetBudgetSelect).toHaveTextContent("Never resets"); @@ -1028,8 +1027,7 @@ describe("TeamInfoView", () => { const user = userEvent.setup({ delay: null }); const resetBudgetSelect = await openSettingsEditorForTeam(user, { budget_duration: null }); - await user.click(resetBudgetSelect); - await user.click(await screen.findByText("weekly")); + await chooseSelectOption(user, resetBudgetSelect, "weekly"); await user.click(screen.getByRole("button", { name: /save changes/i })); diff --git a/ui/litellm-dashboard/src/components/ui/ref-forwarding.test.tsx b/ui/litellm-dashboard/src/components/ui/ref-forwarding.test.tsx index 48b1e8be226..66f7bd27138 100644 --- a/ui/litellm-dashboard/src/components/ui/ref-forwarding.test.tsx +++ b/ui/litellm-dashboard/src/components/ui/ref-forwarding.test.tsx @@ -138,15 +138,13 @@ describe("ui primitives forward refs to their DOM node", () => { }); }); -describe("setupTests ref tripwire", () => { - it("records a violation when a ref is passed to a plain function component", () => { - const Plain = (props: React.ComponentPropsWithoutRef<"span">) => ; +describe("plain function components", () => { + it("receives a ref as a prop instead of dropping it", () => { + const Plain = (props: React.ComponentPropsWithoutRef<"span"> & { ref?: React.Ref }) => ( + + ); const ref = React.createRef(); - render(React.createElement(Plain as never, { ref })); - const consume = (globalThis as { __consumePendingRefWarnings?: () => string[] }).__consumePendingRefWarnings; - expect(consume).toBeDefined(); - const violations = consume!(); - expect(violations).toHaveLength(1); - expect(violations[0]).toContain("Function components cannot be given refs"); + render(ok); + expect(ref.current).toBeInstanceOf(HTMLSpanElement); }); }); diff --git a/ui/litellm-dashboard/src/components/update_model_credentials_modal.tsx b/ui/litellm-dashboard/src/components/update_model_credentials_modal.tsx index b0a19e7ea05..a436ba55c01 100644 --- a/ui/litellm-dashboard/src/components/update_model_credentials_modal.tsx +++ b/ui/litellm-dashboard/src/components/update_model_credentials_modal.tsx @@ -1,10 +1,12 @@ -import { Alert, Modal, Typography } from "antd"; +import { Modal, Typography } from "antd"; +import { TriangleAlert } from "lucide-react"; import { useState } from "react"; import { z } from "zod/v4"; import { modelPatchUpdateCall } from "./networking"; import { toast } from "@/lib/toast"; import { FieldGroup } from "@/components/shared/form/field"; import { FormField } from "@/components/shared/form/FormField"; +import { Alert, AlertTitle } from "@/components/shared/Alert"; import { PasswordInput } from "@/components/shared/PasswordInput"; import { Button } from "@/components/ui/button"; import { UiLoadingSpinner } from "@/components/ui/ui-loading-spinner"; @@ -74,12 +76,14 @@ export default function UpdateModelCredentialsModal({ Update this model's API key. Only the new key is sent; the rest of the deployment configuration is left untouched. - + + + + Only the API key is rotated here. Models that authenticate with an Azure AD token, AWS credentials, or a + Vertex service-account JSON aren't supported yet; update those from the model's LiteLLM Params for + now. + + diff --git a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/RoutingDecisionCard.test.tsx b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/RoutingDecisionCard.test.tsx index a2afe0d7a0d..ed99e414706 100644 --- a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/RoutingDecisionCard.test.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/RoutingDecisionCard.test.tsx @@ -59,7 +59,9 @@ describe("RoutingDecisionCard", () => { }} />, ); - expect(screen.getByText("Heuristic, REASONING override (2 or more reasoning markers)")).toBeInTheDocument(); + expect( + screen.getByText("Heuristic, REASONING override (2 or more reasoning markers, score above the lowest tier)"), + ).toBeInTheDocument(); expect(screen.getByText("0.20")).toBeInTheDocument(); // The score did not decide this tier, so NO band explanation may render at all. // Asserting the absence of one specific band would pass vacuously: 0.20 sits in @@ -188,7 +190,9 @@ describe("RoutingDecisionCard", () => { // `signals` is gone under redaction; the cause alone must suppress the band. render(); expect(screen.queryByText(/SIMPLE|MEDIUM|COMPLEX|at or above/)).not.toBeInTheDocument(); - expect(screen.getByText("Heuristic, REASONING override (2 or more reasoning markers)")).toBeInTheDocument(); + expect( + screen.getByText("Heuristic, REASONING override (2 or more reasoning markers, score above the lowest tier)"), + ).toBeInTheDocument(); }); it("shows the operator's tier name on the badge instead of the canonical one", () => { @@ -212,7 +216,9 @@ describe("RoutingDecisionCard", () => { render( , ); - expect(screen.getByText("Heuristic, Deep override (2 or more reasoning markers)")).toBeInTheDocument(); + expect( + screen.getByText("Heuristic, Deep override (2 or more reasoning markers, score above the lowest tier)"), + ).toBeInTheDocument(); }); it("falls back to the raw cause for a value this build does not know", () => { diff --git a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/RoutingDecisionCard.tsx b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/RoutingDecisionCard.tsx index 0876a539653..cb680668849 100644 --- a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/RoutingDecisionCard.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/RoutingDecisionCard.tsx @@ -72,7 +72,7 @@ function describeCause(decision: RoutingDecision): string { case "heuristic_scorer": return "Heuristic scorer"; case "reasoning_override": - return `Heuristic, ${tierLabel ?? "REASONING"} override (2 or more reasoning markers)`; + return `Heuristic, ${tierLabel ?? "REASONING"} override (2 or more reasoning markers, score above the lowest tier)`; case "llm_classifier": return classifierModel ? `LLM classifier (${classifierModel})` : "LLM classifier"; case "literal_keyword_match": diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 5794c60e97b..4c63586d4d6 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -816,7 +816,8 @@ export interface paths { }; /** * List Shadow Eval Jobs - * @description List shadow eval jobs, newest first. Counts and results ride the detail endpoint only. + * @description List shadow eval jobs, newest first, each key with its attempt count so status is + * accurate. Judged counts, spend, and results ride the detail endpoint only. */ get: operations["list_shadow_eval_jobs_auto_router_shadow_eval_get"]; put?: never; @@ -838,20 +839,21 @@ export interface paths { put?: never; /** * Start Shadow Eval - * @description Start a shadow eval: duplicate a sampled slice of a key's live traffic against a second - * arm, judge the two responses blind, and stratify win rates by tier and by the model that - * served the real arm. + * @description Start a shadow eval: duplicate a sampled slice of one or more keys' live traffic against + * a second arm, judge the two responses blind, and stratify win rates by tier, by the model + * that served the real arm, and by key. * - * A forward job answers whether the key should adopt router_name: it samples the requests + * A forward job answers whether the keys should adopt router_name: it samples the requests * the router did not serve and duplicates them through it. A reverse job answers whether a * key already on the router still gains from it: it samples the requests the router did * serve and duplicates them against baseline_model. A key can hold one active job per * direction, so both questions can run at once. * - * Shadow responses are never served to users. The job samples until it has judged - * max_turns turns, reaches the end of its window, or is stopped; sampling changes - * propagate to pods within about 10 seconds. Shadow and judge calls bill to the - * shadowed key but are excluded from request counts and auto-router adoption metrics. + * Shadow responses are never served to users. Each key samples until it has judged + * max_turns turns of its own traffic, the job's window ends, or the job is stopped, so one + * key running out of budget does not end sampling for the others; sampling changes + * propagate to pods within about 10 seconds. Shadow and judge calls bill to the shadowed + * key but are excluded from request counts and auto-router adoption metrics. */ post: operations["start_shadow_eval_auto_router_shadow_eval_start_post"]; delete?: never; @@ -891,7 +893,12 @@ export interface paths { put?: never; /** * Stop Shadow Eval Job - * @description Stop an active shadow eval job. Attempts are kept; sampling halts within ~10s. + * @description Stop an active shadow eval job, every key it scopes at once. Attempts are kept; + * sampling halts within ~10s. Keys that already stopped on their own budget keep the + * stopped_at they earned. The statement is the whole state machine: it claims the job + * only while a leg still samples inside the window with no stop recorded, so a racing + * operator, a same-instant budget spend, and a repeat stop all read the same 400 with + * the status the job actually holds. */ post: operations["stop_shadow_eval_job_auto_router_shadow_eval__job_id__stop_post"]; delete?: never; @@ -33201,18 +33208,49 @@ export interface components { timeout?: number | null; }; /** - * ShadowEvalJobResponse - * @description A shadow-eval job. Validates directly from the prisma record (job_id reads the - * row's id); status is derived from stopped_at and ends_at, never stored, so no writer - * anywhere can produce an inconsistent one. Aggregate fields are populated by the - * detail endpoint only and stay None on list responses. + * ShadowEvalJobKeyResponse + * @description One key a job shadows, with its own budget and stop state. */ - ShadowEvalJobResponse: { + ShadowEvalJobKeyResponse: { /** * Api Key Id - * @description The hashed virtual key whose traffic this job evaluates, and only that key's + * @description The hashed virtual key whose traffic this entry scopes */ api_key_id: string; + /** + * Attempt Count + * @description This key's sampled attempts so far, judged and errored alike, the same count the sampler budgets against max_turns; populated on list and detail responses. Frozen at stopped_at once the key is stamped, so in-flight attempts landing after a stop never reclassify it + */ + attempt_count?: number | null; + /** + * Key Alias + * @description Alias of the shadowed key, resolved from the key row at read time; None when unset or deleted + */ + key_alias?: string | null; + /** + * Key Name + * @description Masked display name (sk-...) of the shadowed key, resolved at read time like key_alias + */ + key_name?: string | null; + /** + * Max Turns + * @description This key's own sample budget, independent of its siblings' + */ + max_turns: number; + /** + * Stopped At + * @description When this key's slot was stamped free, whether its own budget ran out, the window closed, or an operator stopped the job; status is derived, so a spent budget reads completed even while this is still unset + */ + stopped_at?: string | null; + }; + /** + * ShadowEvalJobResponse + * @description A shadow-eval job over one or more keys, each with its own budget and stop state; + * status is derived from stopped_by, the keys' stop and budget state, and ends_at, + * never stored, so no writer anywhere can produce an inconsistent one. Aggregate + * fields are populated by the detail endpoint only and stay None on list responses. + */ + ShadowEvalJobResponse: { /** Baseline Model */ baseline_model?: string | null; /** @@ -33251,22 +33289,15 @@ export interface components { */ judged_count?: number | null; /** - * Key Alias - * @description Alias of the shadowed key, resolved from the key row at read time; None when unset or deleted + * Keys + * @description The keys whose traffic this job evaluates, and only those keys', each with its own budget */ - key_alias?: string | null; - /** - * Key Name - * @description Masked display name (sk-...) of the shadowed key, resolved at read time like key_alias - */ - key_name?: string | null; + keys: components["schemas"]["ShadowEvalJobKeyResponse"][]; /** * Last Error * @description Most recent attempt error; detail endpoint only */ last_error?: string | null; - /** Max Turns */ - max_turns: number; /** @description Stratified verdicts; detail endpoint only */ results?: components["schemas"]["ShadowEvalResult"] | null; /** Router Name */ @@ -33275,13 +33306,19 @@ export interface components { shadow_percentage: number; /** * Status - * @description A job whose window has passed reads completed even if a later sweep stamped - * stopped_at; stopped means sampling ended before the window did. + * @description Three recorded facts, no history-guessing: a stop is stopped_by (the migration + * backfills it for every job that displayed stopped when the column arrived, so the + * pre-column population is closed), completion is the window passing or every key + * spending its budget, and anything else is running. The all-keys-stamped fallback + * covers only stops written by pre-column pods during a rolling deploy. * @enum {string} */ readonly status: "running" | "completed" | "stopped"; - /** Stopped At */ - stopped_at?: string | null; + /** + * Stopped By + * @description The operator who stopped the job early, recorded by the stop endpoint; 'unknown' backfilled by migration for jobs that displayed stopped when the column arrived; None when the job ended on its own. Its presence is what makes a job read stopped rather than completed + */ + stopped_by?: string | null; }; /** * ShadowEvalResult @@ -33290,9 +33327,14 @@ export interface components { ShadowEvalResult: { /** * By Current Model - * @description Sliced by the model that served the real arm: the key's incumbent models in forward mode, and in reverse the models the router itself picked + * @description Sliced by the model that served the real arm: the keys' incumbent models in forward mode, and in reverse the models the router itself picked */ by_current_model: components["schemas"]["ShadowEvalSlice"][]; + /** + * By Key + * @description One slice per scoped key that has judged verdicts, grouped on the raw key hash. Keys the job scopes but has not judged a turn for yet are absent rather than reported as zero + */ + by_key: components["schemas"]["ShadowEvalSlice"][]; /** By Tier */ by_tier: components["schemas"]["ShadowEvalSlice"][]; /** Overall Shadow Win Rate Pct */ @@ -33500,14 +33542,14 @@ export interface components { }; /** * StartShadowEvalRequest - * @description Start duplicating a key's traffic for blind comparison against an auto-router. + * @description Start duplicating one or more keys' traffic for blind comparison against an auto-router. */ StartShadowEvalRequest: { /** - * Api Key Id - * @description The hashed virtual key whose traffic will be shadowed. Shadow evaluation runs ONLY on this key's traffic; requests made with any other key are not sampled. + * Api Key Ids + * @description The hashed virtual keys whose traffic will be shadowed. Shadow evaluation runs ONLY on these keys' traffic; requests made with any other key are not sampled. Each key carries its own max_turns budget, so one key exhausting its budget leaves the others sampling. At most 100 keys per job, which also bounds every read the job's endpoints make. */ - api_key_id: string; + api_key_ids: string[]; /** * Baseline Model * @description Required when direction is reverse and rejected otherwise: the fixed model the router's own responses are judged against. Must be a plain model rather than another auto-router @@ -33534,7 +33576,7 @@ export interface components { judge_model: string; /** * Max Turns - * @description Sample budget: the job judges at most this many turns, then completes. This is also the spend bound; expected judge cost is roughly max_turns times one judge call + * @description Per-key sample budget: the job judges at most this many turns of EACH scoped key's traffic, so a job over N keys judges at most N times max_turns turns. This is also the spend bound; expected judge cost is roughly that turn ceiling times one judge call * @default 200 */ max_turns: number; @@ -37931,7 +37973,7 @@ export interface operations { list_shadow_eval_jobs_auto_router_shadow_eval_get: { parameters: { query?: { - /** @description Filter to jobs shadowing this key */ + /** @description Filter to jobs that shadow this key, alone or alongside others */ api_key_id?: string | null; /** @description Newest jobs to return */ limit?: number; diff --git a/ui/litellm-dashboard/src/utils/cookieUtils.test.ts b/ui/litellm-dashboard/src/utils/cookieUtils.test.ts index cc4fe64e4c8..a38b96e74c2 100644 --- a/ui/litellm-dashboard/src/utils/cookieUtils.test.ts +++ b/ui/litellm-dashboard/src/utils/cookieUtils.test.ts @@ -1,3 +1,5 @@ +// @vitest-environment jsdom + import { describe, it, expect, beforeEach, vi } from "vitest"; import { clearTokenCookies, getCookie, storeLoginToken } from "./cookieUtils"; import { getToken, setToken } from "./mcpTokenStore"; diff --git a/ui/litellm-dashboard/src/utils/dataUtils.test.ts b/ui/litellm-dashboard/src/utils/dataUtils.test.ts index d975a4a73f4..f696780bc0a 100644 --- a/ui/litellm-dashboard/src/utils/dataUtils.test.ts +++ b/ui/litellm-dashboard/src/utils/dataUtils.test.ts @@ -1,3 +1,5 @@ +// @vitest-environment jsdom + import { describe, it, expect, beforeEach, vi, afterEach } from "vitest"; import { copyToClipboard, formatNumberWithCommas, getSpendString, updateExistingKeys } from "./dataUtils"; diff --git a/ui/litellm-dashboard/src/utils/localStorageUtils.test.ts b/ui/litellm-dashboard/src/utils/localStorageUtils.test.ts index 934c358cb74..2ef5e1d5b43 100644 --- a/ui/litellm-dashboard/src/utils/localStorageUtils.test.ts +++ b/ui/litellm-dashboard/src/utils/localStorageUtils.test.ts @@ -1,3 +1,5 @@ +// @vitest-environment jsdom + import { describe, it, expect, vi, beforeEach } from "vitest"; import { emitLocalStorageChange, diff --git a/ui/litellm-dashboard/src/utils/mcpTokenStore.test.ts b/ui/litellm-dashboard/src/utils/mcpTokenStore.test.ts index 0d096677f0d..e384f41042a 100644 --- a/ui/litellm-dashboard/src/utils/mcpTokenStore.test.ts +++ b/ui/litellm-dashboard/src/utils/mcpTokenStore.test.ts @@ -1,3 +1,5 @@ +// @vitest-environment jsdom + import { afterEach, beforeEach, describe, expect, it } from "vitest"; import { clearAllMcpTokens, getToken, isTokenValid, removeToken, setToken } from "./mcpTokenStore"; diff --git a/ui/litellm-dashboard/src/utils/returnUrlUtils.test.ts b/ui/litellm-dashboard/src/utils/returnUrlUtils.test.ts index 0f3a8b4bf69..3d1e83ad236 100644 --- a/ui/litellm-dashboard/src/utils/returnUrlUtils.test.ts +++ b/ui/litellm-dashboard/src/utils/returnUrlUtils.test.ts @@ -1,3 +1,5 @@ +// @vitest-environment jsdom + import { buildLoginUrlWithReturn, clearStoredReturnUrl, diff --git a/ui/litellm-dashboard/tests/setup.unit.ts b/ui/litellm-dashboard/tests/setup.unit.ts new file mode 100644 index 00000000000..4eb0c473989 --- /dev/null +++ b/ui/litellm-dashboard/tests/setup.unit.ts @@ -0,0 +1,12 @@ +import { vi } from "vitest"; + +vi.mock("@/lib/toast", () => ({ + toast: { + success: vi.fn(), + info: vi.fn(), + warning: vi.fn(), + error: vi.fn(), + fromError: vi.fn(), + dismiss: vi.fn(), + }, +})); diff --git a/ui/litellm-dashboard/tests/setupTests.ts b/ui/litellm-dashboard/tests/setupTests.ts index 3eceef211a5..d007206e681 100644 --- a/ui/litellm-dashboard/tests/setupTests.ts +++ b/ui/litellm-dashboard/tests/setupTests.ts @@ -113,31 +113,7 @@ vi.mock("@/app/(dashboard)/hooks/useAuthorized", () => ({ }), })); -const pendingRefWarnings: string[] = []; -const consumePendingRefWarnings = (): string[] => pendingRefWarnings.splice(0, pendingRefWarnings.length); -(globalThis as { __consumePendingRefWarnings?: () => string[] }).__consumePendingRefWarnings = - consumePendingRefWarnings; - -const originalConsoleError = console.error.bind(console); -vi.spyOn(console, "error").mockImplementation((...args: unknown[]) => { - originalConsoleError(...args); - if (typeof args[0] === "string" && args[0].includes("Function components cannot be given refs")) { - pendingRefWarnings.push(args.map(String).join(" ")); - } -}); - -afterEach(() => { - cleanup(); - const refWarnings = consumePendingRefWarnings(); - if (refWarnings.length > 0) { - throw new Error( - "A ref was passed to a plain function component and silently dropped under React 18, which breaks " + - "ref-based composition (Base UI render triggers, tooltips, focus). Wrap the component in React.forwardRef. " + - "This tripwire lives in tests/setupTests.ts and can be removed after the React 19 upgrade.\n\n" + - refWarnings.join("\n\n"), - ); - } -}); +afterEach(cleanup); // Make toLocaleString deterministic in tests; individual tests can override // This returns ISO-like strings to keep assertions stable. diff --git a/ui/litellm-dashboard/tests/test-utils.tsx b/ui/litellm-dashboard/tests/test-utils.tsx index 2573fbdfeac..66966201a9c 100644 --- a/ui/litellm-dashboard/tests/test-utils.tsx +++ b/ui/litellm-dashboard/tests/test-utils.tsx @@ -1,7 +1,9 @@ import React, { PropsWithChildren } from "react"; -import { render, RenderOptions } from "@testing-library/react"; +import { render, RenderOptions, screen, waitFor } from "@testing-library/react"; +import type userEvent from "@testing-library/user-event"; import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; import { NuqsTestingAdapter, OnUrlUpdateFunction } from "nuqs/adapters/testing"; +import { expect } from "vitest"; // Create a client for testing export const testQueryClient = new QueryClient({ @@ -35,4 +37,29 @@ export const renderWithProviders = (ui: React.ReactElement, options?: RenderOpti return render(ui, { wrapper: Providers, ...renderOptions }); }; +const pointerBlocked = (element: HTMLElement): boolean => { + for (let node: HTMLElement | null = element; node !== null; node = node.parentElement) { + if (node.style.pointerEvents === "none") return true; + } + return false; +}; + +/** + * Opens a Base UI Select and picks an option by its accessible name. + * + * The option is in the DOM one render before the popup finishes entering, and until then its + * positioner still carries `pointer-events: none`, which user-event refuses to click. Waiting on + * the option text alone is a race that React 19's flush timing loses. + */ +export const chooseSelectOption = async ( + user: ReturnType, + trigger: HTMLElement, + optionName: string | RegExp, +) => { + await user.click(trigger); + const option = await screen.findByRole("option", { name: optionName }); + await waitFor(() => expect(pointerBlocked(option)).toBe(false)); + await user.click(option); +}; + export * from "@testing-library/react"; diff --git a/ui/litellm-dashboard/vitest.config.ts b/ui/litellm-dashboard/vitest.config.ts index 7d8825909d8..ca2a25cba9f 100644 --- a/ui/litellm-dashboard/vitest.config.ts +++ b/ui/litellm-dashboard/vitest.config.ts @@ -12,17 +12,81 @@ const staticImageData: Plugin = { }, }; -const config: ViteUserConfig = { +const sharedViteConfig = { plugins: [staticImageData], + resolve: { alias: { "@": resolve(__dirname, "src") } }, + define: { "import.meta.vitest": "undefined" }, + esbuild: { jsx: "automatic", jsxImportSource: "react" } as const, +}; + +const TEST_TS_FILES_THAT_RENDER_REACT: readonly string[] = [ + "src/**/hooks/**/*.test.ts", + "src/**/cost-tracking/_components/**/use_*.test.ts", + "src/**/models-and-endpoints/detailNavigation.test.ts", + "src/**/models-and-endpoints/vertexCredentialsUpload.test.ts", + "src/components/chat/useChatHistory.test.ts", + "src/lib/forms/pickDirty.test.ts", +]; + +const jsdomTier = { + environment: "./tests/jsdomFetchEnv.ts", + setupFiles: ["tests/setupTests.ts"], + globals: true, + css: true, + testTimeout: 60_000, + hookTimeout: 30_000, +}; + +const config: ViteUserConfig = { + ...sharedViteConfig, test: { - environment: "./tests/jsdomFetchEnv.ts", - setupFiles: ["tests/setupTests.ts"], - globals: true, - css: true, // lets you import CSS/modules without extra mocks - testTimeout: 60000, - hookTimeout: 30000, + projects: [ + { + ...sharedViteConfig, + test: { + name: "unit", + environment: "node", + setupFiles: ["tests/setup.unit.ts"], + globals: true, + testTimeout: 60_000, + hookTimeout: 30_000, + include: ["src/**/*.test.ts", "tests/**/*.test.ts"], + exclude: ["node_modules/**", ...TEST_TS_FILES_THAT_RENDER_REACT], + }, + }, + { + ...sharedViteConfig, + test: { + ...jsdomTier, + name: "component", + include: ["src/**/*.test.tsx", "tests/**/*.test.tsx", ...TEST_TS_FILES_THAT_RENDER_REACT], + exclude: ["node_modules/**", "**/*.integration.test.tsx"], + }, + }, + { + ...sharedViteConfig, + test: { + ...jsdomTier, + name: "integration", + include: ["src/**/*.integration.test.tsx", "tests/**/*.integration.test.tsx"], + exclude: ["node_modules/**"], + }, + }, + { + ...sharedViteConfig, + test: { + name: "types", + include: [], + typecheck: { + enabled: true, + include: ["src/**/*.test-d.ts", "src/**/*.test-d.tsx"], + ignoreSourceErrors: true, + }, + }, + }, + ], silent: process.env.CI ? "passed-only" : false, - teardownTimeout: 60000, + teardownTimeout: 60_000, coverage: { provider: "v8", reporter: ["text", "lcov"], @@ -32,37 +96,16 @@ const config: ViteUserConfig = { "**/*.test.*", "**/*.test-d.*", "**/*.spec.*", - "tests/**", - "node_modules/**", ".next/**", "out/**", - "**/*.config.*", "postcss.config.*", "tailwind.config.*", "next.config.*", ], }, - exclude: ["node_modules/**"], - include: ["src/**/*.test.ts", "src/**/*.test.tsx", "tests/**/*.test.ts", "tests/**/*.test.tsx"], - typecheck: { - include: ["src/**/*.test-d.ts", "src/**/*.test-d.tsx"], - ignoreSourceErrors: true, - }, - }, - resolve: { - alias: { - "@": resolve(__dirname, "src"), - }, - }, - define: { - "import.meta.vitest": "undefined", - }, - esbuild: { - jsx: "automatic", - jsxImportSource: "react", }, };